From 1dd70cc2c2b9b0a9f65a704ded9e322afabf736f Mon Sep 17 00:00:00 2001 From: Adam Soltys Date: Fri, 16 Oct 2026 12:57:19 -0700 Subject: Make this the coinosv3-server repo: the services, with coinosv3 pinned Carved out of coinosv3 (names, feed, nwcpush, blossom, vmedia, bridge and their tests, with history). The shared wallet code they import (src/ark, noffer, bolt12, popular-feed) now comes from a coinosv3 submodule pinned at 082fc2e, the commit the services have actually run on, so moving here changes no behaviour; bumping it is its own commit. The two text helpers translation uses arrived on the server later than that commit and are vendored under names/vendor at the versions deployed, until the pin moves past them. Root package.json pins the npm versions installed on the server. compose.yml runs names, nwcpush and blossom from this checkout (read-only) with configs and state under $STATE, outside the repo, and pins the Bun image. Co-Authored-By: Claude Opus 5.5 --- names/server.js | 10 ++--- names/translate.js | 2 +- names/vendor/lang-guess.js | 99 +++++++++++++++++++++++++++++++++++++++++ names/vendor/translate-split.js | 61 +++++++++++++++++++++++++ 4 files changed, 166 insertions(+), 6 deletions(-) create mode 100644 names/vendor/lang-guess.js create mode 100644 names/vendor/translate-split.js (limited to 'names') diff --git a/names/server.js b/names/server.js index dacbda6..9d94b51 100644 --- a/names/server.js +++ b/names/server.js @@ -26,13 +26,13 @@ import { createHash, randomBytes } from 'node:crypto'; import { HDKey } from '@scure/bip32'; import { mnemonicToSeedSync } from '@scure/bip39'; import { lnBackend } from '../bridge/ln.js'; -import { ArkManager } from '../src/ark/manager.js'; +import { ArkManager } from '../coinosv3/src/ark/manager.js'; import { fetchPreview } from './preview.js'; import { translate, translateDeepl, languages } from './translate.js'; -import { decodeAddress } from '../src/ark/proto.js'; -import { decodeNoffer } from '../src/noffer.js'; -import { decodeBolt11 } from '../src/ark/lightning.js'; -import { decodeOffer } from '../src/bolt12.js'; +import { decodeAddress } from '../coinosv3/src/ark/proto.js'; +import { decodeNoffer } from '../coinosv3/src/noffer.js'; +import { decodeBolt11 } from '../coinosv3/src/ark/lightning.js'; +import { decodeOffer } from '../coinosv3/src/bolt12.js'; import { registrationIdentity } from './ownership.js'; const CFG = JSON.parse(readFileSync(process.env.NAMES_CONFIG diff --git a/names/translate.js b/names/translate.js index 36ab8cf..bcc47d9 100644 --- a/names/translate.js +++ b/names/translate.js @@ -3,7 +3,7 @@ // passes through here; the cache keys on the text itself, so a popular post // is translated once whoever asks. import { createHash } from 'node:crypto'; -import { translatable, translatePieces } from '../src/translate-split.js'; +import { translatable, translatePieces } from './vendor/translate-split.js'; const LT = process.env.LT_URL || 'http://libretranslate:5000'; const MAX_CHARS = 5000; diff --git a/names/vendor/lang-guess.js b/names/vendor/lang-guess.js new file mode 100644 index 0000000..6d62e30 --- /dev/null +++ b/names/vendor/lang-guess.js @@ -0,0 +1,99 @@ +// A rough guess at the language a post is written in, cheap enough to run +// on every post of a public feed. Script first (a post in Han or Arabic +// letters is not English), then, among the Latin-script languages the app +// speaks, a handful of function words each. Short or ambiguous text is +// "unknown", and unknown is never held against a post. +const SCRIPTS = [ + ['latin', /[A-Za-zÀ-ɏḀ-ỿ]/g], + ['cyrillic', /[Ѐ-ӿ]/g], + ['arabic', /[؀-ۿݐ-ݿ]/g], + ['devanagari', /[ऀ-ॿ]/g], + ['bengali', /[ঀ-৿]/g], + ['tamil', /[஀-௿]/g], + ['telugu', /[ఀ-౿]/g], + ['kana', /[぀-ヿ]/g], + ['han', /[一-鿿㐀-䶿]/g], + ['hangul', /[가-힯ᄀ-ᇿ]/g], +]; +export const LANG_SCRIPT = { + en: 'latin', es: 'latin', fr: 'latin', de: 'latin', pt: 'latin', id: 'latin', sw: 'latin', tr: 'latin', vi: 'latin', + ru: 'cyrillic', ar: 'arabic', ur: 'arabic', hi: 'devanagari', mr: 'devanagari', bn: 'bengali', ta: 'tamil', te: 'telugu', + ja: 'kana', ko: 'hangul', zh: 'han', + // not app languages, but languages people read posts in (a translation target) + it: 'latin', nl: 'latin', pl: 'latin', +}; +const STOP = { + en: 'the and is to of in that it for you with this on are be have not was but they'.split(' '), + es: 'el la de que y en los es un una por con para las del se no como más pero'.split(' '), + fr: 'le la les de des et est un une pour que qui dans pas sur avec ce il nous vous'.split(' '), + de: 'der die und das ist nicht ein eine ich zu mit den auf für sich dem des auch wir'.split(' '), + pt: 'o a de que e do da em um uma para com não os as se mais por isso você'.split(' '), + id: 'yang dan di ini itu dengan untuk tidak dari ada saya akan ke juga bisa kita sudah atau kalau aku'.split(' '), + tr: 've bir bu için ile de da çok ne gibi daha ama var ben sen mi değil kadar her olan'.split(' '), + vi: 'và của là có không được một trong cho những này với các để người đã tôi khi cũng như'.split(' '), + sw: 'na ya wa kwa ni za la katika hii kuwa kama hata lakini sana yake wote sasa tu huo hapa'.split(' '), + it: 'il di che è e la un una per non sono con del della gli anche ma questo mi più nel'.split(' '), + nl: 'de het een en van is dat niet op te zijn met voor ik je die maar ook wat'.split(' '), + pl: 'i w nie na się z to że jest do co jak ale tak o po czy już'.split(' '), +}; +const STOPSETS = Object.fromEntries(Object.entries(STOP).map(([k, v]) => [k, new Set(v)])); + +// The text a reader actually reads: no links, no nostr references, no tags. +export const proseOf = (text) => String(text || '') + .replace(/https?:\/\/\S+/g, ' ').replace(/nostr:[a-z0-9]+/gi, ' ').replace(/#[\p{L}\p{N}_]+/gu, ' '); + +export function scriptOf(text, min = 3) { + let best = null, bestN = 0, total = 0; + for (const [name, re] of SCRIPTS) { + const n = (text.match(re) || []).length; + total += n; + if (n > bestN) { best = name; bestN = n; } + } + return total < min ? null : best; +} + +// A language code from the app's list, or null when it cannot tell. +export function guessLang(text) { + const prose = proseOf(text); + const script = scriptOf(prose); + if (!script) return null; + if (script === 'kana') return 'ja'; + if (script === 'han') return 'zh'; + if (script !== 'latin') { + const langs = Object.entries(LANG_SCRIPT).filter(([, s]) => s === script).map(([l]) => l); + return langs.length === 1 ? langs[0] : null; // ar/ur, hi/mr share a script: unknown between them + } + const words = prose.toLowerCase().match(/[\p{L}']+/gu) || []; + if (words.length < 4) return null; + const score = {}; + for (const w of words) for (const [l, set] of Object.entries(STOPSETS)) if (set.has(w)) score[l] = (score[l] || 0) + 1; + const ranked = Object.entries(score).sort((a, b) => b[1] - a[1]); + if (!ranked.length || ranked[0][1] < 2) return null; + if (ranked[1] && ranked[0][1] < ranked[1][1] * 1.5) return null; + return ranked[0][0]; +} + +// Does a post read in this language, as far as we can tell? Unknown passes; +// a wrong script or a clear other language fails. +export function inLanguage(text, lang) { + const want = LANG_SCRIPT[lang]; + if (!want) return true; + const prose = proseOf(text); + const script = scriptOf(prose); + if (!script) return true; // nothing readable to judge (a picture post, an emoji) + if (script !== want) return false; + if (want === 'han' && lang === 'zh' && /[぀-ヿ]/.test(prose)) return false; // kana means Japanese + const g = guessLang(prose); + return g === null || g === lang; +} + +// Machine output dressed as a post: JSON, a hash dump, a wall of symbols. +export function isMachinePost(text) { + const s = String(text || '').trim(); + if (!s) return false; + if (/^[\[{]/.test(s)) { try { JSON.parse(s); return true; } catch {} } + const prose = proseOf(s).replace(/\s+/g, ''); + if (prose.length < 24) return false; + const letters = (prose.match(/\p{L}/gu) || []).length; + return letters / prose.length < 0.4; +} diff --git a/names/vendor/translate-split.js b/names/vendor/translate-split.js new file mode 100644 index 0000000..95f6468 --- /dev/null +++ b/names/vendor/translate-split.js @@ -0,0 +1,61 @@ +// What must come back from a translation exactly as it went in: links, +// nostr: mentions, custom emoji shortcodes, hashtags, @handles, invoices and +// other long codes. Translation models mangle them ("https: / / example.com +// / fot.jpg"), so only the prose between them is translated and the text is +// put back together around the untouched tokens. Shared by the registrar +// (names/translate.js) and the browser's own translator (messages.js). +import { scriptOf, LANG_SCRIPT } from './lang-guess.js'; + +const KEEP = /(https?:\/\/\S+|nostr:[a-z0-9]+|:[a-z0-9_+-]{1,40}:|#[\p{L}\p{N}_]+|@[\w.-]+(?:@[\w.-]+)?|\b(?:ln|lightning:|cashu|bitcoin:|nevent|npub|nprofile|note|naddr)[a-z0-9]{20,}|\S{40,})/giu; + +export function splitProse(text) { + const parts = []; + let last = 0; + for (const m of String(text || '').matchAll(KEEP)) { + if (m.index > last) parts.push({ prose: text.slice(last, m.index) }); + parts.push({ keep: m[0] }); + last = m.index + m[0].length; + } + if (last < text.length) parts.push({ prose: text.slice(last) }); + return parts; +} +// a piece worth translating: has words, not just spaces and punctuation +export const hasWords = (s) => /\p{L}{2,}/u.test(s || ''); + +// A hashtag stays a hashtag (it's a link to its topic), but one written in +// another script than the reader's means nothing to them: such a tag gets +// its translation in brackets after it, "#総合部絵チャ (general art chat)". +const tagWords = (tag) => tag.slice(1).replace(/_/g, ' '); +export function glossable(tag, target) { + if (!tag.startsWith('#') || !target) return false; + const s = scriptOf(tagWords(tag), 1); + return !!s && !!LANG_SCRIPT[target] && s !== LANG_SCRIPT[target]; +} +// What a translator gets to see of a post: its prose, then the words of the +// hashtags the reader can't read (also what language detection runs on). +export function translatable(text, target) { + const parts = splitProse(text); + return { + parts, + prose: parts.filter((p) => p.prose && hasWords(p.prose)), + tags: parts.filter((p) => p.keep && glossable(p.keep, target)), + }; +} + +// Translate the prose pieces and the glossed tags (one call for all of them) +// and reassemble. Each piece keeps its own leading/trailing whitespace — +// line breaks matter. +export async function translatePieces(text, translateMany, target = null) { + const { parts, prose, tags } = translatable(text, target); + if (!prose.length && !tags.length) return text; + const outs = await translateMany([...prose.map((p) => p.prose.trim()), ...tags.map((p) => tagWords(p.keep))]); + prose.forEach((p, i) => { + const lead = p.prose.match(/^\s*/)[0], trail = p.prose.match(/\s*$/)[0]; + p.out = lead + String(outs[i] ?? p.prose.trim()) + trail; + }); + tags.forEach((p, i) => { + const g = String(outs[prose.length + i] ?? '').trim(); + p.out = g && g.toLowerCase() !== tagWords(p.keep).toLowerCase() ? p.keep + ' (' + g + ')' : p.keep; + }); + return parts.map((p) => p.out ?? p.keep ?? p.prose).join(''); +} -- cgit v1.2.3