diff options
Diffstat (limited to 'names')
| -rw-r--r-- | names/translate.js | 2 | ||||
| -rw-r--r-- | names/vendor/lang-guess.js | 99 | ||||
| -rw-r--r-- | names/vendor/translate-split.js | 61 |
3 files changed, 1 insertions, 161 deletions
diff --git a/names/translate.js b/names/translate.js index bcc47d9..dd44535 100644 --- a/names/translate.js +++ b/names/translate.js @@ -3,7 +3,7 @@ // passes through here; the cache keys on the text itself, so a popular post // is translated once whoever asks. import { createHash } from 'node:crypto'; -import { translatable, translatePieces } from './vendor/translate-split.js'; +import { translatable, translatePieces } from '../coinosv3/src/translate-split.js'; const LT = process.env.LT_URL || 'http://libretranslate:5000'; const MAX_CHARS = 5000; diff --git a/names/vendor/lang-guess.js b/names/vendor/lang-guess.js deleted file mode 100644 index 6d62e30..0000000 --- a/names/vendor/lang-guess.js +++ /dev/null @@ -1,99 +0,0 @@ -// A rough guess at the language a post is written in, cheap enough to run -// on every post of a public feed. Script first (a post in Han or Arabic -// letters is not English), then, among the Latin-script languages the app -// speaks, a handful of function words each. Short or ambiguous text is -// "unknown", and unknown is never held against a post. -const SCRIPTS = [ - ['latin', /[A-Za-zÀ-ɏḀ-ỿ]/g], - ['cyrillic', /[Ѐ-ӿ]/g], - ['arabic', /[-ۿݐ-ݿ]/g], - ['devanagari', /[ऀ-ॿ]/g], - ['bengali', /[ঀ-]/g], - ['tamil', /[-]/g], - ['telugu', /[ఀ-౿]/g], - ['kana', /[-ヿ]/g], - ['han', /[一-鿿㐀-䶿]/g], - ['hangul', /[가-ᄀ-ᇿ]/g], -]; -export const LANG_SCRIPT = { - en: 'latin', es: 'latin', fr: 'latin', de: 'latin', pt: 'latin', id: 'latin', sw: 'latin', tr: 'latin', vi: 'latin', - ru: 'cyrillic', ar: 'arabic', ur: 'arabic', hi: 'devanagari', mr: 'devanagari', bn: 'bengali', ta: 'tamil', te: 'telugu', - ja: 'kana', ko: 'hangul', zh: 'han', - // not app languages, but languages people read posts in (a translation target) - it: 'latin', nl: 'latin', pl: 'latin', -}; -const STOP = { - en: 'the and is to of in that it for you with this on are be have not was but they'.split(' '), - es: 'el la de que y en los es un una por con para las del se no como más pero'.split(' '), - fr: 'le la les de des et est un une pour que qui dans pas sur avec ce il nous vous'.split(' '), - de: 'der die und das ist nicht ein eine ich zu mit den auf für sich dem des auch wir'.split(' '), - pt: 'o a de que e do da em um uma para com não os as se mais por isso você'.split(' '), - id: 'yang dan di ini itu dengan untuk tidak dari ada saya akan ke juga bisa kita sudah atau kalau aku'.split(' '), - tr: 've bir bu için ile de da çok ne gibi daha ama var ben sen mi değil kadar her olan'.split(' '), - vi: 'và của là có không được một trong cho những này với các để người đã tôi khi cũng như'.split(' '), - sw: 'na ya wa kwa ni za la katika hii kuwa kama hata lakini sana yake wote sasa tu huo hapa'.split(' '), - it: 'il di che è e la un una per non sono con del della gli anche ma questo mi più nel'.split(' '), - nl: 'de het een en van is dat niet op te zijn met voor ik je die maar ook wat'.split(' '), - pl: 'i w nie na się z to że jest do co jak ale tak o po czy już'.split(' '), -}; -const STOPSETS = Object.fromEntries(Object.entries(STOP).map(([k, v]) => [k, new Set(v)])); - -// The text a reader actually reads: no links, no nostr references, no tags. -export const proseOf = (text) => String(text || '') - .replace(/https?:\/\/\S+/g, ' ').replace(/nostr:[a-z0-9]+/gi, ' ').replace(/#[\p{L}\p{N}_]+/gu, ' '); - -export function scriptOf(text, min = 3) { - let best = null, bestN = 0, total = 0; - for (const [name, re] of SCRIPTS) { - const n = (text.match(re) || []).length; - total += n; - if (n > bestN) { best = name; bestN = n; } - } - return total < min ? null : best; -} - -// A language code from the app's list, or null when it cannot tell. -export function guessLang(text) { - const prose = proseOf(text); - const script = scriptOf(prose); - if (!script) return null; - if (script === 'kana') return 'ja'; - if (script === 'han') return 'zh'; - if (script !== 'latin') { - const langs = Object.entries(LANG_SCRIPT).filter(([, s]) => s === script).map(([l]) => l); - return langs.length === 1 ? langs[0] : null; // ar/ur, hi/mr share a script: unknown between them - } - const words = prose.toLowerCase().match(/[\p{L}']+/gu) || []; - if (words.length < 4) return null; - const score = {}; - for (const w of words) for (const [l, set] of Object.entries(STOPSETS)) if (set.has(w)) score[l] = (score[l] || 0) + 1; - const ranked = Object.entries(score).sort((a, b) => b[1] - a[1]); - if (!ranked.length || ranked[0][1] < 2) return null; - if (ranked[1] && ranked[0][1] < ranked[1][1] * 1.5) return null; - return ranked[0][0]; -} - -// Does a post read in this language, as far as we can tell? Unknown passes; -// a wrong script or a clear other language fails. -export function inLanguage(text, lang) { - const want = LANG_SCRIPT[lang]; - if (!want) return true; - const prose = proseOf(text); - const script = scriptOf(prose); - if (!script) return true; // nothing readable to judge (a picture post, an emoji) - if (script !== want) return false; - if (want === 'han' && lang === 'zh' && /[-ヿ]/.test(prose)) return false; // kana means Japanese - const g = guessLang(prose); - return g === null || g === lang; -} - -// Machine output dressed as a post: JSON, a hash dump, a wall of symbols. -export function isMachinePost(text) { - const s = String(text || '').trim(); - if (!s) return false; - if (/^[\[{]/.test(s)) { try { JSON.parse(s); return true; } catch {} } - const prose = proseOf(s).replace(/\s+/g, ''); - if (prose.length < 24) return false; - const letters = (prose.match(/\p{L}/gu) || []).length; - return letters / prose.length < 0.4; -} diff --git a/names/vendor/translate-split.js b/names/vendor/translate-split.js deleted file mode 100644 index 95f6468..0000000 --- a/names/vendor/translate-split.js +++ /dev/null @@ -1,61 +0,0 @@ -// What must come back from a translation exactly as it went in: links, -// nostr: mentions, custom emoji shortcodes, hashtags, @handles, invoices and -// other long codes. Translation models mangle them ("https: / / example.com -// / fot.jpg"), so only the prose between them is translated and the text is -// put back together around the untouched tokens. Shared by the registrar -// (names/translate.js) and the browser's own translator (messages.js). -import { scriptOf, LANG_SCRIPT } from './lang-guess.js'; - -const KEEP = /(https?:\/\/\S+|nostr:[a-z0-9]+|:[a-z0-9_+-]{1,40}:|#[\p{L}\p{N}_]+|@[\w.-]+(?:@[\w.-]+)?|\b(?:ln|lightning:|cashu|bitcoin:|nevent|npub|nprofile|note|naddr)[a-z0-9]{20,}|\S{40,})/giu; - -export function splitProse(text) { - const parts = []; - let last = 0; - for (const m of String(text || '').matchAll(KEEP)) { - if (m.index > last) parts.push({ prose: text.slice(last, m.index) }); - parts.push({ keep: m[0] }); - last = m.index + m[0].length; - } - if (last < text.length) parts.push({ prose: text.slice(last) }); - return parts; -} -// a piece worth translating: has words, not just spaces and punctuation -export const hasWords = (s) => /\p{L}{2,}/u.test(s || ''); - -// A hashtag stays a hashtag (it's a link to its topic), but one written in -// another script than the reader's means nothing to them: such a tag gets -// its translation in brackets after it, "#総合部絵チャ (general art chat)". -const tagWords = (tag) => tag.slice(1).replace(/_/g, ' '); -export function glossable(tag, target) { - if (!tag.startsWith('#') || !target) return false; - const s = scriptOf(tagWords(tag), 1); - return !!s && !!LANG_SCRIPT[target] && s !== LANG_SCRIPT[target]; -} -// What a translator gets to see of a post: its prose, then the words of the -// hashtags the reader can't read (also what language detection runs on). -export function translatable(text, target) { - const parts = splitProse(text); - return { - parts, - prose: parts.filter((p) => p.prose && hasWords(p.prose)), - tags: parts.filter((p) => p.keep && glossable(p.keep, target)), - }; -} - -// Translate the prose pieces and the glossed tags (one call for all of them) -// and reassemble. Each piece keeps its own leading/trailing whitespace — -// line breaks matter. -export async function translatePieces(text, translateMany, target = null) { - const { parts, prose, tags } = translatable(text, target); - if (!prose.length && !tags.length) return text; - const outs = await translateMany([...prose.map((p) => p.prose.trim()), ...tags.map((p) => tagWords(p.keep))]); - prose.forEach((p, i) => { - const lead = p.prose.match(/^\s*/)[0], trail = p.prose.match(/\s*$/)[0]; - p.out = lead + String(outs[i] ?? p.prose.trim()) + trail; - }); - tags.forEach((p, i) => { - const g = String(outs[prose.length + i] ?? '').trim(); - p.out = g && g.toLowerCase() !== tagWords(p.keep).toLowerCase() ? p.keep + ' (' + g + ')' : p.keep; - }); - return parts.map((p) => p.out ?? p.keep ?? p.prose).join(''); -} |