1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
|
// Post translation for coinos v3: a thin, cached front for a LibreTranslate
// container on the same docker network. Notes are public, so nothing private
// passes through here; the cache keys on the text itself, so a popular post
// is translated once whoever asks.
import { createHash } from 'node:crypto';
import { translatable, translatePieces } from '../src/translate-split.js';
const LT = process.env.LT_URL || 'http://libretranslate:5000';
const MAX_CHARS = 5000;
const CACHE_MAX = 20000;
const BUSY_MAX = 4; // LibreTranslate is CPU-bound: a few at once, the rest wait their turn
const cache = new Map(); // sha256(target|text) -> { text, source }
let busy = 0;
const waiting = [];
let langs = null, langsAt = 0;
async function slot() {
if (busy < BUSY_MAX) { busy++; return; }
await new Promise((r) => waiting.push(r));
busy++;
}
function release() { busy--; const next = waiting.shift(); if (next) next(); }
// The languages the container has models for: { code: [targets] }
export async function languages() {
if (langs && Date.now() - langsAt < 3600_000) return langs;
const r = await fetch(LT + '/languages', { signal: AbortSignal.timeout(5000) });
if (!r.ok) throw new Error('languages ' + r.status);
const list = await r.json();
langs = Object.fromEntries(list.map((l) => [l.code, l.targets || []]));
if (langs['zh-Hans']) langs.zh = langs['zh-Hans']; // the app says 'zh'
langsAt = Date.now();
return langs;
}
// The app's locale codes ↔ LibreTranslate's ('zh' is Simplified there).
const norm = (c) => String(c || '').toLowerCase().split(/[-_]/)[0];
const toLt = (c) => (c === 'zh' ? 'zh-Hans' : c);
export async function translate(text, target, source = 'auto') {
text = String(text || '');
target = norm(target);
source = source && source !== 'auto' ? norm(source) : 'auto';
if (!text.trim()) return { text: '', source };
if (text.length > MAX_CHARS) throw Object.assign(new Error('too long'), { status: 413 });
const have = await languages();
if (!have[toLt(target)]) throw Object.assign(new Error('unsupported language'), { status: 422 });
const key = createHash('sha256').update(target + '|' + source + '|' + text).digest('hex');
const hit = cache.get(key);
if (hit) { cache.delete(key); cache.set(key, hit); return hit; }
const { prose, tags } = translatable(text, target);
if (!prose.length && !tags.length) return { text, source: source === 'auto' ? null : source, nothing: true };
await slot();
try {
const call = async (path, body) => {
const r = await fetch(LT + path, {
method: 'POST', headers: { 'content-type': 'application/json' },
body: JSON.stringify(body), signal: AbortSignal.timeout(30_000),
});
const out = await r.json().catch(() => ({}));
if (!r.ok) throw Object.assign(new Error(out.error || path + ' ' + r.status), { status: r.status === 400 ? 422 : 502 });
return out;
};
let from = source, confidence = null;
if (from === 'auto') {
const d = await call('/detect', { q: [...prose.map((p) => p.prose), ...tags.map((p) => p.keep.slice(1).replace(/_/g, ' '))].join('\n') });
from = norm(d[0]?.language) || null;
confidence = d[0]?.confidence ?? null;
if (!from) throw Object.assign(new Error('unknown language'), { status: 422 });
}
if (from === target) return { text, source: from, confidence };
if (!have[toLt(from)]) throw Object.assign(new Error('unsupported language'), { status: 422 });
const translated = await translatePieces(text, async (qs) => {
const r = await call('/translate', { q: qs, source: toLt(from), target: toLt(target), format: 'text' });
return Array.isArray(r.translatedText) ? r.translatedText : [r.translatedText];
}, target);
const out = { text: translated, source: from, confidence };
cache.set(key, out);
if (cache.size > CACHE_MAX) cache.delete(cache.keys().next().value);
return out;
} finally { release(); }
}
|
