diff options
| -rw-r--r-- | .gitignore | 14 | ||||
| -rw-r--r-- | .gitmodules | 3 | ||||
| -rw-r--r-- | LICENSE | 21 | ||||
| -rw-r--r-- | blossom/compose-blossom.yml | 23 | ||||
| -rw-r--r-- | bridge/server.js | 8 | ||||
| -rw-r--r-- | bun.lock | 84 | ||||
| m--------- | coinosv3 | 0 | ||||
| -rw-r--r-- | compose.yml | 62 | ||||
| -rw-r--r-- | feed/cache.js | 4 | ||||
| -rw-r--r-- | names/server.js | 10 | ||||
| -rw-r--r-- | names/translate.js | 2 | ||||
| -rw-r--r-- | names/vendor/lang-guess.js | 99 | ||||
| -rw-r--r-- | names/vendor/translate-split.js | 61 | ||||
| -rw-r--r-- | package.json | 20 | ||||
| -rw-r--r-- | tools/public-feed-cache-test.js | 2 | ||||
| -rw-r--r-- | tools/translate-test.js | 119 |
16 files changed, 377 insertions, 155 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c756898 --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +node_modules/ + +# Service configs hold secrets (DNS provider keys, admin tokens, the forwarder +# mnemonic, VAPID keys, nostr secret keys). Only *.example.json is committed. +*config*.json +!*config.example.json + +# Runtime state +data/ +data-*/ +*.db +*.db-wal +*.db-shm +*.bak* diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..f3f740f --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "coinosv3"] + path = coinosv3 + url = https://github.com/coinos/coinosv3.git @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Adam Soltys and the coinos contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/blossom/compose-blossom.yml b/blossom/compose-blossom.yml deleted file mode 100644 index 28acbad..0000000 --- a/blossom/compose-blossom.yml +++ /dev/null @@ -1,23 +0,0 @@ -# blossom.coinos.io — Blossom blob server for oversized wallet-sync domains. -# Stores nothing but the wallet's encrypted sync envelopes (see server.js). -# Lives beside the names registrar: ~/arkswap is a coinosv3 checkout on cs. -services: - blossom: - image: oven/bun:1 - container_name: blossom - working_dir: /app - command: bun blossom/server.js - environment: - BLOSSOM_DATA: /data - BLOSSOM_PORT: "8796" - BLOSSOM_HOST: blossom.coinos.io - volumes: - - /home/adam/arkswap:/app - - /home/adam/arkswap/data-blossom:/data - ports: - - 127.0.0.1:8796:8796 - networks: [net] - restart: unless-stopped -networks: - net: - external: true diff --git a/bridge/server.js b/bridge/server.js index 6a5c04e..13b1914 100644 --- a/bridge/server.js +++ b/bridge/server.js @@ -37,10 +37,10 @@ import { sha256 } from '@noble/hashes/sha256'; import { HDKey } from '@scure/bip32'; import { mnemonicToSeedSync } from '@scure/bip39'; -import { ArkManager } from '../src/ark/manager.js'; -import { decodeVtxo } from '../src/ark/proto.js'; -import { validateVtxo } from '../src/ark/validate.js'; -import { decodeBolt11 } from '../src/ark/lightning.js'; +import { ArkManager } from '../coinosv3/src/ark/manager.js'; +import { decodeVtxo } from '../coinosv3/src/ark/proto.js'; +import { validateVtxo } from '../coinosv3/src/ark/validate.js'; +import { decodeBolt11 } from '../coinosv3/src/ark/lightning.js'; import { lnBackend } from './ln.js'; // --------------------------------------------------------------------------- diff --git a/bun.lock b/bun.lock new file mode 100644 index 0000000..173d544 --- /dev/null +++ b/bun.lock @@ -0,0 +1,84 @@ +{ + "lockfileVersion": 2, + "configVersion": 1, + "workspaces": { + "": { + "name": "coinosv3-server", + "dependencies": { + "@noble/curves": "1.9.2", + "@noble/hashes": "1.8.0", + "@scure/base": "1.2.6", + "@scure/bip32": "1.7.0", + "@scure/bip39": "1.6.0", + "@scure/btc-signer": "1.8.1", + "nostr-tools": "2.23.5", + "web-push": "3.6.7", + }, + }, + }, + "packages": { + "@noble/ciphers": ["@noble/[email protected]", "", {}, "sha512-bysYuiVfhxNJuldNXlFEitTVdNnYUc+XNJZd7Qm2a5j1vZHgY+fazadNFWFaMK/2vye0JVlxV3gHmC0WDfAOQw=="], + + "@noble/curves": ["@noble/[email protected]", "", { "dependencies": { "@noble/hashes": "1.8.0" } }, "sha512-HxngEd2XUcg9xi20JkwlLCtYwfoFw4JGkuZpT+WlsPD4gB/cxkvTD8fSsoAnphGZhFdZYKeQIPCuFlWPm1uE0g=="], + + "@noble/hashes": ["@noble/[email protected]", "", {}, "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A=="], + + "@scure/base": ["@scure/[email protected]", "", {}, "sha512-g/nm5FgUa//MCj1gV09zTJTaM6KBAHqLN907YVQqf7zC49+DcO4B1so4ZX07Ef10Twr6nuqYEH9GEggFXA4Fmg=="], + + "@scure/bip32": ["@scure/[email protected]", "", { "dependencies": { "@noble/curves": "~1.9.0", "@noble/hashes": "~1.8.0", "@scure/base": "~1.2.5" } }, "sha512-E4FFX/N3f4B80AKWp5dP6ow+flD1LQZo/w8UnLGYZO674jS6YnYeepycOOksv+vLPSpgN35wgKgy+ybfTb2SMw=="], + + "@scure/bip39": ["@scure/[email protected]", "", { "dependencies": { "@noble/hashes": "~1.8.0", "@scure/base": "~1.2.5" } }, "sha512-+lF0BbLiJNwVlev4eKelw1WWLaiKXw7sSl8T6FvBlWkdX+94aGJ4o8XjUdlyhTCjd8c+B3KT3JfS8P0bLRNU6A=="], + + "@scure/btc-signer": ["@scure/[email protected]", "", { "dependencies": { "@noble/curves": "~1.9.0", "@noble/hashes": "~1.8.0", "@scure/base": "~1.2.5", "micro-packed": "~0.7.3" } }, "sha512-8nX9T++dFyKpvqksNHfSi9CgRsGnHAQtCdIQ1y1GmbCGLpV97v4MUyemUUT6uDumKL3oo3m4niyY6A32nmdLuQ=="], + + "agent-base": ["[email protected]", "", {}, "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ=="], + + "asn1.js": ["[email protected]", "", { "dependencies": { "bn.js": "^4.0.0", "inherits": "^2.0.1", "minimalistic-assert": "^1.0.0", "safer-buffer": "^2.1.0" } }, "sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA=="], + + "bn.js": ["[email protected]", "", {}, "sha512-3aRg6/JxfffFD+OlOjOFR3Vo79l39ooBTFucxx+MT3dhCtzn3EmiUPQo+6/OZuI2jbXi3YKgmiTFBgChQMwIRQ=="], + + "buffer-equal-constant-time": ["[email protected]", "", {}, "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA=="], + + "debug": ["[email protected]", "", { "dependencies": { "ms": "^2.1.3" }, "peerDependencies": { "supports-color": "*" }, "optionalPeers": ["supports-color"] }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], + + "ecdsa-sig-formatter": ["[email protected]", "", { "dependencies": { "safe-buffer": "^5.0.1" } }, "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ=="], + + "http_ece": ["[email protected]", "", {}, "sha512-JrF8SSLVmcvc5NducxgyOrKXe3EsyHMgBFgSaIUGmArKe+rwr0uphRkRXvwiom3I+fpIfoItveHrfudL8/rxuA=="], + + "https-proxy-agent": ["[email protected]", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "4" } }, "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw=="], + + "inherits": ["[email protected]", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], + + "jwa": ["[email protected]", "", { "dependencies": { "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg=="], + + "jws": ["[email protected]", "", { "dependencies": { "jwa": "^2.0.1", "safe-buffer": "^5.0.1" } }, "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA=="], + + "micro-packed": ["[email protected]", "", { "dependencies": { "@scure/base": "~1.2.5" } }, "sha512-2Milxs+WNC00TRlem41oRswvw31146GiSaoCT7s3Xi2gMUglW5QBeqlQaZeHr5tJx9nm3i57LNXPqxOOaWtTYg=="], + + "minimalistic-assert": ["[email protected]", "", {}, "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A=="], + + "minimist": ["[email protected]", "", {}, "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="], + + "ms": ["[email protected]", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + + "nostr-tools": ["[email protected]", "", { "dependencies": { "@noble/ciphers": "2.1.1", "@noble/curves": "2.0.1", "@noble/hashes": "2.0.1", "@scure/base": "2.0.0", "@scure/bip32": "2.0.1", "@scure/bip39": "2.0.1", "nostr-wasm": "0.1.0" }, "peerDependencies": { "typescript": ">=5.0.0" }, "optionalPeers": ["typescript"] }, "sha512-Fa7ZlUdjfUW1P4E7H3yBexhOHYi18XNyvd2n7eNHkYR085xADX6Y8V8Vm7nT/XQajaFOBrptXmVIGkJ2E4vfVw=="], + + "nostr-wasm": ["[email protected]", "", {}, "sha512-78BTryCLcLYv96ONU8Ws3Q1JzjlAt+43pWQhIl86xZmWeegYCNLPml7yQ+gG3vR6V5h4XGj+TxO+SS5dsThQIA=="], + + "safe-buffer": ["[email protected]", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="], + + "safer-buffer": ["[email protected]", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="], + + "web-push": ["[email protected]", "", { "dependencies": { "asn1.js": "^5.3.0", "http_ece": "1.2.0", "https-proxy-agent": "^7.0.0", "jws": "^4.0.0", "minimist": "^1.2.5" }, "bin": { "web-push": "src/cli.js" } }, "sha512-OpiIUe8cuGjrj3mMBFWY+e4MMIkW3SVT+7vEIjvD9kejGUypv8GPDf84JdPWskK8zMRIJ6xYGm+Kxr8YkPyA0A=="], + + "nostr-tools/@noble/curves": ["@noble/[email protected]", "", { "dependencies": { "@noble/hashes": "2.0.1" } }, "sha512-vs1Az2OOTBiP4q0pwjW5aF0xp9n4MxVrmkFBxc6EKZc6ddYx5gaZiAsZoq0uRRXWbi3AT/sBqn05eRPtn1JCPw=="], + + "nostr-tools/@noble/hashes": ["@noble/[email protected]", "", {}, "sha512-XlOlEbQcE9fmuXxrVTXCTlG2nlRXa9Rj3rr5Ue/+tX+nmkgbX720YHh0VR3hBF9xDvwnb8D2shVGOwNx+ulArw=="], + + "nostr-tools/@scure/base": ["@scure/[email protected]", "", {}, "sha512-3E1kpuZginKkek01ovG8krQ0Z44E3DHPjc5S2rjJw9lZn3KSQOs8S7wqikF/AH7iRanHypj85uGyxk0XAyC37w=="], + + "nostr-tools/@scure/bip32": ["@scure/[email protected]", "", { "dependencies": { "@noble/curves": "2.0.1", "@noble/hashes": "2.0.1", "@scure/base": "2.0.0" } }, "sha512-4Md1NI5BzoVP+bhyJaY3K6yMesEFzNS1sE/cP+9nuvE7p/b0kx9XbpDHHFl8dHtufcbdHRUUQdRqLIPHN/s7yA=="], + + "nostr-tools/@scure/bip39": ["@scure/[email protected]", "", { "dependencies": { "@noble/hashes": "2.0.1", "@scure/base": "2.0.0" } }, "sha512-PsxdFj/d2AcJcZDX1FXN3dDgitDDTmwf78rKZq1a6c1P1Nan1X/Sxc7667zU3U+AN60g7SxxP0YCVw2H/hBycg=="], + } +} diff --git a/coinosv3 b/coinosv3 new file mode 160000 +Subproject 082fc2e4005387697f77e485ae1c783ed3203e9 diff --git a/compose.yml b/compose.yml new file mode 100644 index 0000000..df0d2d1 --- /dev/null +++ b/compose.yml @@ -0,0 +1,62 @@ +# The coinos v3 services. Each runs this checkout under Bun. +# +# Secrets and state live outside the checkout, under $STATE (set it in a .env +# file next to this one; see README.md): +# $STATE/names/config.json $STATE/names/data/ (names.db) +# $STATE/nwcpush/config.json $STATE/nwcpush/data/ +# $STATE/blossom/data/ +services: + names: + image: oven/bun:1.3.14 + container_name: names + working_dir: /app + command: bun names/server.js + environment: + NAMES_CONFIG: /etc/coinosv3/config.json + volumes: + - .:/app:ro + - ${STATE:?set STATE in .env}/names/config.json:/etc/coinosv3/config.json:ro + - ${STATE}/names/data:/data + - ${CLN_RPC:?set CLN_RPC in .env}:/cln/lightning-rpc + - ${CLN_MUT_RPC:-/dev/null}:/cln-mut/lightning-rpc + ports: + - 127.0.0.1:8798:8798 + networks: [net] + restart: unless-stopped + + nwcpush: + image: oven/bun:1.3.14 + container_name: nwcpush + working_dir: /app + command: bun nwcpush/server.js + environment: + NWCPUSH_CONFIG: /etc/coinosv3/config.json + volumes: + - .:/app:ro + - ${STATE}/nwcpush/config.json:/etc/coinosv3/config.json:ro + - ${STATE}/nwcpush/data:/data + ports: + - 127.0.0.1:8797:8797 + networks: [net] + restart: unless-stopped + + blossom: + image: oven/bun:1.3.14 + container_name: blossom + working_dir: /app + command: bun blossom/server.js + environment: + BLOSSOM_DATA: /data + BLOSSOM_PORT: "8796" + BLOSSOM_HOST: blossom.coinos.io + volumes: + - .:/app:ro + - ${STATE}/blossom/data:/data + ports: + - 127.0.0.1:8796:8796 + networks: [net] + restart: unless-stopped + +networks: + net: + external: true diff --git a/feed/cache.js b/feed/cache.js index 4edc017..bfefe98 100644 --- a/feed/cache.js +++ b/feed/cache.js @@ -1,5 +1,5 @@ -import { PUBLIC_FEED_RELAYS, popularCandidates } from '../src/popular-feed.js'; -import { isMachinePost } from '../src/lang-guess.js'; +import { PUBLIC_FEED_RELAYS, popularCandidates } from '../coinosv3/src/popular-feed.js'; +import { isMachinePost } from '../coinosv3/src/lang-guess.js'; // One snapshot for all languages; language/mute filtering stays in the browser. export async function buildSnapshot(query, now = Date.now) { diff --git a/names/server.js b/names/server.js index dacbda6..9d94b51 100644 --- a/names/server.js +++ b/names/server.js @@ -26,13 +26,13 @@ import { createHash, randomBytes } from 'node:crypto'; import { HDKey } from '@scure/bip32'; import { mnemonicToSeedSync } from '@scure/bip39'; import { lnBackend } from '../bridge/ln.js'; -import { ArkManager } from '../src/ark/manager.js'; +import { ArkManager } from '../coinosv3/src/ark/manager.js'; import { fetchPreview } from './preview.js'; import { translate, translateDeepl, languages } from './translate.js'; -import { decodeAddress } from '../src/ark/proto.js'; -import { decodeNoffer } from '../src/noffer.js'; -import { decodeBolt11 } from '../src/ark/lightning.js'; -import { decodeOffer } from '../src/bolt12.js'; +import { decodeAddress } from '../coinosv3/src/ark/proto.js'; +import { decodeNoffer } from '../coinosv3/src/noffer.js'; +import { decodeBolt11 } from '../coinosv3/src/ark/lightning.js'; +import { decodeOffer } from '../coinosv3/src/bolt12.js'; import { registrationIdentity } from './ownership.js'; const CFG = JSON.parse(readFileSync(process.env.NAMES_CONFIG diff --git a/names/translate.js b/names/translate.js index 36ab8cf..bcc47d9 100644 --- a/names/translate.js +++ b/names/translate.js @@ -3,7 +3,7 @@ // passes through here; the cache keys on the text itself, so a popular post // is translated once whoever asks. import { createHash } from 'node:crypto'; -import { translatable, translatePieces } from '../src/translate-split.js'; +import { translatable, translatePieces } from './vendor/translate-split.js'; const LT = process.env.LT_URL || 'http://libretranslate:5000'; const MAX_CHARS = 5000; diff --git a/names/vendor/lang-guess.js b/names/vendor/lang-guess.js new file mode 100644 index 0000000..6d62e30 --- /dev/null +++ b/names/vendor/lang-guess.js @@ -0,0 +1,99 @@ +// A rough guess at the language a post is written in, cheap enough to run +// on every post of a public feed. Script first (a post in Han or Arabic +// letters is not English), then, among the Latin-script languages the app +// speaks, a handful of function words each. Short or ambiguous text is +// "unknown", and unknown is never held against a post. +const SCRIPTS = [ + ['latin', /[A-Za-zÀ-ɏḀ-ỿ]/g], + ['cyrillic', /[Ѐ-ӿ]/g], + ['arabic', /[-ۿݐ-ݿ]/g], + ['devanagari', /[ऀ-ॿ]/g], + ['bengali', /[ঀ-]/g], + ['tamil', /[-]/g], + ['telugu', /[ఀ-౿]/g], + ['kana', /[-ヿ]/g], + ['han', /[一-鿿㐀-䶿]/g], + ['hangul', /[가-ᄀ-ᇿ]/g], +]; +export const LANG_SCRIPT = { + en: 'latin', es: 'latin', fr: 'latin', de: 'latin', pt: 'latin', id: 'latin', sw: 'latin', tr: 'latin', vi: 'latin', + ru: 'cyrillic', ar: 'arabic', ur: 'arabic', hi: 'devanagari', mr: 'devanagari', bn: 'bengali', ta: 'tamil', te: 'telugu', + ja: 'kana', ko: 'hangul', zh: 'han', + // not app languages, but languages people read posts in (a translation target) + it: 'latin', nl: 'latin', pl: 'latin', +}; +const STOP = { + en: 'the and is to of in that it for you with this on are be have not was but they'.split(' '), + es: 'el la de que y en los es un una por con para las del se no como más pero'.split(' '), + fr: 'le la les de des et est un une pour que qui dans pas sur avec ce il nous vous'.split(' '), + de: 'der die und das ist nicht ein eine ich zu mit den auf für sich dem des auch wir'.split(' '), + pt: 'o a de que e do da em um uma para com não os as se mais por isso você'.split(' '), + id: 'yang dan di ini itu dengan untuk tidak dari ada saya akan ke juga bisa kita sudah atau kalau aku'.split(' '), + tr: 've bir bu için ile de da çok ne gibi daha ama var ben sen mi değil kadar her olan'.split(' '), + vi: 'và của là có không được một trong cho những này với các để người đã tôi khi cũng như'.split(' '), + sw: 'na ya wa kwa ni za la katika hii kuwa kama hata lakini sana yake wote sasa tu huo hapa'.split(' '), + it: 'il di che è e la un una per non sono con del della gli anche ma questo mi più nel'.split(' '), + nl: 'de het een en van is dat niet op te zijn met voor ik je die maar ook wat'.split(' '), + pl: 'i w nie na się z to że jest do co jak ale tak o po czy już'.split(' '), +}; +const STOPSETS = Object.fromEntries(Object.entries(STOP).map(([k, v]) => [k, new Set(v)])); + +// The text a reader actually reads: no links, no nostr references, no tags. +export const proseOf = (text) => String(text || '') + .replace(/https?:\/\/\S+/g, ' ').replace(/nostr:[a-z0-9]+/gi, ' ').replace(/#[\p{L}\p{N}_]+/gu, ' '); + +export function scriptOf(text, min = 3) { + let best = null, bestN = 0, total = 0; + for (const [name, re] of SCRIPTS) { + const n = (text.match(re) || []).length; + total += n; + if (n > bestN) { best = name; bestN = n; } + } + return total < min ? null : best; +} + +// A language code from the app's list, or null when it cannot tell. +export function guessLang(text) { + const prose = proseOf(text); + const script = scriptOf(prose); + if (!script) return null; + if (script === 'kana') return 'ja'; + if (script === 'han') return 'zh'; + if (script !== 'latin') { + const langs = Object.entries(LANG_SCRIPT).filter(([, s]) => s === script).map(([l]) => l); + return langs.length === 1 ? langs[0] : null; // ar/ur, hi/mr share a script: unknown between them + } + const words = prose.toLowerCase().match(/[\p{L}']+/gu) || []; + if (words.length < 4) return null; + const score = {}; + for (const w of words) for (const [l, set] of Object.entries(STOPSETS)) if (set.has(w)) score[l] = (score[l] || 0) + 1; + const ranked = Object.entries(score).sort((a, b) => b[1] - a[1]); + if (!ranked.length || ranked[0][1] < 2) return null; + if (ranked[1] && ranked[0][1] < ranked[1][1] * 1.5) return null; + return ranked[0][0]; +} + +// Does a post read in this language, as far as we can tell? Unknown passes; +// a wrong script or a clear other language fails. +export function inLanguage(text, lang) { + const want = LANG_SCRIPT[lang]; + if (!want) return true; + const prose = proseOf(text); + const script = scriptOf(prose); + if (!script) return true; // nothing readable to judge (a picture post, an emoji) + if (script !== want) return false; + if (want === 'han' && lang === 'zh' && /[-ヿ]/.test(prose)) return false; // kana means Japanese + const g = guessLang(prose); + return g === null || g === lang; +} + +// Machine output dressed as a post: JSON, a hash dump, a wall of symbols. +export function isMachinePost(text) { + const s = String(text || '').trim(); + if (!s) return false; + if (/^[\[{]/.test(s)) { try { JSON.parse(s); return true; } catch {} } + const prose = proseOf(s).replace(/\s+/g, ''); + if (prose.length < 24) return false; + const letters = (prose.match(/\p{L}/gu) || []).length; + return letters / prose.length < 0.4; +} diff --git a/names/vendor/translate-split.js b/names/vendor/translate-split.js new file mode 100644 index 0000000..95f6468 --- /dev/null +++ b/names/vendor/translate-split.js @@ -0,0 +1,61 @@ +// What must come back from a translation exactly as it went in: links, +// nostr: mentions, custom emoji shortcodes, hashtags, @handles, invoices and +// other long codes. Translation models mangle them ("https: / / example.com +// / fot.jpg"), so only the prose between them is translated and the text is +// put back together around the untouched tokens. Shared by the registrar +// (names/translate.js) and the browser's own translator (messages.js). +import { scriptOf, LANG_SCRIPT } from './lang-guess.js'; + +const KEEP = /(https?:\/\/\S+|nostr:[a-z0-9]+|:[a-z0-9_+-]{1,40}:|#[\p{L}\p{N}_]+|@[\w.-]+(?:@[\w.-]+)?|\b(?:ln|lightning:|cashu|bitcoin:|nevent|npub|nprofile|note|naddr)[a-z0-9]{20,}|\S{40,})/giu; + +export function splitProse(text) { + const parts = []; + let last = 0; + for (const m of String(text || '').matchAll(KEEP)) { + if (m.index > last) parts.push({ prose: text.slice(last, m.index) }); + parts.push({ keep: m[0] }); + last = m.index + m[0].length; + } + if (last < text.length) parts.push({ prose: text.slice(last) }); + return parts; +} +// a piece worth translating: has words, not just spaces and punctuation +export const hasWords = (s) => /\p{L}{2,}/u.test(s || ''); + +// A hashtag stays a hashtag (it's a link to its topic), but one written in +// another script than the reader's means nothing to them: such a tag gets +// its translation in brackets after it, "#総合部絵チャ (general art chat)". +const tagWords = (tag) => tag.slice(1).replace(/_/g, ' '); +export function glossable(tag, target) { + if (!tag.startsWith('#') || !target) return false; + const s = scriptOf(tagWords(tag), 1); + return !!s && !!LANG_SCRIPT[target] && s !== LANG_SCRIPT[target]; +} +// What a translator gets to see of a post: its prose, then the words of the +// hashtags the reader can't read (also what language detection runs on). +export function translatable(text, target) { + const parts = splitProse(text); + return { + parts, + prose: parts.filter((p) => p.prose && hasWords(p.prose)), + tags: parts.filter((p) => p.keep && glossable(p.keep, target)), + }; +} + +// Translate the prose pieces and the glossed tags (one call for all of them) +// and reassemble. Each piece keeps its own leading/trailing whitespace — +// line breaks matter. +export async function translatePieces(text, translateMany, target = null) { + const { parts, prose, tags } = translatable(text, target); + if (!prose.length && !tags.length) return text; + const outs = await translateMany([...prose.map((p) => p.prose.trim()), ...tags.map((p) => tagWords(p.keep))]); + prose.forEach((p, i) => { + const lead = p.prose.match(/^\s*/)[0], trail = p.prose.match(/\s*$/)[0]; + p.out = lead + String(outs[i] ?? p.prose.trim()) + trail; + }); + tags.forEach((p, i) => { + const g = String(outs[prose.length + i] ?? '').trim(); + p.out = g && g.toLowerCase() !== tagWords(p.keep).toLowerCase() ? p.keep + ' (' + g + ')' : p.keep; + }); + return parts.map((p) => p.out ?? p.keep ?? p.prose).join(''); +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..6267263 --- /dev/null +++ b/package.json @@ -0,0 +1,20 @@ +{ + "name": "coinosv3-server", + "private": true, + "type": "module", + "description": "Server-side services for coinos v3: the payment-name registrar, NWC push, Blossom sync storage, the public feed and media proxy", + "license": "MIT", + "scripts": { + "test": "bun test names/store.test.js" + }, + "dependencies": { + "@noble/curves": "1.9.2", + "@noble/hashes": "1.8.0", + "@scure/base": "1.2.6", + "@scure/bip32": "1.7.0", + "@scure/bip39": "1.6.0", + "@scure/btc-signer": "1.8.1", + "nostr-tools": "2.23.5", + "web-push": "3.6.7" + } +} diff --git a/tools/public-feed-cache-test.js b/tools/public-feed-cache-test.js index 77f833a..412caef 100644 --- a/tools/public-feed-cache-test.js +++ b/tools/public-feed-cache-test.js @@ -1,6 +1,6 @@ import assert from 'node:assert/strict'; import { buildSnapshot, createSnapshotCache, snapshotResponse } from '../feed/cache.js'; -import { popularCandidates } from '../src/popular-feed.js'; +import { popularCandidates } from '../coinosv3/src/popular-feed.js'; const id = 'a'.repeat(64), author = 'b'.repeat(64); const reaction = (pk, kind = 7, tags = []) => ({ kind, pubkey: pk, created_at: 100, tags: [['e', id], ...tags] }); const post = { kind: 1, id, pubkey: author, created_at: 90, content: 'This is a post about the things we build together.', tags: [] }; diff --git a/tools/translate-test.js b/tools/translate-test.js deleted file mode 100644 index d3fe095..0000000 --- a/tools/translate-test.js +++ /dev/null @@ -1,119 +0,0 @@ -// Translation: posts that read foreign are shown in the app's language -// ("Translated from Spanish · Show original"), automatically by default; -// with the switch off they offer Translate instead; ⋯ → Translate asks for -// any post. The registrar's /translate is stubbed here (window.fetch shim). -// Run: bun tools/translate-test.js -import puppeteer from './puppeteer.js'; -import { cacheKeyFor } from '../src/wallet.js'; -import { generateSecretKey, getPublicKey, finalizeEvent } from 'nostr-tools/pure'; -import { buildHtml } from '../build.js'; -const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); -let ok = true; -const check = (n, c, d = '') => { console.log(` ${c ? '✓' : '✗'} ${n}${d ? ' — ' + d : ''}`); if (!c) ok = false; }; -const AUTHOR = (() => { const sk = generateSecretKey(); return { sk, pk: getPublicKey(sk) }; })(); -const now = Math.floor(Date.now() / 1000); -const post = (content, ago) => finalizeEvent({ kind: 1, created_at: now - ago, tags: [], content }, AUTHOR.sk); -const ES = post('Hola a todos, hoy compré café con bitcoin en el mercado y fue muy fácil para todos los que estaban allí', 100); -const JA = post('ビットコインは素晴らしいです。明日また会いましょう。', 200); -const TAG = post('#総合部絵チャ', 250); -const EN = post('An ordinary post in English about the weather and the market today', 300); -const ANSWERS = { - [ES.content]: { text: 'Hello everyone, today I bought coffee with bitcoin at the market and it was very easy for everyone there', source: 'es' }, - [JA.content]: { text: 'Bitcoin is great. Let’s meet again tomorrow.', source: 'ja' }, - [EN.content]: { text: EN.content, source: 'en' }, - [TAG.content]: { text: '#総合部絵チャ (general art chat)', source: 'ja' }, -}; - -const html = await buildHtml({ minify: true, pwa: false }); -const server = Bun.serve({ port: 5303, fetch: () => new Response(html, { headers: { 'content-type': 'text/html' } }) }); -const browser = await puppeteer.launch({ executablePath: '/usr/bin/google-chrome', headless: 'new', args: ['--no-sandbox'] }); -const page = await browser.newPage(); -await page.evaluateOnNewDocument((answers) => { - window.__tr = []; - try { delete self.Translator; } catch {} // the registrar path, every time - const real = window.fetch; - window.fetch = async (url, opts) => { - if (String(url) === 'https://names.coinos.io/translate') { - const body = JSON.parse(opts.body); - window.__tr.push(body); - const a = answers[body.text]; - await new Promise((r) => setTimeout(r, 150)); - return new Response(JSON.stringify(a || { error: 'nope' }), { status: a ? 200 : 503, headers: { 'content-type': 'application/json' } }); - } - return real(url, opts); - }; - const Real = window.WebSocket; - window.WebSocket = function (...a) { - const ws = new Real(...a); - const send = ws.send.bind(ws); - ws.send = (data) => { - try { const m = JSON.parse(data); if (m[0] === 'REQ') { setTimeout(() => ws.dispatchEvent(new MessageEvent('message', { data: JSON.stringify(['EOSE', m[1]]) })), 0); return; } if (m[0] === 'EVENT') return; } catch {} - return send(data); - }; - return ws; - }; - window.WebSocket.prototype = Real.prototype; Object.assign(window.WebSocket, Real); -}, ANSWERS); -const errs = []; -page.on('pageerror', (e) => errs.push(String(e))); -const click = (t) => page.evaluate((x) => { const e = [...document.querySelectorAll('button')].find((n) => n.textContent.trim().toLowerCase().includes(x)); if (e) { e.click(); return true; } return false; }, t); -const text = () => page.evaluate(() => document.body.innerText); -const waitText = async (x, ms = 25000) => { for (let i = 0; i < ms / 250; i++) { if ((await text()).toLowerCase().includes(x.toLowerCase())) return true; await sleep(250); } return false; }; -const has = async (x) => (await text()).includes(x); -const row = (m) => page.evaluate((m) => { const r = [...document.querySelectorAll('[data-zap-post]')].find((n) => n.textContent.includes(m)); return r ? r.innerText : null; }, m); -const openFeed = async () => { await page.evaluate(() => document.querySelector('.app-nav-button[aria-label="Feed"]')?.click()); await sleep(1200); }; - -try { - await page.setViewport({ width: 390, height: 844 }); - await page.goto('http://localhost:5303/', { waitUntil: 'domcontentloaded' }); - await page.evaluate(() => { localStorage.setItem('btc-wallet-network', 'regtest'); localStorage.setItem('btc-wallet-lang', 'en'); }); - await page.reload({ waitUntil: 'domcontentloaded' }); - await sleep(400); - await click('create a new wallet'); await sleep(600); - await page.waitForSelector('.words .w .t'); - const mn = (await page.$$eval('.words .w .t', (l) => l.map((e) => e.textContent.trim()))).join(' '); - await click('skip verification'); - await waitText('receive', 20000); - const base = cacheKeyFor(mn + '\n'); - await page.evaluate(([k, pk, ns]) => { - localStorage.setItem(k + ':follows', JSON.stringify({ tags: [['p', pk]], c: '', at: Math.floor(Date.now() / 1000) })); - localStorage.setItem(k + ':feedNotes', JSON.stringify(ns)); - localStorage.setItem(k + ':profiles', JSON.stringify({ [pk]: { name: 'Globetrotter', t: Date.now() } })); - localStorage.setItem('btc-wallet-feed:' + k, 'following'); - }, [base, AUTHOR.pk, [ES, JA, TAG, EN]]); - await page.reload({ waitUntil: 'domcontentloaded' }); - await waitText('receive', 20000); - await openFeed(); - - console.log('\n[automatic, the default]'); - check('the Spanish post is shown in English', await waitText('today I bought coffee with bitcoin', 10000) && !(await has('compré café'))); - check('...saying where it came from', /Translated from Spanish/.test(await row('today I bought coffee') || ''), (await row('today I bought coffee') || '').split('\n').slice(-6).join(' | ')); - check('the Japanese post too', await waitText('Bitcoin is great', 5000) && /Translated from Japanese/.test(await row('Bitcoin is great') || '')); - check('a post that is only a Japanese hashtag gets its tag glossed, the tag kept', await waitText('#総合部絵チャ (general art chat)', 5000)); - const asked = await page.evaluate(() => window.__tr.map((b) => b.text.slice(0, 12) + '>' + b.target)); - check('the English post is left alone (never sent)', asked.length === 3 && !asked.some((x) => x.startsWith('An ordinary')), asked.join(', ')); - await page.evaluate(() => { const r = [...document.querySelectorAll('[data-zap-post]')].find((n) => n.textContent.includes('today I bought coffee')); [...r.querySelectorAll('.note-translate-btn')].find((b) => /original/i.test(b.textContent)).click(); }); - await sleep(400); - check('Show original brings the Spanish back', await has('compré café') && !(await has('today I bought coffee'))); - check('...with the offer to translate again', /Translate/.test(await row('compré café') || '')); - - console.log('\n[⋯ → Translate on a post the guess calls English]'); - await page.evaluate(() => { const r = [...document.querySelectorAll('[data-zap-post]')].find((n) => n.textContent.includes('An ordinary post')); [...r.querySelectorAll('button')].find((b) => b.getAttribute('aria-label') === 'More').click(); }); - await sleep(300); - check('the menu offers Translate', await page.evaluate(() => [...document.querySelectorAll('.confirm-pop button')].some((b) => /Translate/.test(b.textContent)))); - await page.evaluate(() => [...document.querySelectorAll('.confirm-pop button')].find((b) => /Translate/.test(b.textContent)).click()); - check('it says the post is already in English', await waitText('already in your language', 4000)); - - console.log('\n[automatic switched off]'); - await page.evaluate(() => localStorage.setItem('coinos-auto-translate', '0')); - await page.reload({ waitUntil: 'domcontentloaded' }); - await waitText('receive', 20000); - await openFeed(); - check('foreign posts stay as written', await waitText('明日また会いましょう', 10000) && !(await has('Bitcoin is great'))); - check('...offering Translate', /Translate/.test(await row('明日また') || '')); - await page.evaluate(() => { const r = [...document.querySelectorAll('[data-zap-post]')].find((n) => n.textContent.includes('明日また')); r.querySelector('.note-translate-btn').click(); }); - check('tapping it translates (remembered from before: no new request)', await waitText('Bitcoin is great', 5000)); - check('no page errors', errs.length === 0, errs.slice(0, 2).join(' | ')); -} finally { await browser.close(); server.stop(true); } -console.log(ok ? '\n✅ posts translate into the reader\'s language' : '\n❌ failed'); -process.exit(ok ? 0 : 1); |
