diff options
| -rw-r--r-- | names/preview.js | 131 | ||||
| -rw-r--r-- | names/server.js | 23 | ||||
| -rw-r--r-- | src/features/messages.js | 80 | ||||
| -rw-r--r-- | src/style.css | 15 | ||||
| -rw-r--r-- | tools/link-preview-test.js | 123 |
5 files changed, 372 insertions, 0 deletions
diff --git a/names/preview.js b/names/preview.js new file mode 100644 index 0000000..25051d9 --- /dev/null +++ b/names/preview.js @@ -0,0 +1,131 @@ +// Link previews for the wallet's feed: the Open Graph card a page describes +// for itself (title, one line of description, a picture, the site's name). +// A browser can't read another site's HTML, so the registrar fetches it +// once on the reader's behalf and hands back only what a card needs. +// +// The server sees more than the internet does (containers, the LAN), so a +// URL is fetched only after its host resolves to public addresses — and +// again after every redirect. Bodies are capped, and only HTML is read. +import { promises as dns } from 'node:dns'; + +const MAX_BODY = 512 * 1024; +const TIMEOUT_MS = 7000; +const MAX_HOPS = 4; +const UA = 'Mozilla/5.0 (compatible; coinos-link-preview/1.0; +https://coinos.io)'; + +const isPrivate4 = (ip) => { + const p = ip.split('.').map(Number); + return p[0] === 10 || p[0] === 127 || p[0] === 0 + || (p[0] === 172 && p[1] >= 16 && p[1] <= 31) + || (p[0] === 192 && p[1] === 168) + || (p[0] === 169 && p[1] === 254) + || (p[0] === 100 && p[1] >= 64 && p[1] <= 127) + || p[0] >= 224; +}; +const isPrivate6 = (ip) => { + const s = ip.toLowerCase(); + if (s === '::1' || s === '::') return true; + if (s.startsWith('::ffff:')) return isPrivate4(s.slice(7)); + return /^(fc|fd|fe[89ab])/.test(s); +}; +export const isPrivateIp = (ip) => (ip.includes(':') ? isPrivate6(ip) : isPrivate4(ip)); + +// A URL the server may fetch: http(s), a default port, a public host. +async function publicUrl(str) { + let u; + try { u = new URL(str); } catch { return null; } + if (!/^https?:$/.test(u.protocol)) return null; + if (u.port && u.port !== '80' && u.port !== '443') return null; + if (u.username || u.password) return null; + const host = u.hostname.replace(/^\[|\]$/g, ''); + if (!host || host === 'localhost' || /\.(local|internal|localhost)$/i.test(host)) return null; + let addrs; + try { addrs = await dns.lookup(host, { all: true }); } catch { return null; } + if (!addrs.length || addrs.some((a) => isPrivateIp(a.address))) return null; + return u; +} + +const decodeEntities = (s) => String(s || '') + .replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCodePoint(parseInt(h, 16))) + .replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(Number(d))) + .replace(/&(amp|quot|apos|lt|gt|nbsp);/g, (_, e) => ({ amp: '&', quot: '"', apos: "'", lt: '<', gt: '>', nbsp: ' ' })[e]); +const tidy = (s, max) => decodeEntities(s).replace(/\s+/g, ' ').trim().slice(0, max); + +// Every <meta> in the head, whichever way its attributes are ordered. +function metaTags(html) { + const out = []; + const re = /<meta\s+([^>]*?)\/?>/gi; + let m; + while ((m = re.exec(html))) { + const attrs = {}; + const ar = /([a-zA-Z:_-]+)\s*=\s*("([^"]*)"|'([^']*)'|([^\s"'>]+))/g; + let a; + while ((a = ar.exec(m[1]))) attrs[a[1].toLowerCase()] = a[3] ?? a[4] ?? a[5] ?? ''; + const key = (attrs.property || attrs.name || attrs.itemprop || '').toLowerCase(); + if (key && attrs.content != null) out.push([key, attrs.content]); + } + return out; +} + +// What a card shows, from a page's HTML. Exported for the tests. +export function parsePreview(html, baseUrl) { + const head = html.slice(0, MAX_BODY); + const meta = new Map(); + for (const [k, v] of metaTags(head)) if (!meta.has(k) && v.trim()) meta.set(k, v); + const pick = (...keys) => { for (const k of keys) if (meta.get(k)) return meta.get(k); return ''; }; + const titleTag = (head.match(/<title[^>]*>([\s\S]*?)<\/title>/i) || [])[1] || ''; + const title = tidy(pick('og:title', 'twitter:title') || titleTag, 200); + const description = tidy(pick('og:description', 'twitter:description', 'description'), 300); + let image = pick('og:image:secure_url', 'og:image', 'twitter:image', 'twitter:image:src'); + if (image) { + try { image = new URL(decodeEntities(image).trim(), baseUrl).href; } catch { image = ''; } + if (!/^https?:/.test(image)) image = ''; + } + let site = tidy(pick('og:site_name'), 80); + if (!site) { try { site = new URL(baseUrl).hostname.replace(/^www\./, ''); } catch {} } + if (!title && !image) return null; + return { url: baseUrl, title, description, image, site }; +} + +// Read at most MAX_BODY bytes of a response. +async function readCapped(res) { + const reader = res.body?.getReader(); + if (!reader) return ''; + const chunks = []; let got = 0; + while (got < MAX_BODY) { + const { value, done } = await reader.read(); + if (done) break; + chunks.push(value); got += value.length; + } + try { await reader.cancel(); } catch {} + return new TextDecoder('utf-8', { fatal: false }).decode(Buffer.concat(chunks.map((c) => Buffer.from(c)))); +} + +// Fetch a page and describe it, or null when there is nothing to show +// (not HTML, unreachable, a private host, no title or picture). +export async function fetchPreview(str) { + let u = await publicUrl(str); + for (let hop = 0; u && hop < MAX_HOPS; hop++) { + let res; + try { + res = await fetch(u.href, { + redirect: 'manual', + signal: AbortSignal.timeout(TIMEOUT_MS), + headers: { 'user-agent': UA, accept: 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.1', 'accept-language': 'en' }, + }); + } catch { return null; } + if (res.status >= 300 && res.status < 400 && res.headers.get('location')) { + let next; + try { next = new URL(res.headers.get('location'), u.href).href; } catch { return null; } + try { await res.body?.cancel(); } catch {} + u = await publicUrl(next); + continue; + } + if (!res.ok) return null; + const type = (res.headers.get('content-type') || '').toLowerCase(); + if (!/text\/html|application\/xhtml/.test(type)) { try { await res.body?.cancel(); } catch {} return null; } + const html = await readCapped(res); + return parsePreview(html, u.href); + } + return null; +} diff --git a/names/server.js b/names/server.js index d99e330..82d00ec 100644 --- a/names/server.js +++ b/names/server.js @@ -27,6 +27,7 @@ import { HDKey } from '@scure/bip32'; import { mnemonicToSeedSync } from '@scure/bip39'; import { lnBackend } from '../bridge/ln.js'; import { ArkManager } from '../src/ark/manager.js'; +import { fetchPreview } from './preview.js'; import { decodeAddress } from '../src/ark/proto.js'; import { decodeNoffer } from '../src/noffer.js'; import { decodeBolt11 } from '../src/ark/lightning.js'; @@ -847,6 +848,7 @@ setInterval(() => { drainWebhooks().catch(() => {}); }, 30_000); // crude per-IP limiter const rate = new Map(); +const previews = new Map(); // url -> { meta, until } function rateOk(ip, limit = 10) { const now = Date.now(); const arr = (rate.get(ip) || []).filter((t) => t > now - 60_000); @@ -982,6 +984,27 @@ Bun.serve({ return json({ error: 'not found' }, 404); } + // A link in a post, as the card its page describes for itself (Open + // Graph). Cached a day either way: a page with nothing to show is asked + // again only after an hour. Public data; the CDN may cache it too. + if (url.pathname === '/preview' && req.method === 'GET') { + if (!rateOk(ip, 120)) return json({ error: 'rate limited' }, 429); + const target = (url.searchParams.get('url') || '').trim(); + if (!/^https?:\/\/\S{4,2000}$/.test(target)) return json({ error: 'bad url' }, 400); + const now = Date.now(); + let hit = previews.get(target); + if (!hit || now > hit.until) { + const meta = await fetchPreview(target).catch(() => null); + hit = { meta, until: now + (meta ? 24 : 1) * 3600_000 }; + previews.set(target, hit); + if (previews.size > 5000) previews.delete(previews.keys().next().value); + } + return new Response(JSON.stringify(hit.meta || {}), { + headers: { 'content-type': 'application/json', 'access-control-allow-origin': '*', + 'cache-control': 'public, max-age=' + (hit.meta ? 86400 : 3600) }, + }); + } + if (url.pathname === '/health') { return json({ ok: true, domain: DOMAIN, names: Object.keys(state.names).length, diff --git a/src/features/messages.js b/src/features/messages.js index cd0631a..e004e91 100644 --- a/src/features/messages.js +++ b/src/features/messages.js @@ -4240,6 +4240,76 @@ export function messagesFeature(ctx) { mediaWarming.set(url, task); return task; }; + // ---- link previews -------------------------------------------------- + // A plain link in a post becomes the card its page describes (Open Graph: + // title, a line, a picture), fetched through the registrar because a + // browser can't read another site's HTML. Warmed with the rest of a post + // so the row arrives whole; remembered across sessions, a few hundred at + // a time, so a feed you've seen paints its cards at once. + const PREVIEW_API = 'https://names.coinos.io/preview?url='; + const PREVIEW_KEEP = 400; + const previews = new Map(); // url -> { title, description, image, site } | null (nothing there) + const previewWarming = new Map(); + try { + for (const [u, m] of JSON.parse(localStorage.getItem('coinos-link-previews') || '[]')) previews.set(u, m); + } catch {} + let previewFlush = null; + const savePreviews = () => { + if (previewFlush) return; + previewFlush = setTimeout(() => { + previewFlush = null; + const rows = [...previews].filter(([, m]) => m).slice(-PREVIEW_KEEP); + try { localStorage.setItem('coinos-link-previews', JSON.stringify(rows)); } catch {} + }, 1000); + }; + const previewable = (url) => /^https?:\/\//i.test(url) && !youtubeId(url) + && !/\.(png|jpe?g|gif|webp|avif|mp4|webm|mov|m4v|mp3|m4a|ogg|wav|pdf|zip)(\?[^\s]*)?$/i.test(url); + const warmPreview = (url) => { + if (!url || previews.has(url) || typeof fetch === 'undefined') return Promise.resolve(); + if (previewWarming.has(url)) return previewWarming.get(url); + const task = (async () => { + let meta = null; + try { + const r = await fetch(PREVIEW_API + encodeURIComponent(url), { signal: AbortSignal.timeout(READY_MS) }); + const j = r.ok ? await r.json() : null; + if (j && (j.title || j.image)) meta = { title: j.title || '', description: j.description || '', image: j.image || '', site: j.site || '' }; + } catch { return; } // asked again another time: the registrar was away, not the page + previews.set(url, meta); + if (meta) { savePreviews(); if (meta.image) await warmMedia(meta.image); } + })().finally(() => previewWarming.delete(url)); + previewWarming.set(url, task); + return task; + }; + // the first two plain links of a post (a wall of links is a list, not cards) + function notePreviewUrls(content) { + const urls = []; + for (const part of String(content || '').split(NOTE_SPLIT)) { + if (!part || urls.length >= 2) continue; + const md = MD_PARTS.exec(part); + const url = md ? md[3] : (/^https?:\/\//i.test(part) ? part : null); + if (url && !(md && md[1]) && previewable(url) && !PACK_LINK_RE.test(url)) urls.push(url); + } + return urls; + } + function linkCard(url, pv) { + let host = ''; + try { host = new URL(url).hostname.replace(/^www\./, ''); } catch {} + const size = pv.image ? (feedPaint ? feedMedia(pv.image) : mediaReady.get(pv.image)) : null; + return h('a', { + class: 'link-card', href: url, target: '_blank', rel: 'noopener noreferrer', + onClick: (e) => e.stopPropagation(), + }, + pv.image ? h('img', { + class: 'link-thumb', src: pv.image, alt: '', loading: 'lazy', + width: size?.width, height: size?.height, + style: size && size.width && size.height ? 'aspect-ratio:' + size.width + '/' + size.height : undefined, + onError: (e) => { e.target.style.display = 'none'; }, + }) : null, + h('div', { class: 'link-meta' }, + h('div', { class: 'link-title' }, pv.title || host), + pv.description ? h('div', { class: 'link-desc small muted' }, pv.description) : null, + h('div', { class: 'link-site small faint' }, pv.site || host))); + } function noteMediaUrls(content) { const urls = []; for (const part of String(content || '').split(NOTE_SPLIT)) { @@ -4266,6 +4336,7 @@ export function messagesFeature(ctx) { } async function noteReady(ev, deadline = Date.now() + READY_MS, depth = 0) { const tasks = [warmAvatar(ev.pubkey, deadline), ...noteMediaUrls(ev.content).map(warmMedia), + ...notePreviewUrls(ev.content).map(warmPreview), ...[...emojiTagMap(ev.tags).values()].map(warmMedia)]; for (const part of String(ev.content || '').split(NOTE_SPLIT)) { if (/^(nostr:|@?)(npub|nprofile)1/i.test(part)) { @@ -4873,6 +4944,15 @@ export function messagesFeature(ctx) { }); } if (youtubeId(url)) return youtubeEmbed(url, youtubeId(url)); + if (!label && previewable(url)) { + // a frozen feed row keeps the card (or the bare link) it first painted + let pv = previews.get(url) || null; + if (feedPaint) { + if (!feedPaint.media.has('pv:' + url)) feedPaint.media.set('pv:' + url, pv); + pv = feedPaint.media.get('pv:' + url); + } + if (pv) return linkCard(url, pv); + } const shown = label || (url.length > 64 ? url.slice(0, 61) + '…' : url); return h('a', { href: url, target: '_blank', rel: 'noopener noreferrer' }, shown); } diff --git a/src/style.css b/src/style.css index 1e5a1c7..11b9cf0 100644 --- a/src/style.css +++ b/src/style.css @@ -833,6 +833,21 @@ html.no-scroll { overflow: hidden; scrollbar-gutter: auto; } padding: 4px 7px; border-radius: 8px; background: rgba(18, 18, 18, .6); text-decoration: none; } .yt-open:hover { background: rgba(18, 18, 18, .85); text-decoration: none; } +/* A link, as the card its page describes: picture on top, title, a line, + the site. The whole card is the link. */ +.link-card { + display: block; margin-top: 6px; border: 1px solid var(--line); border-radius: 10px; + overflow: hidden; color: inherit; text-decoration: none; background: var(--surface2); +} +.link-card:hover { text-decoration: none; border-color: var(--line-strong); } +.link-thumb { display: block; width: 100%; max-height: 220px; object-fit: cover; background: var(--surface2); } +.link-meta { padding: 8px 10px; display: flex; flex-direction: column; gap: 2px; min-width: 0; } +.link-title, .link-desc { + display: -webkit-box; -webkit-line-clamp: 2; -webkit-box-orient: vertical; overflow: hidden; overflow-wrap: anywhere; +} +.link-title { font-weight: 600; line-height: 1.3; } +.link-site { white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } +.quote-card .link-thumb { max-height: 140px; } /* the profile notes feed: page-scrolled (no inner scrollbox); on phones it runs edge to edge — the card walls step outside #app's padding */ .notes-feed { padding: 2px 14px; } diff --git a/tools/link-preview-test.js b/tools/link-preview-test.js new file mode 100644 index 0000000..52d86fe --- /dev/null +++ b/tools/link-preview-test.js @@ -0,0 +1,123 @@ +// A plain link in a post becomes the card its page describes: the registrar +// is asked for the page's Open Graph data (shimmed here), the picture is +// warmed with the rest of the post, and the row paints whole. A page with +// nothing to say stays a bare link; pictures and YouTube keep their own +// boxes. +// Run: bun tools/link-preview-test.js +import puppeteer from 'puppeteer-core'; +import { cacheKeyFor } from '../src/wallet.js'; +import { generateSecretKey, getPublicKey, finalizeEvent } from 'nostr-tools/pure'; +import { privateKeyFromSeedWords } from 'nostr-tools/nip06'; +import { buildHtml } from '../build.js'; +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +let ok = true; +const check = (n, c, d = '') => { console.log(` ${c ? '✓' : '✗'} ${n}${d ? ' — ' + d : ''}`); if (!c) ok = false; }; +const AUTHOR = generateSecretKey(); const APK = getPublicKey(AUTHOR); +const now = Math.floor(Date.now() / 1000); +const PNG = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==', 'base64'); +const POSTS = [ + finalizeEvent({ kind: 1, created_at: now - 30, tags: [], content: 'Well, yeah https://news.example/story/1' }, AUTHOR), + finalizeEvent({ kind: 1, created_at: now - 60, tags: [], content: 'nothing here https://blank.example/page' }, AUTHOR), + finalizeEvent({ kind: 1, created_at: now - 90, tags: [], content: 'a picture http://localhost:5311/pic.png' }, AUTHOR), +]; +const PROFILE = finalizeEvent({ kind: 0, created_at: now - 100, tags: [], content: JSON.stringify({ name: 'Linker', picture: 'http://localhost:5311/face.png' }) }, AUTHOR); + +const html = await buildHtml({ minify: true, pwa: false }); +const previewHits = []; +const server = Bun.serve({ port: 5311, fetch: (req) => { + const u = new URL(req.url); + if (u.pathname === '/preview') { + const target = u.searchParams.get('url'); previewHits.push(target); + const meta = target === 'https://news.example/story/1' + ? { url: target, title: 'Lawmakers doubt the plan', description: 'A story in one line.', image: 'http://localhost:5311/card.png', site: 'News Example' } + : {}; + return new Response(JSON.stringify(meta), { headers: { 'content-type': 'application/json', 'access-control-allow-origin': '*' } }); + } + if (u.pathname.endsWith('.png')) return new Response(PNG, { headers: { 'content-type': 'image/png' } }); + if (u.pathname.startsWith('/punks') || u.pathname === '/verify-worker.js') return new Response(Bun.file('dist' + u.pathname)); + return new Response(html, { headers: { 'content-type': 'text/html' } }); +} }); +const browser = await puppeteer.launch({ executablePath: '/usr/bin/google-chrome', headless: 'new', args: ['--no-sandbox'] }); +const page = await browser.newPage(); +await page.evaluateOnNewDocument((posts, profile) => { + // the registrar's preview endpoint answers from this test's server + const realFetch = window.fetch; + window.fetch = (u, o) => realFetch(String(u).startsWith('https://names.coinos.io/preview') ? String(u).replace('https://names.coinos.io', 'http://localhost:5311') : u, o); + const Real = window.WebSocket; + window.WebSocket = function (...a) { + const ws = new Real(...a); + const send = ws.send.bind(ws); + ws.send = (data) => { + try { + const m = JSON.parse(data); + if (m[0] === 'REQ') { + const f = m[2] || {}; + let evs = []; + if ((f.kinds || []).includes(1) && (f.authors || []).includes(profile.pubkey)) evs = posts; + if ((f.kinds || []).includes(0) && (f.authors || []).includes(profile.pubkey)) evs = [profile]; + setTimeout(() => { + for (const e of evs) ws.dispatchEvent(new MessageEvent('message', { data: JSON.stringify(['EVENT', m[1], e]) })); + ws.dispatchEvent(new MessageEvent('message', { data: JSON.stringify(['EOSE', m[1]]) })); + }, 0); + return; + } + if (m[0] === 'EVENT') return; + } catch {} + return send(data); + }; + return ws; + }; + window.WebSocket.prototype = Real.prototype; Object.assign(window.WebSocket, Real); +}, POSTS, PROFILE); +const errs = []; +page.on('pageerror', (e) => errs.push(String(e))); +const click = (t) => page.evaluate((x) => { const e = [...document.querySelectorAll('button')].find((n) => n.textContent.trim().toLowerCase().includes(x)); if (e) { e.click(); return true; } return false; }, t); +const waitText = async (x, ms = 25000) => { for (let i = 0; i < ms / 250; i++) { if ((await page.evaluate(() => document.body.innerText)).toLowerCase().includes(x)) return true; await sleep(250); } return false; }; + +try { + await page.setViewport({ width: 390, height: 844 }); + await page.goto('http://localhost:5311/', { waitUntil: 'domcontentloaded' }); + await page.evaluate(() => localStorage.setItem('btc-wallet-network', 'regtest')); + await page.reload({ waitUntil: 'domcontentloaded' }); await sleep(400); + await click('create a new wallet'); await sleep(600); + await page.waitForSelector('.words .w .t'); + const mn = (await page.$$eval('.words .w .t', (l) => l.map((e) => e.textContent.trim()))).join(' '); + await click('skip verification'); + await waitText('receive', 20000); + const base = cacheKeyFor(mn + '\n'); + // follow the author, so the feed asks for their posts + await page.evaluate(([k, pk]) => { + localStorage.setItem(k + ':follows', JSON.stringify({ tags: [['p', pk]], c: '', at: Math.floor(Date.now() / 1000) })); + localStorage.setItem(k + ':profiles', JSON.stringify({ [pk]: { name: 'Linker', t: Date.now() } })); + }, [base, APK]); + await page.reload({ waitUntil: 'domcontentloaded' }); + await waitText('receive', 20000); + await page.evaluate(() => { const b = [...document.querySelectorAll('button')].find((e) => /message/i.test(e.getAttribute('aria-label') || '')); if (b) b.click(); }); + await sleep(500); + await page.evaluate(() => { const e = [...document.querySelectorAll('.item')].find((n) => /feed/i.test(n.textContent)); if (e) e.click(); }); + await page.waitForSelector('.notes-feed:not([data-booting]) > .row', { timeout: 30000 }); + await sleep(800); + const rows = await page.evaluate(() => [...document.querySelectorAll('.notes-feed > .row')].map((r) => ({ + text: r.textContent.trim().slice(0, 60), + card: !!r.querySelector('.link-card'), + title: r.querySelector('.link-title')?.textContent, + thumb: r.querySelector('.link-thumb')?.getAttribute('src') || r.querySelector('.link-thumb')?.dataset.lazySrc, + thumbShown: (() => { const i = r.querySelector('.link-thumb'); return !!i && i.complete && i.naturalWidth > 0; })(), + bareLink: !!r.querySelector('a[href^="https://blank.example"]'), + img: !!r.querySelector('img.note-img'), + }))); + const story = rows.find((r) => /well, yeah/i.test(r.text)); + const blank = rows.find((r) => /nothing here/i.test(r.text)); + const pic = rows.find((r) => /a picture/i.test(r.text)); + check('a post with a link paints its card', !!story && story.card, JSON.stringify(story)); + check('...with the page\'s title', story?.title === 'Lawmakers doubt the plan', story?.title); + check('...and its picture, already decoded', !!story?.thumb && story.thumbShown, JSON.stringify({ thumb: story?.thumb, shown: story?.thumbShown })); + check('a page with nothing to show stays a bare link', !!blank && blank.bareLink && !blank.card, JSON.stringify(blank)); + check('a picture link is still a picture, not a card', !!pic && pic.img && !pic.card, JSON.stringify(pic)); + check('the registrar was asked once per link', previewHits.filter((u) => u === 'https://news.example/story/1').length === 1 && !previewHits.some((u) => u.endsWith('.png')), JSON.stringify(previewHits)); + const kept = await page.evaluate(() => JSON.parse(localStorage.getItem('coinos-link-previews') || '[]').length); + check('the card is remembered for next time', kept === 1, kept + ' kept'); + check('no page errors', errs.length === 0, errs.slice(0, 2).join(' | ')); +} finally { await browser.close(); server.stop(true); } +console.log(ok ? '\n✅ links show as the cards their pages describe' : '\n❌ failed'); +process.exit(ok ? 0 : 1); |
