diff options
| -rw-r--r-- | src/features/messages.js | 79 | ||||
| -rw-r--r-- | tools/video-events-test.js | 139 |
2 files changed, 202 insertions, 16 deletions
diff --git a/src/features/messages.js b/src/features/messages.js index f4e01b7..42e2839 100644 --- a/src/features/messages.js +++ b/src/features/messages.js @@ -536,8 +536,13 @@ export function messagesFeature(ctx) { // copies go there too, so other clients reading that list see them. let ownInbox = []; const toOwnInbox = (wrap) => { if (ownInbox.length) publishOn(ownInbox, wrap, { onauth: relayAuth }); }; - // what a feed or a profile lists: notes, and polls (NIP-88) - const FEED_KINDS = [1, 1068]; + // NIP-71 video events: a clip posted as itself — normal (21) and short, + // vertical (22), and the addressable 34235/34236 that divine.video and + // others publish. The file is in an imeta tag, not the text. + const VIDEO_KINDS = [21, 22, 34235, 34236]; + const isVideoEvent = (ev) => !!ev && VIDEO_KINDS.includes(ev.kind); + // what a feed or a profile lists: notes, polls (NIP-88) and video events + const FEED_KINDS = [1, 1068, ...VIDEO_KINDS]; // A repost (NIP-18 kind 6) of someone's note shows in a feed of the // people who reposted it, as the note itself under "<name> reposted". // Only reposts that carry the note (every major client embeds it) and @@ -4091,7 +4096,7 @@ export function messagesFeature(ctx) { const editOf = (ev) => (ev && ev.id && ev.pubkey && (ev.kind === 1 || ev.kind === 1111) ? postEdits().ed.get(editKey(ev.id, ev.pubkey)) : undefined); // what a post says now, and the emoji its words may use - const noteText = (ev) => { const e = editOf(ev); return e ? e.c : ev.content; }; + const noteText = (ev) => { if (isVideoEvent(ev)) return videoEventText(ev); const e = editOf(ev); return e ? e.c : ev.content; }; const noteEmojiTags = (ev) => { const e = editOf(ev); return emojiTagMap(e ? [...ev.tags, ...e.tags] : ev.tags); }; function notePostEdit(ev) { const ids = (ev.tags || []).filter((x) => x[0] === 'e' && /^[0-9a-f]{64}$/.test(x[1] || '')).map((x) => x[1]); @@ -6283,6 +6288,43 @@ export function messagesFeature(ctx) { // moov-first file — with a short cap, since it gates the row. Remembered // on disk so a clip seen once is never probed again. const VIDEO_RE = /\.(mp4|webm|mov|m4v)(\?[^\s]*)?$/i; + // files known to be clips without saying so in their name (divine.video's + // have no extension): an imeta tag's video/* type, or a video event's file + const videoUrls = new Set(); + const isVideoUrl = (url) => VIDEO_RE.test(url) || videoUrls.has(url); + const imetaFields = (tag) => { const f = {}; for (const p of tag.slice(1)) { const i = typeof p === 'string' ? p.indexOf(' ') : -1; if (i > 0) f[p.slice(0, i)] = p.slice(i + 1); } return f; }; + // A video event's file: of the versions its imeta tags offer, the one + // nearest 1080 on its short side (a phone's screen; not the 4K original), + // never an HLS playlist; else its url tag. + function videoEventFile(ev) { + let best = null; + for (const tag of ev.tags || []) { + if (tag[0] !== 'imeta') continue; + const f = imetaFields(tag); + if (!/^https?:\/\//.test(f.url || '') || /mpegurl/i.test(f.m || '') || /\.m3u8(\?|$)/i.test(f.url)) continue; + if (f.m && !/^video\//.test(f.m)) continue; + const dm = /^(\d+)(?:\.0+)?x(\d+)/.exec(f.dim || ''); + const edge = dm ? Math.min(+dm[1], +dm[2]) : 0; + const score = !edge ? -400 : -Math.abs(edge - 1080) - (edge > 1080 ? 1 : 0); // nearest 1080; the smaller on a tie + if (!best || score > best.score) best = { url: f.url, image: f.image, score }; + } + if (!best) { const u = (ev.tags.find((x) => x[0] === 'url') || [])[1]; if (/^https?:\/\//.test(u || '')) best = { url: u }; } + if (best) videoUrls.add(best.url); + return best; + } + // what a video event reads as: its title, its words, and its file (drawn + // as the clip it is) + function videoEventText(ev) { + learnVideoMeta(ev); + const f = videoEventFile(ev); + const title = String((ev.tags.find((x) => x[0] === 'title') || [])[1] || '').trim(); + const words = String(ev.content || '').trim(); + const parts = []; + if (title && title !== words && !words.startsWith(title)) parts.push(title); + if (words) parts.push(words); + if (f && !words.includes(f.url)) parts.push(f.url); + return parts.join('\n\n'); + } const AUDIO_RE = /\.(mp3|m4a|aac|ogg|oga|opus|wav|flac)(\?[^\s]*)?$/i; const VIDEO_PROBE_MS = 3000, VIDEO_DIMS_KEEP = 400, VIDEO_DIMS_KEY = 'coinos-video-dims-v2'; // v2: turned clips stand upright const videoDims = new Map(); // url -> { width, height } @@ -6338,9 +6380,9 @@ export function messagesFeature(ctx) { function learnVideoMeta(ev) { for (const tag of ev?.tags || []) { if (tag[0] !== 'imeta') continue; - const f = {}; - for (const p of tag.slice(1)) { const i = typeof p === 'string' ? p.indexOf(' ') : -1; if (i > 0) f[p.slice(0, i)] = p.slice(i + 1); } - if (!f.url || !(VIDEO_RE.test(f.url) || /^video\//.test(f.m || ''))) continue; + const f = imetaFields(tag); + if (!f.url || !(VIDEO_RE.test(f.url) || /^video\//.test(f.m || '') || (isVideoEvent(ev) && !f.m))) continue; + if (!VIDEO_RE.test(f.url)) videoUrls.add(f.url); const have = videoInfo.get(f.url) || {}; const d = {}; if (!have.size && +f.size > 0) d.size = +f.size; @@ -6608,7 +6650,7 @@ export function messagesFeature(ctx) { for (const part of String(content || '').split(NOTE_SPLIT)) { const md = part && MD_PARTS.exec(part); const url = md ? md[3] : (/^https?:\/\//i.test(part || '') ? part : null); - if (url && !(md && md[1]) && VIDEO_RE.test(url)) urls.push(url); + if (url && !(md && md[1]) && isVideoUrl(url)) urls.push(url); } return urls; } @@ -6784,10 +6826,13 @@ export function messagesFeature(ctx) { // (`staged` holds the note's id too, for a repost) const noteOf = (e) => (e.kind === REPOST ? repostInner(e) : e); const notesIn = new Set([...c.notes, ...(c.deferred || [])].map((e) => noteOf(e)?.id).filter(Boolean)); + // an addressable post (a 34235/34236 clip) once, however many versions of it come + for (const e of [...c.notes, ...(c.deferred || [])]) { const a = noteOf(e) && addrOf(noteOf(e)); if (a) notesIn.add(a); } + const addrNew = (e) => { const n = noteOf(e); const a = n && addrOf(n); return !a || (!notesIn.has(a) && !staged.has(a) && !!staged.add(a)); }; const fresh = (e) => { const n = noteOf(e); return !!n && !notesIn.has(n.id) && !staged.has(n.id); }; - const unstage = (e) => { staged.delete(e.id); const n = noteOf(e); if (n) staged.delete(n.id); }; + const unstage = (e) => { staged.delete(e.id); const n = noteOf(e); if (n) { staged.delete(n.id); const a = addrOf(n); if (a) staged.delete(a); } }; let add = (evs || []).filter((e) => (e.kind === REPOST ? !!repostInner(e) && !hidden(repostInner(e)) && !reboosted(e) : FEED_KINDS.includes(e.kind) && !isReply(e)) && !hidden(e) - && fresh(e) && !known.has(e.id) && !staged.has(e.id) && known.add(e.id) && staged.add(e.id) && staged.add(noteOf(e).id)); + && fresh(e) && !known.has(e.id) && !staged.has(e.id) && addrNew(e) && known.add(e.id) && staged.add(e.id) && staged.add(noteOf(e).id)); if (!add.length) return false; const def = feedDef(c.id); if (def && def.curated) { @@ -8080,7 +8125,7 @@ export function messagesFeature(ctx) { return x >= 0 && x <= w && y >= 0 && y <= hh; } function urlNode(url, { label = null, isImage = false } = {}) { - if (VIDEO_RE.test(url)) { + if (isVideoUrl(url)) { return videoNode(url, { stable: !!feedPaint }); } if (AUDIO_RE.test(url)) return audioNode(url, label); @@ -9856,7 +9901,7 @@ export function messagesFeature(ctx) { // An event of a kind this app has no renderer for (a chess game, kind // 64, reached through a reaction to it) is shown as what its author's // client said it is (NIP-31 alt), never as raw content dressed as a post. - const foreign = ev.kind !== 1 && ev.kind !== ARTICLE && ev.kind !== POLL && ev.kind !== COMMENT && !zapRec; + const foreign = ev.kind !== 1 && ev.kind !== ARTICLE && ev.kind !== POLL && ev.kind !== COMMENT && !isVideoEvent(ev) && !zapRec; const altText = foreign ? ((ev.tags.find((x) => x[0] === 'alt') || [])[1] || t('noteForeignKind', { kind: ev.kind })) : null; const isReply = ev.kind === COMMENT || (!foreign && !zapRec && ev.tags.some((x) => x[0] === 'e')); const canZap = canZapPk(pk); @@ -11346,7 +11391,9 @@ export function messagesFeature(ctx) { // for more picks up where it stopped instead of re-reading the relays. const mediaIdx = new Map(); // key -> { key, images: Map, videos: Map, checked: Set, until, end, loaded } const mediaIdxKey = (feedId) => (mePk() || 'visitor') + '|' + feedId; - const hasMediaOf = (ev, kind) => ev.kind === REPOST ? false : kind === 'videos' ? noteVideoUrls(ev.content).length > 0 + // a post's clips: a video event's file, or the clips linked in a note + const clipUrlsOf = (ev) => (isVideoEvent(ev) ? [videoEventFile(ev)?.url].filter(Boolean) : noteVideoUrls(ev.content)); + const hasMediaOf = (ev, kind) => ev.kind === REPOST ? false : kind === 'videos' ? clipUrlsOf(ev).length > 0 : noteMediaUrls(ev.content).some((u) => !u.startsWith('https://i.ytimg.com/')); function mediaIndex(feedId) { const key = mediaIdxKey(feedId); @@ -11723,7 +11770,7 @@ export function messagesFeature(ctx) { // a grid shows the picture itself — no warning to fold it behind if (curated ? notForPublic(ev) : modOn('sensitive') && sensitiveWhy(ev) !== null && !isMe(ev.pubkey)) continue; if (kind === 'videos') learnVideoMeta(ev); - const urls = kind === 'videos' ? noteVideoUrls(ev.content).filter(verticalClip) + const urls = kind === 'videos' ? clipUrlsOf(ev).filter(verticalClip) : noteMediaUrls(ev.content).filter((u) => !u.startsWith('https://i.ytimg.com/')); for (const url of urls) { const it = { ev, url }; @@ -11745,7 +11792,7 @@ export function messagesFeature(ctx) { const ev = row.kind === REPOST ? repostInner(row) || row : row; if (modOn('sensitive') && sensitiveWhy(ev) !== null && !isMe(ev.pubkey)) continue; if (kind === 'videos') learnVideoMeta(ev); - const urls = kind === 'videos' ? noteVideoUrls(ev.content) + const urls = kind === 'videos' ? clipUrlsOf(ev) : noteMediaUrls(ev.content).filter((u) => !u.startsWith('https://i.ytimg.com/')); for (const url of urls) out.push({ ev, url, row }); } @@ -11832,9 +11879,9 @@ export function messagesFeature(ctx) { } // The words of the post, without the links to the media it carries. function mediaCaption(ev) { - let s = String(ev.content || ''); + let s = isVideoEvent(ev) ? videoEventText(ev) : String(ev.content || ''); s = s.replace(new RegExp(MD_LINK, 'gi'), (m) => (MD_PARTS.exec(m)?.[1] ? '' : m)); - for (const u of [...noteMediaUrls(ev.content), ...noteVideoUrls(ev.content)]) s = s.split(u).join(''); + for (const u of [...noteMediaUrls(s), ...noteVideoUrls(s)]) s = s.split(u).join(''); return plainExcerpt(s); } function setMediaMode(m) { diff --git a/tools/video-events-test.js b/tools/video-events-test.js new file mode 100644 index 0000000..bef0956 --- /dev/null +++ b/tools/video-events-test.js @@ -0,0 +1,139 @@ +// NIP-71 video events in the feed: a clip posted as itself (kind 21, 22, and +// the addressable 34235/34236 divine.video publishes) is asked for with the +// notes and drawn as a post — its title, its words, and the clip from its +// imeta tag (the version nearest 1080, its poster), even a file whose name +// doesn't say it's a video. An addressable clip shows once, not once per +// version. A note linking such a file, typed video/* in its imeta, plays it +// too. And the clips are in the pager. +// Run: bun tools/video-events-test.js +import puppeteer from './puppeteer.js'; +import { cacheKeyFor } from '../src/wallet.js'; +import { generateSecretKey, getPublicKey, finalizeEvent } from 'nostr-tools/pure'; +import { buildHtml } from '../build.js'; +import { existsSync } from 'node:fs'; +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +let ok = true; +const check = (n, c, d = '') => { console.log(` ${c ? '✓' : '✗'} ${n}${d ? ' — ' + d : ''}`); if (!c) ok = false; }; +const CLIP = '/tmp/coinos-events-clip.webm'; +if (!existsSync(CLIP)) Bun.spawnSync(['ffmpeg', '-y', '-loglevel', 'error', '-f', 'lavfi', '-i', 'testsrc=size=180x320:rate=10', '-t', '10', '-c:v', 'libvpx', '-b:v', '100k', CLIP]); +const PNG = Bun.spawnSync(['ffmpeg', '-loglevel', 'error', '-f', 'lavfi', '-i', 'testsrc=size=180x320', '-frames:v', '1', '-f', 'image2', '-c:v', 'png', 'pipe:1']).stdout; + +const SK = generateSecretKey(), AUTHOR = getPublicKey(SK); +const now = Math.floor(Date.now() / 1000); +const html = await buildHtml({ minify: true, pwa: false }); +const server = Bun.serve({ port: 0, fetch: (req) => { + const path = new URL(req.url).pathname; + if (path.startsWith('/v/')) return new Response(Bun.file(CLIP), { headers: { 'content-type': 'video/webm' } }); + if (path.endsWith('.png')) return new Response(PNG, { headers: { 'content-type': 'image/png' } }); + if (path.startsWith('/punks') || path === '/verify-worker.js') return new Response(Bun.file('dist' + path)); + return new Response(html, { headers: { 'content-type': 'text/html' } }); +} }); +const O = server.url.origin; +const ev = (kind, content, tags, ago) => finalizeEvent({ kind, created_at: now - ago, tags, content }, SK); +const EVENTS = [ + // a normal video event: two versions on offer, a 4K and a 720p — the 720p is the one + ev(21, 'about the normal one', [['title', 'A normal clip'], + ['imeta', 'url ' + O + '/v/normal-4k.mp4', 'm video/mp4', 'dim 3840x2160'], + ['imeta', 'url ' + O + '/v/normal-720.mp4', 'm video/mp4', 'dim 1280x720', 'image ' + O + '/normal.png']], 100), + // divine.video's way: addressable, a file with no extension, a poster + ev(34236, '', [['d', 'vine-1'], ['title', 'Short one'], ['imeta', 'url ' + O + '/v/shortfile', 'm video/mp4', 'image ' + O + '/short.png', 'dim 180x320']], 200), + // ...and an older version of the same one + ev(34236, '', [['d', 'vine-1'], ['title', 'old version'], ['imeta', 'url ' + O + '/v/shortfile-old', 'm video/mp4']], 900), + // a note linking a file whose name doesn't say, typed in its imeta + ev(1, 'look at this ' + O + '/v/notefile', [['imeta', 'url ' + O + '/v/notefile', 'm video/mp4', 'dim 180x320']], 300), + ev(1, 'just words, nothing to watch here at all', [], 400), +]; + +const browser = await puppeteer.launch({ executablePath: '/usr/bin/google-chrome', headless: 'new', args: ['--no-sandbox', '--autoplay-policy=no-user-gesture-required'] }); +const page = await browser.newPage(); +await page.evaluateOnNewDocument((evs) => { + const match = (f, e) => (!f.kinds || f.kinds.includes(e.kind)) && (!f.authors || f.authors.includes(e.pubkey)) && (!f.ids || f.ids.includes(e.id)) + && (!f.until || e.created_at <= f.until) && (!f.since || e.created_at >= f.since) && !Object.keys(f).some((k) => k.startsWith('#')); + window.__kindsAsked = new Set(); + class FakeWS { + constructor(url) { this.url = String(url); this.readyState = 0; setTimeout(() => { this.readyState = 1; this.onopen && this.onopen({}); }, 5); } + send(raw) { + const m = JSON.parse(raw); + const reply = (x) => setTimeout(() => this.readyState === 1 && this.onmessage && this.onmessage({ data: JSON.stringify(x) }), 5); + if (m[0] === 'REQ') { + for (const f of m.slice(2)) { + if (f.cache) continue; + if (f.authors) for (const k of f.kinds || []) window.__kindsAsked.add(k); + const hits = evs.filter((e) => match(f, e)).sort((a, b) => b.created_at - a.created_at); + for (const e of f.limit ? hits.slice(0, f.limit) : hits) reply(['EVENT', m[1], e]); + } + reply(['EOSE', m[1]]); + } else if (m[0] === 'EVENT') reply(['OK', m[1].id, true, '']); + else if (m[0] === 'COUNT') reply(['COUNT', m[1], { count: 0 }]); + } + close() { this.readyState = 3; this.onclose && this.onclose({ code: 1000 }); } + addEventListener(t, fn) { this['on' + t] = fn; } + removeEventListener() {} + } + FakeWS.CONNECTING = 0; FakeWS.OPEN = 1; FakeWS.CLOSING = 2; FakeWS.CLOSED = 3; + window.WebSocket = FakeWS; +}, EVENTS); +const errs = []; +page.on('pageerror', (e) => errs.push(String(e))); +const click = (t) => page.evaluate((x) => { const e = [...document.querySelectorAll('button')].find((n) => n.textContent.trim().toLowerCase().includes(x)); if (e) { e.click(); return true; } return false; }, t); +const waitText = async (x, ms = 25000) => { for (let i = 0; i < ms / 250; i++) { if ((await page.evaluate(() => document.body.innerText)).toLowerCase().includes(x)) return true; await sleep(250); } return false; }; +const waitFor = async (fn, arg, ms = 10000) => { for (let i = 0; i < ms / 100; i++) { if (await page.evaluate(fn, arg)) return true; await sleep(100); } return false; }; +const row = (w) => page.evaluate((x) => { + const rs = [...document.querySelectorAll('.notes-feed > [data-key]')].filter((n) => n.textContent.includes(x)); + const r = rs[0]; const v = r && r.querySelector('video.note-video'); + return r ? { n: rs.length, video: v ? (v.getAttribute('src') || '').replace(location.origin, '') : null, poster: v ? (v.getAttribute('poster') || '').replace(location.origin, '') : null, text: r.querySelector('.note-text')?.innerText.trim().slice(0, 120) } : null; +}, w); + +try { + await page.setViewport({ width: 390, height: 844 }); + await page.goto(server.url.href, { waitUntil: 'domcontentloaded' }); + await page.evaluate(() => localStorage.setItem('btc-wallet-network', 'regtest')); + await page.reload({ waitUntil: 'domcontentloaded' }); await sleep(400); + await click('create a new wallet'); await sleep(600); + await page.waitForSelector('.words .w .t'); + const mn = (await page.$$eval('.words .w .t', (l) => l.map((e) => e.textContent.trim()))).join(' '); + await click('skip verification'); + await waitText('receive', 20000); + const base = cacheKeyFor(mn + '\n'); + await page.evaluate(([k, pk]) => { + localStorage.setItem(k + ':follows', JSON.stringify({ tags: [['p', pk]], c: '', at: Math.floor(Date.now() / 1000) })); + localStorage.setItem(k + ':profiles', JSON.stringify({ [pk]: { name: 'Clip Maker', t: Date.now() } })); + localStorage.setItem('btc-wallet-feed:' + k, 'following'); + }, [base, AUTHOR]); + await page.reload({ waitUntil: 'domcontentloaded' }); + await waitText('receive', 20000); + await page.evaluate(() => document.querySelector('.app-nav-button[aria-label="Feed"]').click()); + check('the feed fills', await waitFor(() => [...document.querySelectorAll('.notes-feed > [data-key]')].some((n) => n.textContent.includes('just words')), null, 20000)); + await sleep(1500); + check('the feed asks for video events with the notes', await page.evaluate(() => [21, 22, 34235, 34236].every((k) => window.__kindsAsked.has(k))), JSON.stringify(await page.evaluate(() => [...window.__kindsAsked]))); + + console.log('\n[a kind 21 video event]'); + let r = await row('A normal clip'); + check('it is a post: its title and its words', r && /A normal clip/.test(r.text) && /about the normal one/.test(r.text), JSON.stringify(r)); + check('…with its clip — the version nearest 1080, not the 4K', r && r.video === '/v/normal-720.mp4', JSON.stringify(r)); + check('…its poster from the imeta', r && r.poster === '/normal.png', JSON.stringify(r)); + check('not shown as a kind this app can\'t draw', !(await page.evaluate(() => /kind 21|kind 34236/i.test(document.querySelector('.notes-feed').innerText)))); + + console.log('\n[a divine.video clip (34236): addressable, a file with no extension]'); + r = await row('Short one'); + check('it plays: the file is a clip though its name doesn\'t say', r && r.video === '/v/shortfile', JSON.stringify(r)); + check('…with its poster', r && r.poster === '/short.png', JSON.stringify(r)); + check('…once: its older version isn\'t shown as well', r && r.n === 1 && !(await row('old version')), JSON.stringify([r, await row('old version')])); + + console.log('\n[a note linking such a file, typed in its imeta]'); + r = await row('look at this'); + check('it plays the file', r && r.video === '/v/notefile', JSON.stringify(r)); + + console.log('\n[the pager has them]'); + await page.evaluate(() => { const r = [...document.querySelectorAll('.notes-feed > [data-key]')].find((n) => n.textContent.includes('A normal clip')); r.querySelector('video.note-video').scrollIntoView({ block: 'center' }); }); + await sleep(500); + const at = await page.evaluate(() => { const r = [...document.querySelectorAll('.notes-feed > [data-key]')].find((n) => n.textContent.includes('A normal clip')); const b = r.querySelector('video.note-video').getBoundingClientRect(); return { x: b.x + b.width * 0.2, y: b.y + b.height * 0.3 }; }); + await page.mouse.click(at.x, at.y); + check('a tap opens the pager', await waitFor(() => !!document.querySelector('.media-pager.videos'), null, 4000)); + const slides = await page.evaluate(() => [...document.querySelectorAll('.mp-slide')].map((s) => s.getAttribute('data-k').split('|')[1].replace(location.origin, ''))); + check('…over the three clips, in the feed\'s order', JSON.stringify(slides) === JSON.stringify(['/v/normal-720.mp4', '/v/shortfile', '/v/notefile']), JSON.stringify(slides)); + check('…the caption, the title and words (no file link)', await page.evaluate(() => { const c = document.querySelector('.mp-slide .mp-caption'); return !!c && /A normal clip/.test(c.textContent) && !/\/v\//.test(c.textContent); })); + check('no page errors', errs.length === 0, errs.slice(0, 2).join(' | ')); +} finally { await browser.close(); server.stop(true); } +console.log(ok ? '\n✅ video events are posts with their clips' : '\n❌ failed'); +process.exit(ok ? 0 : 1); |