diff options
| author | Adam Soltys <[email protected]> | 2026-10-03 15:05:52 -0700 |
|---|---|---|
| committer | Adam Soltys <[email protected]> | 2026-10-03 15:05:52 -0700 |
| commit | 8edbd7b7544d829f44d36b430c4fcbbc521224a8 (patch) | |
| tree | 5c248b79a19a744779293b51bd5bdb022df725e5 | |
| parent | 09ce0c7f213802757b13501898c02cfe309bfbf8 (diff) | |
| download | coinosv3-8edbd7b7544d829f44d36b430c4fcbbc521224a8.tar.gz coinosv3-8edbd7b7544d829f44d36b430c4fcbbc521224a8.zip | |
Media views keep their own cache and crawl for media directly
Videos was slow because the grid read from the feed's page of posts:
the feed cache holds 30, so after every reload the grid re-crawled the
relays, ten posts a step, each post first waiting (up to 8 s) for its
avatar, pictures and link cards — none of which the grid shows.
- Each feed now keeps a media index (IndexedDB, src/media-cache.js): the
posts carrying pictures and those carrying clips, 400 of each, and how
far back it has looked. The grid and pager read from it; a reload
paints them at once.
- The crawl reads older posts straight into the index, 100 at a time,
without the readiness wait and without touching the feed's own page;
it resumes from where it stopped. The bottom of a feed is looked for
again after a day. Curated feeds still page the usual way.
- A tile's first frame is kept (a small JPEG in the Cache API, when the
host lets its pixels be read) and painted as a picture from then on: in
the grid, as the pager's poster and the feed's.
- Probe results kept for 2000 clips (was 600).
- Forgetting a wallet wipes the index and the frames.
tools/feed-media-cache-test.js
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01ULQZS9MQa6iAsXnrJUD9Du
| -rw-r--r-- | src/features/messages.js | 151 | ||||
| -rw-r--r-- | src/media-cache.js | 64 | ||||
| -rw-r--r-- | tools/feed-media-cache-test.js | 145 |
3 files changed, 350 insertions, 10 deletions
diff --git a/src/features/messages.js b/src/features/messages.js index c9497e9..3be79fa 100644 --- a/src/features/messages.js +++ b/src/features/messages.js @@ -29,6 +29,7 @@ import { packAddr, packNaddr, parsePackRef, parsePackAddr, parseEmojiSet, } from '../emoji.js'; import { saveInbox } from '../dm-inbox.js'; +import { loadMediaIndex, saveMediaIndex, wipeMediaIndex } from '../media-cache.js'; import { mergeFeedWindow } from '../feed-window.js'; import { createThreadStore } from '../thread-cache.js'; import { createFeedCache, FEED_CACHE_POSTS, FEED_SEED_POSTS } from '../feed-cache.js'; @@ -5485,7 +5486,7 @@ export function messagesFeature(ctx) { // the server won't serve ranges. Its track header gives the size on screen // too. Clips over the budget, above 1080p, or that we couldn't measure // wait for a tap — unless the reader asked for large clips to play. - const VIDEO_INFO_KEY = 'coinos-video-info', VIDEO_INFO_KEEP = 600, VIDEO_PROBE_BYTES = 65536; + const VIDEO_INFO_KEY = 'coinos-video-info', VIDEO_INFO_KEEP = 2000, VIDEO_PROBE_BYTES = 65536; const VIDEO_LARGE_KEY = 'coinos-video-large'; const videoInfo = new Map(); // url -> { size, w, h, image, faststart, ranges, probed, failed } try { for (const [u, d] of JSON.parse(localStorage.getItem(VIDEO_INFO_KEY) || '[]')) videoInfo.set(u, d); } catch {} @@ -6049,6 +6050,7 @@ export function messagesFeature(ctx) { let got = false; await queryStreamOn(relays, filter, async (ev) => { if (c.stopped) return; + if (merge && merge.sink) { if (qualifies(ev)) { merge.sink(ev); got = true; } return; } // the media crawl's: into the index, not the page sawBack(c, [ev]); if (qualifies(ev) && await mergeFeed([ev], merge, c)) got = true; }, maxWait); @@ -6279,7 +6281,7 @@ export function messagesFeature(ctx) { // a grid holds only the posts with media: keep fetching while its // foot is near (within reason — a firehose has no end) const g = document.querySelector('.media-grid'); - if (c.notes.length < 3000 && (!g || g.getBoundingClientRect().bottom < window.innerHeight * 3)) loadOlderFeed().catch(() => {}); + if (!g || g.getBoundingClientRect().bottom < window.innerHeight * 3) crawlMedia(c).catch(() => {}); return; } if (c.win && c.win.bottom) return; // rows below are held back by the window, not by paging @@ -6421,7 +6423,7 @@ export function messagesFeature(ctx) { // Nothing is fetched until the clip is about to play (preload none): a // window of rows used to pull every clip's metadata, which for a file // with its index at the end can be most of the file. - const v = h('video', { src: url, class: 'note-video', controls: true, poster: (videoInfo.get(url) || {}).image || undefined, + const v = h('video', { src: url, class: 'note-video', controls: true, poster: videoThumbs.get(url) || (videoInfo.get(url) || {}).image || undefined, preload: 'none', playsinline: true, muted: loud ? undefined : true, style: stable ? videoBoxStyle(feedMedia(url)) : undefined, // learned now, used by the next paint of any row with this clip @@ -9576,6 +9578,118 @@ export function messagesFeature(ctx) { } return null; }; + // ---- the media index: each feed's pictures and clips, kept apart ------- + // (src/media-cache.js) The grid reads from here, not from the feed's page + // of posts: a reload paints what was found before, at once, and the crawl + // for more picks up where it stopped instead of re-reading the relays. + const mediaIdx = new Map(); // key -> { key, images: Map, videos: Map, checked: Set, until, end, loaded } + const mediaIdxKey = (feedId) => (mePk() || 'visitor') + '|' + feedId; + const hasMediaOf = (ev, kind) => kind === 'videos' ? noteVideoUrls(ev.content).length > 0 + : noteMediaUrls(ev.content).some((u) => !u.startsWith('https://i.ytimg.com/')); + function mediaIndex(feedId) { + const key = mediaIdxKey(feedId); + let r = mediaIdx.get(key); + if (r) return r; + r = { key, images: new Map(), videos: new Map(), checked: new Set(), until: 0, end: false, loaded: false }; + mediaIdx.set(key, r); + loadMediaIndex(key).then(async (rec) => { + if (rec) { + for (const kind of ['images', 'videos']) for (const e of rec[kind] || []) if (!r[kind].has(e.id)) r[kind].set(e.id, e); + if (rec.until && (!r.until || rec.until < r.until)) r.until = rec.until; + // the bottom of a feed is looked for again after a day (a relay + // added, a follow back-filled) + if (rec.end && Date.now() - rec.end < 86_400_000) r.end = r.end || rec.end; + } + // kept frames first, so the first paint of the grid is pictures + await warmVideoThumbs([...r.videos.values()]); + r.loaded = true; + if (mediaMode !== 'all') scheduleRepaint(); + }); + return r; + } + function saveMediaIdx(r) { + clearTimeout(r.saveT); + r.saveT = setTimeout(() => { + const list = (m) => [...m.values()].sort((a, b) => b.created_at - a.created_at); + saveMediaIndex(r.key, { images: list(r.images), videos: list(r.videos), until: r.until, end: r.end }); + }, 1500); + } + // file the posts that carry media; each post is looked at once + function indexMedia(r, evs) { + let n = 0; + for (const e of evs) { + if (r.checked.has(e.id)) continue; + r.checked.add(e.id); + if (!FEED_KINDS.includes(e.kind) || isReply(e)) continue; + for (const kind of ['images', 'videos']) if (!r[kind].has(e.id) && hasMediaOf(e, kind)) { r[kind].set(e.id, e); n++; } + } + if (n) saveMediaIdx(r); + return n; + } + // Older posts for the grid, read straight into the index: a hundred a + // time, no waiting on avatars, pictures and link cards (the grid shows + // none of that), and the feed's own page of posts left as it was. + let mediaCrawling = null; + function crawlMedia(c = feedNow()) { + const def = feedDef(c.id); + if (!def || !feedHasQuery(def)) return Promise.resolve(); + if (def.curated) return loadOlderFeed(); // a curated feed pages its own way + const r = mediaIndex(c.id); + if (!r.loaded || r.end || mediaCrawling) return mediaCrawling || Promise.resolve(); + mediaCrawling = (async () => { + scheduleRepaint(); + const from = [r.until, c.notes.at(-1)?.created_at].filter(Boolean); + const until = (from.length ? Math.min(...from) : Math.floor(Date.now() / 1000) + 1) - 1; + let oldest = Infinity, saw = 0; + const sink = (ev) => { + saw++; + if (ev.created_at < oldest) oldest = ev.created_at; + noteForSpam(ev); + indexMedia(r, [ev]); + }; + try { await feedPass({ until, limit: 100 }, { sink }, c); } catch {} + if (!saw) r.end = Date.now(); + else r.until = Math.min(oldest, until); + saveMediaIdx(r); + })().finally(() => { mediaCrawling = null; scheduleRepaint(); }); + return mediaCrawling; + } + + // A clip's first frame, kept: drawn from the tile's player once it has + // one (when the host allows the pixels to be read), stored small in the + // Cache API, and painted as a plain picture from then on — in the grid, + // as the pager's poster and the feed's. + const VIDEO_THUMB_CACHE = 'coinos-video-thumbs-v1'; + const videoThumbs = new Map(); // url -> object URL + const thumbKey = (url) => 'https://video-thumb.invalid/?u=' + encodeURIComponent(url); + async function warmVideoThumbs(evs) { + if (typeof caches === 'undefined') return; + let found = 0; + try { + const store = await caches.open(VIDEO_THUMB_CACHE); + for (const e of evs) for (const url of noteVideoUrls(e.content)) { + if (videoThumbs.has(url)) continue; + const res = await store.match(thumbKey(url)); + if (res) { videoThumbs.set(url, URL.createObjectURL(await res.blob())); found++; } + } + } catch {} + if (found && mediaMode !== 'all') scheduleRepaint(); + } + function keepVideoThumb(v, url) { + if (videoThumbs.has(url) || !v.videoWidth) return; + try { + const w = 270, hh = Math.round(w * v.videoHeight / v.videoWidth); + const cv = document.createElement('canvas'); + cv.width = w; cv.height = hh; + cv.getContext('2d').drawImage(v, 0, 0, w, hh); + cv.toBlob((blob) => { + if (!blob) return; + videoThumbs.set(url, URL.createObjectURL(blob)); + if (typeof caches !== 'undefined') caches.open(VIDEO_THUMB_CACHE).then((s) => s.put(thumbKey(url), new Response(blob, { headers: { 'content-type': 'image/jpeg' } }))).catch(() => {}); + }, 'image/jpeg', 0.75); + } catch {} // a host that won't let its pixels be read: the tile keeps its player + } + // Videos is for flipping through phone-shaped clips: only those known to // stand taller than they're wide. A clip of unknown shape is measured // (the header probe, or a player when the index sits at the end of a @@ -9598,7 +9712,10 @@ export function messagesFeature(ctx) { } function mediaItems(c, kind) { const out = [], seen = new Set(); - for (const ev of c.notes) { + const r = mediaIndex(c.id); + indexMedia(r, c.notes); + const evs = [...r[kind].values()].sort((a, b) => b.created_at - a.created_at); + for (const ev of evs) { if (hidden(ev)) continue; if (kind === 'videos') learnVideoMeta(ev); const urls = kind === 'videos' ? noteVideoUrls(ev.content).filter(verticalClip) @@ -9636,7 +9753,8 @@ export function messagesFeature(ctx) { function mediaGrid(c) { const items = mediaItems(c, mediaMode); if (!items.length) { - return c.booting || c.status === 'loading' || c.loadingMore || !c.end + const r = mediaIndex(c.id); + return !r.loaded || mediaCrawling || (!r.end && feedHasQuery(feedDef(c.id)) && !feedDef(c.id)?.curated) || c.booting || c.status === 'loading' || c.loadingMore || (mediaMode === 'videos' && (videoProbes.size || videoProbeQueue.length)) ? h('div', { class: 'row gap6', 'data-key': 'media-empty', style: 'justify-content:center;padding:12px 0' }, h('span', { class: 'spinner sm' })) : h('div', { class: 'small faint', 'data-key': 'media-empty', style: 'text-align:center;padding:12px 0' }, @@ -9645,7 +9763,7 @@ export function messagesFeature(ctx) { return h('div', { class: 'media-grid', 'data-key': 'media-grid:' + mediaMode }, ...items.map((it) => { const k = mediaKey(it); - const poster = mediaMode === 'videos' ? posterOf(it.ev, it.url) || (videoInfo.get(it.url) || {}).image : null; + const poster = mediaMode === 'videos' ? videoThumbs.get(it.url) || posterOf(it.ev, it.url) || (videoInfo.get(it.url) || {}).image : null; return h('button', { class: 'media-tile' + (mediaMode === 'videos' ? ' video' : ''), type: 'button', 'data-key': 'tile:' + k, 'aria-label': displayName(it.ev.pubkey), onClick: () => openMediaPager(mediaMode, k), @@ -9681,7 +9799,18 @@ export function messagesFeature(ctx) { const heavy = videoHeavy(url); const still = d && (d.probed ? d.faststart || d.ranges : !heavy); if (still) { - v.muted = true; v.src = v.getAttribute('data-src'); + v.muted = true; + // readable pixels, so the frame can be kept; a host that refuses + // CORS gets a second, plain try (shown, not kept) + v.crossOrigin = 'anonymous'; + const keep = () => { if (v.readyState >= 2) keepVideoThumb(v, url); }; + v.addEventListener('seeked', keep, { once: true }); + v.addEventListener('loadeddata', () => setTimeout(keep, 150), { once: true }); + v.addEventListener('error', () => { + if (!v.hasAttribute('crossorigin')) return; + v.removeAttribute('crossorigin'); v.src = v.getAttribute('data-src'); + }, { once: true }); + v.src = v.getAttribute('data-src'); if (heavy && d.size) v.after(h('span', { class: 'media-tile-badge' }, heavyLabel(url))); return; } @@ -9736,7 +9865,7 @@ export function messagesFeature(ctx) { const video = pager.kind === 'videos'; const media = video ? h('video', { class: 'mp-media', 'data-src': url, loop: true, playsinline: true, preload: 'none', - poster: posterOf(ev, url) || (videoInfo.get(url) || {}).image || undefined }) + poster: videoThumbs.get(url) || posterOf(ev, url) || (videoInfo.get(url) || {}).image || undefined }) : h('img', { class: 'mp-media', 'data-src': url, alt: '', draggable: 'false' }); const caption = mediaCaption(ev); const stop = (fn) => (e) => { e.stopPropagation(); fn(e); }; @@ -9873,7 +10002,7 @@ export function messagesFeature(ctx) { pagerCounts(e.target); // near the end: ask the feed for more, the pager picks them up const slides = scroller.children; - if ([...slides].indexOf(e.target) >= slides.length - 3) loadOlderFeed().catch(() => {}); + if ([...slides].indexOf(e.target) >= slides.length - 3) crawlMedia().catch(() => {}); } }, { root: scroller, threshold: [0.6] }); growPager(); @@ -10024,9 +10153,10 @@ export function messagesFeature(ctx) { // posts on their way, or nothing older left to fetch. (Older pages // prepare offscreen, so the line is mostly seen by a reader who // outruns them.) - c.loadingMore + c.loadingMore || (mediaMode !== 'all' && mediaCrawling) ? h('div', { class: 'row gap6 feed-foot', style: 'justify-content:center;align-items:center;padding:10px 0' }, h('span', { class: 'spinner sm' }), h('span', { class: 'small muted' }, t('feedLoadingMore'))) + : mediaMode !== 'all' ? (mediaIndex(c.id).end ? h('div', { class: 'small faint feed-foot', style: 'text-align:center;padding:10px 0' }, t('feedEnd')) : null) : c.end && visible.length && !(c.win && c.win.bottom) && c.shown >= visible.length ? h('div', { class: 'small faint feed-foot', style: 'text-align:center;padding:10px 0' }, t('feedEnd')) : null, @@ -11803,6 +11933,7 @@ export function messagesFeature(ctx) { id: 'messages', // group-chat state is device state in IndexedDB: it goes with the account wipeCache(bases) { import('../marmot-store.js').then((m) => { for (const b of bases) m.wipeMarmot(b); }).catch(() => {}); }, + forgetAll() { wipeMediaIndex(); mediaIdx.clear(); try { caches.delete(VIDEO_THUMB_CACHE); } catch {} }, nostrSettingsCards() { return [relaysCard(), followHistoryCard(), moderationCard(), videosCard()]; }, // The app came back after being backgrounded. A phone freezes a hidden // tab: the relay sockets are cut and every post made in the meantime is diff --git a/src/media-cache.js b/src/media-cache.js new file mode 100644 index 0000000..ee9e118 --- /dev/null +++ b/src/media-cache.js @@ -0,0 +1,64 @@ +// The feed's pictures and videos, kept apart from its posts. The feed cache +// holds a page of posts; a grid of clips needs hundreds of posts' worth, and +// re-reading the relays for them on every visit was most of why Videos was +// slow. So each feed keeps, per kind, the posts that carry media (compact: +// no tags but imeta), plus how far back it has looked — a reload paints the +// grid from here at once and the crawl resumes where it stopped. +// +// IndexedDB: a few hundred posts per feed and kind is too much for +// localStorage, and none of it is precious (it can always be re-read). + +const DB = 'coinos-media'; +const STORE = 'kv'; +export const MEDIA_KEEP = 400; // posts per feed and kind + +let dbP = null; +function db() { + if (!dbP) + dbP = new Promise((resolve, reject) => { + if (typeof indexedDB === 'undefined') return reject(new Error('IndexedDB unavailable')); + const req = indexedDB.open(DB, 1); + req.onupgradeneeded = () => { + if (!req.result.objectStoreNames.contains(STORE)) req.result.createObjectStore(STORE); + }; + req.onsuccess = () => { + req.result.onversionchange = () => { req.result.close(); dbP = null; }; + resolve(req.result); + }; + req.onerror = () => { dbP = null; reject(req.error); }; + }); + return dbP; +} +async function tx(mode, fn) { + const d = await db(); + return new Promise((resolve, reject) => { + const t = d.transaction(STORE, mode); + let out; + try { out = fn(t.objectStore(STORE)); } catch (e) { reject(e); return; } + t.oncomplete = () => resolve(out && 'result' in out ? out.result : out); + t.onerror = () => reject(t.error); + t.onabort = () => reject(t.error); + }); +} + +// what a grid needs of a post: who, when, the words, and its imeta +export const compactMediaEvent = (e) => ({ + id: e.id, pubkey: e.pubkey, created_at: e.created_at, kind: e.kind, content: String(e.content || '').slice(0, 2000), + tags: (e.tags || []).filter((x) => x[0] === 'imeta'), sig: e.sig, +}); + +// { images: [...], videos: [...] (newest first), until: oldest created_at looked at, end: when the bottom was hit } +export async function loadMediaIndex(key) { + try { return (await tx('readonly', (s) => s.get(key))) || null; } catch { return null; } +} +export async function saveMediaIndex(key, rec) { + try { + await tx('readwrite', (s) => { + s.put({ ...rec, images: (rec.images || []).slice(0, MEDIA_KEEP).map(compactMediaEvent), + videos: (rec.videos || []).slice(0, MEDIA_KEEP).map(compactMediaEvent), at: Date.now() }, key); + }); + } catch {} +} +export function wipeMediaIndex() { + return tx('readwrite', (s) => { s.clear(); }).catch(() => {}); +} diff --git a/tools/feed-media-cache-test.js b/tools/feed-media-cache-test.js new file mode 100644 index 0000000..08c679e --- /dev/null +++ b/tools/feed-media-cache-test.js @@ -0,0 +1,145 @@ +// The media index: Videos finds clips far down a feed by crawling the +// relays straight into its own cache (100 posts a time, not held up by a +// slow picture), a reload paints the grid from that cache at once and +// resumes the crawl where it stopped, and tile frames are kept as pictures. +// Run: bun tools/feed-media-cache-test.js +import puppeteer from 'puppeteer-core'; +import { cacheKeyFor } from '../src/wallet.js'; +import { generateSecretKey, getPublicKey, finalizeEvent } from 'nostr-tools/pure'; +import { buildHtml } from '../build.js'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +let ok = true; +const check = (n, c, d = '') => { console.log(` ${c ? '✓' : '✗'} ${n}${d ? ' — ' + d : ''}`); if (!c) ok = false; }; +const dir = mkdtempSync(tmpdir() + '/mcache-'); +const ff = (args) => { const p = Bun.spawnSync(['ffmpeg', '-y', '-loglevel', 'error', ...args]); if (p.exitCode !== 0) throw new Error('ffmpeg failed'); }; +for (const n of ['v1', 'v2', 'v3']) ff(['-f', 'lavfi', '-i', 'testsrc=size=360x640:rate=10', '-t', '2', '-c:v', 'libx264', '-pix_fmt', 'yuv420p', '-movflags', '+faststart', `${dir}/${n}.mp4`]); +ff(['-f', 'lavfi', '-i', 'testsrc=size=800x600', '-frames:v', '1', `${dir}/slow.jpg`]); + +const html = await buildHtml({ minify: true, pwa: false }); +const server = Bun.serve({ port: 0, idleTimeout: 30, fetch: async (req) => { + const path = new URL(req.url).pathname; + const cors = { 'access-control-allow-origin': '*', 'access-control-expose-headers': 'content-range, content-length' }; + if (path.startsWith('/v/')) { + const f = Bun.file(dir + path.slice(2)); + const m = /bytes=(\d+)-(\d*)/.exec(req.headers.get('range') || ''); + if (m) { + const a = +m[1], b = Math.min(m[2] ? +m[2] : f.size - 1, f.size - 1); + return new Response(f.slice(a, b + 1), { status: 206, headers: { ...cors, 'content-type': 'video/mp4', 'content-range': `bytes ${a}-${b}/${f.size}`, 'accept-ranges': 'bytes' } }); + } + return new Response(f, { headers: { ...cors, 'content-type': 'video/mp4' } }); + } + // a picture that takes ten seconds: the crawl must not wait on it + if (path === '/i/slow.jpg') { await sleep(10000); return new Response(Bun.file(dir + '/slow.jpg'), { headers: { 'content-type': 'image/jpeg' } }); } + if (path.startsWith('/punks') || path === '/verify-worker.js') return new Response(Bun.file('dist' + path)); + return new Response(html, { headers: { 'content-type': 'text/html' } }); +} }); +const base = server.url.origin; +const SK = generateSecretKey(), PK = getPublicKey(SK); +const now = Math.floor(Date.now() / 1000); +const post = (i, content) => finalizeEvent({ kind: 1, created_at: now - 100 - i * 60, tags: [], content }, SK); +// 40 text posts on top, then the media further down, more text in between +const all = []; +for (let i = 0; i < 40; i++) all.push(post(i, 'just words, number ' + i)); +all.push(post(40, 'a slow picture ' + base + '/i/slow.jpg')); +all.push(post(41, 'first clip ' + base + '/v/v1.mp4')); +for (let i = 42; i < 60; i++) all.push(post(i, 'more words ' + i)); +all.push(post(60, 'second clip ' + base + '/v/v2.mp4')); +for (let i = 61; i < 150; i++) all.push(post(i, 'deep words ' + i)); +all.push(post(150, 'third clip, far down ' + base + '/v/v3.mp4')); + +const browser = await puppeteer.launch({ executablePath: process.env.CHROME || '/usr/bin/google-chrome', headless: 'new', args: ['--no-sandbox', '--autoplay-policy=no-user-gesture-required'] }); +const page = await browser.newPage(); +await page.evaluateOnNewDocument(([evs, pk]) => { + window.__reqs = []; + const Real = window.WebSocket; + window.WebSocket = function (...a) { + const ws = new Real(...a); + const send = ws.send.bind(ws); + ws.send = (data) => { + try { + const m = JSON.parse(data); + if (m[0] === 'REQ') { + const f = m[2] || {}; + let out = []; + if ((f.kinds || []).includes(1) && (f.authors || []).includes(pk) && !f['#e'] && !f.ids) { + window.__reqs.push({ until: f.until, limit: f.limit }); + out = evs.filter((e) => !f.until || e.created_at <= f.until).slice(0, f.limit || 30); + } + setTimeout(() => { + for (const ev of out) ws.dispatchEvent(new MessageEvent('message', { data: JSON.stringify(['EVENT', m[1], ev]) })); + ws.dispatchEvent(new MessageEvent('message', { data: JSON.stringify(['EOSE', m[1]]) })); + }, 0); + return; + } + if (m[0] === 'EVENT') return; + } catch {} + return send(data); + }; + return ws; + }; + window.WebSocket.prototype = Real.prototype; Object.assign(window.WebSocket, Real); +}, [all, PK]); +const errs = []; +page.on('pageerror', (e) => errs.push(String(e))); +const click = (t) => page.evaluate((x) => { const e = [...document.querySelectorAll('button')].find((n) => n.textContent.trim().toLowerCase().includes(x)); if (e) { e.click(); return true; } return false; }, t); +const waitText = async (x, ms = 25000) => { for (let i = 0; i < ms / 250; i++) { if ((await page.evaluate(() => document.body.innerText)).toLowerCase().includes(x)) return true; await sleep(250); } return false; }; +const waitFor = async (fn, ms = 10000) => { for (let i = 0; i < ms / 100; i++) { if (await page.evaluate(fn)) return true; await sleep(100); } return false; }; +const tiles = () => page.evaluate(() => document.querySelectorAll('.media-grid .media-tile.video').length); +const openFeed = async () => { + await page.evaluate(() => [...document.querySelectorAll('.app-nav-button, button')].find((b) => b.textContent.trim() === 'Feed')?.click()); + await waitFor(() => !!document.querySelector('.media-modes'), 20000); +}; + +try { + await page.setViewport({ width: 390, height: 844 }); + await page.goto(base, { waitUntil: 'domcontentloaded' }); + await page.evaluate(() => localStorage.setItem('btc-wallet-network', 'regtest')); + await page.reload({ waitUntil: 'domcontentloaded' }); await sleep(400); + await click('create a new wallet'); await sleep(600); + await page.waitForSelector('.words .w .t'); + const mn = (await page.$$eval('.words .w .t', (l) => l.map((e) => e.textContent.trim()))).join(' '); + await click('skip verification'); + await waitText('receive', 20000); + const key = cacheKeyFor(mn + '\n'); + await page.evaluate(([k, pk]) => { + localStorage.setItem(k + ':follows', JSON.stringify({ tags: [['p', pk]], c: '', at: Math.floor(Date.now() / 1000) })); + localStorage.setItem(k + ':profiles', JSON.stringify({ [pk]: { name: 'Deep Poster', t: Date.now() } })); + localStorage.setItem('btc-wallet-feed:' + k, 'following'); + }, [key, PK]); + await page.reload({ waitUntil: 'domcontentloaded' }); + await waitText('receive', 20000); + await openFeed(); + await waitFor(() => window.__reqs.length > 0, 10000); + await sleep(1500); + + console.log('\n[a crawl straight into the media index]'); + const t0 = Date.now(); + await page.evaluate(() => { window.__reqs = []; document.querySelector('.media-mode[data-mode="videos"]').click(); }); + check('all three clips are found, the deepest 150 posts down', await waitFor(() => document.querySelectorAll('.media-grid .media-tile.video').length === 3, 15000), String(await tiles())); + const took = Date.now() - t0; + check('...without waiting on the slow picture along the way', took < 8000, took + ' ms'); + const reqs = await page.evaluate(() => window.__reqs); + const steps = [...new Set(reqs.map((r) => r.until))]; // each step asks every relay + check('...asking a hundred posts at a time, each step older than the last', reqs.length >= 1 && reqs.every((r) => r.limit === 100 && r.until) + && steps.every((u, i) => !i || u < steps[i - 1]), JSON.stringify(steps)); + check('tile frames are kept as pictures', await waitFor(async () => (await (await caches.open('coinos-video-thumbs-v1')).keys()).length === 3, 8000)); + await sleep(2000); // the index's write-behind + + console.log('\n[a reload]'); + await page.reload({ waitUntil: 'domcontentloaded' }); + await waitText('receive', 20000); + await page.evaluate(() => { window.__reqs = []; }); + const t1 = Date.now(); + await openFeed(); + check('the grid paints from the index at once', await waitFor(() => document.querySelectorAll('.media-grid .media-tile.video').length === 3, 4000), (Date.now() - t1) + ' ms'); + check('...its tiles pictures from the kept frames, no player needed', await waitFor(() => [...document.querySelectorAll('.media-grid .media-tile.video img')].filter((i) => i.src.startsWith('blob:')).length === 3, 4000)); + await sleep(2500); + const after = await page.evaluate(() => window.__reqs.filter((r) => r.limit === 100)); + const deepest = all.find((e) => /far down/.test(e.content)).created_at; + check('...and nothing is crawled again from the top', after.every((r) => r.until <= deepest), JSON.stringify(after)); + check('no page errors', errs.length === 0, errs.slice(0, 2).join(' | ')); +} finally { await browser.close(); server.stop(true); } +console.log(ok ? '\n✅ media index: crawled once, painted at once after a reload' : '\n❌ failed'); +process.exit(ok ? 0 : 1); |
