From d718fcebcf40672f856805d69cbb2a84296919c6 Mon Sep 17 00:00:00 2001 From: Adam Soltys Date: Fri, 16 Oct 2026 23:03:55 -0700 Subject: The video pager's slides near the reader hold their clip's first frame — the server's, full size — so no slide comes up black. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some slides were black until the player had its frame, then the picture appeared all at once (Adam, 2026-10-16: "i'm still seeing mostly black between some videos and then they suddenly appear when it starts playing"). The frame drawn in the browser (ebaec42) needed the next clip's cached opening, a file led by its index and a host that allows the read — and only the next clip had one at all: not the one before (its player let go on leaving), not a flight's landing, not the clip two along. vmedia gains /vmedia/first.jpg?u=: frame 0 itself, longest side 1280 (the player's size), any host, any layout, kept and cached at the CDN like the stills. The pager now gives every slide within three of the reader, the clips ahead and the slide landed on (not the one the pager opened on mid-clip) their clip's first frame: the server's, fetched and decoded aside and put on the slide when ready, so the flick glides onto the very picture the player shows first; a flight's landing slide fetches its frame as the flight sets off. Where the server has none the next clip's frame is drawn here as before, and a clip beyond keeps the post's still — soft, but not black — replaced by a first frame when one comes; never a soft still under the clip on screen. Data Saver is unchanged (posters at build). pager-first-frame-test (https clips, vmedia answered by interception): the next clip holds the server's frame, asked once; a clip the server has none for keeps the still, then has its frame drawn here once it is the clip ahead; the clip left behind holds its frame for the swipe back and plays again on it. pager-no-poster, pager-eager, pager-seek-hide, pager-swipe-fly, video-tile-poster, video-fallback, pager-controls still pass. vmedia deployed to cs first (first frames of live clips: 47–144 KB, 0.4–1.7 s to make). Co-Authored-By: Claude Fable 5.1 --- src/features/messages.js | 57 +++++++++++++++++-- tools/pager-first-frame-test.js | 122 ++++++++++++++++++++++++++++++++++++++++ vmedia/server.js | 32 ++++++++++- 3 files changed, 204 insertions(+), 7 deletions(-) create mode 100644 tools/pager-first-frame-test.js diff --git a/src/features/messages.js b/src/features/messages.js index 1371b04..ce578e4 100644 --- a/src/features/messages.js +++ b/src/features/messages.js @@ -7781,6 +7781,9 @@ export function messagesFeature(ctx) { const vmediaOk = (url) => /^https:\/\//i.test(url || ''); // v=4: stills past a dark opening frame (a new URL, so the CDN's old black ones aren't served) const serverThumb = (url) => (vmediaOk(url) ? VMEDIA + 'thumb.jpg?v=4&u=' + encodeURIComponent(url) : undefined); + // the clip's very first frame at full size (the pager's slides ahead hold it) + const serverFirst = (url) => (vmediaOk(url) ? VMEDIA + 'first.jpg?u=' + encodeURIComponent(url) : undefined); + const serverFirstFailed = new Set(); // clips the server had no first frame for (this session) const h264Of = (url) => VMEDIA + 'h264.mp4?u=' + encodeURIComponent(url); const noPicture = new Set(), h264Failed = new Set(); const serverThumbFailed = new Set(); // clips vmedia had no picture for (this session) @@ -12819,14 +12822,13 @@ export function messagesFeature(ctx) { slides.forEach((s, j) => { const k = ahead.indexOf(s); if (Math.abs(j - i) <= 2 || k >= 0) slidePoster(s); + // the slides around the reader (and the one landed on) hold their clip's first frame + if (eager && (Math.abs(j - i) <= 3 || k >= 0) && !(j === i && pager.at && slide.getAttribute('data-k') === pager.start)) slideFrame(s, k); if (j === i) return; const v = s.querySelector('video.mp-media'); if (eager && k >= 0) { pagerRelease(s); - const src = playSrc(v.getAttribute('data-src')); - headStart(src, k === 0 ? 1 : 0.5); - // the next clip holds its first frame: the flick glides onto it - if (k === 0 && !v.getAttribute('poster')) pagerPoster(src).then((p) => { if (p && pager && s.isConnected && !v.getAttribute('poster')) v.setAttribute('poster', p); }); + headStart(playSrc(v.getAttribute('data-src')), k === 0 ? 1 : 0.5); return; } if (k === 0) { @@ -12840,6 +12842,51 @@ export function messagesFeature(ctx) { }); for (const s of slides.slice(i + 2, i + 4)) probeVideo(s.querySelector('video.mp-media').getAttribute('data-src')); } + // A slide near the reader holds its clip's first frame as its poster, so + // a flick glides onto the very picture the player shows first and the + // player takes over with no visible change — and a slide left behind + // (its player let go) keeps it for the swipe back. Some slides came up + // black until the player had its frame: the frame drawn here needed the + // next clip's cached opening, a file led by its index and a host that + // allows the read, and only the next clip had one at all — not the one + // before, not a flight's landing (Adam, 2026-10-16: "i'm still seeing + // mostly black between some videos and then they suddenly appear when + // it starts playing"). The server's first frame (vmedia first.jpg: frame + // 0 at full size, any host, any layout, cached at the CDN) is the first + // choice, fetched and decoded aside and put on the slide when it is + // ready; the frame drawn here is the fallback for the next clip; a clip + // with neither keeps the still there is (a kept frame, the post's + // picture), soft but not black, replaced by a first frame when one comes. + function slideFrame(s, k) { + const v = s.querySelector('video.mp-media'); + if (!v || v._frame === 'first' || v._frameBusy) return; + const url = v.getAttribute('data-src'); + const put = (p, kind) => { + if (!p || !pager || !s.isConnected || v._frame === 'first') return false; + if (kind === 'soft' && (v._frame || pager.cur === s)) return false; // never a soft still under the clip on screen + v.setAttribute('poster', p); v._frame = kind; + return true; + }; + const soft = () => put(videoThumbs.get(url) || posterOf(s._ev, url), 'soft'); + const first = serverFirst(url); + if (first && !serverFirstFailed.has(url)) { + v._frameBusy = true; + const im = new Image(); + im.decoding = 'async'; + const ok = () => { v._frameBusy = false; put(first, 'first'); }; + im.onload = () => (im.decode ? im.decode().then(ok, ok) : ok()); + im.onerror = () => { v._frameBusy = false; serverFirstFailed.add(url); slideFrame(s, k); }; + im.src = first; + return; + } + if (k === 0 && !v._grabbed) { // the next clip: its opening is in the cache, the frame comes from there + v._grabbed = true; + v._frameBusy = true; + pagerPoster(playSrc(url)).then((p) => { v._frameBusy = false; if (!put(p, 'first')) soft(); }); + return; + } + if (first) soft(); // (the server had none: the still, if any, rather than black) + } // The opening of a clip about to be reached, fetched into the browser's // HTTP cache with a range request: the player's own request for the file // is answered from there as far as it goes, and streams on from the host. @@ -13074,6 +13121,8 @@ export function messagesFeature(ctx) { const dest = pagerAt(pager.scroller, to, true); const screens = Math.abs(dest - pager.scroller.scrollTop) / Math.max(1, pager.scroller.clientHeight); pager.flying = true; + const land = pager.kind === 'videos' && !frugalData() && rowSlide(to); // the landing's first frame, fetched on the way + if (land) slideFrame(land, -1); pagerGlide(pager.scroller, 'y', dest, vel, { ms: screens > 1.05 ? Math.min(most, 100 + pace * screens) : 0, onDone: () => { if (!pager) return; pager.flying = false; const s = pagerHere(); if (s && s !== pager.cur) pagerArrive(s); }, diff --git a/tools/pager-first-frame-test.js b/tools/pager-first-frame-test.js new file mode 100644 index 0000000..f88a074 --- /dev/null +++ b/tools/pager-first-frame-test.js @@ -0,0 +1,122 @@ +// The video pager's slides near the reader hold their clip's first frame +// before the reader gets there: the server's (vmedia first.jpg — frame 0 at +// full size, any host), put on the slide once fetched and decoded; where the +// server has none, the next clip's frame is drawn here from its cached +// opening, and a clip beyond keeps the post's still rather than black, until +// a first frame comes. The slide left behind keeps its frame for the swipe +// back. Run: bun tools/pager-first-frame-test.js +import puppeteer from './puppeteer.js'; +import { cacheKeyFor } from '../src/wallet.js'; +import { generateSecretKey, getPublicKey, finalizeEvent } from 'nostr-tools/pure'; +import { buildHtml } from '../build.js'; +import { existsSync } from 'node:fs'; +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +let ok = true; +const check = (n, c, d = '') => { console.log(` ${c ? '✓' : '✗'} ${n}${d ? ' — ' + d : ''}`); if (!c) ok = false; }; +const png = Bun.spawnSync(['ffmpeg', '-loglevel', 'error', '-f', 'lavfi', '-i', 'testsrc=size=320x180', '-frames:v', '1', '-f', 'image2', '-c:v', 'png', 'pipe:1']).stdout; +const jpg = (c, size) => Bun.spawnSync(['ffmpeg', '-loglevel', 'error', '-f', 'lavfi', '-i', 'color=c=' + c + ':s=' + size, '-frames:v', '1', '-f', 'image2', '-c:v', 'mjpeg', 'pipe:1']).stdout; +const FIRST = { wide: jpg('orange', '1280x720'), tall: jpg('green', '720x1280') }; // (last: the server has none) +const clip = (name, size) => { + const f = '/tmp/coinos-vpager-' + name + '.webm'; + if (!existsSync(f)) Bun.spawnSync(['ffmpeg', '-y', '-loglevel', 'error', '-f', 'lavfi', '-i', 'testsrc=size=' + size + ':rate=10', '-t', '30', '-c:v', 'libvpx', '-b:v', '100k', f]); + return Buffer.from(Bun.spawnSync(['cat', f]).stdout); +}; +const CLIPS = { wide: clip('wide', '320x180'), tall: clip('tall', '180x320'), last: clip('last', '240x240') }; +const SK = generateSecretKey(), AUTHOR = getPublicKey(SK); +const now = Math.floor(Date.now() / 1000); +const html = await buildHtml({ minify: true, pwa: false }); +const server = Bun.serve({ port: 0, fetch: (req) => { + const path = new URL(req.url).pathname; + if (path === '/still.png') return new Response(png, { headers: { 'content-type': 'image/png' } }); + if (path.startsWith('/punks') || path === '/verify-worker.js') return new Response(Bun.file('dist' + path)); + return new Response(html, { headers: { 'content-type': 'text/html' } }); +} }); +const U = (n) => 'https://media.test/' + n + '.webm'; +const post = (content, url, ago) => finalizeEvent({ kind: 1, created_at: now - ago, tags: [['imeta', 'url ' + url, 'image ' + server.url.origin + '/still.png', 'm video/webm']], content }, SK); +const seeded = [post('a wide one ' + U('wide'), U('wide'), 100), post('a tall one ' + U('tall'), U('tall'), 200), post('the last clip ' + U('last'), U('last'), 400)]; + +const browser = await puppeteer.launch({ executablePath: '/usr/bin/google-chrome', headless: 'new', args: ['--no-sandbox', '--autoplay-policy=no-user-gesture-required'] }); +const page = await browser.newPage(); +await page.setRequestInterception(true); +const asked = []; +page.on('request', (r) => { + const u = new URL(r.url()); + if (u.pathname === '/vmedia/first.jpg') { + const name = /\/(\w+)\.webm$/.exec(u.searchParams.get('u') || '')?.[1]; + asked.push('first:' + name); + if (!FIRST[name]) return r.respond({ status: 404, body: 'No picture', headers: { 'access-control-allow-origin': '*' } }); + return r.respond({ status: 200, contentType: 'image/jpeg', body: FIRST[name], headers: { 'access-control-allow-origin': '*' } }); + } + if (u.pathname.startsWith('/vmedia/')) { asked.push('vmedia:' + u.pathname); return r.respond({ status: 404, body: 'no' }); } + if (u.host === 'media.test') { + const name = /\/(\w+)\.webm$/.exec(u.pathname)[1], f = CLIPS[name]; + const range = /^bytes=(\d+)-(\d*)$/.exec(r.headers().range || ''); + const cors = { 'access-control-allow-origin': '*', 'accept-ranges': 'bytes', 'access-control-expose-headers': 'content-range, content-length' }; + if (!range) return r.respond({ status: 200, contentType: 'video/webm', body: f, headers: cors }); + const a = +range[1], b = range[2] ? Math.min(+range[2], f.length - 1) : f.length - 1; + return r.respond({ status: 206, contentType: 'video/webm', body: f.subarray(a, b + 1), headers: { ...cors, 'content-range': 'bytes ' + a + '-' + b + '/' + f.length } }); + } + r.continue(); +}); +await page.evaluateOnNewDocument(() => { + const Real = window.WebSocket; + window.WebSocket = function (...a) { const ws = new Real(...a); const send = ws.send.bind(ws); ws.send = (d) => { try { const m = JSON.parse(d); if (m[0] === 'REQ') { setTimeout(() => ws.dispatchEvent(new MessageEvent('message', { data: JSON.stringify(['EOSE', m[1]]) })), 0); return; } if (m[0] === 'EVENT') return; } catch {} return send(d); }; return ws; }; + window.WebSocket.prototype = Real.prototype; Object.assign(window.WebSocket, Real); +}); +const errs = []; +page.on('pageerror', (e) => errs.push(String(e))); +const click = (t) => page.evaluate((x) => { const e = [...document.querySelectorAll('button')].find((n) => n.textContent.trim().toLowerCase().includes(x)); if (e) { e.click(); return true; } return false; }, t); +const waitText = async (x, ms = 25000) => { for (let i = 0; i < ms / 250; i++) { if ((await page.evaluate(() => document.body.innerText)).toLowerCase().includes(x)) return true; await sleep(250); } return false; }; +const waitFor = async (fn, arg, ms = 10000) => { for (let i = 0; i < ms / 100; i++) { if (await page.evaluate(fn, arg)) return true; await sleep(100); } return false; }; +const kind = (p) => (!p ? 'none' : /\/vmedia\/first\.jpg/.test(p) ? 'server' : /^blob:/.test(p) ? 'drawn' : /still\.png/.test(p) ? 'still' : p); +const slides = () => page.evaluate(() => [...document.querySelectorAll('.mp-slide')].map((s) => { const v = s.querySelector('video.mp-media'); const r = s.getBoundingClientRect(); return { clip: s.getAttribute('data-k').split('/').pop().replace('.webm', ''), poster: v.getAttribute('poster'), playing: !v.paused, here: Math.abs(r.top) < 4 }; })) + .then((l) => l.map((x) => ({ ...x, poster: kind(x.poster) }))); +const posterOf = async (clip) => (await slides()).find((x) => x.clip === clip)?.poster; +const waitPoster = (clip, k, ms = 8000) => waitFor(([c, k]) => { const s = [...document.querySelectorAll('.mp-slide')].find((x) => x.getAttribute('data-k').endsWith('/' + c + '.webm')); const p = s && s.querySelector('video.mp-media').getAttribute('poster'); return !!p && (k === 'server' ? /\/vmedia\/first\.jpg/.test(p) : k === 'drawn' ? /^blob:/.test(p) : /still\.png/.test(p)); }, [clip, k], ms); + +try { + await page.setViewport({ width: 390, height: 844 }); + await page.goto(server.url.href, { waitUntil: 'domcontentloaded' }); + await page.evaluate(() => localStorage.setItem('btc-wallet-network', 'regtest')); + await page.reload({ waitUntil: 'domcontentloaded' }); + await sleep(400); + await click('create a new wallet'); await sleep(600); + await page.waitForSelector('.words .w .t'); + const mn = (await page.$$eval('.words .w .t', (l) => l.map((e) => e.textContent.trim()))).join(' '); + await click('skip verification'); + await waitText('receive', 20000); + const base = cacheKeyFor(mn + '\n'); + await page.evaluate(([k, pk, ns]) => { + localStorage.setItem(k + ':follows', JSON.stringify({ tags: [['p', pk]], c: '', at: Math.floor(Date.now() / 1000) })); + localStorage.setItem(k + ':feedNotes', JSON.stringify(ns)); + localStorage.setItem(k + ':profiles', JSON.stringify({ [pk]: { name: 'Clip Poster', t: Date.now() } })); + localStorage.setItem('btc-wallet-feed:' + k, 'following'); + }, [base, AUTHOR, seeded]); + await page.reload({ waitUntil: 'domcontentloaded' }); + await waitText('receive', 20000); + await page.evaluate(() => document.querySelector('.app-nav-button[aria-label="Feed"]').click()); + check('the clips are in the feed', await waitFor(() => document.querySelectorAll('.notes-feed video.note-video').length >= 3, null, 20000)); + await page.evaluate(() => { const r = [...document.querySelectorAll('.notes-feed [data-key]')].find((n) => n.textContent.includes('a wide one')); r.querySelector('video.note-video').scrollIntoView({ block: 'center' }); }); + await sleep(600); + const at = await page.evaluate(() => { const r = [...document.querySelectorAll('.notes-feed [data-key]')].find((n) => n.textContent.includes('a wide one')); const b = r.querySelector('video.note-video').getBoundingClientRect(); return { x: b.x + b.width * 0.2, y: b.y + b.height * 0.3 }; }); + await page.mouse.click(at.x, at.y); + check('the pager opens', await waitFor(() => !!document.querySelector('.media-pager.videos .mp-slide'), null, 4000)); + check('the clip on screen plays', await waitFor(() => [...document.querySelectorAll('.mp-slide.mp-cur video.mp-media')].some((v) => !v.paused), null, 8000)); + + console.log('\n[the slides ahead hold their first frames]'); + check('the next clip holds the server\'s first frame', await waitPoster('tall', 'server'), JSON.stringify(await slides())); + check('…the server was asked for it once', asked.filter((x) => x === 'first:tall').length === 1, asked.join(' ')); + check('the clip after, which the server has no frame for, keeps the post\'s still', await waitPoster('last', 'still'), JSON.stringify(await slides())); + check('…no thumbnail route was tried for any of them', !asked.some((x) => /thumb\.jpg/.test(x)), asked.join(' ')); + await page.keyboard.press('ArrowDown'); + check('on to the next: it plays', await waitFor(() => { const s = document.querySelector('.mp-slide.mp-cur'); return !!s && s.getAttribute('data-k').endsWith('/tall.webm') && !s.querySelector('video').paused; }, null, 6000), JSON.stringify(await slides())); + check('…the server\'s frame still under it', (await posterOf('tall')) === 'server', String(await posterOf('tall'))); + check('the one left behind holds its first frame for the swipe back', await waitPoster('wide', 'server'), JSON.stringify(await slides())); + check('the next, now the clip ahead, has its frame drawn here in place of the still', await waitPoster('last', 'drawn', 12000), JSON.stringify(await slides())); + check('…the server asked only once more for it, not again and again', asked.filter((x) => x === 'first:last').length === 1, asked.join(' ')); + await page.keyboard.press('ArrowUp'); + check('back on the first: it plays again, its frame under it', await waitFor(() => { const s = document.querySelector('.mp-slide.mp-cur'); return !!s && s.getAttribute('data-k').endsWith('/wide.webm') && !s.querySelector('video').paused && /first\.jpg/.test(s.querySelector('video').getAttribute('poster') || ''); }, null, 6000), JSON.stringify(await slides())); + check('no page errors', errs.length === 0, errs.slice(0, 2).join(' | ')); +} catch (e) { console.error(e); ok = false; } finally { await browser.close(); server.stop(true); } +console.log(ok ? '\n✅ the slides near the reader hold their clips\' first frames' : '\n❌ failed'); +process.exit(ok ? 0 : 1); diff --git a/vmedia/server.js b/vmedia/server.js index b2f5bae..0f11200 100644 --- a/vmedia/server.js +++ b/vmedia/server.js @@ -4,6 +4,9 @@ // /vmedia/thumb.jpg?u= a still from the clip's first half second, // 480px wide — the poster a post's imeta didn't give. A 4K or 134 MB // clip waits for a tap, so without one the feed showed a black box. +// /vmedia/first.jpg?u= the clip's very first frame, at its size +// (longest side 1280): the very picture a player shows first, so the +// video pager's slide ahead holds it before the player has it. // /vmedia/h264.mp4?u= the clip as H.264/AAC at 720p, for a browser // that can't decode the original's picture (an iPhone's HEVC on Chrome // for Linux plays its sound over black). The first ask streams the @@ -74,6 +77,9 @@ export const thumbArgs = (src, at, lift = false) => ['-hide_banner', '-loglevel' // that fades in from black has a black frame at half a second '-frames:v', '1', '-vf', "scale='min(480,iw)':-2," + (lift ? 'normalize=strength=0.7:smoothing=0,' : '') + "signalstats,metadata=mode=print:key=lavfi.signalstats.YAVG:file='pipe\\:2'", '-q:v', '5', '-f', 'image2', '-c:v', 'mjpeg', 'pipe:1']; +// frame 0 itself, as big as the player shows it (the pager's slide ahead) +export const firstArgs = (src) => ['-hide_banner', '-loglevel', 'error', ...NET, '-i', src, + '-frames:v', '1', '-vf', SCALE_720, '-q:v', '3', '-f', 'image2', '-c:v', 'mjpeg', 'pipe:1']; export const clipArgs = (src, out) => ['-hide_banner', '-loglevel', 'error', '-y', ...NET, '-i', src, '-map', '0:v:0', '-map', '0:a:0?', '-t', String(MAX_SECONDS), '-vf', SCALE_720, '-c:v', 'libx264', '-preset', 'veryfast', '-crf', '26', '-profile:v', 'high', '-pix_fmt', 'yuv420p', '-threads', '4', @@ -131,6 +137,26 @@ async function makeThumb(src, key) { failed.set(key, Date.now()); return null; } +// The very first frame, full size, for the video pager: the slide ahead +// holds the picture its player will show first, so a flick glides onto it +// and the player takes over with no visible change. Dark is dark: this one +// is meant to match the player, not to show what the clip is. +async function makeFirst(src, key) { + const out = `${THUMBS}/${key}-f.jpg`; + const { p, done } = run(firstArgs(src), 25_000); + const [buf] = await Promise.all([new Response(p.stdout).arrayBuffer(), done]); + if (p.exitCode === 0 && buf.byteLength > 500) { await Bun.write(out, buf); return out; } + failed.set(key, Date.now()); + return null; +} +const FIRST_V = 1; +async function first(src) { + const key = keyOf('f' + FIRST_V + '|' + src), out = `${THUMBS}/${key}-f.jpg`; + if (existsSync(out)) { touch(out); return out; } + if (failedLately(key)) return null; + if (!thumbJobs.has(key)) thumbJobs.set(key, thumbSlots(() => makeFirst(src, key)).finally(() => thumbJobs.delete(key))); + return thumbJobs.get(key); +} const THUMB_V = 4; // v4: past a dark opening frame (stills made before are made again) async function thumb(src) { const key = keyOf('v' + THUMB_V + '|' + src), out = `${THUMBS}/${key}.jpg`; @@ -349,7 +375,7 @@ export async function handle(req) { const u = new URL(req.url); if (!['GET', 'HEAD'].includes(req.method)) return nope(405, 'GET only'); const route = u.pathname.replace(/^\/vmedia/, ''); - if (route !== '/thumb.jpg' && route !== '/h264.mp4' && route !== '/pic.webp') return nope(404, 'Not found'); + if (route !== '/thumb.jpg' && route !== '/first.jpg' && route !== '/h264.mp4' && route !== '/pic.webp') return nope(404, 'Not found'); const src = await sourceOk(u.searchParams.get('u') || ''); if (!src) return nope(400, 'Not a public https URL'); const ip = req.headers.get('cf-connecting-ip') || req.headers.get('x-real-ip') || 'unknown'; @@ -357,8 +383,8 @@ export async function handle(req) { const shape = u.searchParams.get('s') || 'col'; return SHAPES[shape] ? picResponse(req, src, shape, ip) : nope(400, 'Unknown shape'); } - if (route === '/thumb.jpg') { - const p = await thumb(src); + if (route === '/thumb.jpg' || route === '/first.jpg') { + const p = await (route === '/first.jpg' ? first(src) : thumb(src)); return p ? fileResponse(req, p, 'image/jpeg') : nope(404, 'No picture'); } return clipResponse(req, src, ip); -- cgit v1.2.3