diff options
| -rw-r--r-- | src/features/messages.js | 85 | ||||
| -rw-r--r-- | src/nostr.js | 25 | ||||
| -rw-r--r-- | tools/profile-depth-test.js | 45 |
3 files changed, 141 insertions, 14 deletions
diff --git a/src/features/messages.js b/src/features/messages.js index 91fc22d..e36750e 100644 --- a/src/features/messages.js +++ b/src/features/messages.js @@ -15,7 +15,7 @@ import { mediaServers } from '../media-servers.js'; // listens for both identities and decrypts with whichever keys are present. import { - subscribeOn, subscribeAuthed, publishOn, queryOn, queryManyOn, queryStreamOn, fetchInboxRelays, relayAlive, liveRelayList, readRelayList, resetRelay, wakeRelays, + subscribeOn, subscribeAuthed, publishOn, queryOn, queryManyOn, queryStreamOn, queryDepthOn, fetchInboxRelays, relayAlive, liveRelayList, readRelayList, resetRelay, wakeRelays, npubOf, neventOf, parseNostrPubkey, parseNostrRef, generateSecretKey, getPublicKey, finalizeEvent, nip44, PROFILE_RELAYS, openWrapsOffthread, unwrapDMsOffthread, verifyEventsAsync, setSyncConfig, DEFAULT_SYNC_RELAYS, } from '../nostr.js'; @@ -5361,7 +5361,8 @@ export function messagesFeature(ctx) { const batch = [...incoming.values()]; incoming.clear(); chain = chain.then(async () => { const byId = new Map(c.notes.map((e) => [e.id, e])); - const fresh = batch.filter((e) => !byId.has(e.id)); + // older than the page already settled: kept back until paging gets there + const fresh = batch.filter((e) => !byId.has(e.id) && !holdBack(c, e)); if (!fresh.length) return; for (const e of fresh) byId.set(e.id, e); const merged = newestArticles([...byId.values()]).sort((a, b) => b.created_at - a.created_at); @@ -5384,16 +5385,19 @@ export function messagesFeature(ctx) { }; const asked = new Set([...NOTE_RELAYS, ...PROFILE_ARCHIVE_RELAYS]); (async () => { - await Promise.all([ - queryStreamOn([...asked], filter, take, 4500), + const depths = await Promise.all([ + queryDepthOn([...asked], filter, take, 4500), // the author's own relays, as soon as their list is known relaysOf(pk).then((own) => { const more = (own || []).filter((r) => !asked.has(r)); - return more.length ? queryStreamOn(more, filter, take, 4500) : null; - }).catch(() => {}), + return more.length ? queryDepthOn(more, filter, take, 4500) : []; + }).catch(() => []), + primalNotes(pk, filter.limit, 0, take), ]); clearTimeout(timer); await flush(); + await chain; + settlePage(c, depths.flat(), filter.limit); if (any) persistPage('notes', pk, c.notes.slice(0, 20).map(slimNote)); // cache stays bounded })().catch(() => {}).finally(() => { c.status = 'ready'; @@ -5512,22 +5516,75 @@ export function messagesFeature(ctx) { whenHide = setTimeout(() => whenEl && whenEl.classList.remove('on'), 250); }); } + // How far back a page of someone's posts can be trusted. Relays each hold + // their own part of an author's history, so the oldest post of a merged + // answer says little: ours held 6 of Ben Arc's posts reaching back to + // April while his own relay's newest 30 ended in late September, and + // "older than the oldest" then asked for posts before April — five months + // never shown, his Napstr launch among them (Adam, 2026-11-20). A source + // that sent a full page has more below its oldest post; the page is + // complete down to the NEWEST such floor. Posts below it wait in c.held + // and join the page that reaches them. No source full: nothing is left. + const fullPage = (n, limit) => n >= Math.ceil(limit / 2); + const holdBack = (c, e) => { + if (!c.floor || e.created_at >= c.floor) return false; + (c.held ||= new Map()).set(e.id, e); + return true; + }; + function settlePage(c, depths, limit) { + const floors = depths.filter((d) => d && fullPage(d.n, limit) && Number.isFinite(d.min)).map((d) => d.min); + const floor = floors.length ? Math.max(...floors) : 0; + // a refresh of a page already scrolled deeper never raises the floor + c.floor = c.floor && floor ? Math.min(c.floor, floor) : (c.floor && !floor ? c.floor : floor); + const all = new Map([...(c.held || new Map()), ...c.notes.map((e) => [e.id, e])]); + c.held = new Map(); + const shown = []; + for (const e of all.values()) (c.floor && e.created_at < c.floor ? c.held.set(e.id, e) : shown.push(e)); + c.notes = newestArticles(shown).sort((a, b) => b.created_at - a.created_at); + if (!floors.length && !c.floor) c.end = true; + else if (floors.length) c.end = false; + return floors.length > 0; + } + // Primal's cache keeps what relays have pruned (of the nine posts around + // the Napstr launch, the relays we ask held none). It answers posts and + // replies as separate feeds; each is a source with its own depth. The + // events are the signed originals, checked like a relay's. + async function primalNotes(pk, limit, until, take) { + const ask = async (notes) => { + const args = { pubkey: pk, notes, limit }; + if (until) args.until = until; + const evs = await primalCache('feed', args, 5000).catch(() => []); + const good = await verifiedPrimal((evs || []).filter((e) => e.pubkey === pk), PROFILE_KINDS); + for (const e of good) take(e); + return { url: 'primal:' + notes, n: good.length, min: good.length ? Math.min(...good.map((e) => e.created_at)) : Infinity }; + }; + return Promise.all([ask('authored'), ask('replies')]); + } async function loadOlderNotes(pk) { const c = notesCache.get(pk); if (!c || c.status !== 'ready' || c.loadingMore || c.end) return; const oldest = c.notes[c.notes.length - 1]; - if (!oldest) { c.end = true; return; } + if (!oldest && !c.floor) { c.end = true; return; } c.loadingMore = true; render(); try { - const evs = await queryOn(await notesRelays(pk), - { kinds: PROFILE_KINDS, authors: [pk], limit: 30, until: oldest.created_at - 1 }, 4500); + const limit = 30; + const until = (c.floor || oldest.created_at) - 1; + const got = new Map(); + const take = (e) => { if (e && e.pubkey === pk && e.created_at <= until) got.set(e.id, e); }; + const depths = await Promise.all([ + queryDepthOn(await notesRelays(pk), { kinds: PROFILE_KINDS, authors: [pk], limit, until }, take, 4500), + primalNotes(pk, limit, until, take), + ]); + const before = c.notes.length; const seen = new Set(c.notes.map((e) => e.id)); - const older = (evs || []) - .filter((e) => !seen.has(e.id) && seen.add(e.id)) - .sort((a, b) => b.created_at - a.created_at); - if (older.length) { await notesReady(older); c.notes = newestArticles([...c.notes, ...older]); } - else c.end = true; // the relays have nothing older + for (const e of got.values()) if (!seen.has(e.id)) (c.held ||= new Map()).set(e.id, e); + // this page reaches down to the new floor (or to the bottom: no source had more) + c.floor = 0; + const more = settlePage(c, depths.flat(), limit); + if (!more) { c.floor = 0; c.end = true; } + const added = c.notes.slice(before); + if (added.length) await notesReady(added.slice(0, FEED_PAGE)); } catch {} finally { c.loadingMore = false; if (ui.profilePk === pk) render(); diff --git a/src/nostr.js b/src/nostr.js index 4b7f5c3..d2c9e44 100644 --- a/src/nostr.js +++ b/src/nostr.js @@ -602,6 +602,31 @@ export async function queryStreamOn(relays, filter, onEvent, maxWait = 1500) { }); } catch {} finally { await Promise.all(pending); } } +// queryStreamOn that also says how deep each relay's answer went: resolves +// with [{ url, n, min }] — how many events that relay sent for the filter and +// the oldest created_at among them. A caller paging backwards needs it: +// relays hold different parts of an author's history, and the oldest event +// of the merged answer may be one relay's stray copy from months earlier. +export async function queryDepthOn(relays, filter, onEvent, maxWait = 1500) { + probeRelays(relays); + const pending = [], stats = [], seen = new Set(); + try { + const live = readRelays(relays); + await Promise.all(live.map((url) => { + const s = { url, n: 0, min: Infinity }; + stats.push(s); + return askRelay(url, [filter], maxWait, (event) => { + if (!event) return; + s.n++; + if (event.created_at < s.min) s.min = event.created_at; + if (seen.has(event.id)) return; + seen.add(event.id); + pending.push(verifyOneAsync(event).then((ok) => ok && onEvent(event)).catch(() => {})); + }); + })); + } catch {} finally { await Promise.all(pending); } + return stats; +} export async function publishOn(relays, evt, { onauth = null, onrefused = null } = {}) { try { const live = liveRelays(relays); diff --git a/tools/profile-depth-test.js b/tools/profile-depth-test.js new file mode 100644 index 0000000..5aea719 --- /dev/null +++ b/tools/profile-depth-test.js @@ -0,0 +1,45 @@ +// A profile's older posts, page after page, with no months skipped. Relays +// hold different parts of an author's history: paging "older than the oldest +// post on the page" jumped from late September to April on Ben Arc's profile +// because one relay held six stray copies reaching back that far, and the +// posts around his Napstr launch (2026-08-21) are on none of the relays we +// ask — only in Primal's cache. Reads real relays (relays: 'read'; nothing +// is published). +// Run: bun tools/profile-depth-test.js +import puppeteer from './puppeteer.js'; +import { buildHtml } from '../build.js'; +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +let ok = true; +const check = (n, c, d = '') => { console.log(` ${c ? '✓' : '✗'} ${n}${d ? ' — ' + d : ''}`); if (!c) ok = false; }; +const NPUB = process.env.NPUB || 'npub1c878wu04lfqcl5avfy3p5x83ndpvedaxv0dg7pxthakq3jqdyzcs2n8avm'; +const WANT = process.env.WANT || 'For all the seeders out there'; + +const html = await buildHtml({ minify: true, pwa: false }); +const server = Bun.serve({ port: 0, fetch: () => new Response(html, { headers: { 'content-type': 'text/html' } }) }); +const browser = await puppeteer.launch({ relays: 'read', executablePath: '/usr/bin/google-chrome', headless: 'new', args: ['--no-sandbox'] }); +const page = await browser.newPage(); +const errs = []; +page.on('pageerror', (e) => errs.push(String(e.stack || e).slice(0, 600))); +try { + await page.setViewport({ width: 390, height: 844 }); + await page.goto(server.url.href + NPUB, { waitUntil: 'domcontentloaded' }); + await page.waitForSelector('.notes-feed:not(.note-ghosts) > [data-key]', { timeout: 20000 }); + const seen = new Map(); // key -> text + let found = false, still = 0, last = 0; + const t0 = Date.now(); + for (let i = 0; i < 400 && !found && still < 40; i++) { + const rows = await page.evaluate(() => [...document.querySelectorAll('.notes-feed > [data-key]')].map((r) => [r.dataset.key, r.textContent.slice(0, 300)])); + for (const [k, t] of rows) seen.set(k, t); + found = [...seen.values()].some((t) => t.includes(WANT)); + still = seen.size === last ? still + 1 : 0; + last = seen.size; + await page.evaluate(() => window.scrollBy(0, 1400)); + await sleep(350); + } + console.log(` ${seen.size} posts read in ${((Date.now() - t0) / 1000).toFixed(1)} s`); + check('scrolling back reaches the Napstr launch post', found, WANT); + check('...past more than a first page', seen.size > 40, seen.size + ' posts'); + check('no page errors', !errs.length, errs.join(' | ')); +} catch (e) { console.error(e); ok = false; } finally { await browser.close(); server.stop(); } +console.log(ok ? '\nPASS' : '\nFAIL'); +process.exit(ok ? 0 : 1); |
