aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--src/features/messages.js85
-rw-r--r--src/nostr.js25
-rw-r--r--tools/profile-depth-test.js45
3 files changed, 141 insertions, 14 deletions
diff --git a/src/features/messages.js b/src/features/messages.js
index 91fc22d..e36750e 100644
--- a/src/features/messages.js
+++ b/src/features/messages.js
@@ -15,7 +15,7 @@ import { mediaServers } from '../media-servers.js';
// listens for both identities and decrypts with whichever keys are present.
import {
- subscribeOn, subscribeAuthed, publishOn, queryOn, queryManyOn, queryStreamOn, fetchInboxRelays, relayAlive, liveRelayList, readRelayList, resetRelay, wakeRelays,
+ subscribeOn, subscribeAuthed, publishOn, queryOn, queryManyOn, queryStreamOn, queryDepthOn, fetchInboxRelays, relayAlive, liveRelayList, readRelayList, resetRelay, wakeRelays,
npubOf, neventOf, parseNostrPubkey, parseNostrRef, generateSecretKey, getPublicKey, finalizeEvent, nip44,
PROFILE_RELAYS, openWrapsOffthread, unwrapDMsOffthread, verifyEventsAsync, setSyncConfig, DEFAULT_SYNC_RELAYS,
} from '../nostr.js';
@@ -5361,7 +5361,8 @@ export function messagesFeature(ctx) {
const batch = [...incoming.values()]; incoming.clear();
chain = chain.then(async () => {
const byId = new Map(c.notes.map((e) => [e.id, e]));
- const fresh = batch.filter((e) => !byId.has(e.id));
+ // older than the page already settled: kept back until paging gets there
+ const fresh = batch.filter((e) => !byId.has(e.id) && !holdBack(c, e));
if (!fresh.length) return;
for (const e of fresh) byId.set(e.id, e);
const merged = newestArticles([...byId.values()]).sort((a, b) => b.created_at - a.created_at);
@@ -5384,16 +5385,19 @@ export function messagesFeature(ctx) {
};
const asked = new Set([...NOTE_RELAYS, ...PROFILE_ARCHIVE_RELAYS]);
(async () => {
- await Promise.all([
- queryStreamOn([...asked], filter, take, 4500),
+ const depths = await Promise.all([
+ queryDepthOn([...asked], filter, take, 4500),
// the author's own relays, as soon as their list is known
relaysOf(pk).then((own) => {
const more = (own || []).filter((r) => !asked.has(r));
- return more.length ? queryStreamOn(more, filter, take, 4500) : null;
- }).catch(() => {}),
+ return more.length ? queryDepthOn(more, filter, take, 4500) : [];
+ }).catch(() => []),
+ primalNotes(pk, filter.limit, 0, take),
]);
clearTimeout(timer);
await flush();
+ await chain;
+ settlePage(c, depths.flat(), filter.limit);
if (any) persistPage('notes', pk, c.notes.slice(0, 20).map(slimNote)); // cache stays bounded
})().catch(() => {}).finally(() => {
c.status = 'ready';
@@ -5512,22 +5516,75 @@ export function messagesFeature(ctx) {
whenHide = setTimeout(() => whenEl && whenEl.classList.remove('on'), 250);
});
}
+ // How far back a page of someone's posts can be trusted. Relays each hold
+ // their own part of an author's history, so the oldest post of a merged
+ // answer says little: ours held 6 of Ben Arc's posts reaching back to
+ // April while his own relay's newest 30 ended in late September, and
+ // "older than the oldest" then asked for posts before April — five months
+ // never shown, his Napstr launch among them (Adam, 2026-11-20). A source
+ // that sent a full page has more below its oldest post; the page is
+ // complete down to the NEWEST such floor. Posts below it wait in c.held
+ // and join the page that reaches them. No source full: nothing is left.
+ const fullPage = (n, limit) => n >= Math.ceil(limit / 2);
+ const holdBack = (c, e) => {
+ if (!c.floor || e.created_at >= c.floor) return false;
+ (c.held ||= new Map()).set(e.id, e);
+ return true;
+ };
+ function settlePage(c, depths, limit) {
+ const floors = depths.filter((d) => d && fullPage(d.n, limit) && Number.isFinite(d.min)).map((d) => d.min);
+ const floor = floors.length ? Math.max(...floors) : 0;
+ // a refresh of a page already scrolled deeper never raises the floor
+ c.floor = c.floor && floor ? Math.min(c.floor, floor) : (c.floor && !floor ? c.floor : floor);
+ const all = new Map([...(c.held || new Map()), ...c.notes.map((e) => [e.id, e])]);
+ c.held = new Map();
+ const shown = [];
+ for (const e of all.values()) (c.floor && e.created_at < c.floor ? c.held.set(e.id, e) : shown.push(e));
+ c.notes = newestArticles(shown).sort((a, b) => b.created_at - a.created_at);
+ if (!floors.length && !c.floor) c.end = true;
+ else if (floors.length) c.end = false;
+ return floors.length > 0;
+ }
+ // Primal's cache keeps what relays have pruned (of the nine posts around
+ // the Napstr launch, the relays we ask held none). It answers posts and
+ // replies as separate feeds; each is a source with its own depth. The
+ // events are the signed originals, checked like a relay's.
+ async function primalNotes(pk, limit, until, take) {
+ const ask = async (notes) => {
+ const args = { pubkey: pk, notes, limit };
+ if (until) args.until = until;
+ const evs = await primalCache('feed', args, 5000).catch(() => []);
+ const good = await verifiedPrimal((evs || []).filter((e) => e.pubkey === pk), PROFILE_KINDS);
+ for (const e of good) take(e);
+ return { url: 'primal:' + notes, n: good.length, min: good.length ? Math.min(...good.map((e) => e.created_at)) : Infinity };
+ };
+ return Promise.all([ask('authored'), ask('replies')]);
+ }
async function loadOlderNotes(pk) {
const c = notesCache.get(pk);
if (!c || c.status !== 'ready' || c.loadingMore || c.end) return;
const oldest = c.notes[c.notes.length - 1];
- if (!oldest) { c.end = true; return; }
+ if (!oldest && !c.floor) { c.end = true; return; }
c.loadingMore = true;
render();
try {
- const evs = await queryOn(await notesRelays(pk),
- { kinds: PROFILE_KINDS, authors: [pk], limit: 30, until: oldest.created_at - 1 }, 4500);
+ const limit = 30;
+ const until = (c.floor || oldest.created_at) - 1;
+ const got = new Map();
+ const take = (e) => { if (e && e.pubkey === pk && e.created_at <= until) got.set(e.id, e); };
+ const depths = await Promise.all([
+ queryDepthOn(await notesRelays(pk), { kinds: PROFILE_KINDS, authors: [pk], limit, until }, take, 4500),
+ primalNotes(pk, limit, until, take),
+ ]);
+ const before = c.notes.length;
const seen = new Set(c.notes.map((e) => e.id));
- const older = (evs || [])
- .filter((e) => !seen.has(e.id) && seen.add(e.id))
- .sort((a, b) => b.created_at - a.created_at);
- if (older.length) { await notesReady(older); c.notes = newestArticles([...c.notes, ...older]); }
- else c.end = true; // the relays have nothing older
+ for (const e of got.values()) if (!seen.has(e.id)) (c.held ||= new Map()).set(e.id, e);
+ // this page reaches down to the new floor (or to the bottom: no source had more)
+ c.floor = 0;
+ const more = settlePage(c, depths.flat(), limit);
+ if (!more) { c.floor = 0; c.end = true; }
+ const added = c.notes.slice(before);
+ if (added.length) await notesReady(added.slice(0, FEED_PAGE));
} catch {} finally {
c.loadingMore = false;
if (ui.profilePk === pk) render();
diff --git a/src/nostr.js b/src/nostr.js
index 4b7f5c3..d2c9e44 100644
--- a/src/nostr.js
+++ b/src/nostr.js
@@ -602,6 +602,31 @@ export async function queryStreamOn(relays, filter, onEvent, maxWait = 1500) {
});
} catch {} finally { await Promise.all(pending); }
}
+// queryStreamOn that also says how deep each relay's answer went: resolves
+// with [{ url, n, min }] — how many events that relay sent for the filter and
+// the oldest created_at among them. A caller paging backwards needs it:
+// relays hold different parts of an author's history, and the oldest event
+// of the merged answer may be one relay's stray copy from months earlier.
+export async function queryDepthOn(relays, filter, onEvent, maxWait = 1500) {
+ probeRelays(relays);
+ const pending = [], stats = [], seen = new Set();
+ try {
+ const live = readRelays(relays);
+ await Promise.all(live.map((url) => {
+ const s = { url, n: 0, min: Infinity };
+ stats.push(s);
+ return askRelay(url, [filter], maxWait, (event) => {
+ if (!event) return;
+ s.n++;
+ if (event.created_at < s.min) s.min = event.created_at;
+ if (seen.has(event.id)) return;
+ seen.add(event.id);
+ pending.push(verifyOneAsync(event).then((ok) => ok && onEvent(event)).catch(() => {}));
+ });
+ }));
+ } catch {} finally { await Promise.all(pending); }
+ return stats;
+}
export async function publishOn(relays, evt, { onauth = null, onrefused = null } = {}) {
try {
const live = liveRelays(relays);
diff --git a/tools/profile-depth-test.js b/tools/profile-depth-test.js
new file mode 100644
index 0000000..5aea719
--- /dev/null
+++ b/tools/profile-depth-test.js
@@ -0,0 +1,45 @@
+// A profile's older posts, page after page, with no months skipped. Relays
+// hold different parts of an author's history: paging "older than the oldest
+// post on the page" jumped from late September to April on Ben Arc's profile
+// because one relay held six stray copies reaching back that far, and the
+// posts around his Napstr launch (2026-08-21) are on none of the relays we
+// ask — only in Primal's cache. Reads real relays (relays: 'read'; nothing
+// is published).
+// Run: bun tools/profile-depth-test.js
+import puppeteer from './puppeteer.js';
+import { buildHtml } from '../build.js';
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+let ok = true;
+const check = (n, c, d = '') => { console.log(` ${c ? '✓' : '✗'} ${n}${d ? ' — ' + d : ''}`); if (!c) ok = false; };
+const NPUB = process.env.NPUB || 'npub1c878wu04lfqcl5avfy3p5x83ndpvedaxv0dg7pxthakq3jqdyzcs2n8avm';
+const WANT = process.env.WANT || 'For all the seeders out there';
+
+const html = await buildHtml({ minify: true, pwa: false });
+const server = Bun.serve({ port: 0, fetch: () => new Response(html, { headers: { 'content-type': 'text/html' } }) });
+const browser = await puppeteer.launch({ relays: 'read', executablePath: '/usr/bin/google-chrome', headless: 'new', args: ['--no-sandbox'] });
+const page = await browser.newPage();
+const errs = [];
+page.on('pageerror', (e) => errs.push(String(e.stack || e).slice(0, 600)));
+try {
+ await page.setViewport({ width: 390, height: 844 });
+ await page.goto(server.url.href + NPUB, { waitUntil: 'domcontentloaded' });
+ await page.waitForSelector('.notes-feed:not(.note-ghosts) > [data-key]', { timeout: 20000 });
+ const seen = new Map(); // key -> text
+ let found = false, still = 0, last = 0;
+ const t0 = Date.now();
+ for (let i = 0; i < 400 && !found && still < 40; i++) {
+ const rows = await page.evaluate(() => [...document.querySelectorAll('.notes-feed > [data-key]')].map((r) => [r.dataset.key, r.textContent.slice(0, 300)]));
+ for (const [k, t] of rows) seen.set(k, t);
+ found = [...seen.values()].some((t) => t.includes(WANT));
+ still = seen.size === last ? still + 1 : 0;
+ last = seen.size;
+ await page.evaluate(() => window.scrollBy(0, 1400));
+ await sleep(350);
+ }
+ console.log(` ${seen.size} posts read in ${((Date.now() - t0) / 1000).toFixed(1)} s`);
+ check('scrolling back reaches the Napstr launch post', found, WANT);
+ check('...past more than a first page', seen.size > 40, seen.size + ' posts');
+ check('no page errors', !errs.length, errs.join(' | '));
+} catch (e) { console.error(e); ok = false; } finally { await browser.close(); server.stop(); }
+console.log(ok ? '\nPASS' : '\nFAIL');
+process.exit(ok ? 0 : 1);