Feed: coming back settles once — a few new posts above you, a lot means the top.

The catch-up on arriving at the feed asked every relay for its newest posts
with no lower bound, and everything not already in the fifty-post cache went
behind the pill, chunk by chunk: the count climbed in steps and stopped at
the pending list's cap of 200, whether there were 12 new posts or 500. Now
the pass is buffered and settled once, after every relay has answered. Posts
older than the newest we had are gap fills and merge silently. Up to twenty
genuinely new posts go in above what you're looking at with the page held
still (holdScroll: the row at the top of the viewport stays put, measured
after the paint so a browser that anchored on its own isn't corrected
twice), so scrolling up reveals them in order, newest at the top. More than
twenty means you've been away: the feed goes to the top, at the newest post,
as on a fresh open. The window grows only by posts landing inside it, so a
catch-up doesn't paint two hundred rows. The pill is now only for posts that
arrive live while you read further down. The feed-live test covers the three
arrivals, and its stale checks (auto-flush on scrolling up was removed on
purpose; a fresh <img> reports complete a task late) now say what the app
does.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

2 files changed, 171 insertions, 33 deletions

diff --git a/src/features/messages.js b/src/features/messages.js
index 958cd8d..5224970 100644
--- a/src/features/messages.js
+++ b/src/features/messages.js
@@ -3654,12 +3654,19 @@ export function messagesFeature(ctx) {
   const feedStaged = new Set();
   async function mergeFeed(evs, opts = {}) {
     const c = feedNow();
-    const known = new Set([...c.notes, ...(c.pending || [])].map((e) => e.id));
+    const known = new Set([...c.notes, ...(c.pending || []), ...(c.catchup || [])].map((e) => e.id));
     const add = (evs || []).filter((e) => e.kind === 1 && !isReply(e) && !isMuted(e.pubkey)
       && !known.has(e.id) && !feedStaged.has(e.id) && known.add(e.id) && feedStaged.add(e.id));
     if (!add.length) return false;
     await notesReady(add);
     for (const e of add) feedStaged.delete(e.id);
+    // A catch-up is settled once, after every relay has answered (see
+    // settleCatchup) — not chunk by chunk, which is what made the pill count
+    // up in steps and land on the same round number every time.
+    if (opts.catchup) {
+      c.catchup = [...(c.catchup || []), ...add];
+      return true;
+    }
     // A post that arrived on its own waits behind the pill instead of
     // shoving the page down — wherever you are on it. Only what you asked
     // for (the first load, a page of older posts) goes straight in.
@@ -3725,13 +3732,75 @@ export function messagesFeature(ctx) {
     if (!feedAuthors().length) { if (feed) feed.status = 'ready'; return; }
     if (!opts.force && Date.now() - feedAt < 30_000) return;
     feedAt = Date.now();
-    const live = opts.live != null ? !!opts.live : !!(feed && feed.notes.length);
-    try { await feedPass({}, { live }); } catch {} finally {
+    const catchup = opts.live != null ? !!opts.live : !!(feed && feed.notes.length);
+    const topBefore = feed && feed.notes[0] ? feed.notes[0].created_at : 0;
+    try { await feedPass({}, { catchup }); } catch {} finally {
       if (feed) feed.status = 'ready';
-      scheduleRepaint();
+      if (catchup) settleCatchup(topBefore); else scheduleRepaint();
     }
     watchFeed();
   }
+  // ---- coming back to the feed ---------------------------------------------
+  // The catch-up asks every relay for its newest posts, with no lower bound,
+  // so most of what it returns is not new at all — it's older posts that
+  // simply weren't in the fifty we keep. Those are gap fills and go in
+  // silently. What's actually new is what's newer than the newest post we
+  // had when we asked, and how many there are decides what happens:
+  //   a few — they go in above what you're looking at, and the page is held
+  //           still, so scrolling up reveals them in order, newest at the top
+  //   a lot — you've been gone a while: the feed goes to the top, at the
+  //           newest post, the way it would on a fresh open
+  const FEED_CATCHUP_MAX = 20;
+  function settleCatchup(topBefore) {
+    const c = feed;
+    if (!c) return;
+    const add = c.catchup || [];
+    c.catchup = [];
+    if (!add.length) { scheduleRepaint(); return; }
+    const seen = new Set(c.notes.map((e) => e.id));
+    const fresh = add.filter((e) => !seen.has(e.id) && e.created_at > topBefore).length;
+    // the window grows only by what lands inside it — gap fills below its
+    // bottom edge are paged in later, not painted now
+    const edge = (c.notes[Math.min(c.shown || FEED_PAGE, c.notes.length) - 1] || {}).created_at || 0;
+    const inside = add.filter((e) => !seen.has(e.id) && e.created_at > edge).length;
+    c.notes = [...c.notes, ...add.filter((e) => !seen.has(e.id))]
+      .sort((a, b) => b.created_at - a.created_at).slice(0, FEED_KEEP);
+    try { wallet.saveFeatureState(FEED_CACHE, c.notes.slice(0, FEED_STORE).map(slimNote)); } catch {}
+    const onScreen = ui.chatOpen && ui.msgView === 'feed';
+    if (!onScreen || fresh > FEED_CATCHUP_MAX) {
+      c.shown = FEED_PAGE;
+      render();
+      // after the paint, so a browser that anchors the scroll through the
+      // repaint can't leave you where you were
+      if (onScreen) { try { window.scrollTo({ top: 0 }); } catch {} }
+      return;
+    }
+    // the window grows by what went in above its edge, so the posts you'd
+    // scrolled to are still there below; no entrance animation — these
+    // open above the fold
+    c.shown = Math.min((c.shown || FEED_PAGE) + inside, c.notes.length);
+    holdScroll(render);
+  }
+  // Repaint with the page held still: the post at the top of the viewport
+  // stays where it was, however much was inserted above it. Measured after
+  // the paint, so a browser that anchored the scroll itself isn't corrected
+  // twice.
+  function holdScroll(paint) {
+    let key = null, top = 0;
+    try {
+      for (const r of document.querySelectorAll('.notes-feed > [data-key]')) {
+        const b = r.getBoundingClientRect();
+        if (b.bottom > FEED_TOP_PX) { key = r.getAttribute('data-key'); top = b.top; break; }
+      }
+    } catch {}
+    paint();
+    if (!key) return;
+    try {
+      const r = document.querySelector('.notes-feed > [data-key="' + CSS.escape(key) + '"]');
+      const d = r ? r.getBoundingClientRect().top - top : 0;
+      if (Math.abs(d) > 1) window.scrollBy(0, d);
+    } catch {}
+  }
   // While the feed is what's on screen, new posts arrive by themselves — on
   // the same relays the pass above reads, one subscription each.
   function watchFeed() {
diff --git a/tools/feed-live-test.js b/tools/feed-live-test.js
index 01ec384..c5aea48 100644
--- a/tools/feed-live-test.js
+++ b/tools/feed-live-test.js
@@ -54,6 +54,12 @@ await page.evaluateOnNewDocument(() => {
         if (m[0] === 'REQ' && m[2] && Array.isArray(m[2].kinds) && m[2].kinds.includes(1)) {
           window.__feedReqs++;
           subs.push([ws, m[1]]);
+          // A catch-up query (no since, no until) is answered from the queue
+          // the test filled, ahead of the real relay's end-of-stream.
+          if (!m[2].since && !m[2].until && (window.__catchup || []).length) {
+            const evs = window.__catchup; window.__catchup = [];
+            setTimeout(() => { for (const ev of evs) ws.dispatchEvent(new MessageEvent('message', { data: JSON.stringify(['EVENT', m[1], ev]) })); }, 0);
+          }
         }
       } catch {}
       return send(data);
@@ -145,21 +151,25 @@ try {
   check('...and the pill is done', !opened.pill);
 
   console.log('\n[scrolling back up on your own]');
+  // Deliberately NOT let in by the scroll itself: that re-rendered the feed
+  // under a finger on its way to the pill. The pill waits for its tap.
   await page.evaluate(() => window.scrollTo(0, 900));
   await sleep(400);
   await page.evaluate((ev) => window.__inject(ev), signed('ANOTHER ONE', 0));
-  await sleep(800);
+  await sleep(2600);
   check('a pill again', !!(await page.$('.feed-new-pill')));
   await page.evaluate(() => window.scrollTo(0, 0));
   await sleep(900);
-  check('...and scrolling to the top lets it in by itself',
-    await page.evaluate(() => /ANOTHER ONE/.test(document.querySelector('.notes-feed')?.innerText || '')
-      && !document.querySelector('.feed-new-pill')));
+  check('...that scrolling to the top leaves alone', await page.evaluate(() => !!document.querySelector('.feed-new-pill')
+    && !/ANOTHER ONE/.test(document.querySelector('.notes-feed')?.innerText || '')));
+  await page.evaluate(() => document.querySelector('.feed-new-pill').click());
+  await sleep(1200);
+  check('...until it is tapped', await page.evaluate(() => /ANOTHER ONE/.test(document.querySelector('.notes-feed')?.innerText || '') && !document.querySelector('.feed-new-pill')));
 
   console.log('\n[a post arrives whole]');
-  // A row used to paint first and fill in its picture after. Watch the feed
-  // from the instant the post is injected: the first time the row exists,
-  // its image must already be decoded.
+  // A row used to paint first and fill in its picture after. The post waits
+  // at the door for its picture before it is even counted; and when the pill
+  // lets it in, the first time the row exists its image is already decoded.
   await page.evaluate(() => {
     window.__firstSight = null;
     const feed = document.querySelector('.notes-feed');
@@ -169,17 +179,26 @@ try {
       if (!row) return;
       const img = row.querySelector('img.note-img');
       window.__firstSight = { img: !!img, ready: !!(img && img.complete && img.naturalWidth > 0), at: performance.now() };
+      // a fresh <img> node reports complete a beat after it is inserted, even
+      // from cache — what matters is that it is drawn on its first frame
+      const t = performance.now();
+      const poll = () => { if (img && img.complete && img.naturalWidth > 0) window.__firstSight.readyMs = Math.round(performance.now() - t); else if (performance.now() - t < 400) requestAnimationFrame(poll); };
+      requestAnimationFrame(poll);
     };
     new MutationObserver(look).observe(feed, { childList: true, subtree: true });
   });
   const t0 = await page.evaluate(() => performance.now());
   await page.evaluate((ev) => window.__inject(ev), signed('PICTURE POST http://localhost:5296/slow.png?' + Date.now(), 0));
+  await sleep(350);
+  check('a post is not even counted before its picture is in', !(await page.$('.feed-new-pill')));
+  await page.waitForSelector('.feed-new-pill', { timeout: 4000 }).catch(() => {});
+  const counted = await page.evaluate(() => performance.now());
+  check('...it is counted once the slow host answered', counted - t0 >= 700, Math.round(counted - t0) + 'ms');
+  await page.evaluate(() => document.querySelector('.feed-new-pill')?.click());
   await page.waitForFunction(() => !!window.__firstSight, { timeout: 4000 }).catch(() => {});
   await sleep(120); // a hair in: the animation has been started and not finished
   const sight = await page.evaluate(() => window.__firstSight);
   check('the post is in the feed', !!sight && sight.img, JSON.stringify(sight));
-  // ...and it opened rather than appeared: caught mid-animation, shorter
-  // than it will be, and full height once the animation is done
   const mid = await page.evaluate(() => {
     const row = [...document.querySelectorAll('.notes-feed > .row')].find((n) => /PICTURE POST/.test(n.innerText || ''));
     return row ? { anims: row.getAnimations().length, h: row.getBoundingClientRect().height } : null;
@@ -191,27 +210,77 @@ try {
   });
   check('...and it opened rather than appeared', !!mid && !!settled && mid.anims > 0 && mid.h < settled.h
     && settled.anims === 0 && settled.h > 0 && !settled.overflow, JSON.stringify({ mid, settled }));
-  check('...and its picture was already decoded the first time the row existed', !!sight && sight.ready);
-  check('...having waited for the slow host', !!sight && sight.at - t0 >= 700, sight ? Math.round(sight.at - t0) + 'ms' : '');
+  // decoded before the row is inserted; the fresh <img> node reports it a
+  // task later, so the row's first frames are what is checked here
+  check('...and its picture was already decoded when the row appeared', !!sight && (sight.ready || sight.readyMs <= 60), JSON.stringify(sight));
 
-  console.log('\n[coming back after being away]');
-  // the throttle would otherwise swallow a refresh this soon after the last
-  // the app was hidden for a while, then focused again
-  await page.evaluate(() => {
-    Object.defineProperty(document, 'visibilityState', { value: 'hidden', configurable: true });
-    document.dispatchEvent(new Event('visibilitychange'));
-  });
-  // longer than the five seconds that count as a blink between apps
-  await sleep(6500);
-  const reqsBefore = await page.evaluate(() => window.__feedReqs || 0);
-  await page.evaluate(() => {
-    Object.defineProperty(document, 'visibilityState', { value: 'visible', configurable: true });
-    document.dispatchEvent(new Event('visibilitychange'));
-  });
-  await sleep(3500);
-  const reqsAfter = await page.evaluate(() => window.__feedReqs || 0);
-  check('coming back re-asks the relays without being told',
-    reqsAfter > reqsBefore, `${reqsBefore} → ${reqsAfter} kind-1 requests`);
+  // ---- coming back: the app was hidden for a while, then focused again. The
+  // catch-up re-asks the relays (force, past the throttle); the test answers.
+  const away = async (makeQueue) => {
+    await page.evaluate(() => {
+      Object.defineProperty(document, 'visibilityState', { value: 'hidden', configurable: true });
+      document.dispatchEvent(new Event('visibilitychange'));
+    });
+    await sleep(6500); // longer than the five seconds that count as a blink between apps
+    const reqsBefore = await page.evaluate(() => window.__feedReqs || 0);
+    const queue = makeQueue(); // signed now, after the wait: these are newer than everything on the page
+    await page.evaluate((q) => {
+      window.__catchup = q;
+      
+      Object.defineProperty(document, 'visibilityState', { value: 'visible', configurable: true });
+      document.dispatchEvent(new Event('visibilitychange'));
+    }, queue);
+    await sleep(6500); // the catch-up settles once every relay has answered (5s at most)
+    return { reqsBefore, reqsAfter: await page.evaluate(() => window.__feedReqs || 0) };
+  };
+  const label = (n) => n.innerText.split('\n').find((l) => /post number|NEW|ONE|POST|CATCH|TOP|AWAY/.test(l)) || n.innerText.slice(0, 40);
+  const anchor = () => page.evaluate((label) => {
+    const r = [...document.querySelectorAll('.notes-feed > .row')].find((n) => n.getBoundingClientRect().bottom > 120);
+    return r ? { text: eval(label)(r), top: Math.round(r.getBoundingClientRect().top), y: Math.round(window.scrollY) } : null;
+  }, label.toString());
+  const rowAt = (text) => page.evaluate((label, text) => {
+    const r = [...document.querySelectorAll('.notes-feed > .row')].find((n) => eval(label)(n) === text);
+    return r ? { text, top: Math.round(r.getBoundingClientRect().top), y: Math.round(window.scrollY) } : null;
+  }, label.toString(), text);
+  const topRow = () => page.evaluate(() => (document.querySelector('.notes-feed > .row') || {}).innerText || '');
+
+  console.log('\n[coming back after being away, a few new posts]');
+  await page.evaluate(() => window.scrollTo(0, 900));
+  await sleep(500);
+  const held = await anchor();
+  const r1 = await away(() => [signed('CATCH-UP 1', 0), signed('CATCH-UP 2', 1), signed('CATCH-UP 3', 2),
+    signed('an old post that was not in the cache', 5000), signed('another old one', 5100)]);
+  check('coming back re-asks the relays without being told', r1.reqsAfter > r1.reqsBefore, `${r1.reqsBefore} → ${r1.reqsAfter} kind-1 requests`);
+  const a1 = await rowAt(held.text);
+  const feedText = () => page.evaluate(() => document.querySelector('.notes-feed')?.innerText || '');
+  check('a few new posts go straight in — no pill', !(await page.$('.feed-new-pill')) && /CATCH-UP 1/.test(await feedText()));
+  check('...and the post you were reading has not moved', !!held && !!a1 && a1.text === held.text && Math.abs(a1.top - held.top) <= 2,
+    JSON.stringify({ held, now: a1 }));
+  // they sort below everything cached: page down to them
+  await page.evaluate(() => window.scrollTo(0, document.documentElement.scrollHeight));
+  await sleep(600);
+  check('...older posts that merely missed the cache went in quietly too', /not in the cache/.test(await feedText()));
+  await page.evaluate(() => window.scrollTo(0, 0));
+  await sleep(300);
+  check('...scrolling up reaches the newest post', /CATCH-UP 1/.test(await topRow()), (await topRow()).slice(0, 40));
+
+  console.log('\n[coming back while sitting at the top]');
+  const wasTop = await anchor();
+  await away(() => [signed('AT-THE-TOP A', 0), signed('AT-THE-TOP B', 1)]);
+  const a2 = await rowAt(wasTop.text);
+  check('the post you had at the top stays under your eyes', !!wasTop && !!a2 && a2.text === wasTop.text && Math.abs(a2.top - wasTop.top) <= 2 && a2.y > 0,
+    JSON.stringify({ was: wasTop, now: a2 }));
+  check('...with the new ones above it, newest first', /AT-THE-TOP A/.test(await topRow()) && !(await page.$('.feed-new-pill')));
+
+  console.log('\n[coming back after a long time away]');
+  await page.evaluate(() => window.scrollTo(0, 900));
+  await sleep(400);
+  // all in the same second — a post per second back would reach past the
+  // top of the page after the thirteen seconds the two waits above took
+  await away(() => Array.from({ length: 25 }, (_, i) => signed('LONG AWAY ' + (i + 1), 0)));
+  const a3 = await anchor();
+  check('many new posts: the feed goes to the top', !!a3 && a3.y < 120, JSON.stringify(a3));
+  check('...at the newest post, with no pill', /LONG AWAY 1\b/.test(await topRow()) && !(await page.$('.feed-new-pill')), (await topRow()).slice(0, 40));
 
   check('no page errors', errs.length === 0, errs.slice(0, 2).join(' | '));
 } finally { await browser.close(); server.stop(true); }