Negentropy (NIP-77): DM and notification catch-up fetch only what this device lacks.

A relay that speaks it compares the ids it holds for a filter with the ids we hold and answers with the difference — a few hundred bytes, one round — instead of re-serving its newest events every time we ask. Measured on relay.coinos.io: the DM subscription re-served 200 gift wraps (468 KB) on every boot and every resubscribe, the Notifications refresh 120 events (105 KB) a visit; a reconcile holding the newest 100 of 120 found the 20 missing in 1.8 KB and 41 ms.

src/negentropy.js: the protocol (version 1, as strfry speaks it), a port of Doug Hoyte's reference client (MIT): NegSet of [created_at, id] items, Negentropy.initiate / reconcile → { next, have, need }, WebCrypto SHA-256. Byte-identical opening message to the reference's; seven pairings of ours and theirs as client and relay agreed on every have/need.

src/nostr.js: reconcileOn(url, filter, items) on its own socket (the pool's parser has no ear for NEG-* frames) → { need, have } or null; a relay that doesn't speak it (NOTICE "negentropy disabled", CLOSED, a socket shut on the opening message, silence) is remembered for a day in coinos-neg-off, so the plain query goes straight out there next time; NEG-ERR blocked/closed is not remembered. syncOn(relays, filter, items): reconciliation where it works, the plain query where it doesn't, the ids we lack fetched by id in slices, each event once, verified.

messages.js — DMs: relay by relay, one that speaks it gets the live subscription with limit 0 (opened first, so nothing slips between) and a reconcile of its newest 400 against wrapItems (the wraps held, full id and time, 600 kept beside the 16-char prefix log; a wrap the prefix log knows but the list doesn't is added as the relay re-serves it, once); nos.lol, with negentropy disabled, keeps the plain subscription with its backlog, and so does every relay while the one-time heal pass runs. Notifications: the refresh reconciles the relay's newest 120 against notifHeld (every notification event seen, 300 kept — not just the rows, or a folded zap twin would come back every visit), persisted beside the rows; the watch is unchanged. Not applied to feeds: the outbox relays mostly lack it (nos.lol, damus, primal, nostr.wine, nostr.mom, nostrplebs say no; relay.coinos.io, purplepag.es, ditto, oxtr, nostr21, offchain yes) and a first older page has nothing to reconcile against.

Tests: tools/negentropy-test.js (the reference's bytes, one round for a small difference, eight for a 4 KB frame, nothing to say when equal); tools/negentropy-sync-test.js (an in-page relay speaking NEG with the library on its side: the wrap subscription opens limit 0 and the three wraps lacked are fetched by id, nos.lol's "disabled" gets the plain subscription and is remembered, the boot refresh fetches the three notifications by id and the page's refresh and a refresh after reload fetch nothing). notif-report-fold, notif-faces, dm-filter, dm-row-resort, dm-nip42 pass.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
Claude-Session: https://claude.ai/code/session_01J1Kon8bxJoHRpJdSzshCkp

1 files changed, 108 insertions, 17 deletions

diff --git a/src/features/messages.js b/src/features/messages.js
index a40f7d5..ad5c32c 100644
--- a/src/features/messages.js
+++ b/src/features/messages.js
@@ -17,6 +17,7 @@ import {
   npubOf, neventOf, parseNostrPubkey, parseNostrRef, generateSecretKey, getPublicKey, finalizeEvent, nip44,
   PROFILE_RELAYS, openWrapsOffthread, unwrapDMsOffthread, verifyEventsAsync, setSyncConfig, DEFAULT_SYNC_RELAYS,
 } from '../nostr.js';
+import { reconcileOn, syncOn, negentropyOff } from '../nostr.js'; // NIP-77: fetch only what we lack
 import {
   channelKey, channelStream, channelEpoch, channelIsPrivate, controlKey, guestbookKey, openWrap, wrapRumor, rumorWithId,
   foldControl, foldGuestbook, observeAuthor, eventMs, msTags, makeEdition,
@@ -688,23 +689,57 @@ export function messagesFeature(ctx) {
     healSince = 0;
   }
   const healing = (wrap) => !!healSince && wrap.created_at >= healSince && !healed.has(wrap.id) && !!healed.add(wrap.id);
-  function rememberWrap(id) {
-    const k = wrapKey(id);
-    if (!wrapLog) wrapLog = [];
-    if (!seenWraps.has(k)) wrapLog.push(k);
-    seenWraps.add(k);
+  // The wraps this device holds — full id and time — for reconciliation
+  // (NIP-77) with a relay that speaks it: a subscribe then fetches only the
+  // wraps we lack instead of the relay's newest two hundred (half a
+  // megabyte, every boot and every resubscribe), and the live subscription
+  // opens with limit 0, nothing re-served. The prefix log above stays the
+  // "skip this one" memory; a wrap it knows but this list doesn't (logged
+  // before the list existed) is added as the relay re-serves it, once.
+  const WRAP_ITEMS_MAX = 600; // the relay reconciles its newest 400: that many held, with room (46 KB of a phone's near-full localStorage)
+  let wrapItems = null; // id -> created_at, oldest first
+  function loadWrapItems() {
+    if (wrapItems) return;
+    wrapItems = new Map();
+    try { for (const [id, ts] of st().wrapItems || []) if (/^[0-9a-f]{64}$/.test(id)) wrapItems.set(id, ts | 0); } catch {}
+  }
+  function noteWrapItem(id, ts) {
+    if (!id || !ts) return;
+    loadWrapItems();
+    if (wrapItems.has(id)) return;
+    wrapItems.set(id, ts);
+    while (wrapItems.size > WRAP_ITEMS_MAX) wrapItems.delete(wrapItems.keys().next().value);
+    scheduleWrapSave();
+  }
+  const wrapItemsHeld = () => { loadWrapItems(); return [...wrapItems].map(([id, ts]) => [ts, id]); };
+  function scheduleWrapSave() {
     clearTimeout(wrapsTimer);
     wrapsTimer = setTimeout(() => {
       try {
         // another tab may have logged wraps since this one loaded: keep theirs
         const s = st();
-        const mine = new Set(wrapLog);
-        s.wraps = [...(s.wraps || []).filter((k) => !mine.has(k)), ...wrapLog].slice(-WRAPS_MAX);
-        wrapLog = s.wraps;
+        if (wrapLog) {
+          const mine = new Set(wrapLog);
+          s.wraps = [...(s.wraps || []).filter((k) => !mine.has(k)), ...wrapLog].slice(-WRAPS_MAX);
+          wrapLog = s.wraps;
+        }
+        if (wrapItems) {
+          const all = new Map(s.wrapItems || []);
+          for (const [id, ts] of wrapItems) all.set(id, ts);
+          s.wrapItems = [...all].slice(-WRAP_ITEMS_MAX);
+        }
         save(s);
       } catch {}
     }, 1500);
   }
+  function rememberWrap(id, ts) {
+    const k = wrapKey(id);
+    if (!wrapLog) wrapLog = [];
+    if (!seenWraps.has(k)) wrapLog.push(k);
+    seenWraps.add(k);
+    if (ts) noteWrapItem(id, ts);
+    scheduleWrapSave();
+  }
   let dmStarted = false;
   let allUnsubs = [];
   let dmUnsubs = []; // the DM wrap subscriptions alone, so they can be rebuilt
@@ -3055,13 +3090,14 @@ export function messagesFeature(ctx) {
   async function handleInboxWrap(wrap) {
     let heal = false;
     if (seenWraps.has(wrap.id) || seenWraps.has(wrapKey(wrap.id))) {
+      noteWrapItem(wrap.id, wrap.created_at); // known by its prefix alone: now held in full
       if (!healing(wrap)) return;
       heal = true;
     }
     seenWraps.add(wrap.id);
     const r = await openInboxWrap(wrap, heal);
     if (r === 'stale') return;
-    if (r === true) { rememberWrap(wrap.id); return; }
+    if (r === true) { rememberWrap(wrap.id, wrap.created_at); return; }
     if (pendingWraps.size >= PENDING_MAX) pendingWraps.delete(pendingWraps.keys().next().value);
     pendingWraps.set(wrap.id, { wrap, heal, tries: r !== 'later' && decryptorsComplete() ? 1 : 0 });
     scheduleDrain(8000);
@@ -3101,9 +3137,9 @@ export function messagesFeature(ctx) {
         if (complete && !decryptorsComplete()) break;
         const r = await openInboxWrap(p.wrap, p.heal);
         if (r === 'stale') break; // the account changed under this pass
-        if (r === true) { pendingWraps.delete(id); rememberWrap(id); }
+        if (r === true) { pendingWraps.delete(id); rememberWrap(id, p.wrap && p.wrap.created_at); }
         else if (r === 'later') break; // the signer went quiet — nothing learned, try the rest later
-        else if (complete && ++p.tries >= PENDING_TRIES) { pendingWraps.delete(id); rememberWrap(id); }
+        else if (complete && ++p.tries >= PENDING_TRIES) { pendingWraps.delete(id); rememberWrap(id, p.wrap && p.wrap.created_at); }
         if (performance.now() - chunkStart > 8) {
           await new Promise((r) => setTimeout(r));
           chunkStart = performance.now();
@@ -3160,12 +3196,42 @@ export function messagesFeature(ctx) {
     }
     bumpMsgRev();
     if (swept) save(s);
-    const dmSub = (relays) => {
-      const u = subscribeOn(relays, { kinds: [1059], '#p': pks, limit: 400 }, (wrap) => {
-        handleInboxWrap(wrap).catch(() => {});
-      }, { onclose: noteSubClosed });
+    const onWrap = (wrap) => { handleInboxWrap(wrap).catch(() => {}); };
+    const plainDmSub = (relays) => {
+      const u = subscribeOn(relays, { kinds: [1059], '#p': pks, limit: 400 }, onWrap, { onclose: noteSubClosed });
       allUnsubs.push(u); dmUnsubs.push(u);
     };
+    // Relay by relay. One that speaks negentropy gets the live subscription
+    // with limit 0 — opened first, so nothing slips between — and a
+    // reconciliation of the wraps we hold against its newest four hundred,
+    // the ones we lack fetched by id. One that doesn't (nos.lol has it
+    // disabled; remembered for a day) is asked the old way, its newest
+    // wraps re-served and skipped by the prefix log. So is every relay
+    // while the one-time heal is running: that pass wants them re-served.
+    const dmSub = (relays) => {
+      for (const relay of relays) {
+        if (healSince || negentropyOff(relay)) { plainDmSub([relay]); continue; }
+        const u = subscribeOn([relay], { kinds: [1059], '#p': pks, limit: 0 }, onWrap, { onclose: noteSubClosed });
+        allUnsubs.push(u); dmUnsubs.push(u);
+        reconcileWraps(relay).then((ok) => {
+          if (ok || !dmStarted || !dmUnsubs.includes(u)) return;
+          // it doesn't speak it after all: the plain subscription, with its backlog
+          try { u(); } catch {}
+          allUnsubs = allUnsubs.filter((x) => x !== u); dmUnsubs = dmUnsubs.filter((x) => x !== u);
+          plainDmSub([relay]);
+        }).catch(() => {});
+      }
+    };
+    async function reconcileWraps(relay) {
+      const r = await reconcileOn(relay, { kinds: [1059], '#p': pks, limit: 400 }, wrapItemsHeld());
+      if (!r) return false;
+      dlog('dm: ' + relay + ' reconciled: ' + r.need.length + ' wraps to fetch, ' + r.have.length + ' held it lacks');
+      for (let i = 0; i < Math.min(r.need.length, 600) && dmStarted; i += 100) {
+        const evs = await queryOn([relay], { ids: r.need.slice(i, i + 100) }, 6000);
+        for (const w of evs) onWrap(w);
+      }
+      return true;
+    }
     dmSub(DM_RELAYS);
     // Senders deliver to the relays our kind-10050 advertises — a list other
     // clients may have published with relays beyond our defaults. Read those
@@ -15092,6 +15158,28 @@ export function messagesFeature(ctx) {
   let notif = null;      // { status, items }
   const NOTIF_V = 2;
   let notifAt = 0, notifUnsub = null;
+  // Every notification event seen, id and time, for reconciliation (NIP-77)
+  // with the relays that speak it: a refresh then fetches only what's new,
+  // not the newest hundred and twenty again — a hundred kilobytes a visit.
+  // All of them, not the rows kept: a zap receipt folded into its twin, or
+  // a hidden author's reply, would otherwise come back on every refresh.
+  const NOTIF_SEEN_MAX = 300; // the relay reconciles its newest 120
+  let notifHeld = null; // id -> created_at, oldest first
+  function notifHeldItems() {
+    if (!notifHeld) {
+      notifHeld = new Map();
+      try { const s = st(); if (s.notifHeldPk === identityPk()) for (const [id, ts] of s.notifHeld || []) notifHeld.set(id, ts | 0); } catch {}
+      for (const x of notifNow().items) if (x && x.id && x.ts && !notifHeld.has(x.id)) notifHeld.set(x.id, x.ts);
+    }
+    return [...notifHeld].map(([id, ts]) => [ts, id]);
+  }
+  function noteNotifHeld(evs) {
+    notifHeldItems();
+    let grew = false;
+    for (const ev of evs || []) if (ev && ev.id && !notifHeld.has(ev.id)) { notifHeld.set(ev.id, ev.created_at | 0); grew = true; }
+    while (notifHeld.size > NOTIF_SEEN_MAX) notifHeld.delete(notifHeld.keys().next().value);
+    return grew;
+  }
   const notifNoteAsking = new Set();
   const notifSeen = () => st().notifSeen || 0;
   // Notifications are about the IDENTITY, not the wallet: the stored list
@@ -15151,6 +15239,7 @@ export function messagesFeature(ctx) {
     for (const id of wanted) if (notifNotes.has(id)) notes[id] = savedNotifNote(notifNotes.get(id));
     const s2 = st();
     s2.notifs = notif.items; s2.notifsPk = identityPk(); s2.notifsV = NOTIF_V;
+    if (notifHeld) { s2.notifHeld = [...notifHeld]; s2.notifHeldPk = identityPk(); }
     s2.notifNoteCache = notes; s2.notifNotesPk = identityPk();
     save(s2);
   }
@@ -15263,9 +15352,11 @@ export function messagesFeature(ctx) {
     notifAt = Date.now();
     const g = session;
     try {
-      const evs = await queryOn(zapRelays(), { kinds: NOTIF_KINDS, '#p': my, limit: 120 }, 5000);
+      const evs = await syncOn(zapRelays(), { kinds: NOTIF_KINDS, '#p': my, limit: 120 }, notifHeldItems(), { maxWait: 5000 });
       if (g !== session) return; // asked for the previous account
+      const grew = noteNotifHeld(evs);
       if (mergeNotifs(evs)) scheduleRepaint();
+      if (grew) persistNotifs();
     } catch {} finally {
       if (g === session) {
         if (notif) notif.status = 'ready';
@@ -15817,7 +15908,7 @@ export function messagesFeature(ctx) {
       noteCountsReady.clear();
       // what happened to THEIR posts stays with them — the next identity
       // starts its list empty and asks the relays under its own key
-      stopNotifWatch(); notif = null; notifAt = 0; notifNotes.clear();
+      stopNotifWatch(); notif = null; notifHeld = null; notifAt = 0; notifNotes.clear();
       session++;
       zapWho.clear(); whoOpenIds.clear();
       clearTimeout(zapSaveT); zapSaveT = null; zapSeed = null;