Negentropy (NIP-77): DM and notification catch-up fetch only what this device lacks.

A relay that speaks it compares the ids it holds for a filter with the ids we hold and answers with the difference — a few hundred bytes, one round — instead of re-serving its newest events every time we ask. Measured on relay.coinos.io: the DM subscription re-served 200 gift wraps (468 KB) on every boot and every resubscribe, the Notifications refresh 120 events (105 KB) a visit; a reconcile holding the newest 100 of 120 found the 20 missing in 1.8 KB and 41 ms.

src/negentropy.js: the protocol (version 1, as strfry speaks it), a port of Doug Hoyte's reference client (MIT): NegSet of [created_at, id] items, Negentropy.initiate / reconcile → { next, have, need }, WebCrypto SHA-256. Byte-identical opening message to the reference's; seven pairings of ours and theirs as client and relay agreed on every have/need.

src/nostr.js: reconcileOn(url, filter, items) on its own socket (the pool's parser has no ear for NEG-* frames) → { need, have } or null; a relay that doesn't speak it (NOTICE "negentropy disabled", CLOSED, a socket shut on the opening message, silence) is remembered for a day in coinos-neg-off, so the plain query goes straight out there next time; NEG-ERR blocked/closed is not remembered. syncOn(relays, filter, items): reconciliation where it works, the plain query where it doesn't, the ids we lack fetched by id in slices, each event once, verified.

messages.js — DMs: relay by relay, one that speaks it gets the live subscription with limit 0 (opened first, so nothing slips between) and a reconcile of its newest 400 against wrapItems (the wraps held, full id and time, 600 kept beside the 16-char prefix log; a wrap the prefix log knows but the list doesn't is added as the relay re-serves it, once); nos.lol, with negentropy disabled, keeps the plain subscription with its backlog, and so does every relay while the one-time heal pass runs. Notifications: the refresh reconciles the relay's newest 120 against notifHeld (every notification event seen, 300 kept — not just the rows, or a folded zap twin would come back every visit), persisted beside the rows; the watch is unchanged. Not applied to feeds: the outbox relays mostly lack it (nos.lol, damus, primal, nostr.wine, nostr.mom, nostrplebs say no; relay.coinos.io, purplepag.es, ditto, oxtr, nostr21, offchain yes) and a first older page has nothing to reconcile against.

Tests: tools/negentropy-test.js (the reference's bytes, one round for a small difference, eight for a 4 KB frame, nothing to say when equal); tools/negentropy-sync-test.js (an in-page relay speaking NEG with the library on its side: the wrap subscription opens limit 0 and the three wraps lacked are fetched by id, nos.lol's "disabled" gets the plain subscription and is remembered, the boot refresh fetches the three notifications by id and the page's refresh and a refresh after reload fetch nothing). notif-report-fold, notif-faces, dm-filter, dm-row-resort, dm-nip42 pass.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
Claude-Session: https://claude.ai/code/session_01J1Kon8bxJoHRpJdSzshCkp

1 files changed, 94 insertions, 0 deletions

diff --git a/src/nostr.js b/src/nostr.js
index b76e865..8dcab01 100644
--- a/src/nostr.js
+++ b/src/nostr.js
@@ -18,6 +18,7 @@ import { normalizeURL } from 'nostr-tools/utils';
 import { wrapEvent as nip17WrapEvent } from 'nostr-tools/nip17';
 import { SimplePool } from 'nostr-tools/pool';
 import { dlog } from './debug.js';
+import { NegSet, Negentropy } from './negentropy.js';
 import { randomBytes, bytesToHex } from '@noble/hashes/utils';
 import { base64urlnopad } from '@scure/base';
 
@@ -365,6 +366,99 @@ export async function queryOn(relays, filter, maxWait = 1500) {
 }
 // Several filters in ONE REQ per relay (NIP-01 allows a list) — one
 // subscription against the relay's REQ budget instead of one per filter.
+// ---- negentropy (NIP-77): fetch only what we lack ---------------------------
+// A relay that speaks it (strfry with it enabled: relay.coinos.io does;
+// nos.lol, damus and primal answer "negentropy disabled") compares the ids
+// it holds for a filter with the ids we hold and tells us the difference —
+// a few hundred bytes — instead of re-sending the newest N events every
+// time we ask. The events we lack are then fetched by id. A relay that
+// doesn't speak it is remembered for a day, so the plain query goes
+// straight out there next time rather than after a wasted socket.
+const NEG_OFF_KEY = 'coinos-neg-off';
+const NEG_OFF_MS = 24 * 60 * 60_000;
+const negOff = new Map(); // normalized url -> until
+try {
+  for (const [u, until] of Object.entries(JSON.parse(localStorage.getItem(NEG_OFF_KEY) || '{}'))) if (until > Date.now()) negOff.set(u, until);
+} catch {}
+function negRefused(url) {
+  negOff.set(relayKey(url), Date.now() + NEG_OFF_MS);
+  try { localStorage.setItem(NEG_OFF_KEY, JSON.stringify(Object.fromEntries([...negOff].slice(-100)))); } catch {}
+}
+export const negentropyOff = (url) => (negOff.get(relayKey(url)) || 0) > Date.now();
+// Reconcile `items` ([[created_at, id], ...]: the events we hold) with one
+// relay under `filter`. Resolves { need, have } — the ids it holds that we
+// don't, and ours it lacks — or null when it can't be done there: the relay
+// doesn't speak it (a NOTICE, a CLOSED, a socket shut on the opening
+// message, or silence — remembered for a day), refuses this query (NEG-ERR
+// blocked/closed — not remembered), is sick, or we're offline. Its own
+// socket: the pool's parser has no ear for NEG-* frames.
+export async function reconcileOn(url, filter, items, { timeout = 8000 } = {}) {
+  const n = relayKey(url);
+  if (sick.has(n) || negentropyOff(url) || offline()) return null;
+  let ws;
+  try { ws = new WebSocket(url); } catch { return null; }
+  const neg = new Negentropy(new NegSet(items));
+  const id = 'neg' + Math.random().toString(36).slice(2, 10);
+  return new Promise((resolve) => {
+    let done = false, heard = false;
+    const need = [], have = [];
+    const finish = (r, refused) => {
+      if (done) return;
+      done = true; clearTimeout(t);
+      if (refused) negRefused(url);
+      try { if (ws.readyState === 1) ws.send(JSON.stringify(['NEG-CLOSE', id])); } catch {}
+      try { ws.close(); } catch {}
+      resolve(r);
+    };
+    const t = setTimeout(() => finish(null, !heard), timeout);
+    ws.onerror = () => finish(null, false);
+    ws.onclose = () => finish(null, !heard && !offline() && !hiddenTab());
+    ws.onopen = async () => {
+      try { ws.send(JSON.stringify(['NEG-OPEN', id, filter, await neg.initiate()])); } catch { finish(null, false); }
+    };
+    ws.onmessage = async (e) => {
+      let m; try { m = JSON.parse(e.data); } catch { return; }
+      if (!Array.isArray(m)) return;
+      if (m[0] === 'NEG-MSG' && m[1] === id) {
+        heard = true;
+        try {
+          const r = await neg.reconcile(m[2]);
+          need.push(...r.need); have.push(...r.have);
+          if (r.next) ws.send(JSON.stringify(['NEG-MSG', id, r.next]));
+          else finish({ need, have }, false);
+        } catch (err) { dlog('nostr: negentropy with ' + url + ' failed: ' + (err?.message || err)); finish(null, false); }
+      } else if (m[0] === 'NEG-ERR' && m[1] === id) { dlog('nostr: negentropy refused by ' + url + ': ' + m[2]); finish(null, false); }
+      else if (m[0] === 'NOTICE' && /negentropy|NEG-OPEN|bad msg/i.test(String(m[1] || ''))) finish(null, true);
+      else if (m[0] === 'CLOSED' && m[1] === id) finish(null, true);
+    };
+  });
+}
+// What `filter` matches on these relays, each event once, verified: by
+// reconciliation where a relay speaks it (only the events we don't hold
+// come, by id) and by the plain query where it doesn't. `items` are the
+// events held, [[created_at, id], ...].
+export async function syncOn(relays, filter, items, { maxWait = 5000 } = {}) {
+  probeRelays(relays);
+  const live = readRelays(relays);
+  if (!live.length) return [];
+  const plain = [], byRelay = new Map(), seen = new Set();
+  await Promise.all(live.map(async (url) => {
+    const r = await reconcileOn(url, filter, items).catch(() => null);
+    if (!r) { plain.push(url); return; }
+    for (const id of r.need) {
+      if (seen.has(id)) continue; // one copy is enough: the first relay that has it
+      seen.add(id);
+      if (!byRelay.has(url)) byRelay.set(url, []);
+      byRelay.get(url).push(id);
+    }
+  }));
+  const jobs = [];
+  if (plain.length) jobs.push(queryOn(plain, filter, maxWait));
+  for (const [url, ids] of byRelay) for (let i = 0; i < Math.min(ids.length, 1000); i += 200) jobs.push(queryOn([url], { ids: ids.slice(i, i + 200) }, maxWait));
+  const out = new Map();
+  for (const evs of await Promise.all(jobs)) for (const e of evs || []) if (e && e.id && !out.has(e.id)) out.set(e.id, e);
+  return [...out.values()];
+}
 export async function queryManyOn(relays, filters, maxWait = 1500) {
   probeRelays(relays);
   try {