// Catch-up by reconciliation (NIP-77). A relay that speaks negentropy — // relay.coinos.io does — is no longer asked to re-serve its newest events: // the DM subscription opens live with limit 0 and the wraps we lack are // fetched by id after a reconcile; the Notifications refresh reconciles // its newest 120 against what we hold and fetches only the new ones, and // a refresh holding everything fetches nothing. A relay that answers // "negentropy disabled" (nos.lol) gets the plain subscription with its // backlog, and is remembered as such for a day: the next ask there goes // straight out as a REQ. // Run: bun tools/negentropy-sync-test.js import puppeteer from './puppeteer.js'; import { generateSecretKey, getPublicKey, finalizeEvent } from 'nostr-tools/pure'; import { privateKeyFromSeedWords } from 'nostr-tools/nip06'; import { buildHtml } from '../build.js'; const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); let ok = true; const check = (n, c, d = '') => { console.log(` ${c ? '✓' : '✗'} ${n}${d ? ' — ' + d : ''}`); if (!c) ok = false; }; const key = () => { const sk = generateSecretKey(); return { sk, pk: getPublicKey(sk) }; }; const FRIEND = key(), JOHNNY = key(), SENDER = key(); const now = Math.floor(Date.now() / 1000); const html = await buildHtml({ minify: true, pwa: false }); // the client library plays the relay's side in the page const negSrc = (await Bun.file('src/negentropy.js').text()).replace(/^export /gm, '') + '\nreturn { NegSet, Negentropy };'; const server = Bun.serve({ port: 5337, fetch: (req) => { const path = new URL(req.url).pathname; if (path.startsWith('/punks') || path === '/verify-worker.js' || path.startsWith('/locales')) return new Response(Bun.file('dist' + path)); return new Response(html, { headers: { 'content-type': 'text/html' } }); } }); const browser = await puppeteer.launch({ executablePath: '/usr/bin/google-chrome', headless: 'new', args: ['--no-sandbox'] }); const page = await browser.newPage(); const errs = []; page.on('pageerror', (e) => errs.push(String(e))); const click = (t) => page.evaluate((x) => { const e = [...document.querySelectorAll('button')].find((n) => n.textContent.trim().toLowerCase().includes(x)); if (e) { e.click(); return true; } return false; }, t); const text = () => page.evaluate(() => document.body.innerText); const waitText = async (x, ms = 25000) => { for (let i = 0; i < ms / 250; i++) { if ((await text()).toLowerCase().includes(x)) return true; await sleep(250); } return false; }; const waitFor = async (fn, arg, ms = 10000, step = 100) => { for (let i = 0; i < ms / step; i++) { if (await page.evaluate(fn, arg)) return true; await sleep(step); } return false; }; const sent = () => page.evaluate(() => window.__sent); const rows = () => page.evaluate(() => [...document.querySelectorAll('.alert-row')].map((r) => r.innerText.replace(/\s+/g, ' ').trim())); const host = (u) => u.replace(/^wss:\/\//, '').replace(/\/$/, ''); const shape = (m) => { const f = m[0] === 'REQ' ? m[2] : m[0] === 'NEG-OPEN' ? m[2] : null; return m[0] + (f ? ' kinds=' + (f.kinds || []).join(',') + (f.ids ? ' ids=' + f.ids.length : '') + ('limit' in f ? ' limit=' + f.limit : '') + (f.since ? ' since=' + (f.since - Math.floor(Date.now() / 1000)) : '') : ''); }; try { await page.setViewport({ width: 390, height: 844 }); await page.goto('http://localhost:5337/', { waitUntil: 'domcontentloaded' }); await page.evaluate(() => localStorage.setItem('btc-wallet-network', 'regtest')); await page.reload({ waitUntil: 'domcontentloaded' }); await sleep(400); await click('create a new wallet'); await sleep(600); await page.waitForSelector('.words .w .t'); const mn = (await page.$$eval('.words .w .t', (l) => l.map((e) => e.textContent.trim()))).join(' '); await click('skip verification'); await waitText('receive', 20000); const ME = getPublicKey(privateKeyFromSeedWords(mn)); const MINE = finalizeEvent({ kind: 1, created_at: now - 600, tags: [], content: 'my post, replied to' }, FRIEND.sk); const reply = (who, content, ago) => finalizeEvent({ kind: 1, created_at: now - ago, tags: [['e', MINE.id, '', 'root'], ['p', ME, '', 'mention']], content }, who.sk); const NOTIFS = [reply(JOHNNY, 'johnny says hello there', 100), finalizeEvent({ kind: 7, created_at: now - 90, tags: [['e', MINE.id], ['p', ME]], content: '+' }, JOHNNY.sk), reply(FRIEND, 'a reply from a friend', 70)]; // gift wraps to me (sealed to nobody: they will not open, which is beside the point here) const WRAPS = [1, 2, 3].map((i) => finalizeEvent({ kind: 1059, created_at: now - i * 3600, tags: [['p', ME]], content: 'wrapped ' + i }, SENDER.sk)); const PROFILES = [finalizeEvent({ kind: 0, created_at: now - 1000, tags: [], content: JSON.stringify({ name: 'Johnny' }) }, JOHNNY.sk), finalizeEvent({ kind: 0, created_at: now - 1000, tags: [], content: JSON.stringify({ name: 'Friend' }) }, FRIEND.sk)]; await page.evaluateOnNewDocument((fx, negSrc) => { const { NegSet, Negentropy } = new Function(negSrc)(); window.__sent = []; const Real = window.WebSocket; const say = (ws, m) => { if (ws.readyState === 1) ws.dispatchEvent(new MessageEvent('message', { data: JSON.stringify(m) })); }; const store = [...fx.notifs, ...fx.wraps, fx.mine, ...fx.profiles]; const matches = (f) => store.filter((e) => (!f.kinds || f.kinds.includes(e.kind)) && (!f.authors || f.authors.includes(e.pubkey)) && (!f.ids || f.ids.includes(e.id)) && (!f.since || e.created_at >= f.since) && (!f.until || e.created_at <= f.until) && (!f['#p'] || e.tags.some((t) => t[0] === 'p' && f['#p'].includes(t[1]))) && (!f['#e'] || e.tags.some((t) => t[0] === 'e' && f['#e'].includes(t[1])))); const newest = (evs, f) => { const s = [...evs].sort((a, b) => b.created_at - a.created_at); return 'limit' in f ? s.slice(0, f.limit) : s; }; window.WebSocket = function (...a) { const ws = new Real(...a); const url = String(a[0]); const coinos = /relay\.coinos\.io/.test(url); const negs = new Map(); // subid -> relay side const send = ws.send.bind(ws); ws.send = (data) => { try { const m = JSON.parse(data); window.__sent.push({ url, m }); if (m[0] === 'REQ') { const hit = newest(matches(m[2]), m[2]); setTimeout(() => { for (const e of hit) say(ws, ['EVENT', m[1], e]); say(ws, ['EOSE', m[1]]); }, 30); return; } if (m[0] === 'NEG-OPEN') { if (!coinos) { setTimeout(() => say(ws, ['NOTICE', 'ERROR: bad msg: negentropy disabled']), 30); return; } const items = newest(matches(m[2]), m[2]).map((e) => [e.created_at, e.id]); const n = new Negentropy(new NegSet(items)); negs.set(m[1], n); n.reconcile(m[3]).then((r) => say(ws, ['NEG-MSG', m[1], r.next])); return; } if (m[0] === 'NEG-MSG') { const n = negs.get(m[1]); if (n) n.reconcile(m[2]).then((r) => say(ws, ['NEG-MSG', m[1], r.next])); return; } if (m[0] === 'NEG-CLOSE') { negs.delete(m[1]); return; } if (m[0] === 'EVENT') { setTimeout(() => say(ws, ['OK', m[1].id, true, '']), 0); return; } } catch {} return send(data); }; return ws; }; window.WebSocket.prototype = Real.prototype; Object.assign(window.WebSocket, Real); }, { me: ME, mine: MINE, notifs: NOTIFS, wraps: WRAPS, profiles: PROFILES }, negSrc); await page.reload({ waitUntil: 'domcontentloaded' }); await waitText('receive', 20000); console.log('\n[the DM subscription]'); await waitFor(() => window.__sent.some((x) => /coinos/.test(x.url) && x.m[0] === 'REQ' && (x.m[2].ids || []).length), null, 15000); await sleep(500); let log = await sent(); const at = (re) => log.filter((x) => re.test(x.url)).map((x) => shape(x.m)); const coinosDm = at(/coinos/).filter((s) => /kinds=1059/.test(s)); check('relay.coinos.io: the live wrap subscription opens with limit 0', coinosDm.some((s) => s === 'REQ kinds=1059 limit=0'), coinosDm.join(' | ')); check('…and a reconcile of the newest 400', coinosDm.some((s) => s === 'NEG-OPEN kinds=1059 limit=400')); check('…the relay is never asked to re-serve its backlog', !coinosDm.some((s) => /^REQ kinds=1059 limit=400/.test(s))); const idsAsk = log.find((x) => /coinos/.test(x.url) && x.m[0] === 'REQ' && (x.m[2].ids || []).length); check('…and the wraps we lack are fetched by id', !!idsAsk && WRAPS.every((w) => idsAsk.m[2].ids.includes(w.id)), idsAsk ? idsAsk.m[2].ids.length + ' ids' : 'no ask by id'); const nosDm = at(/nos\.lol/).filter((s) => /kinds=1059/.test(s)); check('nos.lol says negentropy is disabled: it gets the plain subscription, backlog and all', nosDm.includes('NEG-OPEN kinds=1059 limit=400') && nosDm.includes('REQ kinds=1059 limit=400'), nosDm.join(' | ')); check('…and is remembered as such', await page.evaluate(() => /nos\.lol/.test(localStorage.getItem('coinos-neg-off') || ''))); console.log('\n[Notifications]'); const before = log.length; await page.evaluate(() => document.querySelector('.app-nav-button[aria-label="Notifications"]')?.click()); let got = []; for (let i = 0; i < 60; i++) { got = await rows(); if (got.length >= 3) break; await sleep(250); } check('the three notifications show', got.length === 3, got.join(' | ')); log = (await sent()).slice(before); const coinosN = log.filter((x) => /coinos/.test(x.url)).map((x) => shape(x.m)).filter((s) => /kinds=1,/.test(s) || /ids=/.test(s)); console.log(' coinos:', log.filter((x) => /coinos/.test(x.url)).map((x) => shape(x.m) + (x.m[0] === 'REQ' && x.m[2].ids ? ':' + x.m[2].ids.map((i) => i.slice(0, 6)).join(',') : '') + (x.m[0] === 'NEG-MSG' ? ' ' + x.m[2].length + 'ch' : '')).join(' | ')); console.log(' notif ids:', NOTIFS.map((n) => n.id.slice(0, 6)).join(','), 'mine', MINE.id.slice(0, 6)); check('relay.coinos.io: reconciled, its newest 120', coinosN.some((s) => /^NEG-OPEN kinds=1,1111,6,7.* limit=120$/.test(s)), coinosN.join(' | ')); check('…not asked for them outright', !coinosN.some((s) => /^REQ kinds=1,1111,6,7.* limit=120$/.test(s))); // (the first refresh ran at boot, for the unread count: that is the one that fetched them) const all = await sent(); const nAsk = all.find((x) => /coinos/.test(x.url) && x.m[0] === 'REQ' && (x.m[2].ids || []).length && NOTIFS.some((n) => x.m[2].ids.includes(n.id))); check('…the three we lacked fetched by id, once, at the boot refresh', !!nAsk && nAsk.m[2].ids.length === 3 && all.indexOf(nAsk) < before, nAsk ? nAsk.m[2].ids.length + ' ids at ' + all.indexOf(nAsk) + ' (page opened at ' + before + ')' : 'none'); check('…and the page\'s own refresh, holding them, fetched nothing', !log.some((x) => /coinos/.test(x.url) && x.m[0] === 'REQ' && (x.m[2].ids || []).some((id) => NOTIFS.some((n) => n.id === id)))); const nosN = log.filter((x) => /nos\.lol/.test(x.url)).map((x) => shape(x.m)); check('nos.lol: the plain query, straight out, no reconcile tried', nosN.some((s) => /^REQ kinds=1,1111,6,7.* limit=120$/.test(s)) && !nosN.some((s) => s.startsWith('NEG-OPEN')), nosN.join(' | ')); console.log('\n[a refresh holding everything, after a reload]'); await page.reload({ waitUntil: 'domcontentloaded' }); await waitText('receive', 20000); const b2 = (await sent()).length; await page.evaluate(() => document.querySelector('.app-nav-button[aria-label="Notifications"]')?.click()); for (let i = 0; i < 60; i++) { got = await rows(); if (got.length >= 3) break; await sleep(250); } check('the notifications are there from the cache', got.length === 3, got.join(' | ')); await waitFor((n) => window.__sent.slice(n).some((x) => /coinos/.test(x.url) && x.m[0] === 'NEG-CLOSE'), b2, 10000); await sleep(800); log = (await sent()).slice(b2); const again = log.filter((x) => /coinos/.test(x.url)).map((x) => shape(x.m)).filter((s) => /kinds=1,/.test(s) || /ids=/.test(s)); check('relay.coinos.io reconciled again…', again.some((s) => /^NEG-OPEN kinds=1,1111,6,7.* limit=120$/.test(s)), again.join(' | ')); check('…and nothing was fetched: we held it all', !log.some((x) => /coinos/.test(x.url) && x.m[0] === 'REQ' && (x.m[2].ids || []).some((id) => NOTIFS.some((n) => n.id === id)))); check('no page errors', errs.length === 0, errs.slice(0, 2).join(' | ')); } catch (e) { console.error(e); ok = false; } finally { await browser.close(); server.stop(true); } console.log(ok ? '\n✅ catch-up by reconciliation where the relay speaks it, the plain query where it does not' : '\n❌ failed'); process.exit(ok ? 0 : 1);