From 1beae75ebd34142cc63342fbf7c8a798af45bb5a Mon Sep 17 00:00:00 2001 From: Adam Soltys Date: Sat, 27 Jun 2026 14:42:11 -0700 Subject: Add regtest SP tweak indexer; accept Core's array-witness in inputPubKey MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tools/sp-indexer.js: a minimal BIP-352 oracle (Bun) that walks Core and serves, per block, one entry per eligible tx — the tweak (input_hash*A) plus that tx's taproot outputs — for the wallet to scan on-device. Shares src/silentpay.js so the tweak math is the vector-validated code. Added to the regtest compose as the 'sp' service (oven/bun, pointed at bc:18443). inputPubKey now accepts a witness as either serialized hex (vectors) or an array of stack items (Core getblock RPC). Verified end-to-end on regtest: a real SP payment is indexed and the scan recovers the output, value, and spend tweak. Co-Authored-By: Claude Opus 4.8 (1M context) --- tools/sp-indexer.js | 118 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 tools/sp-indexer.js (limited to 'tools') diff --git a/tools/sp-indexer.js b/tools/sp-indexer.js new file mode 100644 index 0000000..12f76f7 --- /dev/null +++ b/tools/sp-indexer.js @@ -0,0 +1,118 @@ +// Minimal BIP-352 silent-payment tweak indexer (an "oracle") for regtest dev. +// +// It walks the chain from a Bitcoin Core node and, per block, emits one entry per +// silent-payment-eligible transaction: the tweak (input_hash·A over the tx's +// eligible inputs) plus that transaction's taproot outputs. A wallet fetches +// these, computes ecdh = scan_priv·tweak on-device, derives P_k = B_spend + t_k·G, +// and matches it against the entry's outputs — the scan key never leaves the +// client. Discovered outputs are ordinary taproot UTXOs, so spends/confirmations +// are tracked by the normal Electrum backend; this server only does discovery. +// +// Shares the SP crypto with the wallet (../src/silentpay.js) so the tweak math is +// the same code validated against the BIP-352 vectors. In-memory, regtest-sized; +// re-indexes on restart. Run with bun. + +import { inputPubKey, silentPaymentTweak } from '../src/silentpay.js'; + +const RPC = process.env.CORE_RPC || 'http://127.0.0.1:18443'; +const RPC_USER = process.env.RPC_USER || 'admin1'; +const RPC_PASS = process.env.RPC_PASS || '123'; +const PORT = Number(process.env.SP_PORT || 8888); +const START = Number(process.env.START_HEIGHT || 0); +const POLL_MS = Number(process.env.POLL_MS || 2000); + +const auth = 'Basic ' + Buffer.from(`${RPC_USER}:${RPC_PASS}`).toString('base64'); +let rpcId = 0; +async function rpc(method, params = []) { + const res = await fetch(RPC, { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: auth }, + body: JSON.stringify({ jsonrpc: '1.0', id: ++rpcId, method, params }), + }); + if (!res.ok) throw new Error(`rpc ${method} HTTP ${res.status}`); + const j = await res.json(); + if (j.error) throw new Error(`rpc ${method}: ${j.error.message}`); + return j.result; +} + +// height -> { hash, items: [{ txid, tweak(hex), outputs: [{ vout, value(sat), xonly(hex) }] }] } +const blocks = new Map(); +let indexed = -1; // highest indexed height + +// Is a scriptPubKey (hex) a P2TR output? OP_1 PUSH32 = 5120<64 hex>. +const taprootXonly = (hex) => (hex && hex.length === 68 && hex.startsWith('5120') ? hex.slice(4) : null); + +function scanTx(tx) { + // Eligible-input set: every input with its recovered pubkey (null if ineligible + // or coinbase); the tweak needs all outpoints but only eligible pubkeys. + if (tx.vin.some((i) => i.coinbase)) return null; // coinbase is never a silent payment + const inputs = tx.vin.map((i) => ({ + txid: i.txid, + vout: i.vout, + pubkey: inputPubKey({ + scriptSig: (i.scriptSig && i.scriptSig.hex) || '', + witness: i.txinwitness || [], + scriptPubKey: (i.prevout && i.prevout.scriptPubKey && i.prevout.scriptPubKey.hex) || '', + }) || undefined, + })); + const outputs = []; + for (const o of tx.vout) { + const x = taprootXonly(o.scriptPubKey && o.scriptPubKey.hex); + if (x) outputs.push({ vout: o.n, value: Math.round(o.value * 1e8), xonly: x }); + } + if (!outputs.length) return null; // no taproot output → can't be an SP payment + let tweak; + try { tweak = silentPaymentTweak(inputs); } catch { return null; } + if (!tweak) return null; // no eligible inputs + return { txid: tx.txid, tweak: Buffer.from(tweak).toString('hex'), outputs }; +} + +async function indexBlock(height) { + const hash = await rpc('getblockhash', [height]); + const block = await rpc('getblock', [hash, 3]); // verbosity 3: full txs with prevouts + const items = []; + for (const tx of block.tx) { const it = scanTx(tx); if (it) items.push(it); } + blocks.set(height, { hash, items }); + indexed = height; +} + +async function sync() { + try { + const tip = await rpc('getblockcount', []); + for (let h = Math.max(START, indexed + 1); h <= tip; h++) { + await indexBlock(h); + if (h % 100 === 0 || h === tip) console.log(`indexed height ${h}/${tip} (${blocks.get(h).items.length} sp-tx)`); + } + } catch (e) { + console.error('sync error:', e.message); + } +} + +const CORS = { 'access-control-allow-origin': '*', 'access-control-allow-headers': '*' }; +const json = (obj, status = 200) => new Response(JSON.stringify(obj), { status, headers: { 'content-type': 'application/json', ...CORS } }); + +Bun.serve({ + port: PORT, + async fetch(req) { + const url = new URL(req.url); + const p = url.pathname; + if (req.method === 'OPTIONS') return new Response(null, { headers: CORS }); + if (p === '/height') return json({ height: indexed }); + let m; + if ((m = p.match(/^\/block\/(\d+)$/))) { + const h = Number(m[1]); + const b = blocks.get(h); + if (!b) return json({ error: 'not indexed', height: h, tip: indexed }, 404); + return json({ height: h, hash: b.hash, items: b.items }); + } + if ((m = p.match(/^\/tweaks\/(\d+)$/))) { + const b = blocks.get(Number(m[1])); + return json(b ? b.items.map((i) => i.tweak) : []); + } + return json({ error: 'not found' }, 404); + }, +}); + +console.log(`sp-indexer: core=${RPC} start=${START} serving :${PORT}`); +await sync(); +setInterval(sync, POLL_MS); -- cgit v1.2.3 From 68edd804f1696aa412da2357a469c013f0a3e92d Mon Sep 17 00:00:00 2001 From: Adam Soltys Date: Sat, 27 Jun 2026 15:01:06 -0700 Subject: Wire silent-payment receiving into the wallet (scan + balance + UI) wallet.js: scanSilentPayments() pulls the indexer's /scan range, matches outputs on-device, and verifies each is unspent via the chain backend; spUtxos + balance are tracked in their own bucket (one-time taproot keys), persisted in the cache. silentPaymentsAvailable()/spBalance() expose state. app.js: the Receive tab shows the reusable sp address (QR + copy), a 'Scan for silent payments' button, and the found amount; Settings gains a per-network SP indexer picker (under Data source) with custom/self-host support. sp-indexer.js: added /scan/{from} so a client fetches a whole range in one request instead of one fetch per block. Receiving verified end-to-end on regtest (scan finds the output, 49,999,800 sat confirmed). Spending SP outputs is the remaining piece. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/app.js | 72 +++++++++++++++++++++++++++++++++++++--- src/i18n.js | 9 +++++ src/wallet.js | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++++- tools/sp-indexer.js | 12 +++++++ 4 files changed, 183 insertions(+), 6 deletions(-) (limited to 'tools') diff --git a/src/app.js b/src/app.js index caddeb5..daf9782 100644 --- a/src/app.js +++ b/src/app.js @@ -8,7 +8,7 @@ import { Wallet, newMnemonic, isValidMnemonic, accountXpubFor, cacheKeyFor, utxo import { qrSvg } from './qr.js'; import { scanQr } from './scan.js'; import { getSyncConfig, setSyncConfig } from './nostr.js'; -import { getExplorerConfig, setExplorerConfig, explorerPresets, getDataMode, setDataMode, electrumPresets, getElectrumServerConfig, setElectrumServerConfig, getNetwork, setNetwork, NETWORKS } from './api.js'; +import { getExplorerConfig, setExplorerConfig, explorerPresets, getDataMode, setDataMode, electrumPresets, getElectrumServerConfig, setElectrumServerConfig, getNetwork, setNetwork, NETWORKS, getSpIndexerConfig, setSpIndexerConfig, spIndexerPresets } from './api.js'; import { isSilentPaymentAddress } from './silentpay.js'; import { t, LANGS, getLang, setLang, isRTL, loadLocale } from './i18n.js'; import { @@ -1307,6 +1307,7 @@ function settingsTab() { ), networkCard(), explorerCard(), + spIndexerCard(), wallet.watchOnly || !wallet.mnemonic ? null : syncCard() ); } @@ -2224,12 +2225,73 @@ function receiveTab() { } const fresh = wallet.freshReceive(); + const spAddr = wallet.silentPaymentsAvailable() ? wallet.silentPaymentAddress() : null; return h( 'div', - { class: 'card col', style: 'align-items:center;gap:14px' }, - h('div', { html: qrSvg(fresh.address) }), - h('div', { class: 'addr-box', style: 'width:100%' }, fresh.address), - copyBtn(fresh.address, t('copyAddress')) + { class: 'col', style: 'gap:16px' }, + h('div', { class: 'card col', style: 'align-items:center;gap:14px' }, + h('div', { html: qrSvg(fresh.address) }), + h('div', { class: 'addr-box', style: 'width:100%' }, fresh.address), + copyBtn(fresh.address, t('copyAddress')) + ), + spAddr ? silentPaymentReceiveCard(spAddr) : null + ); +} + +// Reusable silent-payment receive address (sp1…/tsp1…) + on-device scan. The +// address is static and reusable; senders derive a unique on-chain output each +// time, so there's no address reuse and nothing links the payments. +function silentPaymentReceiveCard(spAddr) { + const bal = wallet.spBalance(); + return h( + 'div', + { class: 'card col', style: 'gap:12px' }, + h('div', { class: 'row between' }, + h('h3', { style: 'margin:0' }, t('spReceiveTitle')), + h('span', { class: 'small muted' }, 'BIP-352') + ), + h('p', { class: 'small muted', style: 'margin:0' }, t('spReceiveDesc')), + h('div', { style: 'align-self:center' }, h('div', { html: qrSvg(spAddr) })), + h('div', { class: 'addr-box break', style: 'width:100%;font-size:12px' }, spAddr), + copyBtn(spAddr, t('copyAddress')), + bal.count + ? h('div', { class: 'small', style: 'text-align:center' }, + t('spReceived', { amount: fmtAmount(bal.confirmed + bal.pending) + ' ' + unitLabel(), n: bal.count })) + : null, + h('button', { class: 'btn-block', disabled: wallet.spScanning, onClick: doSpScan }, + wallet.spScanning ? t('spScanning') : t('spScan')) + ); +} + +function doSpScan() { + if (wallet.spScanning || wallet.offline) return; + wallet.scanSilentPayments().catch(() => {}).finally(render); + render(); +} + +// Silent-payment tweak indexer picker (per network) — the endpoint used to scan +// for received silent payments. Only shown for wallets that can derive SP keys. +function spIndexerCard() { + if (!wallet.silentPaymentKeys || !wallet.silentPaymentKeys()) return null; + const net = getNetwork(); + const cfg = getSpIndexerConfig(net); + return h( + 'div', + { class: 'card col' }, + h('h3', {}, t('spIndexerTitle')), + h('p', { class: 'small muted', style: 'margin:0' }, t('spIndexerDesc')), + h('select', { onChange: (e) => { setSpIndexerConfig({ server: e.target.value, url: cfg.url }, net); render(); } }, + spIndexerPresets(net).map((o) => h('option', { value: o.id, selected: o.id === cfg.server }, o.label))), + cfg.server === 'custom' + ? h('label', { class: 'field' }, + h('span', { class: 'lab' }, t('spIndexerUrl')), + h('input', { + type: 'text', class: 'mono-input', placeholder: 'https://your-indexer:8888', + autocapitalize: 'none', autocomplete: 'off', spellcheck: 'false', value: cfg.url, + onChange: (e) => { setSpIndexerConfig({ server: 'custom', url: e.target.value.trim() }, net); render(); }, + }), + h('div', { class: 'small faint' }, t('spIndexerHint'))) + : null ); } diff --git a/src/i18n.js b/src/i18n.js index 2fe840d..5f029a3 100644 --- a/src/i18n.js +++ b/src/i18n.js @@ -301,6 +301,15 @@ const STR = { electrumUrl: 'Electrum server (wss://)', electrumUrlHint: 'A WebSocket Electrum server, e.g. wss://your-node:50004', silentPaymentNote: 'Silent payment', + spReceiveTitle: 'Silent payment address', + spReceiveDesc: 'A reusable address that leaks nothing on-chain — each sender derives a unique, unlinkable output. Scanning finds payments on-device using the indexer set under Data source.', + spScan: 'Scan for silent payments', + spScanning: 'Scanning…', + spReceived: 'Found {n} silent payment output(s): {amount}', + spIndexerTitle: 'Silent payment indexer', + spIndexerDesc: 'Where to scan for received silent payments. It only serves public tweak data — your scan key stays on-device — but a public one can see which blocks hold your outputs, so self-host for full privacy.', + spIndexerUrl: 'Indexer URL', + spIndexerHint: 'A BIP-352 tweak indexer (e.g. your own).', deviceSync: 'Device sync', deviceSyncDesc: 'Keep this wallet in sync across your devices by storing an encrypted copy on a Nostr relay — only a device with your seed can read it. Turn it off to keep everything on this device.', syncAcross: 'Sync across devices', diff --git a/src/wallet.js b/src/wallet.js index 219b0f5..092a005 100644 --- a/src/wallet.js +++ b/src/wallet.js @@ -21,7 +21,7 @@ import * as btc from '@scure/btc-signer'; import { p2wpkh } from '@scure/btc-signer/payment'; import { concatBytes } from '@scure/btc-signer/utils.js'; -import { Api, pool, getBackend, explorerWeb, electrumCandidates } from './api.js'; +import { Api, pool, getBackend, explorerWeb, electrumCandidates, spIndexerUrl } from './api.js'; import { ElectrumApi } from './electrum.js'; import { NostrSync, getSyncConfig } from './nostr.js'; import { isSilentPaymentAddress, decodeSilentPaymentAddress, silentPaymentScripts, silentPaymentPlaceholder, deriveSilentPaymentKeys, encodeSilentPaymentAddress, silentPaymentScan } from './silentpay.js'; @@ -102,6 +102,12 @@ export class Wallet { this.feeRates = null; this.confirmed = 0; this.pending = 0; + // Silent-payment (BIP-352) receiving: outputs we found by scanning the + // indexer. Kept in their own bucket (one-time taproot keys, not BIP84 + // addresses); { txid, vout, value, confirmed, tweak, xonly, address }. + this.spUtxos = []; + this.lastSpScan = 0; // highest block height scanned for silent payments + this.spScanning = false; this.scanning = false; this.loaded = false; // true once a scan/snapshot has populated balances once @@ -178,6 +184,8 @@ export class Wallet { this.txs = []; this.confirmed = 0; this.pending = 0; + this.spUtxos = []; + this.lastSpScan = 0; this.loaded = false; this.nextReceiveIndex = 0; this.nextChangeIndex = 0; @@ -277,6 +285,88 @@ export class Wallet { return k ? encodeSilentPaymentAddress(k.scanPub, k.spendPub, { testnet: this.netName !== 'mainnet' }) : null; } + // Is SP receiving available here? (we can derive keys AND an indexer is set) + silentPaymentsAvailable() { + return !!(this.silentPaymentKeys() && spIndexerUrl(this.netName)); + } + + // Total received via silent payments: { confirmed, pending, count } over the + // unspent SP outputs we've found. + spBalance() { + let confirmed = 0, pending = 0; + for (const u of this.spUtxos) { if (u.confirmed) confirmed += u.value; else pending += u.value; } + return { confirmed, pending, count: this.spUtxos.length }; + } + + // The taproot address for an x-only output key (for backend UTXO lookups). + _spAddress(xonly) { + return btc.Address(this.netCfg.net).encode(btc.OutScript.decode(btc.OutScript.encode({ type: 'tr', pubkey: hex.decode(xonly) }))); + } + + // Scan the configured SP indexer from the last-scanned height to the tip, find + // outputs paying our scan/spend keys, and verify each is unspent via the chain + // backend (the indexer only does discovery). Updates this.spUtxos. + async scanSilentPayments({ rescan = false } = {}) { + const keys = this.silentPaymentKeys(); + const indexer = spIndexerUrl(this.netName); + if (!keys || !indexer || this.offline) return { unavailable: true }; + if (this.spScanning) return { busy: true }; + this.spScanning = true; + this.emit(); + try { + if (rescan) { this.spUtxos = []; this.lastSpScan = 0; } + const from = (this.lastSpScan || 0) + 1; + const res = await (await fetch(`${indexer}/scan/${from}`)).json(); + const tip = res.tip || 0; + const have = new Set(this.spUtxos.map((u) => `${u.txid}:${u.vout}`)); + const found = []; + for (const it of res.items || []) { + const matches = silentPaymentScan({ + scanPriv: keys.scanPriv, spendPub: keys.spendPub, + tweak: hex.decode(it.tweak), outputs: it.outputs.map((o) => o.xonly), + }); + for (const m of matches) { + const o = it.outputs.find((o) => o.xonly === m.output); + const id = `${it.txid}:${o.vout}`; + if (have.has(id)) continue; + have.add(id); + found.push({ txid: it.txid, vout: o.vout, value: o.value, xonly: m.output, tweak: m.tweak }); + } + } + if (tip > this.lastSpScan) this.lastSpScan = tip; + // Verify newly-found outputs are unspent + get confirmation status. + for (const f of found) { + const address = this._spAddress(f.xonly); + try { + const us = await this.api.addressUtxos(address); + const u = us.find((x) => x.txid === f.txid && x.vout === f.vout); + if (u) this.spUtxos.push({ ...f, address, confirmed: !!(u.status && u.status.confirmed) }); + } catch {} + } + await this._refreshSpUtxos(); + this.saveCache(); + this.emit(); + return { found: found.length, scanned: tip }; + } finally { + this.spScanning = false; + this.emit(); + } + } + + // Re-check tracked SP outputs against the backend: drop spent ones, refresh + // confirmation status. + async _refreshSpUtxos() { + const next = []; + for (const u of this.spUtxos) { + try { + const us = await this.api.addressUtxos(u.address); + const live = us.find((x) => x.txid === u.txid && x.vout === u.vout); + if (live) next.push({ ...u, confirmed: !!(live.status && live.status.confirmed) }); + } catch { next.push(u); } // backend hiccup: keep as-is + } + this.spUtxos = next; + } + node(chain, index) { // Relative derivation from the account node (chain/index are non-hardened). return this.account().deriveChild(chain).deriveChild(index); @@ -1818,6 +1908,8 @@ export class Wallet { nextReceiveIndex: this.nextReceiveIndex, nextChangeIndex: this.nextChangeIndex, feeRates: this.feeRates, + spUtxos: this.spUtxos, + lastSpScan: this.lastSpScan, }; } @@ -1830,6 +1922,8 @@ export class Wallet { this.nextReceiveIndex = d.nextReceiveIndex || 0; this.nextChangeIndex = d.nextChangeIndex || 0; this.feeRates = d.feeRates || this.feeRates; + this.spUtxos = d.spUtxos || []; + this.lastSpScan = d.lastSpScan || 0; this.addrMap = new Map(); for (const a of [...this.receive, ...this.change]) { this.addrMap.set(a.address, { chain: a.chain, index: a.index }); diff --git a/tools/sp-indexer.js b/tools/sp-indexer.js index 12f76f7..d573ab9 100644 --- a/tools/sp-indexer.js +++ b/tools/sp-indexer.js @@ -109,6 +109,18 @@ Bun.serve({ const b = blocks.get(Number(m[1])); return json(b ? b.items.map((i) => i.tweak) : []); } + // All SP-eligible tx items from height `from` to the tip, in one response — + // each tagged with its height. Lets a client scan a range without one fetch + // per block. (Regtest-scale; mainnet would page or use BIP-158 filters.) + if ((m = p.match(/^\/scan\/(\d+)$/))) { + const from = Number(m[1]); + const items = []; + for (let h = from; h <= indexed; h++) { + const b = blocks.get(h); + if (b) for (const it of b.items) items.push({ height: h, ...it }); + } + return json({ tip: indexed, items }); + } return json({ error: 'not found' }, 404); }, }); -- cgit v1.2.3 From 4c95ed2826b4dd11555c0500426747571c148dec Mon Sep 17 00:00:00 2001 From: Adam Soltys Date: Sat, 27 Jun 2026 16:03:40 -0700 Subject: Push silent payments over WebSocket instead of polling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The indexer now serves a /ws endpoint and publishes each newly-indexed block's SP items to subscribers. The wallet subscribes (_spConnect) and scans pushed items on arrival (_scanSpItems) — folding receipts into the balance instantly — catching up via /scan on connect/reconnect. Polling for SP is now only a fallback for when the socket isn't live. Verified on regtest: connect, mine a block, wallet receives the push. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/wallet.js | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++- tools/sp-indexer.js | 19 ++++++++++++++- 2 files changed, 83 insertions(+), 2 deletions(-) (limited to 'tools') diff --git a/src/wallet.js b/src/wallet.js index bff8910..212c471 100644 --- a/src/wallet.js +++ b/src/wallet.js @@ -369,6 +369,66 @@ export class Wallet { this.spUtxos = next; } + // Live silent-payment push: subscribe to the indexer's WebSocket so each new + // block's SP items are scanned on arrival instead of polling. Catches up via a + // /scan on connect (covers anything missed while disconnected) and reconnects. + _spConnect() { + if (this.offline) return; + const base = spIndexerUrl(this.netName); + if (!base || !this.silentPaymentKeys()) return; + const url = base.replace(/^http/, 'ws') + '/ws'; + if (this._spWs && this._spWsUrl === url) return; + this._spDisconnect(); + this._spWsUrl = url; + let ws; + try { ws = new WebSocket(url); } catch { return; } + this._spWs = ws; + ws.onopen = () => { this.spLive = true; this.scanSilentPayments({ silent: true }).catch(() => {}); }; + ws.onmessage = (ev) => { + let msg; + try { msg = JSON.parse(typeof ev.data === 'string' ? ev.data : new TextDecoder().decode(ev.data)); } catch { return; } + if (msg && msg.type === 'block' && Array.isArray(msg.items)) this._scanSpItems(msg.items, msg.height).catch(() => {}); + }; + ws.onerror = () => { try { ws.close(); } catch {} }; + ws.onclose = () => { + if (this._spWs !== ws) return; + this.spLive = false; this._spWs = null; + if (!this.offline) this._spReconnect = setTimeout(() => this._spConnect(), 3000); + }; + } + + _spDisconnect() { + clearTimeout(this._spReconnect); + this.spLive = false; + const ws = this._spWs; + this._spWs = null; this._spWsUrl = null; + if (ws) { try { ws.onclose = null; ws.close(); } catch {} } + } + + // Scan a pushed block's items (no fetch) and fold any of ours into spUtxos. The + // block is confirmed (just mined), so matches are recorded confirmed; the + // backstop _refreshSpUtxos reconciles later spends. + async _scanSpItems(items, height) { + const keys = this.silentPaymentKeys(); + if (!keys) return; + const have = new Set(this.spUtxos.map((u) => `${u.txid}:${u.vout}`)); + let added = false; + for (const it of items) { + const matches = silentPaymentScan({ scanPriv: keys.scanPriv, spendPub: keys.spendPub, tweak: hex.decode(it.tweak), outputs: it.outputs.map((o) => o.xonly) }); + for (const m of matches) { + const o = it.outputs.find((o) => o.xonly === m.output); + if (have.has(`${it.txid}:${o.vout}`)) continue; + have.add(`${it.txid}:${o.vout}`); + this.spUtxos.push({ txid: it.txid, vout: o.vout, value: o.value, xonly: m.output, tweak: m.tweak, address: this._spAddress(m.output), confirmed: true }); + added = true; + } + } + if (height && height > this.lastSpScan) this.lastSpScan = height; + if (added) this._recomputeBalanceFromChains(); + this.saveCache(); + if (added) this.emit(); + } + node(chain, index) { // Relative derivation from the account node (chain/index are non-hardened). return this.account().deriveChild(chain).deriveChild(index); @@ -1635,8 +1695,11 @@ export class Wallet { if (Date.now() - (this._lastPoll || 0) < due) return; this._lastPoll = Date.now(); this.refreshLive(); - this.scanSilentPayments({ silent: true }).catch(() => {}); // pick up new SP receipts (incremental, cheap) + // SP receipts arrive via the indexer WebSocket (_spConnect); only poll for + // them as a fallback when that socket isn't live. + if (!this.spLive) this.scanSilentPayments({ silent: true }).catch(() => {}); }, 1000); + this._spConnect(); // live push of silent-payment receipts // No periodic full re-scan: the WS (and the frontier poll above) keep the // balance current, and refreshLive's _reconcileHeld catches spends of held // coins. The only thing neither covers is a deposit to a reused old address, @@ -1899,6 +1962,7 @@ export class Wallet { stopRealtime() { this._wsWant = false; this.live = false; + this._spDisconnect(); this._rejectPending('realtime stopped'); // fail any in-flight Electrum data calls this._callQueue = []; clearTimeout(this._wsRetry); diff --git a/tools/sp-indexer.js b/tools/sp-indexer.js index d573ab9..bae1e15 100644 --- a/tools/sp-indexer.js +++ b/tools/sp-indexer.js @@ -76,13 +76,20 @@ async function indexBlock(height) { indexed = height; } +let live = false; // true once caught up to the tip — then new blocks are pushed over WS + async function sync() { try { const tip = await rpc('getblockcount', []); for (let h = Math.max(START, indexed + 1); h <= tip; h++) { await indexBlock(h); + // Push each newly-indexed block (once we're live) to subscribed wallets, so + // they scan on arrival instead of polling. The payload carries the items so + // there's no follow-up fetch. + if (live) server.publish('blocks', JSON.stringify({ type: 'block', height: h, items: blocks.get(h).items })); if (h % 100 === 0 || h === tip) console.log(`indexed height ${h}/${tip} (${blocks.get(h).items.length} sp-tx)`); } + if (!live) { live = true; console.log('caught up — pushing new blocks over websocket'); } } catch (e) { console.error('sync error:', e.message); } @@ -91,12 +98,17 @@ async function sync() { const CORS = { 'access-control-allow-origin': '*', 'access-control-allow-headers': '*' }; const json = (obj, status = 200) => new Response(JSON.stringify(obj), { status, headers: { 'content-type': 'application/json', ...CORS } }); -Bun.serve({ +const server = Bun.serve({ port: PORT, async fetch(req) { const url = new URL(req.url); const p = url.pathname; if (req.method === 'OPTIONS') return new Response(null, { headers: CORS }); + // WebSocket: subscribers get a {type:'block', height, items} push per new block. + if (p === '/ws') { + if (server.upgrade(req)) return undefined; + return new Response('expected websocket', { status: 400 }); + } if (p === '/height') return json({ height: indexed }); let m; if ((m = p.match(/^\/block\/(\d+)$/))) { @@ -123,6 +135,11 @@ Bun.serve({ } return json({ error: 'not found' }, 404); }, + websocket: { + open(ws) { ws.subscribe('blocks'); ws.send(JSON.stringify({ type: 'height', height: indexed })); }, + message() {}, // clients don't send anything + close(ws) { ws.unsubscribe('blocks'); }, + }, }); console.log(`sp-indexer: core=${RPC} start=${START} serving :${PORT}`); -- cgit v1.2.3 From 588159a38bf2a2e9cae4d5b8a8ff7faae8b9ad2a Mon Sep 17 00:00:00 2001 From: Adam Soltys Date: Sat, 27 Jun 2026 16:18:20 -0700 Subject: Indexer: SQLite persistence + instant ZMQ hashblock detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Persist blocks/items to SQLite (SP_DB; defaults to in-memory) so the indexer resumes instead of re-walking the chain on restart — verified resume-from the stored height. - Subscribe to Core's hashblock ZMQ (SP_ZMQ) via a tiny pure-JS ZMTP/3.0 SUB client (no native dep, bundle stays self-contained) to trigger a sync the instant a block lands — measured ~200ms end-to-end vs the poll interval. Poll remains a backstop. Co-Authored-By: Claude Opus 4.8 (1M context) --- tools/sp-indexer.js | 118 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 92 insertions(+), 26 deletions(-) (limited to 'tools') diff --git a/tools/sp-indexer.js b/tools/sp-indexer.js index bae1e15..9bbedd6 100644 --- a/tools/sp-indexer.js +++ b/tools/sp-indexer.js @@ -1,4 +1,4 @@ -// Minimal BIP-352 silent-payment tweak indexer (an "oracle") for regtest dev. +// BIP-352 silent-payment tweak indexer (an "oracle"). // // It walks the chain from a Bitcoin Core node and, per block, emits one entry per // silent-payment-eligible transaction: the tweak (input_hash·A over the tx's @@ -9,9 +9,12 @@ // are tracked by the normal Electrum backend; this server only does discovery. // // Shares the SP crypto with the wallet (../src/silentpay.js) so the tweak math is -// the same code validated against the BIP-352 vectors. In-memory, regtest-sized; -// re-indexes on restart. Run with bun. +// the same code validated against the BIP-352 vectors. Persists to SQLite (set +// SP_DB to a file to survive restarts; defaults to in-memory). New blocks are +// pushed to subscribed wallets over WebSocket (/ws). Run with bun. +import { Database } from 'bun:sqlite'; +import net from 'node:net'; import { inputPubKey, silentPaymentTweak } from '../src/silentpay.js'; const RPC = process.env.CORE_RPC || 'http://127.0.0.1:18443'; @@ -20,6 +23,7 @@ const RPC_PASS = process.env.RPC_PASS || '123'; const PORT = Number(process.env.SP_PORT || 8888); const START = Number(process.env.START_HEIGHT || 0); const POLL_MS = Number(process.env.POLL_MS || 2000); +const DB_PATH = process.env.SP_DB || ':memory:'; const auth = 'Basic ' + Buffer.from(`${RPC_USER}:${RPC_PASS}`).toString('base64'); let rpcId = 0; @@ -35,9 +39,19 @@ async function rpc(method, params = []) { return j.result; } -// height -> { hash, items: [{ txid, tweak(hex), outputs: [{ vout, value(sat), xonly(hex) }] }] } -const blocks = new Map(); -let indexed = -1; // highest indexed height +// ---- persistent store (SQLite) -------------------------------------------- +// blocks(height, hash, items-json); meta(indexed). Survives restarts so we don't +// re-walk the chain — important on mainnet where the initial index is slow. +const db = new Database(DB_PATH); +db.exec('PRAGMA journal_mode = WAL'); +db.run('CREATE TABLE IF NOT EXISTS blocks (height INTEGER PRIMARY KEY, hash TEXT, items TEXT)'); +db.run('CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v INTEGER)'); +const qGetMeta = db.query('SELECT v FROM meta WHERE k = ?'); +const qSetMeta = db.query('INSERT OR REPLACE INTO meta (k, v) VALUES (?, ?)'); +const qPutBlock = db.query('INSERT OR REPLACE INTO blocks (height, hash, items) VALUES (?, ?, ?)'); +const qGetBlock = db.query('SELECT hash, items FROM blocks WHERE height = ?'); +const qScanFrom = db.query('SELECT height, items FROM blocks WHERE height >= ? ORDER BY height'); +let indexed = qGetMeta.get('indexed')?.v ?? (START - 1); // Is a scriptPubKey (hex) a P2TR output? OP_1 PUSH32 = 5120<64 hex>. const taprootXonly = (hex) => (hex && hex.length === 68 && hex.startsWith('5120') ? hex.slice(4) : null); @@ -72,8 +86,10 @@ async function indexBlock(height) { const block = await rpc('getblock', [hash, 3]); // verbosity 3: full txs with prevouts const items = []; for (const tx of block.tx) { const it = scanTx(tx); if (it) items.push(it); } - blocks.set(height, { hash, items }); + qPutBlock.run(height, hash, JSON.stringify(items)); + qSetMeta.run('indexed', height); indexed = height; + return items; } let live = false; // true once caught up to the tip — then new blocks are pushed over WS @@ -82,12 +98,12 @@ async function sync() { try { const tip = await rpc('getblockcount', []); for (let h = Math.max(START, indexed + 1); h <= tip; h++) { - await indexBlock(h); - // Push each newly-indexed block (once we're live) to subscribed wallets, so - // they scan on arrival instead of polling. The payload carries the items so + const items = await indexBlock(h); + // Push each newly-indexed block (once live) to subscribed wallets, so they + // scan on arrival instead of polling. The payload carries the items so // there's no follow-up fetch. - if (live) server.publish('blocks', JSON.stringify({ type: 'block', height: h, items: blocks.get(h).items })); - if (h % 100 === 0 || h === tip) console.log(`indexed height ${h}/${tip} (${blocks.get(h).items.length} sp-tx)`); + if (live) server.publish('blocks', JSON.stringify({ type: 'block', height: h, items })); + if (h % 100 === 0 || h === tip) console.log(`indexed height ${h}/${tip} (${items.length} sp-tx)`); } if (!live) { live = true; console.log('caught up — pushing new blocks over websocket'); } } catch (e) { @@ -112,25 +128,20 @@ const server = Bun.serve({ if (p === '/height') return json({ height: indexed }); let m; if ((m = p.match(/^\/block\/(\d+)$/))) { - const h = Number(m[1]); - const b = blocks.get(h); - if (!b) return json({ error: 'not indexed', height: h, tip: indexed }, 404); - return json({ height: h, hash: b.hash, items: b.items }); + const b = qGetBlock.get(Number(m[1])); + if (!b) return json({ error: 'not indexed', height: Number(m[1]), tip: indexed }, 404); + return json({ height: Number(m[1]), hash: b.hash, items: JSON.parse(b.items) }); } if ((m = p.match(/^\/tweaks\/(\d+)$/))) { - const b = blocks.get(Number(m[1])); - return json(b ? b.items.map((i) => i.tweak) : []); + const b = qGetBlock.get(Number(m[1])); + return json(b ? JSON.parse(b.items).map((i) => i.tweak) : []); } // All SP-eligible tx items from height `from` to the tip, in one response — // each tagged with its height. Lets a client scan a range without one fetch - // per block. (Regtest-scale; mainnet would page or use BIP-158 filters.) + // per block. (A future BIP-158/Bloom filter will cut this for large ranges.) if ((m = p.match(/^\/scan\/(\d+)$/))) { - const from = Number(m[1]); const items = []; - for (let h = from; h <= indexed; h++) { - const b = blocks.get(h); - if (b) for (const it of b.items) items.push({ height: h, ...it }); - } + for (const row of qScanFrom.all(Number(m[1]))) for (const it of JSON.parse(row.items)) items.push({ height: row.height, ...it }); return json({ tip: indexed, items }); } return json({ error: 'not found' }, 404); @@ -142,6 +153,61 @@ const server = Bun.serve({ }, }); -console.log(`sp-indexer: core=${RPC} start=${START} serving :${PORT}`); +// Instant block detection via Core's `hashblock` ZMQ — trigger a sync the moment +// a block arrives, instead of waiting for the poll. A minimal ZMTP/3.0 SUB client +// (no native dep, so the bundle stays self-contained). The poll below remains a +// backstop if the socket drops. +function subscribeHashblock(endpoint, onBlock) { + const [host, port] = endpoint.replace(/^tcp:\/\//, '').split(':'); + const sock = net.connect(Number(port), host); + let buf = Buffer.alloc(0); + let greeted = false; + const sendFrame = (flags, body) => { + const head = body.length < 256 + ? Buffer.from([flags, body.length]) + : Buffer.concat([Buffer.from([flags | 0x02]), (() => { const b = Buffer.alloc(8); b.writeBigUInt64BE(BigInt(body.length)); return b; })()]); + sock.write(Buffer.concat([head, body])); + }; + sock.on('connect', () => { + const g = Buffer.alloc(64); + g[0] = 0xff; g[9] = 0x7f; g[10] = 3; g[11] = 0; // signature + ZMTP version 3.0 + g.write('NULL', 12); // security mechanism, null-padded to 20 bytes + sock.write(g); + }); + sock.on('data', (d) => { + buf = Buffer.concat([buf, d]); + if (!greeted) { + if (buf.length < 64) return; + buf = buf.subarray(64); + greeted = true; + // READY (command) advertising Socket-Type=SUB, then subscribe to "hashblock". + sendFrame(0x04, Buffer.concat([ + Buffer.from([5]), Buffer.from('READY'), + Buffer.from([11]), Buffer.from('Socket-Type'), Buffer.from([0, 0, 0, 3]), Buffer.from('SUB'), + ])); + sendFrame(0x00, Buffer.concat([Buffer.from([0x01]), Buffer.from('hashblock')])); + } + while (buf.length >= 2) { + const flags = buf[0]; + const long = flags & 0x02; + const headerLen = long ? 9 : 2; + if (buf.length < headerLen) break; + const size = long ? Number(buf.readBigUInt64BE(1)) : buf[1]; + if (buf.length < headerLen + size) break; + const body = buf.subarray(headerLen, headerLen + size); + buf = buf.subarray(headerLen + size); + if (!(flags & 0x04) && body.length >= 9 && body.subarray(0, 9).toString() === 'hashblock') onBlock(); + } + }); + sock.on('error', (e) => console.error('zmq error:', e.message)); + sock.on('close', () => setTimeout(() => subscribeHashblock(endpoint, onBlock), 3000)); // reconnect +} + +console.log(`sp-indexer: core=${RPC} db=${DB_PATH} start=${START} resume-from=${indexed + 1} serving :${PORT}`); await sync(); -setInterval(sync, POLL_MS); +if (process.env.SP_ZMQ) { + let t; + subscribeHashblock(process.env.SP_ZMQ, () => { clearTimeout(t); t = setTimeout(sync, 50); }); // debounce a burst + console.log(`zmq: subscribed to hashblock at ${process.env.SP_ZMQ}`); +} +setInterval(sync, POLL_MS); // backstop (and the only trigger when SP_ZMQ is unset) -- cgit v1.2.3 From b0b9a71ef4912350ecf188bf6a75d21e97adc06b Mon Sep 17 00:00:00 2001 From: Adam Soltys Date: Sat, 27 Jun 2026 17:11:15 -0700 Subject: Mempool (unconfirmed) silent-payment discovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Indexer: scan the mempool (getrawmempool + getrawtransaction; prevouts fetched explicitly since verbosity-2 omits them for mempool txs), track SP-eligible unconfirmed txs, serve /mempool, and push {type:'mempool'} on change. Switched Core RPC to node:http (no keep-alive) + serialized calls — Bun's fetch was dropping sockets under the added mempool load. Wallet: handle the mempool push and /mempool catch-up (_scanSpMempool) — pending SP receipts rebuild on each update (also drops RBF'd/vanished ones); block pushes promote a pending entry to confirmed. So a silent payment shows the instant it's broadcast, like a normal receive, with no mining. Verified on regtest: an unconfirmed SP is detected and served; clean logs. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/wallet.js | 54 ++++++++++++++++++++++------- tools/sp-indexer.js | 98 ++++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 130 insertions(+), 22 deletions(-) (limited to 'tools') diff --git a/src/wallet.js b/src/wallet.js index 45300c3..5b5ed90 100644 --- a/src/wallet.js +++ b/src/wallet.js @@ -385,11 +385,18 @@ export class Wallet { let ws; try { ws = new WebSocket(url); } catch { return; } this._spWs = ws; - ws.onopen = () => { this.spLive = true; this.scanSilentPayments({ silent: true }).catch(() => {}); }; + ws.onopen = () => { + this.spLive = true; + this.scanSilentPayments({ silent: true }).catch(() => {}); // confirmed catch-up + // Pending (mempool) catch-up. + fetch(`${base}/mempool`).then((r) => r.json()).then((d) => this._scanSpMempool(d.items || [])).catch(() => {}); + }; ws.onmessage = (ev) => { let msg; try { msg = JSON.parse(typeof ev.data === 'string' ? ev.data : new TextDecoder().decode(ev.data)); } catch { return; } - if (msg && msg.type === 'block' && Array.isArray(msg.items)) this._scanSpItems(msg.items, msg.height).catch(() => {}); + if (!msg || !Array.isArray(msg.items)) return; + if (msg.type === 'block') this._scanSpItems(msg.items, msg.height).catch(() => {}); + else if (msg.type === 'mempool') this._scanSpMempool(msg.items).catch(() => {}); }; ws.onerror = () => { try { ws.close(); } catch {} }; ws.onclose = () => { @@ -407,28 +414,49 @@ export class Wallet { if (ws) { try { ws.onclose = null; ws.close(); } catch {} } } - // Scan a pushed block's items (no fetch) and fold any of ours into spUtxos. The - // block is confirmed (just mined), so matches are recorded confirmed; the - // backstop _refreshSpUtxos reconciles later spends. + // Scan a confirmed block's pushed items and upsert ours into spUtxos as + // confirmed (a previously-pending mempool entry is promoted to confirmed). async _scanSpItems(items, height) { const keys = this.silentPaymentKeys(); if (!keys) return; - const have = new Set(this.spUtxos.map((u) => `${u.txid}:${u.vout}`)); - let added = false; + let changed = false; for (const it of items) { const matches = silentPaymentScan({ scanPriv: keys.scanPriv, spendPub: keys.spendPub, tweak: hex.decode(it.tweak), outputs: it.outputs.map((o) => o.xonly) }); for (const m of matches) { const o = it.outputs.find((o) => o.xonly === m.output); - if (have.has(`${it.txid}:${o.vout}`)) continue; - have.add(`${it.txid}:${o.vout}`); - this.spUtxos.push({ txid: it.txid, vout: o.vout, value: o.value, xonly: m.output, tweak: m.tweak, address: this._spAddress(m.output), confirmed: true }); - added = true; + const existing = this.spUtxos.find((u) => u.txid === it.txid && u.vout === o.vout); + if (existing) { if (!existing.confirmed) { existing.confirmed = true; changed = true; } } + else { this.spUtxos.push({ txid: it.txid, vout: o.vout, value: o.value, xonly: m.output, tweak: m.tweak, address: this._spAddress(m.output), confirmed: true }); changed = true; } } } if (height && height > this.lastSpScan) this.lastSpScan = height; - if (added) this._recomputeBalanceFromChains(); + if (changed) this._recomputeBalanceFromChains(); this.saveCache(); - if (added) this.emit(); + if (changed) this.emit(); + } + + // Scan the current mempool's pushed items and rebuild our pending (unconfirmed) + // SP set — so payments show the moment they're broadcast. Confirmed entries are + // kept; the unconfirmed set is replaced, which also drops vanished/RBF'd ones. + async _scanSpMempool(items) { + const keys = this.silentPaymentKeys(); + if (!keys) return; + const confirmed = this.spUtxos.filter((u) => u.confirmed); + const confirmedIds = new Set(confirmed.map((u) => `${u.txid}:${u.vout}`)); + const pending = []; + for (const it of items) { + const matches = silentPaymentScan({ scanPriv: keys.scanPriv, spendPub: keys.spendPub, tweak: hex.decode(it.tweak), outputs: it.outputs.map((o) => o.xonly) }); + for (const m of matches) { + const o = it.outputs.find((o) => o.xonly === m.output); + if (confirmedIds.has(`${it.txid}:${o.vout}`)) continue; // already confirmed + pending.push({ txid: it.txid, vout: o.vout, value: o.value, xonly: m.output, tweak: m.tweak, address: this._spAddress(m.output), confirmed: false }); + } + } + const prevPending = this.spUtxos.filter((u) => !u.confirmed).length; + this.spUtxos = [...confirmed, ...pending]; + if (pending.length !== prevPending || pending.length) this._recomputeBalanceFromChains(); + this.saveCache(); + this.emit(); } node(chain, index) { diff --git a/tools/sp-indexer.js b/tools/sp-indexer.js index 9bbedd6..7e34405 100644 --- a/tools/sp-indexer.js +++ b/tools/sp-indexer.js @@ -15,6 +15,7 @@ import { Database } from 'bun:sqlite'; import net from 'node:net'; +import http from 'node:http'; import { inputPubKey, silentPaymentTweak } from '../src/silentpay.js'; const RPC = process.env.CORE_RPC || 'http://127.0.0.1:18443'; @@ -27,16 +28,41 @@ const DB_PATH = process.env.SP_DB || ':memory:'; const auth = 'Basic ' + Buffer.from(`${RPC_USER}:${RPC_PASS}`).toString('base64'); let rpcId = 0; -async function rpc(method, params = []) { - const res = await fetch(RPC, { - method: 'POST', - headers: { 'content-type': 'application/json', authorization: auth }, - body: JSON.stringify({ jsonrpc: '1.0', id: ++rpcId, method, params }), +// Serialize all RPC so the block sync and mempool poll don't hammer Core +// concurrently (which was dropping sockets). +let rpcLock = Promise.resolve(); +function rpc(method, params = [], retry = 2) { + const p = rpcLock.then(() => rpcCall(method, params, retry)); + rpcLock = p.then(() => {}, () => {}); + return p; +} +const RPC_URL = new URL(RPC); +function rpcCall(method, params = [], retry = 2) { + return new Promise((resolve, reject) => { + const body = JSON.stringify({ jsonrpc: '1.0', id: ++rpcId, method, params }); + const req = http.request({ + hostname: RPC_URL.hostname, port: RPC_URL.port || 80, path: RPC_URL.pathname || '/', method: 'POST', + agent: false, // fresh connection each call — no keep-alive sockets to go stale + headers: { 'content-type': 'application/json', authorization: auth, 'content-length': Buffer.byteLength(body) }, + }, (res) => { + let data = ''; + res.on('data', (c) => (data += c)); + res.on('end', () => { + try { const j = JSON.parse(data); if (j.error) reject(new Error(`rpc ${method}: ${j.error.message}`)); else resolve(j.result); } + catch (e) { reject(new Error(`rpc ${method}: ${e.message}`)); } + }); + }); + req.on('error', (e) => (retry > 0 ? resolve(rpcCall(method, params, retry - 1)) : reject(e))); + req.write(body); + req.end(); }); - if (!res.ok) throw new Error(`rpc ${method} HTTP ${res.status}`); - const j = await res.json(); - if (j.error) throw new Error(`rpc ${method}: ${j.error.message}`); - return j.result; +} + +// The scriptPubKey of a previous output (for mempool txs, whose getrawtransaction +// doesn't carry prevouts). Cached within a mempool pass via the parent tx. +async function prevoutScriptPubKey(txid, vout) { + const ptx = await rpc('getrawtransaction', [txid, true]); + return (ptx.vout[vout] && ptx.vout[vout].scriptPubKey.hex) || ''; } // ---- persistent store (SQLite) -------------------------------------------- @@ -106,11 +132,61 @@ async function sync() { if (h % 100 === 0 || h === tip) console.log(`indexed height ${h}/${tip} (${items.length} sp-tx)`); } if (!live) { live = true; console.log('caught up — pushing new blocks over websocket'); } + await mempoolSync(); // blocks just changed the mempool — reconcile pending } catch (e) { console.error('sync error:', e.message); } } +// ---- mempool (unconfirmed) SP discovery ----------------------------------- +// So a payment shows the moment it's broadcast, not only once mined. We track +// which mempool txs are SP-eligible (getrawtransaction verbosity 2 carries the +// prevouts needed for the tweak) and push the set whenever it changes. +const mempoolSeen = new Map(); // txid -> item | null (null = seen, not an SP tx) +let mempoolItems = []; + +// Like scanTx but for a mempool txid: getrawtransaction has no prevouts, so we +// fetch each input's prevout scriptPubKey (only for txs with a taproot output). +async function scanMempoolTx(txid) { + const tx = await rpc('getrawtransaction', [txid, true]); + if (tx.vin.some((i) => i.coinbase)) return null; + const outputs = []; + for (const o of tx.vout) { const x = taprootXonly(o.scriptPubKey && o.scriptPubKey.hex); if (x) outputs.push({ vout: o.n, value: Math.round(o.value * 1e8), xonly: x }); } + if (!outputs.length) return null; // not SP-relevant — skip the prevout lookups + const inputs = []; + for (const i of tx.vin) { + let spk = ''; + try { spk = await prevoutScriptPubKey(i.txid, i.vout); } catch {} + inputs.push({ txid: i.txid, vout: i.vout, pubkey: inputPubKey({ scriptSig: (i.scriptSig && i.scriptSig.hex) || '', witness: i.txinwitness || [], scriptPubKey: spk }) || undefined }); + } + let tweak; + try { tweak = silentPaymentTweak(inputs); } catch { return null; } + if (!tweak) return null; + return { txid: tx.txid, tweak: Buffer.from(tweak).toString('hex'), outputs }; +} + +async function mempoolSync() { + try { + const txids = await rpc('getrawmempool', []); + const inPool = new Set(txids); + let changed = false; + for (const txid of txids) { + if (mempoolSeen.has(txid)) continue; + mempoolSeen.set(txid, null); // mark seen so we don't refetch + let it = null; + try { it = await scanMempoolTx(txid); } catch {} + if (it) { mempoolSeen.set(txid, it); changed = true; } + } + for (const txid of mempoolSeen.keys()) if (!inPool.has(txid)) { if (mempoolSeen.get(txid)) changed = true; mempoolSeen.delete(txid); } + if (changed) { + mempoolItems = [...mempoolSeen.values()].filter(Boolean); + if (live) server.publish('blocks', JSON.stringify({ type: 'mempool', items: mempoolItems })); + } + } catch (e) { + console.error('mempool sync error:', e.message); + } +} + const CORS = { 'access-control-allow-origin': '*', 'access-control-allow-headers': '*' }; const json = (obj, status = 200) => new Response(JSON.stringify(obj), { status, headers: { 'content-type': 'application/json', ...CORS } }); @@ -144,6 +220,9 @@ const server = Bun.serve({ for (const row of qScanFrom.all(Number(m[1]))) for (const it of JSON.parse(row.items)) items.push({ height: row.height, ...it }); return json({ tip: indexed, items }); } + // Current unconfirmed (mempool) SP-eligible txs — for the pending catch-up on + // connect; live updates arrive as {type:'mempool'} WS pushes. + if (p === '/mempool') return json({ items: mempoolItems }); return json({ error: 'not found' }, 404); }, websocket: { @@ -211,3 +290,4 @@ if (process.env.SP_ZMQ) { console.log(`zmq: subscribed to hashblock at ${process.env.SP_ZMQ}`); } setInterval(sync, POLL_MS); // backstop (and the only trigger when SP_ZMQ is unset) +setInterval(mempoolSync, Number(process.env.MEMPOOL_POLL_MS || 2500)); // unconfirmed SP discovery -- cgit v1.2.3 From ef183514e078169bf171e29b88c85aa16c8ecc75 Mon Sep 17 00:00:00 2001 From: Adam Soltys Date: Sat, 27 Jun 2026 17:12:22 -0700 Subject: Gate mempool discovery behind SP_MEMPOOL (default on; off spares a busy node) Co-Authored-By: Claude Opus 4.8 (1M context) --- tools/sp-indexer.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'tools') diff --git a/tools/sp-indexer.js b/tools/sp-indexer.js index 7e34405..09f9a19 100644 --- a/tools/sp-indexer.js +++ b/tools/sp-indexer.js @@ -25,6 +25,10 @@ const PORT = Number(process.env.SP_PORT || 8888); const START = Number(process.env.START_HEIGHT || 0); const POLL_MS = Number(process.env.POLL_MS || 2000); const DB_PATH = process.env.SP_DB || ':memory:'; +// Mempool (unconfirmed) discovery is great for fast feedback but adds RPC load on +// Core (a getrawtransaction + prevout lookups per new taproot-output tx). On by +// default; set SP_MEMPOOL=0 to disable (e.g. to spare a busy production node). +const MEMPOOL = process.env.SP_MEMPOOL !== '0'; const auth = 'Basic ' + Buffer.from(`${RPC_USER}:${RPC_PASS}`).toString('base64'); let rpcId = 0; @@ -132,7 +136,7 @@ async function sync() { if (h % 100 === 0 || h === tip) console.log(`indexed height ${h}/${tip} (${items.length} sp-tx)`); } if (!live) { live = true; console.log('caught up — pushing new blocks over websocket'); } - await mempoolSync(); // blocks just changed the mempool — reconcile pending + if (MEMPOOL) await mempoolSync(); // blocks just changed the mempool — reconcile pending } catch (e) { console.error('sync error:', e.message); } @@ -290,4 +294,4 @@ if (process.env.SP_ZMQ) { console.log(`zmq: subscribed to hashblock at ${process.env.SP_ZMQ}`); } setInterval(sync, POLL_MS); // backstop (and the only trigger when SP_ZMQ is unset) -setInterval(mempoolSync, Number(process.env.MEMPOOL_POLL_MS || 2500)); // unconfirmed SP discovery +if (MEMPOOL) setInterval(mempoolSync, Number(process.env.MEMPOOL_POLL_MS || 2500)); // unconfirmed SP discovery -- cgit v1.2.3 From 56a2736be51d2f35d68ca88b477a948efe3398ce Mon Sep 17 00:00:00 2001 From: Adam Soltys Date: Sat, 27 Jun 2026 17:17:58 -0700 Subject: Catch-up scaling: dust filter + per-block Bloom filters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit silentpay.js: silentPaymentCandidate() (our P_0 for a tweak) + buildBloom/bloomHas (double-hashed sha256, ~0.8% FP). Validated: candidate == P_0 on all no-label vectors. Indexer /scan now returns, per block, the tweaks + a Bloom over that block's taproot output keys (no outputs), with an optional dustLimit. The wallet derives a candidate per tweak, tests the filter, and only fetches /block on a hit — so a big historical scan costs candidate math + a handful of fetches instead of downloading and scanning every block. Live WS pushes still carry full items. SP_DUST_LIMIT defaults to 0 (receive everything; the indexer honors any value). Verified end-to-end on regtest: the filter flow finds the output. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/silentpay.js | 40 ++++++++++++++++++++++++++++++++++++++++ src/wallet.js | 40 +++++++++++++++++++++++++++------------- tools/sp-indexer.js | 23 ++++++++++++++++------- 3 files changed, 83 insertions(+), 20 deletions(-) (limited to 'tools') diff --git a/src/silentpay.js b/src/silentpay.js index 16de66b..1732ced 100644 --- a/src/silentpay.js +++ b/src/silentpay.js @@ -292,3 +292,43 @@ export function encodeSilentPaymentAddress(scanPub, spendPub, { testnet = false const words = [0, ...bech32m.toWords(concatBytes(scanPub, spendPub))]; return bech32m.encode(hrp, words, 1023); } + +// ---- catch-up scaling: candidate keys + per-block Bloom filters ----------- +// For a large historical scan, downloading every block's outputs is wasteful. +// The indexer gives, per block, the tweaks + a Bloom filter over that block's +// taproot output keys. The client derives its candidate key P_0 from each tweak +// and tests the filter; only on a hit does it pull the block's outputs and run +// the full scan. (P_0 always exists if we're paid, so testing it alone suffices.) + +// Our candidate output key (P_0 = B_spend + t_0·G, x-only hex) for one tweak. +export function silentPaymentCandidate({ scanPriv, spendPub, tweak }) { + const ecdh = Point.fromHex(tweak).multiply(modN(toInt(scanPriv))).toRawBytes(true); + const t0 = modN(toInt(tagged('BIP0352/SharedSecret', concatBytes(ecdh, be32(0))))); + return bytesToHex(Point.fromHex(spendPub).add(G.multiply(t0)).toRawBytes(true).slice(1)); +} + +// k bit indices for a 32-byte x-only key (hex) in an m-bit filter, via +// double-hashing one sha256(key): idx_i = (h1 + i·h2) mod m. +function* bloomIndices(keyHex, m, k) { + const h = sha256(hexToBytes(keyHex)); + const u32 = (o) => (h[o] | (h[o + 1] << 8) | (h[o + 2] << 16) | (h[o + 3] * 0x1000000)) >>> 0; + const h1 = u32(0); + const h2 = u32(4) | 1; // odd, so it's coprime-ish with m + for (let i = 0; i < k; i++) yield ((h1 + i * h2) >>> 0) % m; +} + +// Build a Bloom filter over x-only keys (hex). { m (bits), k (hashes), bits (hex) }. +export function buildBloom(keysHex, bitsPerKey = 10, k = 7) { + const m = Math.max(8, keysHex.length * bitsPerKey); + const bytes = new Uint8Array(Math.ceil(m / 8)); + for (const key of keysHex) for (const idx of bloomIndices(key, m, k)) bytes[idx >> 3] |= 1 << (idx & 7); + return { m, k, bits: bytesToHex(bytes) }; +} + +// Is an x-only key (hex) possibly in the filter? (false = definitely not.) +export function bloomHas(filter, keyHex) { + if (!filter || !filter.m) return true; // no filter ⇒ can't rule out → scan it + const bytes = hexToBytes(filter.bits); + for (const idx of bloomIndices(keyHex, filter.m, filter.k)) if (!(bytes[idx >> 3] & (1 << (idx & 7)))) return false; + return true; +} diff --git a/src/wallet.js b/src/wallet.js index 5b5ed90..7b4c8ae 100644 --- a/src/wallet.js +++ b/src/wallet.js @@ -24,7 +24,7 @@ import { concatBytes } from '@scure/btc-signer/utils.js'; import { Api, pool, getBackend, explorerWeb, electrumCandidates, spIndexerUrl } from './api.js'; import { ElectrumApi } from './electrum.js'; import { NostrSync, getSyncConfig } from './nostr.js'; -import { isSilentPaymentAddress, decodeSilentPaymentAddress, silentPaymentScripts, silentPaymentPlaceholder, deriveSilentPaymentKeys, encodeSilentPaymentAddress, silentPaymentScan, silentPaymentOutputPrivKey } from './silentpay.js'; +import { isSilentPaymentAddress, decodeSilentPaymentAddress, silentPaymentScripts, silentPaymentPlaceholder, deriveSilentPaymentKeys, encodeSilentPaymentAddress, silentPaymentScan, silentPaymentOutputPrivKey, silentPaymentCandidate, bloomHas } from './silentpay.js'; import { schnorr } from '@noble/curves/secp256k1'; // No look-ahead: stop scanning a chain at the first unused address. This wallet @@ -39,6 +39,11 @@ const USED_HIT_DELAY_MS = 500; const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +// Silent-payment catch-up dust limit (sats): the indexer drops tweaks for txs +// whose taproot outputs are all below this, cutting the candidate math on a big +// scan. 0 = receive everything (default, so no small payment is ever missed). +const SP_DUST_LIMIT = 0; + const NETS = { mainnet: { net: btc.NETWORK, coin: 0 }, testnet: { net: btc.TEST_NETWORK, coin: 1 }, @@ -319,21 +324,30 @@ export class Wallet { try { if (rescan) { this.spUtxos = []; this.lastSpScan = 0; } const from = (this.lastSpScan || 0) + 1; - const res = await (await fetch(`${indexer}/scan/${from}`)).json(); + // Catch-up: per block we get tweaks + a Bloom filter (no outputs). Derive a + // candidate key per tweak, test the filter, and only fetch the full block on + // a hit — so most blocks cost just the candidate math, no download/scan. + const res = await (await fetch(`${indexer}/scan/${from}?dustLimit=${SP_DUST_LIMIT}`)).json(); const tip = res.tip || 0; const have = new Set(this.spUtxos.map((u) => `${u.txid}:${u.vout}`)); const found = []; - for (const it of res.items || []) { - const matches = silentPaymentScan({ - scanPriv: keys.scanPriv, spendPub: keys.spendPub, - tweak: hex.decode(it.tweak), outputs: it.outputs.map((o) => o.xonly), - }); - for (const m of matches) { - const o = it.outputs.find((o) => o.xonly === m.output); - const id = `${it.txid}:${o.vout}`; - if (have.has(id)) continue; - have.add(id); - found.push({ txid: it.txid, vout: o.vout, value: o.value, xonly: m.output, tweak: m.tweak }); + for (const block of res.blocks || []) { + let hit = false; + for (const tw of block.tweaks) { + const cand = silentPaymentCandidate({ scanPriv: keys.scanPriv, spendPub: keys.spendPub, tweak: hex.decode(tw) }); + if (bloomHas(block.filter, cand)) { hit = true; break; } + } + if (!hit) continue; + const blk = await (await fetch(`${indexer}/block/${block.height}`)).json(); + for (const it of blk.items || []) { + const matches = silentPaymentScan({ scanPriv: keys.scanPriv, spendPub: keys.spendPub, tweak: hex.decode(it.tweak), outputs: it.outputs.map((o) => o.xonly) }); + for (const m of matches) { + const o = it.outputs.find((o) => o.xonly === m.output); + const id = `${it.txid}:${o.vout}`; + if (have.has(id)) continue; + have.add(id); + found.push({ txid: it.txid, vout: o.vout, value: o.value, xonly: m.output, tweak: m.tweak }); + } } } if (tip > this.lastSpScan) this.lastSpScan = tip; diff --git a/tools/sp-indexer.js b/tools/sp-indexer.js index 09f9a19..934c55c 100644 --- a/tools/sp-indexer.js +++ b/tools/sp-indexer.js @@ -16,7 +16,7 @@ import { Database } from 'bun:sqlite'; import net from 'node:net'; import http from 'node:http'; -import { inputPubKey, silentPaymentTweak } from '../src/silentpay.js'; +import { inputPubKey, silentPaymentTweak, buildBloom } from '../src/silentpay.js'; const RPC = process.env.CORE_RPC || 'http://127.0.0.1:18443'; const RPC_USER = process.env.RPC_USER || 'admin1'; @@ -216,13 +216,22 @@ const server = Bun.serve({ const b = qGetBlock.get(Number(m[1])); return json(b ? JSON.parse(b.items).map((i) => i.tweak) : []); } - // All SP-eligible tx items from height `from` to the tip, in one response — - // each tagged with its height. Lets a client scan a range without one fetch - // per block. (A future BIP-158/Bloom filter will cut this for large ranges.) + // Catch-up over a height range: per block, the tweaks (dust-filtered) + a + // Bloom filter over that block's taproot output keys — NOT the outputs. The + // client derives its candidate key per tweak, tests the filter, and only + // fetches /block for the rare hits. dustLimit (sats) drops txs whose taproot + // outputs are all below it (fewer tweaks to check). if ((m = p.match(/^\/scan\/(\d+)$/))) { - const items = []; - for (const row of qScanFrom.all(Number(m[1]))) for (const it of JSON.parse(row.items)) items.push({ height: row.height, ...it }); - return json({ tip: indexed, items }); + const dust = Number(url.searchParams.get('dustLimit') || 0); + const blocks = []; + for (const row of qScanFrom.all(Number(m[1]))) { + let items = JSON.parse(row.items); + if (dust > 0) items = items.filter((it) => it.outputs.some((o) => o.value >= dust)); + if (!items.length) continue; + const outKeys = items.flatMap((it) => it.outputs.map((o) => o.xonly)); + blocks.push({ height: row.height, tweaks: items.map((it) => it.tweak), filter: buildBloom(outKeys) }); + } + return json({ tip: indexed, blocks }); } // Current unconfirmed (mempool) SP-eligible txs — for the pending catch-up on // connect; live updates arrive as {type:'mempool'} WS pushes. -- cgit v1.2.3 From 97b32b97edea28b72f9be3dd82c468f57f401926 Mon Sep 17 00:00:00 2001 From: Adam Soltys Date: Sat, 27 Jun 2026 17:38:03 -0700 Subject: Skip the mempool backlog on startup for large (mainnet) mempools First mempool pass over a >200-tx mempool would fire thousands of getrawtransaction calls at the node in a burst. Mark the existing backlog seen and scan only new arrivals instead; small (regtest) mempools still scan fully. Co-Authored-By: Claude Opus 4.8 (1M context) --- tools/sp-indexer.js | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'tools') diff --git a/tools/sp-indexer.js b/tools/sp-indexer.js index 934c55c..5611bb5 100644 --- a/tools/sp-indexer.js +++ b/tools/sp-indexer.js @@ -169,10 +169,18 @@ async function scanMempoolTx(txid) { return { txid: tx.txid, tweak: Buffer.from(tweak).toString('hex'), outputs }; } +let mempoolInit = false; async function mempoolSync() { try { const txids = await rpc('getrawmempool', []); const inPool = new Set(txids); + if (!mempoolInit) { + mempoolInit = true; + // Don't scan a large (mainnet) backlog on startup — that's thousands of + // getrawtransaction calls in a burst on the node. Mark it seen and watch + // only new arrivals. A small (regtest) mempool is cheap, so scan it. + if (txids.length > 200) { for (const txid of txids) mempoolSeen.set(txid, null); return; } + } let changed = false; for (const txid of txids) { if (mempoolSeen.has(txid)) continue; -- cgit v1.2.3