Global rate-aware request proxy + lighter polling

429 handling: route every outgoing request (address, utxo, txs, fees,
broadcast) through one scheduler. A 429 from ANY request now pauses ALL
subsequent requests with exponential backoff (2s→30s), easing off after a
streak of successes — instead of only cooling down the same host/kind.

Fewer requests: derive balance from /address chain_stats (no /utxo needed
just to total), and on background polls fetch only /address. The heavier
/utxo + /txs are pulled only on first load or when the balance/addresses
actually change. Idle polls drop from ~3N requests to ~N.

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>

2 files changed, 110 insertions, 84 deletions

diff --git a/src/api.js b/src/api.js
index 3e10351..8ceb870 100644
--- a/src/api.js
+++ b/src/api.js
@@ -1,40 +1,35 @@
 // Wrapper around Esplora-compatible block explorer APIs (mempool.space and
-// Blockstream — same REST shape). Every network call goes through here so that
-// "offline mode" can be enforced in one place.
-//
-// Rate-limiting strategy: public explorers return 429 (which the browser shows
-// as a CORS error, since a 429 response has no CORS headers) when you burst a
-// scan's ~40 address lookups. We defend on three fronts:
-//   1. Throttle: cap concurrency + space out request starts.
-//   2. Spread: round-robin across multiple backends, halving per-host load.
-//   3. Cooldown: when a host 429s, park it for a few seconds and use the others
-//      instead of hammering it with retries (which only makes 429s worse).
+// Blockstream — same REST shape). Every outgoing request goes through one
+// rate-aware proxy (#run) so that:
+//   - offline mode can be enforced in a single place,
+//   - requests are serialized and spaced (one at a time), and
+//   - a 429 from ANY request immediately backs off ALL subsequent requests
+//     (global exponential pause), easing back as requests succeed.
 
 const BACKENDS = {
   mainnet: ['https://mempool.space/api', 'https://blockstream.info/api'],
   testnet: ['https://mempool.space/testnet/api', 'https://blockstream.info/testnet/api'],
 };
 
-const COOLDOWN_MS = 8000;
-
 export class Api {
   constructor(net = 'mainnet') {
     this.net = net;
     this.offline = false;
 
-    this._hosts = (BACKENDS[net] || BACKENDS.mainnet).map((base) => ({
-      base,
-      cooldownUntil: 0,
-    }));
+    this._hosts = BACKENDS[net] || BACKENDS.mainnet;
     this._rr = 0;
 
-    // Global throttle: one request at a time, well spaced. Combined with
-    // round-robin across two backends this keeps each host's rate very low.
-    this._maxConcurrent = 1;
-    this._minGapMs = 400;
+    // Serialized scheduler: one request at a time, min gap between starts.
     this._active = 0;
+    this._maxConcurrent = 1;
+    this._minGapMs = 500;
     this._nextStart = 0;
     this._queue = [];
+
+    // Global rate-limit state, shared across every request and host.
+    this._pauseUntil = 0; // no request may start before this time
+    this._penalty = 0; // current backoff level
+    this._okStreak = 0; // consecutive successes (used to ease the penalty)
   }
 
   explorerTx(txid) {
@@ -42,7 +37,7 @@ export class Api {
     return `https://mempool.space/${path}/${txid}`;
   }
 
-  // Schedule a task respecting max concurrency + a minimum gap between starts.
+  // ---- rate-aware global scheduler --------------------------------------
   #schedule(task) {
     return new Promise((resolve, reject) => {
       this._queue.push({ task, resolve, reject });
@@ -55,7 +50,8 @@ export class Api {
       const { task, resolve, reject } = this._queue.shift();
       this._active++;
       const now = Date.now();
-      const startAt = Math.max(now, this._nextStart);
+      // Respect both the per-request spacing AND any global backoff pause.
+      const startAt = Math.max(now, this._nextStart, this._pauseUntil);
       this._nextStart = startAt + this._minGapMs;
       setTimeout(() => {
         Promise.resolve()
@@ -69,45 +65,60 @@ export class Api {
     }
   }
 
-  // Pick a non-cooling-down host (round-robin), or null if all are cooling.
-  #pickHost() {
-    const now = Date.now();
-    const avail = this._hosts.filter((h) => h.cooldownUntil <= now);
-    if (!avail.length) return null;
-    return avail[this._rr++ % avail.length];
+  // A 429 anywhere → pause every outgoing request, growing the pause each time.
+  #penalize() {
+    this._penalty = Math.min(this._penalty + 1, 6);
+    this._okStreak = 0;
+    const backoff = Math.min(2000 * 2 ** (this._penalty - 1), 30000); // 2s→30s
+    this._pauseUntil = Date.now() + backoff;
   }
 
-  async #get(path, asText = false) {
+  #reward() {
+    if (this._penalty === 0) return;
+    if (++this._okStreak >= 4) {
+      this._penalty--;
+      this._okStreak = 0;
+    }
+  }
+
+  // The single choke-point: fetch through the scheduler with 429 handling.
+  async #run(url, opts) {
     if (this.offline) throw new Error('offline');
-    let lastErr;
-    for (let attempt = 0; attempt < 8; attempt++) {
-      let host = this.#pickHost();
-      if (!host) {
-        // Everyone is cooling down — wait until the soonest is ready.
-        const soonest = Math.min(...this._hosts.map((h) => h.cooldownUntil));
-        await sleep(Math.max(300, soonest - Date.now()));
-        host = this.#pickHost() || this._hosts[0];
+    return this.#schedule(async () => {
+      const res = await fetch(url, opts);
+      if (res.status === 429) {
+        this.#penalize();
+        const e = new Error('rate limited');
+        e.rateLimited = true;
+        throw e;
       }
+      this.#reward();
+      return res;
+    });
+  }
+
+  #host() {
+    return this._hosts[this._rr++ % this._hosts.length];
+  }
+
+  async #get(path, asText = false) {
+    let lastErr;
+    for (let attempt = 0; attempt < 6; attempt++) {
       try {
-        return await this.#schedule(async () => {
-          const res = await fetch(host.base + path);
-          if (res.status === 429) {
-            host.cooldownUntil = Date.now() + COOLDOWN_MS;
-            throw new Error('rate limited');
-          }
-          if (!res.ok) throw new Error(`${res.status} ${res.statusText}: ${path}`);
-          return asText ? res.text() : res.json();
-        });
+        const res = await this.#run(this.#host() + path);
+        if (!res.ok) throw new Error(`${res.status} ${res.statusText}: ${path}`);
+        return asText ? await res.text() : await res.json();
       } catch (e) {
         lastErr = e;
-        await sleep(300); // brief pause, then try another host
+        // On a 429 the global pause already delays the retry; for other errors
+        // add a small wait before trying the other host.
+        if (!e.rateLimited) await sleep(300);
       }
     }
     throw lastErr;
   }
 
-  // chain_stats / mempool_stats let us decide whether an address has ever been
-  // used without downloading its full history.
+  // chain_stats / mempool_stats tell us whether an address has ever been used.
   addressInfo(address) {
     return this.#get(`/address/${address}`);
   }
@@ -116,22 +127,15 @@ export class Api {
     return this.#get(`/address/${address}/utxo`);
   }
 
-  // Returns the most recent transactions touching the address.
   addressTxs(address) {
     return this.#get(`/address/${address}/txs`);
   }
 
   async feeRates() {
-    const now = Date.now();
-    for (const h of this._hosts) {
-      if (h.cooldownUntil > now) continue;
-      const isMempool = h.base.includes('mempool');
+    for (const base of this._hosts) {
+      const isMempool = base.includes('mempool');
       try {
-        const res = await fetch(h.base + (isMempool ? '/v1/fees/recommended' : '/fee-estimates'));
-        if (res.status === 429) {
-          h.cooldownUntil = Date.now() + COOLDOWN_MS;
-          continue;
-        }
+        const res = await this.#run(base + (isMempool ? '/v1/fees/recommended' : '/fee-estimates'));
         if (!res.ok) continue;
         const data = await res.json();
         if (isMempool && data && data.halfHourFee) return data;
@@ -143,19 +147,12 @@ export class Api {
     return { fastestFee: 20, halfHourFee: 10, hourFee: 5, economyFee: 2, minimumFee: 1 };
   }
 
-
   async broadcast(hexTx) {
-    if (this.offline) throw new Error('offline');
     let lastErr;
-    for (const h of this._hosts) {
+    for (const base of this._hosts) {
       try {
-        const res = await fetch(h.base + '/tx', { method: 'POST', body: hexTx });
+        const res = await this.#run(base + '/tx', { method: 'POST', body: hexTx });
         const body = await res.text();
-        if (res.status === 429) {
-          h.cooldownUntil = Date.now() + COOLDOWN_MS;
-          lastErr = new Error('rate limited');
-          continue;
-        }
         if (!res.ok) {
           lastErr = new Error(body || `${res.status} broadcast failed`);
           continue;
@@ -186,7 +183,8 @@ function sleep(ms) {
   return new Promise((r) => setTimeout(r, ms));
 }
 
-// Run async jobs with a small concurrency cap so we stay friendly to the API.
+// Run async jobs with a small concurrency cap. (The global scheduler is the
+// real limiter; this just bounds how many are queued at once.)
 export async function pool(items, worker, concurrency = 4) {
   const results = new Array(items.length);
   let next = 0;
diff --git a/src/wallet.js b/src/wallet.js
index 4f6cf0d..2f9fed7 100644
--- a/src/wallet.js
+++ b/src/wallet.js
@@ -64,6 +64,8 @@ export class Wallet {
     this.utxos = []; // { txid, vout, value, address, chain, index, confirmed }
     this.txs = []; // aggregated history
     this.feeRates = null;
+    this.confirmed = 0;
+    this.pending = 0;
 
     this.scanning = false;
     this.loaded = false; // true once a scan/snapshot has populated balances once
@@ -115,6 +117,8 @@ export class Wallet {
     this.addrMap = new Map();
     this.utxos = [];
     this.txs = [];
+    this.confirmed = 0;
+    this.pending = 0;
     this.loaded = false;
     this.nextReceiveIndex = 0;
     this.nextChangeIndex = 0;
@@ -183,9 +187,13 @@ export class Wallet {
     while (gap < GAP_LIMIT) {
       const { address } = this.derive(chain, i);
       const info = await this.api.addressInfo(address);
-      const used =
-        info.chain_stats.tx_count > 0 || info.mempool_stats.tx_count > 0;
-      found.push({ chain, index: i, address, used });
+      const cs = info.chain_stats || {};
+      const ms = info.mempool_stats || {};
+      const used = (cs.tx_count || 0) > 0 || (ms.tx_count || 0) > 0;
+      // Balance straight from chain_stats — no need to fetch /utxo just to total.
+      const confirmed = (cs.funded_txo_sum || 0) - (cs.spent_txo_sum || 0);
+      const pending = (ms.funded_txo_sum || 0) - (ms.spent_txo_sum || 0);
+      found.push({ chain, index: i, address, used, confirmed, pending });
       this.addrMap.set(address, { chain, index: i });
       if (used) {
         gap = 0;
@@ -202,6 +210,8 @@ export class Wallet {
   // refreshes only re-render when something actually changed.
   _sig() {
     return JSON.stringify({
+      bc: this.confirmed,
+      bp: this.pending,
       u: this.utxos.map((u) => `${u.txid}:${u.vout}:${u.value}:${u.confirmed ? 1 : 0}`),
       t: this.txs.map((t) => `${t.txid}:${t.confirmed ? 1 : 0}`),
       r: this.nextReceiveIndex,
@@ -221,19 +231,23 @@ export class Wallet {
       this.emit();
     }
     try {
+      const prevBal = `${this.confirmed}|${this.pending}|${this.nextReceiveIndex}|${this.nextChangeIndex}`;
       this.addrMap = new Map();
       this.receive = await this.scanChain(0);
       this.change = await this.scanChain(1);
       this.nextReceiveIndex = firstUnused(this.receive);
       this.nextChangeIndex = firstUnused(this.change);
+      this._recomputeBalanceFromChains();
 
-      // Fees are only needed at load — skip on routine background polls.
-      if (!silent || !this.feeRates) {
-        this.feeRates = await this.api.feeRates();
+      // Only pull /utxo and /txs (the heavy part) on first load or when the
+      // balance/addresses actually changed. Idle polls stay /address-only.
+      const balanceChanged =
+        `${this.confirmed}|${this.pending}|${this.nextReceiveIndex}|${this.nextChangeIndex}` !== prevBal;
+      if (!silent || balanceChanged || !this.loaded) {
+        if (!silent || !this.feeRates) this.feeRates = await this.api.feeRates();
+        await this.refreshUtxos();
+        await this.refreshHistory();
       }
-
-      await this.refreshUtxos();
-      await this.refreshHistory();
       this.loaded = true;
       this.saveCache();
     } finally {
@@ -309,16 +323,28 @@ export class Wallet {
   }
 
   // --- balances -----------------------------------------------------------
-  get confirmed() {
-    return this.utxos.reduce((s, u) => s + (u.confirmed ? u.value : 0), 0);
-  }
-  get pending() {
-    return this.utxos.reduce((s, u) => s + (u.confirmed ? 0 : u.value), 0);
-  }
+  // confirmed/pending are plain fields, set from chain_stats during scan (and
+  // from UTXOs when restoring an offline snapshot). total is derived.
   get total() {
     return this.confirmed + this.pending;
   }
 
+  _recomputeBalanceFromChains() {
+    let c = 0;
+    let p = 0;
+    for (const a of [...this.receive, ...this.change]) {
+      c += a.confirmed || 0;
+      p += a.pending || 0;
+    }
+    this.confirmed = c;
+    this.pending = p;
+  }
+
+  _recomputeBalanceFromUtxos() {
+    this.confirmed = this.utxos.reduce((s, u) => s + (u.confirmed ? u.value : 0), 0);
+    this.pending = this.utxos.reduce((s, u) => s + (u.confirmed ? 0 : u.value), 0);
+  }
+
   freshReceive() {
     return this.derive(0, this.nextReceiveIndex);
   }
@@ -564,6 +590,7 @@ export class Wallet {
       for (const a of [...this.receive, ...this.change]) {
         this.addrMap.set(a.address, { chain: a.chain, index: a.index });
       }
+      this._recomputeBalanceFromChains();
       this.loaded = true;
       this.emit();
       return true;
@@ -632,6 +659,7 @@ export class Wallet {
     }
     utxos.sort((a, b) => b.value - a.value);
     this.utxos = utxos;
+    this._recomputeBalanceFromUtxos();
     this.loaded = true;
     this.emit();
     return { imported: utxos.length, unmatched };