From 769589f431725a275751bfc84a0d7e6bacee6d0d Mon Sep 17 00:00:00 2001 From: Adam Soltys Date: Mon, 15 Jun 2026 11:48:19 -0700 Subject: Global rate-aware request proxy + lighter polling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 429 handling: route every outgoing request (address, utxo, txs, fees, broadcast) through one scheduler. A 429 from ANY request now pauses ALL subsequent requests with exponential backoff (2s→30s), easing off after a streak of successes — instead of only cooling down the same host/kind. Fewer requests: derive balance from /address chain_stats (no /utxo needed just to total), and on background polls fetch only /address. The heavier /utxo + /txs are pulled only on first load or when the balance/addresses actually change. Idle polls drop from ~3N requests to ~N. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/api.js | 136 ++++++++++++++++++++++++++++++------------------------------- 1 file changed, 67 insertions(+), 69 deletions(-) (limited to 'src/api.js') diff --git a/src/api.js b/src/api.js index 3e10351..8ceb870 100644 --- a/src/api.js +++ b/src/api.js @@ -1,40 +1,35 @@ // Wrapper around Esplora-compatible block explorer APIs (mempool.space and -// Blockstream — same REST shape). Every network call goes through here so that -// "offline mode" can be enforced in one place. -// -// Rate-limiting strategy: public explorers return 429 (which the browser shows -// as a CORS error, since a 429 response has no CORS headers) when you burst a -// scan's ~40 address lookups. We defend on three fronts: -// 1. Throttle: cap concurrency + space out request starts. -// 2. Spread: round-robin across multiple backends, halving per-host load. -// 3. Cooldown: when a host 429s, park it for a few seconds and use the others -// instead of hammering it with retries (which only makes 429s worse). +// Blockstream — same REST shape). Every outgoing request goes through one +// rate-aware proxy (#run) so that: +// - offline mode can be enforced in a single place, +// - requests are serialized and spaced (one at a time), and +// - a 429 from ANY request immediately backs off ALL subsequent requests +// (global exponential pause), easing back as requests succeed. const BACKENDS = { mainnet: ['https://mempool.space/api', 'https://blockstream.info/api'], testnet: ['https://mempool.space/testnet/api', 'https://blockstream.info/testnet/api'], }; -const COOLDOWN_MS = 8000; - export class Api { constructor(net = 'mainnet') { this.net = net; this.offline = false; - this._hosts = (BACKENDS[net] || BACKENDS.mainnet).map((base) => ({ - base, - cooldownUntil: 0, - })); + this._hosts = BACKENDS[net] || BACKENDS.mainnet; this._rr = 0; - // Global throttle: one request at a time, well spaced. Combined with - // round-robin across two backends this keeps each host's rate very low. - this._maxConcurrent = 1; - this._minGapMs = 400; + // Serialized scheduler: one request at a time, min gap between starts. this._active = 0; + this._maxConcurrent = 1; + this._minGapMs = 500; this._nextStart = 0; this._queue = []; + + // Global rate-limit state, shared across every request and host. + this._pauseUntil = 0; // no request may start before this time + this._penalty = 0; // current backoff level + this._okStreak = 0; // consecutive successes (used to ease the penalty) } explorerTx(txid) { @@ -42,7 +37,7 @@ export class Api { return `https://mempool.space/${path}/${txid}`; } - // Schedule a task respecting max concurrency + a minimum gap between starts. + // ---- rate-aware global scheduler -------------------------------------- #schedule(task) { return new Promise((resolve, reject) => { this._queue.push({ task, resolve, reject }); @@ -55,7 +50,8 @@ export class Api { const { task, resolve, reject } = this._queue.shift(); this._active++; const now = Date.now(); - const startAt = Math.max(now, this._nextStart); + // Respect both the per-request spacing AND any global backoff pause. + const startAt = Math.max(now, this._nextStart, this._pauseUntil); this._nextStart = startAt + this._minGapMs; setTimeout(() => { Promise.resolve() @@ -69,45 +65,60 @@ export class Api { } } - // Pick a non-cooling-down host (round-robin), or null if all are cooling. - #pickHost() { - const now = Date.now(); - const avail = this._hosts.filter((h) => h.cooldownUntil <= now); - if (!avail.length) return null; - return avail[this._rr++ % avail.length]; + // A 429 anywhere → pause every outgoing request, growing the pause each time. + #penalize() { + this._penalty = Math.min(this._penalty + 1, 6); + this._okStreak = 0; + const backoff = Math.min(2000 * 2 ** (this._penalty - 1), 30000); // 2s→30s + this._pauseUntil = Date.now() + backoff; } - async #get(path, asText = false) { + #reward() { + if (this._penalty === 0) return; + if (++this._okStreak >= 4) { + this._penalty--; + this._okStreak = 0; + } + } + + // The single choke-point: fetch through the scheduler with 429 handling. + async #run(url, opts) { if (this.offline) throw new Error('offline'); - let lastErr; - for (let attempt = 0; attempt < 8; attempt++) { - let host = this.#pickHost(); - if (!host) { - // Everyone is cooling down — wait until the soonest is ready. - const soonest = Math.min(...this._hosts.map((h) => h.cooldownUntil)); - await sleep(Math.max(300, soonest - Date.now())); - host = this.#pickHost() || this._hosts[0]; + return this.#schedule(async () => { + const res = await fetch(url, opts); + if (res.status === 429) { + this.#penalize(); + const e = new Error('rate limited'); + e.rateLimited = true; + throw e; } + this.#reward(); + return res; + }); + } + + #host() { + return this._hosts[this._rr++ % this._hosts.length]; + } + + async #get(path, asText = false) { + let lastErr; + for (let attempt = 0; attempt < 6; attempt++) { try { - return await this.#schedule(async () => { - const res = await fetch(host.base + path); - if (res.status === 429) { - host.cooldownUntil = Date.now() + COOLDOWN_MS; - throw new Error('rate limited'); - } - if (!res.ok) throw new Error(`${res.status} ${res.statusText}: ${path}`); - return asText ? res.text() : res.json(); - }); + const res = await this.#run(this.#host() + path); + if (!res.ok) throw new Error(`${res.status} ${res.statusText}: ${path}`); + return asText ? await res.text() : await res.json(); } catch (e) { lastErr = e; - await sleep(300); // brief pause, then try another host + // On a 429 the global pause already delays the retry; for other errors + // add a small wait before trying the other host. + if (!e.rateLimited) await sleep(300); } } throw lastErr; } - // chain_stats / mempool_stats let us decide whether an address has ever been - // used without downloading its full history. + // chain_stats / mempool_stats tell us whether an address has ever been used. addressInfo(address) { return this.#get(`/address/${address}`); } @@ -116,22 +127,15 @@ export class Api { return this.#get(`/address/${address}/utxo`); } - // Returns the most recent transactions touching the address. addressTxs(address) { return this.#get(`/address/${address}/txs`); } async feeRates() { - const now = Date.now(); - for (const h of this._hosts) { - if (h.cooldownUntil > now) continue; - const isMempool = h.base.includes('mempool'); + for (const base of this._hosts) { + const isMempool = base.includes('mempool'); try { - const res = await fetch(h.base + (isMempool ? '/v1/fees/recommended' : '/fee-estimates')); - if (res.status === 429) { - h.cooldownUntil = Date.now() + COOLDOWN_MS; - continue; - } + const res = await this.#run(base + (isMempool ? '/v1/fees/recommended' : '/fee-estimates')); if (!res.ok) continue; const data = await res.json(); if (isMempool && data && data.halfHourFee) return data; @@ -143,19 +147,12 @@ export class Api { return { fastestFee: 20, halfHourFee: 10, hourFee: 5, economyFee: 2, minimumFee: 1 }; } - async broadcast(hexTx) { - if (this.offline) throw new Error('offline'); let lastErr; - for (const h of this._hosts) { + for (const base of this._hosts) { try { - const res = await fetch(h.base + '/tx', { method: 'POST', body: hexTx }); + const res = await this.#run(base + '/tx', { method: 'POST', body: hexTx }); const body = await res.text(); - if (res.status === 429) { - h.cooldownUntil = Date.now() + COOLDOWN_MS; - lastErr = new Error('rate limited'); - continue; - } if (!res.ok) { lastErr = new Error(body || `${res.status} broadcast failed`); continue; @@ -186,7 +183,8 @@ function sleep(ms) { return new Promise((r) => setTimeout(r, ms)); } -// Run async jobs with a small concurrency cap so we stay friendly to the API. +// Run async jobs with a small concurrency cap. (The global scheduler is the +// real limiter; this just bounds how many are queued at once.) export async function pool(items, worker, concurrency = 4) { const results = new Array(items.length); let next = 0; -- cgit v1.2.3