From 609fcebe634b499c1133ea26f162c24e1c76d6d4 Mon Sep 17 00:00:00 2001 From: Adam Soltys Date: Sun, 17 May 2026 23:21:39 -0700 Subject: serialize limit check + reservation under per-asset mutex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous check at debit() read ${type}:limit, compared amount, and deferred any decrement to the next freezeCheck tick (every 10s). That made the check non-atomic: a burst of concurrent withdrawals could each pass the same stale limit value before any reconciliation happened. Combined with the /gateway abuse, this allowed the hot wallet to be drained. withLimitLock() serializes the check + db.decrBy inside a per-type async mutex, and freezeCheck also takes the lock when overwriting limits so its 10s reconciliation can't race in-flight checks. Single-process only; distributed lock or TigerBeetle reservation needed if app ever scales out. Trade-off: decremented limits are not refunded on payment failure — they get reset by the next freezeCheck within 10s. Acceptable false-deny window in exchange for correctness against bursts. Co-Authored-By: Claude Opus 4.7 (1M context) --- lib/payments.ts | 107 ++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 81 insertions(+), 26 deletions(-) diff --git a/lib/payments.ts b/lib/payments.ts index be639534..9d658294 100644 --- a/lib/payments.ts +++ b/lib/payments.ts @@ -33,6 +33,38 @@ import { PaymentType } from "$lib/types"; const bc = rpc(config.bitcoin); const lq = rpc(config.liquid); + +// Throttled warn — emit at most once per WARN_THROTTLE_MS per (key, message) pair +// so a downed external service (lq, bc, etc.) doesn't flood the log every loop tick. +const WARN_THROTTLE_MS = 5 * 60 * 1000; +const warnLastEmitted: Record = {}; +const warnThrottled = (key: string, message: string) => { + const k = `${key}|${message}`; + const now = Date.now(); + if ((warnLastEmitted[k] || 0) + WARN_THROTTLE_MS > now) return; + warnLastEmitted[k] = now; + warn(`${key}:`, message); +}; + +// Per-asset-type async mutex. The May 18 2026 drain exploited the fact that +// the limit check + payment broadcast were non-atomic: a burst of concurrent +// withdrawals would each pass the same stale ${type}:limit value before any +// of them decremented it. This lock serializes the check + reservation per +// asset type. freezeCheck also takes the lock when overwriting limits so its +// periodic reconciliation can't race with an in-flight check. +const limitLocks: Record> = {}; +async function withLimitLock(type: string, fn: () => Promise): Promise { + const prev = limitLocks[type] || Promise.resolve(); + let release: () => void = () => {}; + const next = new Promise((res) => { release = res; }); + limitLocks[type] = prev.then(() => next); + await prev; + try { + return await fn(); + } finally { + release(); + } +} const { URL } = process.env; const dust = 547; @@ -65,27 +97,28 @@ export const debit = async ({ fail("address blocked"); } - const serverLimit = await g(`${type}:limit`); const userLimit = await g("limit"); const frozen = (await g("hardfreeze")) || ((await g("freeze")) && type !== PaymentType.internal); - if (frozen || (amount > userLimit && !whitelisted) || amount > serverLimit) { - warn( - "Blocking", - user.username, - amount, - hash, - user.id, - type, - frozen, - userLimit, - serverLimit, - ); + if (frozen || (amount > userLimit && !whitelisted)) { + warn("Blocking", user.username, amount, hash, user.id, type, frozen, userLimit); fail("Problem sending payment"); } + // Atomic check + reserve against the per-asset-type server limit. Decrement + // immediately so concurrent calls can't reuse the same budget; freezeCheck + // reconciles to actual on-chain balance every 10s. + await withLimitLock(type, async () => { + const serverLimit = Number.parseInt((await g(`${type}:limit`)) ?? "0", 10) || 0; + if (amount > serverLimit) { + warn("Blocking", user.username, amount, hash, user.id, type, "serverLimit", serverLimit); + fail("Problem sending payment"); + } + await db.decrBy(`${type}:limit`, amount); + }); + let ref; const { id: uid, currency } = user; if (!aid) aid = uid; @@ -891,29 +924,51 @@ const reverse = async (p) => { }; const freezeCheck = async () => { + // Each asset is fetched independently so e.g. lq being unreachable doesn't + // prevent lightning and bitcoin limits from refreshing. + let lnbalance: number | undefined; try { const funds = await ln.listfunds(); - const lnbalance = Math.round( + lnbalance = Math.round( funds.channels.reduce((a, b) => a + b.our_amount_msat, 0) / 1000, ); + } catch (e: any) { + warnThrottled("freezeCheck lightning", e.message); + } + + let bcbalance: number | undefined; + try { + bcbalance = Math.round((await bc.getBalance()) * SATS); + } catch (e: any) { + warnThrottled("freezeCheck bitcoin", e.message); + } - const bcbalance = Math.round((await bc.getBalance()) * SATS); + let lqbalance: number | undefined; + try { const { bitcoin } = await lq.getBalance(); - const lqbalance = Math.round(bitcoin * SATS); + lqbalance = Math.round(bitcoin * SATS); + } catch (e: any) { + warnThrottled("freezeCheck liquid", e.message); + } + if (lnbalance !== undefined) { const lnthreshold = await g("lightning:threshold"); + const lim = Math.max(lnbalance - lnthreshold, 0); + for (const t of ["lightning", "fund", "ecash", "bolt12"]) { + await withLimitLock(t, async () => { await s(`${t}:limit`, lim); }); + } + } + if (bcbalance !== undefined) { const bcthreshold = await g("bitcoin:threshold"); + await withLimitLock("bitcoin", async () => { + await s("bitcoin:limit", Math.max(bcbalance - bcthreshold, 0)); + }); + } + if (lqbalance !== undefined) { const lqthreshold = await g("liquid:threshold"); - - await s("lightning:limit", Math.max(lnbalance - lnthreshold, 0)); - await s("bitcoin:limit", Math.max(bcbalance - bcthreshold, 0)); - await s("liquid:limit", Math.max(lqbalance - lqthreshold, 0)); - - await s("fund:limit", Math.max(lnbalance - lnthreshold, 0)); - await s("ecash:limit", Math.max(lnbalance - lnthreshold, 0)); - await s("bolt12:limit", Math.max(lnbalance - lnthreshold, 0)); - } catch (e: any) { - warn("freezeCheck failed:", e.message); + await withLimitLock("liquid", async () => { + await s("liquid:limit", Math.max(lqbalance - lqthreshold, 0)); + }); } setTimeout(freezeCheck, 10000); -- cgit v1.2.3