diff options
| -rw-r--r-- | names/server.js | 117 |
1 files changed, 96 insertions, 21 deletions
diff --git a/names/server.js b/names/server.js index a60ec94..58e5228 100644 --- a/names/server.js +++ b/names/server.js @@ -401,6 +401,21 @@ async function forward(name, sat, hash, pubkey) { log(`forward: ${name}'s service returned an unusable invoice — refusing to pay it`); return false; } + // A coinos wallet's invoice is minted by our own ASP on this very node, so + // a plain pay is a self-payment that never reaches the hold plugin: every + // retry of a 124k forward failed that way for 12 hours on 2026-10-21. + // Go out and back in over a channel pair instead, then wait for the + // wallet to claim. + const dec = await ln.call('decode', { string: pr }); + if (dec.payee === (await ln.call('getinfo')).id) { + const pick = await circularPick(); + if (!pick || pick.cap < sat) return false; + const { paymentHash } = await payCircular(pr, pick); + const r = await ln.call('waitsendpay', { payment_hash: paymentHash, timeout: 240 }); + if (r.status !== 'complete') throw new Error(`circular delivery ${r.status}`); + log(`delivered ${sat} sat to ${name} over lightning via a circular route (no float needed)`); + return true; + } await ln.pay(pr, { maxfeeSat: Math.max(2, Math.ceil(sat * 0.005)) }); log(`delivered ${sat} sat to ${name} over lightning (no float needed)`); return true; @@ -602,7 +617,7 @@ async function settleLoop() { } catch (e) { log(`forward failed for ${name} (${sat} sat): ${e.message} — queued`); state.pending = state.pending || []; - state.pending.push({ name, sat, hash: inv.payment_hash, pubkey: state.names[name]?.pubkey, ts: Date.now() }); + state.pending.push({ name, sat, hash: inv.payment_hash, pubkey: state.names[name]?.pubkey, ts: Date.now(), lastError: e.message }); persist(); } } @@ -617,18 +632,28 @@ async function settleLoop() { // chunkSat caps one attempt: an ASP receive is funded from the vtxo pool, // and asking for more than its fresh buckets hold fails whole ("vtxo pool is // empty") where two smaller asks would land. Ticks accumulate to the target. -const TOPUP = { lowWaterSat: 100_000, targetSat: 500_000, chunkSat: 200_000, ...(CFG.floatTopup || {}) }; +const TOPUP = { lowWaterSat: 100_000, targetSat: 500_000, chunkSat: 200_000, minChunkSat: 20_000, ...(CFG.floatTopup || {}) }; let topupCoolUntil = 0; -async function autoTopupFloat() { - if (!fwd || !ln || Date.now() < topupCoolUntil) return; - const floatSat = fwd.balance().spendableSat; - const queuedSat = (state.pending || []).reduce((n, p) => n + p.sat, 0); - if (floatSat >= TOPUP.lowWaterSat && floatSat >= queuedSat) return; - topupCoolUntil = Date.now() + 10 * 60_000; // one attempt, then cool off - const info = await ln.call('getinfo'); +// A chunk that doesn't settle is usually one the ASP's vtxo pool can't fund +// whole (it ran dry for 35 hours on 2026-10-20/22 and 79 top-ups of 200k all +// failed while smaller ones could have landed): halve the next ask, down to +// minChunkSat, and go back to full size after one settles. /health reports +// the streak so the desk monitor can page us when it doesn't recover. +let topupChunk = TOPUP.chunkSat; +const topupStatus = { failStreak: 0, lastAt: null, lastOk: null, lastError: null }; +function topupFailed(why) { + topupStatus.failStreak++; + topupStatus.lastAt = Date.now(); + topupStatus.lastOk = false; + topupStatus.lastError = why; + topupChunk = Math.max(TOPUP.minChunkSat, Math.floor(topupChunk / 2)); +} + +// The widest same-peer pair of live channels with room for amountSat: out on +// one, back in on the other. +async function circularPick() { const chans = ((await ln.call('listpeerchannels')).channels || []) .filter((c) => c.state === 'CHANNELD_NORMAL' && c.peer_connected); - // the widest same-peer pair: out on one channel, back on the other let pick = null; for (const out of chans) { for (const back of chans) { @@ -639,14 +664,22 @@ async function autoTopupFloat() { if (!pick || cap > pick.cap) pick = { out, back, cap }; } } - if (!pick || pick.cap < 20_000) { log('float top-up: no circular route capacity'); return; } - const amountSat = Math.min(Math.max(TOPUP.targetSat, queuedSat + TOPUP.lowWaterSat) - floatSat, pick.cap, TOPUP.chunkSat); - const a = await fwd.createLnInvoice(amountSat, 'float auto top-up'); - const dec = await ln.call('decode', { string: a.invoice }); + return pick; +} + +// Pay an invoice that lives on this very node — the ASP's, so a float top-up +// or a coinos wallet's own Lightning receive — over a circular route: a plain +// pay of it is a self-payment the hold plugin never sees ("Destination said it +// doesn't know invoice"). Starts the payment and returns; the caller decides +// how to tell it settled. Costs the peer's forwarding fee on the way back in. +async function payCircular(bolt11, pick) { + const info = await ln.call('getinfo'); + const dec = await ln.call('decode', { string: bolt11 }); + const amtMsat = Number(dec.amount_msat?.msat ?? dec.amount_msat); + if (!amtMsat) throw new Error('circular pay needs an amount'); const gossip = await ln.call('listchannels', { short_channel_id: pick.back.short_channel_id }); const ret = (gossip.channels || []).find((c) => c.source === pick.back.peer_id); if (!ret) throw new Error('no gossip for the return channel'); - const amtMsat = amountSat * 1000; const feeMsat = Number(ret.base_fee_millisatoshi) + Math.ceil((amtMsat * ret.fee_per_millionth) / 1e6); const finalDelay = dec.min_final_cltv_expiry || 18; await ln.call('sendpay', { @@ -654,8 +687,22 @@ async function autoTopupFloat() { { id: pick.out.peer_id, channel: pick.out.short_channel_id, amount_msat: amtMsat + feeMsat, delay: finalDelay + ret.delay, style: 'tlv' }, { id: info.id, channel: pick.back.short_channel_id, amount_msat: amtMsat, delay: finalDelay, style: 'tlv' }, ], - payment_hash: a.paymentHash, bolt11: a.invoice, payment_secret: dec.payment_secret, + payment_hash: dec.payment_hash, bolt11, payment_secret: dec.payment_secret, }); + return { feeMsat, paymentHash: dec.payment_hash }; +} + +async function autoTopupFloat() { + if (!fwd || !ln || Date.now() < topupCoolUntil) return; + const floatSat = fwd.balance().spendableSat; + const queuedSat = (state.pending || []).reduce((n, p) => n + p.sat, 0); + if (floatSat >= TOPUP.lowWaterSat && floatSat >= queuedSat) return; + topupCoolUntil = Date.now() + 10 * 60_000; // one attempt, then cool off + const pick = await circularPick(); + if (!pick || pick.cap < TOPUP.minChunkSat) { log('float top-up: no circular route capacity'); topupFailed('no circular route capacity'); return; } + const amountSat = Math.min(Math.max(TOPUP.targetSat, queuedSat + TOPUP.lowWaterSat) - floatSat, pick.cap, topupChunk); + const a = await fwd.createLnInvoice(amountSat, 'float auto top-up'); + const { feeMsat } = await payCircular(a.invoice, pick); // the HTLC sits held until the float wallet claims — drive it through for (let i = 0; i < 30; i++) { await new Promise((r) => setTimeout(r, 2000)); @@ -668,8 +715,11 @@ async function autoTopupFloat() { for (const p of state.pending || []) p.next = 0; // the queue can go again now persist(); topupCoolUntil = 0; + topupChunk = TOPUP.chunkSat; + Object.assign(topupStatus, { failStreak: 0, lastAt: Date.now(), lastOk: true, lastError: null }); } else { - log('float self-top-up did not settle — cooling off'); + log(`float self-top-up of ${amountSat} sat did not settle — cooling off, next ask ${Math.max(TOPUP.minChunkSat, Math.floor(topupChunk / 2))} sat`); + topupFailed(`${amountSat} sat did not settle`); } } @@ -734,13 +784,31 @@ async function consolidateFloat() { // retry queued forwards (e.g. after the float is topped up). Each failure // backs off exponentially (1m → 1h cap): a stuck forward must not fire an // offer request — and push-wake the recipient's devices — every minute -// forever. +// forever. One tick at a time: a top-up or a circular delivery can outlast +// the interval, and an overlapping tick would forward the same queued +// payment twice. +const foreignServer = (p) => /different ark server/.test(p.lastError || ''); +function pendingSummary() { + const q = state.pending || []; + const live = q.filter((p) => !foreignServer(p)), foreign = q.filter(foreignServer); + return { + pendingSat: live.reduce((n, p) => n + p.sat, 0), + oldestPendingAt: live.reduce((m, p) => (p.ts && (!m || p.ts < m) ? p.ts : m), null), + undeliverable: { count: foreign.length, sat: foreign.reduce((n, p) => n + p.sat, 0) }, + }; +} + +let retryTicking = false; setInterval(async () => { - if (!fwd) return; + if (!fwd || retryTicking) return; + retryTicking = true; + try { await retryTick(); } finally { retryTicking = false; } +}, 60_000); +async function retryTick() { // always sync: it reads ark mail AND drives pending actions (a board // waiting out its confirmations, an ln-recv mid-claim, a queued top-up) await fwd.sync().catch(() => {}); - await autoTopupFloat().catch((e) => log('float self-top-up failed: ' + e.message)); + await autoTopupFloat().catch((e) => { log('float self-top-up failed: ' + e.message); topupFailed(e.message); }); await floatUpkeep().catch((e) => log('float upkeep failed: ' + e.message)); if ((await smallCoins().catch(() => [])).length > CONSOLIDATE_AT) { await consolidateFloat().catch((e) => log('float consolidate failed: ' + e.message)); @@ -754,13 +822,14 @@ setInterval(async () => { catch (e) { log(`forward retry failed for ${p.name} (${p.sat} sat, try ${(p.tries || 0) + 1}): ${e.message}`); p.tries = (p.tries || 0) + 1; + p.lastError = e.message; p.next = Date.now() + Math.min(3_600_000, 60_000 * 2 ** Math.min(p.tries - 1, 6)); still.push(p); } } state.pending = still; persist(); -}, 60_000); +} // --------------------------------------------------------------------------- // claims @@ -1119,6 +1188,12 @@ Bun.serve({ offers: !!ln, floatSat: fwd ? fwd.balance().spendableSat : null, floatAddress: fwd ? fwd.address() : null, pendingForwards: (state.pending || []).length, + // for the desk monitor (scripts/float-monitor.ts): stuck money and a + // float that can't refill are what page us. A payment for a wallet + // that moved to another ark server can't go by float at all, so it's + // counted apart — it needs a person, not a top-up. + ...pendingSummary(), + floatTopup: { ...topupStatus, nextChunkSat: topupChunk }, }); } |