names: deliver to coinos wallets over a circular route, shrink failing float top-ups, report stuck money in /health

The ASP's vtxo pool sat empty for 35 hours (2026-10-20/22: 614k in the
rounds wallet against 1.12M needed to refill), and two forwards of ~124k
for jarryd sat in the queue the whole time:

- viaClink paid the wallet's own invoice with a plain pay, but a coinos
  wallet's invoice is minted by our ASP on this very node: a self-payment
  the hold plugin never sees ('Destination said it doesn't know invoice').
  It now goes out and back in over a channel pair (payCircular, shared with
  the float top-up) and waits for the wallet to claim, so a payment is
  delivered whenever the wallet is open, float or no float.
- a float top-up that doesn't settle halves the next ask (down to 20k),
  back to full size after one lands: 79 tries of 200k failed where the
  pool's 100k coins could have funded half of it.
- the retry tick can't overlap itself any more (a circular delivery waits
  up to 4 min; an overlapping tick would forward the same payment twice).
- /health reports pendingSat, oldestPendingAt, the top-up failure streak
  and last error, and payments for wallets on another ark server apart
  (undeliverable): the desk float monitor pages on these.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

1 files changed, 96 insertions, 21 deletions

diff --git a/names/server.js b/names/server.js
index a60ec94..58e5228 100644
--- a/names/server.js
+++ b/names/server.js
@@ -401,6 +401,21 @@ async function forward(name, sat, hash, pubkey) {
       log(`forward: ${name}'s service returned an unusable invoice — refusing to pay it`);
       return false;
     }
+    // A coinos wallet's invoice is minted by our own ASP on this very node, so
+    // a plain pay is a self-payment that never reaches the hold plugin: every
+    // retry of a 124k forward failed that way for 12 hours on 2026-10-21.
+    // Go out and back in over a channel pair instead, then wait for the
+    // wallet to claim.
+    const dec = await ln.call('decode', { string: pr });
+    if (dec.payee === (await ln.call('getinfo')).id) {
+      const pick = await circularPick();
+      if (!pick || pick.cap < sat) return false;
+      const { paymentHash } = await payCircular(pr, pick);
+      const r = await ln.call('waitsendpay', { payment_hash: paymentHash, timeout: 240 });
+      if (r.status !== 'complete') throw new Error(`circular delivery ${r.status}`);
+      log(`delivered ${sat} sat to ${name} over lightning via a circular route (no float needed)`);
+      return true;
+    }
     await ln.pay(pr, { maxfeeSat: Math.max(2, Math.ceil(sat * 0.005)) });
     log(`delivered ${sat} sat to ${name} over lightning (no float needed)`);
     return true;
@@ -602,7 +617,7 @@ async function settleLoop() {
     } catch (e) {
       log(`forward failed for ${name} (${sat} sat): ${e.message} — queued`);
       state.pending = state.pending || [];
-      state.pending.push({ name, sat, hash: inv.payment_hash, pubkey: state.names[name]?.pubkey, ts: Date.now() });
+      state.pending.push({ name, sat, hash: inv.payment_hash, pubkey: state.names[name]?.pubkey, ts: Date.now(), lastError: e.message });
       persist();
     }
   }
@@ -617,18 +632,28 @@ async function settleLoop() {
 // chunkSat caps one attempt: an ASP receive is funded from the vtxo pool,
 // and asking for more than its fresh buckets hold fails whole ("vtxo pool is
 // empty") where two smaller asks would land. Ticks accumulate to the target.
-const TOPUP = { lowWaterSat: 100_000, targetSat: 500_000, chunkSat: 200_000, ...(CFG.floatTopup || {}) };
+const TOPUP = { lowWaterSat: 100_000, targetSat: 500_000, chunkSat: 200_000, minChunkSat: 20_000, ...(CFG.floatTopup || {}) };
 let topupCoolUntil = 0;
-async function autoTopupFloat() {
-  if (!fwd || !ln || Date.now() < topupCoolUntil) return;
-  const floatSat = fwd.balance().spendableSat;
-  const queuedSat = (state.pending || []).reduce((n, p) => n + p.sat, 0);
-  if (floatSat >= TOPUP.lowWaterSat && floatSat >= queuedSat) return;
-  topupCoolUntil = Date.now() + 10 * 60_000; // one attempt, then cool off
-  const info = await ln.call('getinfo');
+// A chunk that doesn't settle is usually one the ASP's vtxo pool can't fund
+// whole (it ran dry for 35 hours on 2026-10-20/22 and 79 top-ups of 200k all
+// failed while smaller ones could have landed): halve the next ask, down to
+// minChunkSat, and go back to full size after one settles. /health reports
+// the streak so the desk monitor can page us when it doesn't recover.
+let topupChunk = TOPUP.chunkSat;
+const topupStatus = { failStreak: 0, lastAt: null, lastOk: null, lastError: null };
+function topupFailed(why) {
+  topupStatus.failStreak++;
+  topupStatus.lastAt = Date.now();
+  topupStatus.lastOk = false;
+  topupStatus.lastError = why;
+  topupChunk = Math.max(TOPUP.minChunkSat, Math.floor(topupChunk / 2));
+}
+
+// The widest same-peer pair of live channels with room for amountSat: out on
+// one, back in on the other.
+async function circularPick() {
   const chans = ((await ln.call('listpeerchannels')).channels || [])
     .filter((c) => c.state === 'CHANNELD_NORMAL' && c.peer_connected);
-  // the widest same-peer pair: out on one channel, back on the other
   let pick = null;
   for (const out of chans) {
     for (const back of chans) {
@@ -639,14 +664,22 @@ async function autoTopupFloat() {
       if (!pick || cap > pick.cap) pick = { out, back, cap };
     }
   }
-  if (!pick || pick.cap < 20_000) { log('float top-up: no circular route capacity'); return; }
-  const amountSat = Math.min(Math.max(TOPUP.targetSat, queuedSat + TOPUP.lowWaterSat) - floatSat, pick.cap, TOPUP.chunkSat);
-  const a = await fwd.createLnInvoice(amountSat, 'float auto top-up');
-  const dec = await ln.call('decode', { string: a.invoice });
+  return pick;
+}
+
+// Pay an invoice that lives on this very node — the ASP's, so a float top-up
+// or a coinos wallet's own Lightning receive — over a circular route: a plain
+// pay of it is a self-payment the hold plugin never sees ("Destination said it
+// doesn't know invoice"). Starts the payment and returns; the caller decides
+// how to tell it settled. Costs the peer's forwarding fee on the way back in.
+async function payCircular(bolt11, pick) {
+  const info = await ln.call('getinfo');
+  const dec = await ln.call('decode', { string: bolt11 });
+  const amtMsat = Number(dec.amount_msat?.msat ?? dec.amount_msat);
+  if (!amtMsat) throw new Error('circular pay needs an amount');
   const gossip = await ln.call('listchannels', { short_channel_id: pick.back.short_channel_id });
   const ret = (gossip.channels || []).find((c) => c.source === pick.back.peer_id);
   if (!ret) throw new Error('no gossip for the return channel');
-  const amtMsat = amountSat * 1000;
   const feeMsat = Number(ret.base_fee_millisatoshi) + Math.ceil((amtMsat * ret.fee_per_millionth) / 1e6);
   const finalDelay = dec.min_final_cltv_expiry || 18;
   await ln.call('sendpay', {
@@ -654,8 +687,22 @@ async function autoTopupFloat() {
       { id: pick.out.peer_id, channel: pick.out.short_channel_id, amount_msat: amtMsat + feeMsat, delay: finalDelay + ret.delay, style: 'tlv' },
       { id: info.id, channel: pick.back.short_channel_id, amount_msat: amtMsat, delay: finalDelay, style: 'tlv' },
     ],
-    payment_hash: a.paymentHash, bolt11: a.invoice, payment_secret: dec.payment_secret,
+    payment_hash: dec.payment_hash, bolt11, payment_secret: dec.payment_secret,
   });
+  return { feeMsat, paymentHash: dec.payment_hash };
+}
+
+async function autoTopupFloat() {
+  if (!fwd || !ln || Date.now() < topupCoolUntil) return;
+  const floatSat = fwd.balance().spendableSat;
+  const queuedSat = (state.pending || []).reduce((n, p) => n + p.sat, 0);
+  if (floatSat >= TOPUP.lowWaterSat && floatSat >= queuedSat) return;
+  topupCoolUntil = Date.now() + 10 * 60_000; // one attempt, then cool off
+  const pick = await circularPick();
+  if (!pick || pick.cap < TOPUP.minChunkSat) { log('float top-up: no circular route capacity'); topupFailed('no circular route capacity'); return; }
+  const amountSat = Math.min(Math.max(TOPUP.targetSat, queuedSat + TOPUP.lowWaterSat) - floatSat, pick.cap, topupChunk);
+  const a = await fwd.createLnInvoice(amountSat, 'float auto top-up');
+  const { feeMsat } = await payCircular(a.invoice, pick);
   // the HTLC sits held until the float wallet claims — drive it through
   for (let i = 0; i < 30; i++) {
     await new Promise((r) => setTimeout(r, 2000));
@@ -668,8 +715,11 @@ async function autoTopupFloat() {
     for (const p of state.pending || []) p.next = 0; // the queue can go again now
     persist();
     topupCoolUntil = 0;
+    topupChunk = TOPUP.chunkSat;
+    Object.assign(topupStatus, { failStreak: 0, lastAt: Date.now(), lastOk: true, lastError: null });
   } else {
-    log('float self-top-up did not settle — cooling off');
+    log(`float self-top-up of ${amountSat} sat did not settle — cooling off, next ask ${Math.max(TOPUP.minChunkSat, Math.floor(topupChunk / 2))} sat`);
+    topupFailed(`${amountSat} sat did not settle`);
   }
 }
 
@@ -734,13 +784,31 @@ async function consolidateFloat() {
 // retry queued forwards (e.g. after the float is topped up). Each failure
 // backs off exponentially (1m → 1h cap): a stuck forward must not fire an
 // offer request — and push-wake the recipient's devices — every minute
-// forever.
+// forever. One tick at a time: a top-up or a circular delivery can outlast
+// the interval, and an overlapping tick would forward the same queued
+// payment twice.
+const foreignServer = (p) => /different ark server/.test(p.lastError || '');
+function pendingSummary() {
+  const q = state.pending || [];
+  const live = q.filter((p) => !foreignServer(p)), foreign = q.filter(foreignServer);
+  return {
+    pendingSat: live.reduce((n, p) => n + p.sat, 0),
+    oldestPendingAt: live.reduce((m, p) => (p.ts && (!m || p.ts < m) ? p.ts : m), null),
+    undeliverable: { count: foreign.length, sat: foreign.reduce((n, p) => n + p.sat, 0) },
+  };
+}
+
+let retryTicking = false;
 setInterval(async () => {
-  if (!fwd) return;
+  if (!fwd || retryTicking) return;
+  retryTicking = true;
+  try { await retryTick(); } finally { retryTicking = false; }
+}, 60_000);
+async function retryTick() {
   // always sync: it reads ark mail AND drives pending actions (a board
   // waiting out its confirmations, an ln-recv mid-claim, a queued top-up)
   await fwd.sync().catch(() => {});
-  await autoTopupFloat().catch((e) => log('float self-top-up failed: ' + e.message));
+  await autoTopupFloat().catch((e) => { log('float self-top-up failed: ' + e.message); topupFailed(e.message); });
   await floatUpkeep().catch((e) => log('float upkeep failed: ' + e.message));
   if ((await smallCoins().catch(() => [])).length > CONSOLIDATE_AT) {
     await consolidateFloat().catch((e) => log('float consolidate failed: ' + e.message));
@@ -754,13 +822,14 @@ setInterval(async () => {
     catch (e) {
       log(`forward retry failed for ${p.name} (${p.sat} sat, try ${(p.tries || 0) + 1}): ${e.message}`);
       p.tries = (p.tries || 0) + 1;
+      p.lastError = e.message;
       p.next = Date.now() + Math.min(3_600_000, 60_000 * 2 ** Math.min(p.tries - 1, 6));
       still.push(p);
     }
   }
   state.pending = still;
   persist();
-}, 60_000);
+}
 
 // ---------------------------------------------------------------------------
 // claims
@@ -1119,6 +1188,12 @@ Bun.serve({
         offers: !!ln, floatSat: fwd ? fwd.balance().spendableSat : null,
         floatAddress: fwd ? fwd.address() : null,
         pendingForwards: (state.pending || []).length,
+        // for the desk monitor (scripts/float-monitor.ts): stuck money and a
+        // float that can't refill are what page us. A payment for a wallet
+        // that moved to another ark server can't go by float at all, so it's
+        // counted apart — it needs a person, not a top-up.
+        ...pendingSummary(),
+        floatTopup: { ...topupStatus, nextChunkSat: topupChunk },
       });
     }