From 2b6d742672400d51269520447783f707e8f6f91a Mon Sep 17 00:00:00 2001 From: Adam Soltys Date: Thu, 27 Aug 2026 11:42:19 -0700 Subject: health: tolerate a transient receive-listener lag, fail only if it's stuck MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The listener-backlog probe fired a failure whenever a paid invoice existed beyond the durable cursor — but that is almost always the listener mid- processing that very payment, a normal sub-second lag, not a wedged listener. Those transient catches produced spurious "listener backlog" health failures that then recovered on their own. Grace-check instead: on detecting a paid invoice beyond the cursor, wait briefly and re-read the cursor; if it advanced, the listener is alive and this was transient (pass). Only fail if the cursor did not move at all — the actual "stuck listener" condition the probe exists to catch. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01DKkev76U4YRnRAqM6jqbae --- lib/health.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/health.ts b/lib/health.ts index 96d15030..2a876909 100644 --- a/lib/health.ts +++ b/lib/health.ts @@ -97,10 +97,21 @@ async function checkLightningHealth(): Promise { "listener backlog probe", ); if (Number(waiting?.pay_index) > payIndex) { - listenerBacklogPayIndex = Number(waiting.pay_index); - throw new Error( - `lightning listener backlog: stored=${payIndex} next=${waiting.pay_index}`, - ); + // A paid invoice exists beyond the durable cursor. That alone is NOT a + // wedged listener — it is almost always the listener mid-processing this + // very payment, before it has advanced the cursor (normal sub-second + // latency). Only a listener that is genuinely STUCK fails to advance. + // Grace-check: give it a moment, then re-read the cursor. If it moved, + // the listener is alive and this was transient. Only fail if it did not. + const next = Number(waiting.pay_index); + await new Promise((r) => setTimeout(r, 2500)); + const after = Number((await g("pay_index")) || 0); + if (after <= payIndex) { + listenerBacklogPayIndex = next; + throw new Error( + `lightning listener backlog: stored=${payIndex} next=${next} (cursor did not advance in 2.5s)`, + ); + } } } catch (e: any) { if (Number(e?.code ?? e?.errno) !== 904) throw e; -- cgit v1.2.3