ln: bound receive-listener wait, phase-aware watchdog, re-enable health check

Give waitanyinvoice an explicit 30s server-side timeout so a dead/half-open
socket can no longer hold the listener indefinitely; CLN's 904 timeout during a
quiet period is treated as a clean re-arm without counting as a failure. Split
the listener into waiting vs processing phases so the watchdog never resets a
healthy socket merely because a ledger write is slow, and never starts a second
concurrent processor — a stalled processing phase is surfaced loudly instead.
Process the settlement and confirm the durable payment pointer BEFORE advancing
pay_index, so a store load/restart between the cursor write and credit() cannot
drop a received payment. Expose listener status and add a backlog probe to the
health check, and re-enable startHealthCheck().

Co-Authored-By: Claude Opus 4.8 <[email protected]>
Claude-Session: https://claude.ai/code/session_01XiwgmQKwKnRxUmVDd4xSf7

1 files changed, 39 insertions, 3 deletions

diff --git a/lib/health.ts b/lib/health.ts
index d9c38fa5..bb05b0db 100644
--- a/lib/health.ts
+++ b/lib/health.ts
@@ -1,4 +1,6 @@
 import ln, { LightningUnavailableError } from "$lib/ln";
+import { g } from "$lib/db";
+import { getLightningListenerStatus } from "$lib/lightning";
 import { err, l, warn } from "$lib/logging";
 
 const HEALTH_CHECK_INTERVAL = 2 * 60 * 1000; // 2 minutes
@@ -9,6 +11,7 @@ let consecutiveFailures = 0;
 let lastSuccessTime = Date.now();
 let lastFailureReason = "";
 let checkTimer: ReturnType<typeof setTimeout> | null = null;
+let listenerBacklogPayIndex: number | null = null;
 
 export function getHealthStatus() {
   return {
@@ -17,18 +20,30 @@ export function getHealthStatus() {
     lastFailureReason,
     healthy: consecutiveFailures < MAX_CONSECUTIVE_FAILURES,
     stalledCheck: checkTimer === null && consecutiveFailures > 0,
+    listenerBacklogPayIndex,
+    listener: getLightningListenerStatus(),
   };
 }
 
-function withTimeout<T>(promise: Promise<T>, ms: number, label: string): Promise<T> {
+function withTimeout<T>(
+  promise: Promise<T>,
+  ms: number,
+  label: string,
+): Promise<T> {
   return new Promise((resolve, reject) => {
     const timer = setTimeout(
       () => reject(new Error(`${label} timed out after ${ms}ms`)),
       ms,
     );
     promise.then(
-      (v) => { clearTimeout(timer); resolve(v); },
-      (e) => { clearTimeout(timer); reject(e); },
+      (v) => {
+        clearTimeout(timer);
+        resolve(v);
+      },
+      (e) => {
+        clearTimeout(timer);
+        reject(e);
+      },
     );
   });
 }
@@ -70,6 +85,27 @@ async function checkLightningHealth(): Promise<boolean> {
       // Ignore cleanup errors
     }
 
+    // A responsive CLN node is not sufficient: the dedicated receive listener
+    // can be wedged while getinfo/invoice still work. Probe one index beyond the
+    // durable cursor. Code 904 means no backlog and is the healthy outcome.
+    const payIndex = Number((await g("pay_index")) || 0);
+    listenerBacklogPayIndex = null;
+    try {
+      const waiting = await withTimeout(
+        ln.waitanyinvoice(payIndex, 1),
+        5_000,
+        "listener backlog probe",
+      );
+      if (Number(waiting?.pay_index) > payIndex) {
+        listenerBacklogPayIndex = Number(waiting.pay_index);
+        throw new Error(
+          `lightning listener backlog: stored=${payIndex} next=${waiting.pay_index}`,
+        );
+      }
+    } catch (e: any) {
+      if (Number(e?.code ?? e?.errno) !== 904) throw e;
+    }
+
     const duration = Date.now() - startTime;
     // l(`health check: passed in ${duration}ms, node: ${info.id.slice(0, 16)}...`);