From 0a1450ada9c20499fa86a619ee36176f717d5414 Mon Sep 17 00:00:00 2001
From: Adam Soltys <adam@coinos.io>
Date: Sun, 14 Jun 2026 12:45:44 -0700
Subject: lightning: watchdog only recycles on real backlog, not normal idle

Port of master 7b71c96b, refining the zombie-listener watchdog staging
adopted in 7c34b431. Treating "waitanyinvoice blocked > 5min while cl is
alive" as a stall false-fires during normal quiet periods (~26x in 6h on
master 2026-06-07) and needlessly recycles the cl listen socket. Instead,
when blocked past the threshold, probe cl on the MAIN socket with
waitanyinvoice(pay_index, 2s server-side timeout): an invoice returned
means a real backlog the wedged listener missed -> recycle; a timeout
(CLN 904) means the listener is just idle -> leave it alone.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 lib/lightning.ts | 36 ++++++++++++++++++++++++++----------
 1 file changed, 26 insertions(+), 10 deletions(-)

diff --git a/lib/lightning.ts b/lib/lightning.ts
index 2f5f4f00..a7d6671e 100644
--- a/lib/lightning.ts
+++ b/lib/lightning.ts
@@ -128,8 +128,9 @@ export async function listenForLightning() {
 }
 
 // Watchdog (run periodically): restart a dead listener, or recycle a zombie one
-// that's been blocked in waitanyinvoice far past any normal quiet period while cl
-// is actually alive (the dead-listen-socket case).
+// whose waitanyinvoice is wedged on a dead listen socket. A long block alone is
+// normal when idle, so it only recycles when cl actually has a paid invoice
+// waiting (a real backlog) — see the body for the probe.
 export async function ensureListenerAlive() {
   if (!listenerActive) {
     warn("lightning listener: not active, restarting");
@@ -137,18 +138,33 @@ export async function ensureListenerAlive() {
     return;
   }
   if (Date.now() - waitStartedAt < LISTENER_STALL_MS) return;
-  let clAlive = false;
+
+  // A long-blocked waitanyinvoice is NORMAL during quiet periods — the only way
+  // to tell a zombie socket from a healthy idle wait is whether cl actually has
+  // a paid invoice the listener is failing to pick up. (The original "blocked >
+  // 5min while cl alive" check false-fired every few minutes during normal quiet
+  // stretches and needlessly recycled the socket — see 2026-06-07.) So only act
+  // on a real backlog: probe cl on the MAIN socket (not the wedged listen one)
+  // with waitanyinvoice at the stored pay_index and a short server-side timeout.
+  // An invoice returned -> genuine backlog the wedged listener missed -> recycle.
+  // A timeout (CLN code 904, nothing waiting) -> listener is idle and healthy.
+  let backlog = false;
   try {
-    const info = await ln.getinfo();
-    clAlive = !!info?.id;
-  } catch (_) {
-    clAlive = false;
+    const payIndex = (await g("pay_index")) || 0;
+    const probe = await ln.waitanyinvoice(payIndex, 2); // 2s server-side timeout
+    if (probe?.pay_index) backlog = true;
+  } catch (e: any) {
+    // CLN returns an error (code 904) on timeout with no waiting invoice — that
+    // means NO backlog (healthy). Any other error: stay conservative, don't
+    // recycle on ambiguous signals.
+    backlog = false;
   }
-  if (!clAlive) return; // cl itself down — the retry/exit path handles that
+  if (!backlog) return; // healthy idle wait, not a zombie.
+
   err(
-    `lightning listener: stalled ${Math.round(
+    `lightning listener: backlog detected while listener blocked ${Math.round(
       (Date.now() - waitStartedAt) / 1000,
-    )}s while cl is alive — recycling listen socket`,
+    )}s — recycling listen socket`,
   );
   listenerEpoch++;
   try {
-- 
cgit v1.2.3