From 7c34b4312c8aafc7643a5192d54daea14cfc1559 Mon Sep 17 00:00:00 2001
From: Adam Soltys <asoltys@gmail.com>
Date: Mon, 8 Jun 2026 22:49:26 +0000
Subject: ln/lightning: watchdog recycles zombie receive-listener socket
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Port of master bd1ece3 to staging. When cl restarts (e.g. autoheal after a
healthcheck timeout), the app's waitanyinvoice long-poll can hang on the now-dead
listen socket: listenerActive stays true forever, nothing restarts it, and
incoming lightning payments silently stop crediting until a manual app restart.
The per-call RPC timeout can't catch this — waitanyinvoice is meant to block.

ensureListenerAlive (run every 120s) now also detects a stalled listener — active
but blocked in waitanyinvoice for >5min — probes cl via the main socket (getinfo),
and only if cl is actually alive treats the listen socket as a zombie: bumps a
listenerEpoch, calls lnListen.reset() (new: drops the proxy socket so the next
call reconnects), and re-arms. The epoch guard makes the wedged invocation bow out
when its waitanyinvoice finally rejects, so we never run two listeners or
double-credit.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 lib/lightning.ts | 39 +++++++++++++++++++++++++++++++++++++++
 lib/ln.ts        |  1 +
 2 files changed, 40 insertions(+)

(limited to 'lib')

diff --git a/lib/lightning.ts b/lib/lightning.ts
index 4eb70e00..9e259ed5 100644
--- a/lib/lightning.ts
+++ b/lib/lightning.ts
@@ -9,8 +9,13 @@ import { getInvoice, getPayment, getUser } from "$lib/utils";
 
 const LISTENER_RETRY_DELAY = 5000; // 5 seconds
 const MAX_LISTENER_RETRIES = 10;
+// If the listener has been blocked in waitanyinvoice this long without returning,
+// AND cl is reachable, treat the listen socket as a zombie and recycle it.
+const LISTENER_STALL_MS = 5 * 60 * 1000; // 5 minutes
 let listenerRetries = 0;
 let listenerActive = false;
+let waitStartedAt = Date.now();
+let listenerEpoch = 0;
 
 export async function listenForLightning() {
   if (listenerActive) {
@@ -19,6 +24,8 @@ export async function listenForLightning() {
   }
 
   listenerActive = true;
+  waitStartedAt = Date.now();
+  const myEpoch = listenerEpoch;
 
   try {
     const payIndex = (await g("pay_index")) || 0;
@@ -36,6 +43,9 @@ export async function listenForLightning() {
       payment_preimage: preimage,
     } = inv;
 
+    // watchdog recycled us while blocked — a fresh listener owns the stream now.
+    if (myEpoch !== listenerEpoch) return;
+
     await s("pay_index", pay_index);
 
     // Reset retry counter on successful receive
@@ -84,6 +94,7 @@ export async function listenForLightning() {
     }
   } catch (e: any) {
     listenerActive = false;
+    if (myEpoch !== listenerEpoch) return; // watchdog already re-armed a fresh listener
     const errorCode = e?.code ?? e?.errno ?? "unknown";
     const errorMsg = e?.message ?? String(e);
 
@@ -116,6 +127,34 @@ export async function listenForLightning() {
   }
 }
 
+export async function ensureListenerAlive() {
+  if (!listenerActive) {
+    warn("lightning listener: not active, restarting");
+    listenForLightning();
+    return;
+  }
+  if (Date.now() - waitStartedAt < LISTENER_STALL_MS) return;
+  let clAlive = false;
+  try {
+    const info = await ln.getinfo();
+    clAlive = !!info?.id;
+  } catch (_) {
+    clAlive = false;
+  }
+  if (!clAlive) return; // cl itself down — the retry/exit path handles that
+  err(
+    `lightning listener: stalled ${Math.round(
+      (Date.now() - waitStartedAt) / 1000,
+    )}s while cl is alive — recycling listen socket`,
+  );
+  listenerEpoch++;
+  try {
+    (lnListen as any).reset?.();
+  } catch (_) {}
+  listenerActive = false;
+  setTimeout(listenForLightning);
+}
+
 export async function replay(index) {
   const inv = await lnListen.waitanyinvoice(index - 1);
   const {
diff --git a/lib/ln.ts b/lib/ln.ts
index 2cbd5120..c125eec4 100644
--- a/lib/ln.ts
+++ b/lib/ln.ts
@@ -98,6 +98,7 @@ function lightningProxy(rpcPath: string): any {
       get(_t, prop: string | symbol) {
         if (prop === "toString") return () => "[LightningProxy]";
         if (prop === Symbol.toStringTag) return "LightningProxy";
+        if (prop === "reset") return () => { client = null; }; // watchdog: drop a zombie socket
 
         return async (...args: any[]) => {
           const c = ensure();
-- 
cgit v1.2.3