health check

1 files changed, 103 insertions, 42 deletions

diff --git a/lib/lightning.ts b/lib/lightning.ts
index f8d0f6a4..1f3b1665 100644
--- a/lib/lightning.ts
+++ b/lib/lightning.ts
@@ -1,66 +1,127 @@
 import { db, g, s } from "$lib/db";
-import ln from "$lib/ln";
-import { err, warn } from "$lib/logging";
+import ln, { lnListen, LightningUnavailableError } from "$lib/ln";
+import { err, l, warn } from "$lib/logging";
 import { handleZap } from "$lib/nostr";
 import { credit } from "$lib/payments";
 import { PaymentType } from "$lib/types";
 import { getInvoice, getPayment, getUser } from "$lib/utils";
 
-export async function listenForLightning() {
-  const inv = await ln.waitanyinvoice((await g("pay_index")) || 0);
-  const {
-    local_offer_id,
-    bolt11,
-    bolt12,
-    description,
-    pay_index,
-    payment_hash,
-    amount_received_msat,
-    payment_preimage: preimage,
-  } = inv;
+const LISTENER_RETRY_DELAY = 5000; // 5 seconds
+const MAX_LISTENER_RETRIES = 10;
+let listenerRetries = 0;
+let listenerActive = false;
 
-  await s("pay_index", pay_index);
-  setTimeout(listenForLightning);
+export async function listenForLightning() {
+  if (listenerActive) {
+    warn("lightning listener: already active, skipping duplicate call");
+    return;
+  }
 
-  const received = Math.round(amount_received_msat / 1000);
+  listenerActive = true;
 
   try {
-    if (!preimage) return;
+    const payIndex = (await g("pay_index")) || 0;
+    l(`lightning listener: waiting for invoice (pay_index=${payIndex})`);
 
-    const invoice = await getInvoice(bolt11 ?? local_offer_id ?? bolt12);
-    if (!invoice) return warn("received lightning with no invoice", bolt11);
+    const inv = await lnListen.waitanyinvoice(payIndex);
+    const {
+      local_offer_id,
+      bolt11,
+      bolt12,
+      description,
+      pay_index,
+      payment_hash,
+      amount_received_msat,
+      payment_preimage: preimage,
+    } = inv;
 
-    const p = await getPayment(bolt11 || bolt12);
-    if (p) return warn("already processed", bolt11 || bolt12);
+    await s("pay_index", pay_index);
 
-    if (invoice?.memo) {
-      try {
-        if (JSON.parse(description).kind === 9734) {
-          const { pubkey } = await getUser(invoice.uid);
-          handleZap(inv, pubkey);
+    // Reset retry counter on successful receive
+    if (listenerRetries > 0) {
+      l(`lightning listener: recovered after ${listenerRetries} retries`);
+    }
+    listenerRetries = 0;
+    listenerActive = false;
+
+    // Schedule next listen
+    setTimeout(listenForLightning);
+
+    const received = Math.round(amount_received_msat / 1000);
+
+    try {
+      if (!preimage) return;
+
+      const invoice = await getInvoice(bolt11 ?? local_offer_id ?? bolt12);
+      if (!invoice) return warn("received lightning with no invoice", bolt11);
+
+      const p = await getPayment(bolt11 || bolt12);
+      if (p) return warn("already processed", bolt11 || bolt12);
+
+      if (invoice?.memo) {
+        try {
+          if (JSON.parse(description).kind === 9734) {
+            const { pubkey } = await getUser(invoice.uid);
+            handleZap(inv, pubkey);
+          }
+        } catch (e) {
+          if (!e.message.includes("Unexpected"))
+            warn("failed to handle zap", e.message);
         }
-      } catch (e) {
-        if (!e.message.includes("Unexpected"))
-          warn("failed to handle zap", e.message);
       }
+
+      await credit({
+        hash: bolt11 || bolt12,
+        amount: received,
+        memo: invoice.memo,
+        ref: preimage,
+        type: bolt12 ? PaymentType.bolt12 : PaymentType.lightning,
+        payment_hash,
+      });
+    } catch (e) {
+      console.log(e);
+      err("problem receiving lightning payment", e.message);
     }
+  } catch (e: any) {
+    listenerActive = false;
+    const errorCode = e?.code ?? e?.errno ?? "unknown";
+    const errorMsg = e?.message ?? String(e);
 
-    await credit({
-      hash: bolt11 || bolt12,
-      amount: received,
-      memo: invoice.memo,
-      ref: preimage,
-      type: bolt12 ? PaymentType.bolt12 : PaymentType.lightning,
-      payment_hash,
-    });
-  } catch (e) {
-    console.log(e);
-    err("problem receiving lightning payment", e.message);
+    err(
+      `lightning listener: error waiting for invoice`,
+      `code=${errorCode}`,
+      `error=${errorMsg}`
+    );
+
+    if (e instanceof LightningUnavailableError) {
+      err("lightning listener: RPC socket unavailable");
+    }
+
+    listenerRetries++;
+
+    if (listenerRetries >= MAX_LISTENER_RETRIES) {
+      err(
+        `lightning listener: ${MAX_LISTENER_RETRIES} consecutive failures, ` +
+          `last error: ${errorMsg}`
+      );
+      err("lightning listener: exiting process to trigger container restart");
+
+      setTimeout(() => {
+        process.exit(1);
+      }, 1000);
+      return;
+    }
+
+    warn(
+      `lightning listener: retry ${listenerRetries}/${MAX_LISTENER_RETRIES} ` +
+        `in ${LISTENER_RETRY_DELAY / 1000}s`
+    );
+    setTimeout(listenForLightning, LISTENER_RETRY_DELAY);
   }
 }
 
 export async function replay(index) {
-  const inv = await ln.waitanyinvoice(index - 1);
+  const inv = await lnListen.waitanyinvoice(index - 1);
   const {
     local_offer_id,
     bolt11,