1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
|
import ln, { LightningUnavailableError } from "$lib/ln";
import { g } from "$lib/db";
import { getLightningListenerStatus } from "$lib/lightning";
import { err, l, warn } from "$lib/logging";
const HEALTH_CHECK_INTERVAL = 2 * 60 * 1000; // 2 minutes
const RPC_TIMEOUT = 30_000; // 30 seconds - fail fast if RPC hangs
const MAX_CONSECUTIVE_FAILURES = 3;
let consecutiveFailures = 0;
let lastSuccessTime = Date.now();
let lastFailureReason = "";
let checkTimer: ReturnType<typeof setTimeout> | null = null;
let listenerBacklogPayIndex: number | null = null;
export function getHealthStatus() {
return {
consecutiveFailures,
lastSuccessTime,
lastFailureReason,
healthy: consecutiveFailures < MAX_CONSECUTIVE_FAILURES,
stalledCheck: checkTimer === null && consecutiveFailures > 0,
listenerBacklogPayIndex,
listener: getLightningListenerStatus(),
};
}
function withTimeout<T>(
promise: Promise<T>,
ms: number,
label: string,
): Promise<T> {
return new Promise((resolve, reject) => {
const timer = setTimeout(
() => reject(new Error(`${label} timed out after ${ms}ms`)),
ms,
);
promise.then(
(v) => {
clearTimeout(timer);
resolve(v);
},
(e) => {
clearTimeout(timer);
reject(e);
},
);
});
}
async function checkLightningHealth(): Promise<boolean> {
const startTime = Date.now();
try {
// l("health check: testing lightning connection...");
const info = await withTimeout(ln.getinfo(), RPC_TIMEOUT, "getinfo");
if (!info?.id) {
throw new Error("getinfo returned invalid response (no node id)");
}
const testLabel = `health_check_${Date.now()}`;
const invoice = await withTimeout(
ln.invoice({
amount_msat: "1000msat",
label: testLabel,
description: "health check",
expiry: 60,
}),
RPC_TIMEOUT,
"invoice",
);
if (!invoice?.bolt11) {
throw new Error("invoice creation returned invalid response");
}
try {
await withTimeout(
ln.delinvoice({ label: testLabel, status: "unpaid" }),
RPC_TIMEOUT,
"delinvoice",
);
} catch (e) {
// Ignore cleanup errors
}
// A responsive CLN node is not sufficient: the dedicated receive listener
// can be wedged while getinfo/invoice still work. Probe one index beyond the
// durable cursor. Code 904 means no backlog and is the healthy outcome.
const payIndex = Number((await g("pay_index")) || 0);
listenerBacklogPayIndex = null;
try {
const waiting = await withTimeout(
ln.waitanyinvoice(payIndex, 1),
15_000,
"listener backlog probe",
);
if (Number(waiting?.pay_index) > payIndex) {
// A paid invoice exists beyond the durable cursor. That alone is NOT a
// wedged listener — it is almost always the listener mid-processing this
// very payment, before it has advanced the cursor (normal sub-second
// latency). Only a listener that is genuinely STUCK fails to advance.
// Grace-check: give it a moment, then re-read the cursor. If it moved,
// the listener is alive and this was transient. Only fail if it did not.
const next = Number(waiting.pay_index);
await new Promise((r) => setTimeout(r, 2500));
const after = Number((await g("pay_index")) || 0);
if (after <= payIndex) {
listenerBacklogPayIndex = next;
throw new Error(
`lightning listener backlog: stored=${payIndex} next=${next} (cursor did not advance in 2.5s)`,
);
}
}
} catch (e: any) {
if (Number(e?.code ?? e?.errno) !== 904) throw e;
}
const duration = Date.now() - startTime;
// l(`health check: passed in ${duration}ms, node: ${info.id.slice(0, 16)}...`);
return true;
} catch (e: any) {
const duration = Date.now() - startTime;
const errorCode = e?.code ?? e?.errno ?? "unknown";
const errorMsg = e?.message ?? String(e);
err(
`health check: FAILED after ${duration}ms`,
`code=${errorCode}`,
`error=${errorMsg}`,
);
if (e instanceof LightningUnavailableError) {
err("health check: lightning RPC socket unavailable");
}
lastFailureReason = `${errorCode}: ${errorMsg}`;
return false;
}
}
export async function runHealthCheck() {
checkTimer = null;
try {
const healthy = await checkLightningHealth();
if (healthy) {
if (consecutiveFailures > 0) {
l(
`health check: recovered after ${consecutiveFailures} consecutive failures`,
);
}
consecutiveFailures = 0;
lastSuccessTime = Date.now();
lastFailureReason = "";
} else {
consecutiveFailures++;
warn(
`health check: failure ${consecutiveFailures}/${MAX_CONSECUTIVE_FAILURES}`,
);
if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) {
err(
`health check: ${MAX_CONSECUTIVE_FAILURES} consecutive failures, ` +
`last success was ${Math.round((Date.now() - lastSuccessTime) / 1000)}s ago`,
);
err(`health check: last failure reason: ${lastFailureReason}`);
err("health check: exiting process to trigger container restart");
setTimeout(() => {
process.exit(1);
}, 1000);
return;
}
}
} catch (e: any) {
err("health check: unexpected error during health check", e?.message);
consecutiveFailures++;
}
// Schedule next check
checkTimer = setTimeout(runHealthCheck, HEALTH_CHECK_INTERVAL);
}
export function startHealthCheck() {
// l(
// `health check: starting (interval=${HEALTH_CHECK_INTERVAL / 1000}s, timeout=${RPC_TIMEOUT / 1000}s, max_failures=${MAX_CONSECUTIVE_FAILURES})`,
// );
// Run first check after a delay to let the app initialize
checkTimer = setTimeout(runHealthCheck, 30000);
}
|
