1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
|
import ln, { LightningUnavailableError } from "$lib/ln";
import { err, l, warn } from "$lib/logging";
const HEALTH_CHECK_INTERVAL = 2 * 60 * 1000; // 2 minutes
const RPC_TIMEOUT = 30_000; // 30 seconds - fail fast if RPC hangs
const MAX_CONSECUTIVE_FAILURES = 3;
let consecutiveFailures = 0;
let lastSuccessTime = Date.now();
let lastFailureReason = "";
let checkTimer: ReturnType<typeof setTimeout> | null = null;
export function getHealthStatus() {
return {
consecutiveFailures,
lastSuccessTime,
lastFailureReason,
healthy: consecutiveFailures < MAX_CONSECUTIVE_FAILURES,
stalledCheck: checkTimer === null && consecutiveFailures > 0,
};
}
function withTimeout<T>(promise: Promise<T>, ms: number, label: string): Promise<T> {
return new Promise((resolve, reject) => {
const timer = setTimeout(
() => reject(new Error(`${label} timed out after ${ms}ms`)),
ms,
);
promise.then(
(v) => { clearTimeout(timer); resolve(v); },
(e) => { clearTimeout(timer); reject(e); },
);
});
}
async function checkLightningHealth(): Promise<boolean> {
const startTime = Date.now();
try {
// l("health check: testing lightning connection...");
const info = await withTimeout(ln.getinfo(), RPC_TIMEOUT, "getinfo");
if (!info?.id) {
throw new Error("getinfo returned invalid response (no node id)");
}
const testLabel = `health_check_${Date.now()}`;
const invoice = await withTimeout(
ln.invoice({
amount_msat: "1000msat",
label: testLabel,
description: "health check",
expiry: 60,
}),
RPC_TIMEOUT,
"invoice",
);
if (!invoice?.bolt11) {
throw new Error("invoice creation returned invalid response");
}
try {
await withTimeout(
ln.delinvoice({ label: testLabel, status: "unpaid" }),
RPC_TIMEOUT,
"delinvoice",
);
} catch (e) {
// Ignore cleanup errors
}
const duration = Date.now() - startTime;
// l(`health check: passed in ${duration}ms, node: ${info.id.slice(0, 16)}...`);
return true;
} catch (e: any) {
const duration = Date.now() - startTime;
const errorCode = e?.code ?? e?.errno ?? "unknown";
const errorMsg = e?.message ?? String(e);
err(
`health check: FAILED after ${duration}ms`,
`code=${errorCode}`,
`error=${errorMsg}`,
);
if (e instanceof LightningUnavailableError) {
err("health check: lightning RPC socket unavailable");
}
lastFailureReason = `${errorCode}: ${errorMsg}`;
return false;
}
}
export async function runHealthCheck() {
checkTimer = null;
try {
const healthy = await checkLightningHealth();
if (healthy) {
if (consecutiveFailures > 0) {
l(
`health check: recovered after ${consecutiveFailures} consecutive failures`,
);
}
consecutiveFailures = 0;
lastSuccessTime = Date.now();
lastFailureReason = "";
} else {
consecutiveFailures++;
warn(
`health check: failure ${consecutiveFailures}/${MAX_CONSECUTIVE_FAILURES}`,
);
if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) {
err(
`health check: ${MAX_CONSECUTIVE_FAILURES} consecutive failures, ` +
`last success was ${Math.round((Date.now() - lastSuccessTime) / 1000)}s ago`,
);
err(`health check: last failure reason: ${lastFailureReason}`);
err("health check: exiting process to trigger container restart");
setTimeout(() => {
process.exit(1);
}, 1000);
return;
}
}
} catch (e: any) {
err("health check: unexpected error during health check", e?.message);
consecutiveFailures++;
}
// Schedule next check
checkTimer = setTimeout(runHealthCheck, HEALTH_CHECK_INTERVAL);
}
export function startHealthCheck() {
// l(
// `health check: starting (interval=${HEALTH_CHECK_INTERVAL / 1000}s, timeout=${RPC_TIMEOUT / 1000}s, max_failures=${MAX_CONSECUTIVE_FAILURES})`,
// );
// Run first check after a delay to let the app initialize
checkTimer = setTimeout(runHealthCheck, 30000);
}
|
