From be896868de55ef82b268d8c9f92c3f0ba857fed8 Mon Sep 17 00:00:00 2001 From: Adam Soltys Date: Thu, 4 Jun 2026 12:14:15 -0700 Subject: analytics: missing-history audit + lightning->liquid arb finder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit audit-missing-history.ts: read-only audit classifying every user's payment history as clean / recoverable (record in arc not db) / lost (gone from both). Full-scan finding: of 6,330 users with history, 77% are "arc-only" which is the NORMAL archived state — the payments route reads via gf() (archive fallback) so those render fine in the app. True loss is 58 users / 59 records (0.02%). So there is no widespread history loss; "missing history" reports are almost always either dropped receive-callbacks (record never written, like apk403's 1100 sats) or one of the rare 59. ln-liquid-arb.ts: finds users funding via lightning and withdrawing via liquid (the pattern draining the liquid hot wallet / forcing threshold cuts). Reports per-user through-flow + net liquid drainers. Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/audit-missing-history.ts | 106 ++++++++++++++++++++++++++++++++ scripts/ln-liquid-arb.ts | 126 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 232 insertions(+) create mode 100644 scripts/audit-missing-history.ts create mode 100644 scripts/ln-liquid-arb.ts diff --git a/scripts/audit-missing-history.ts b/scripts/audit-missing-history.ts new file mode 100644 index 00000000..30c2c8fc --- /dev/null +++ b/scripts/audit-missing-history.ts @@ -0,0 +1,106 @@ +#!/usr/bin/env bun +// Audit how widespread missing payment history is and classify WHY. +// +// For each user with a non-empty :payments list, check each referenced +// payment: across db and arc, and bucket the user: +// - clean: all records resolve (db or arc) +// - recoverable: some/all records missing from db but present in arc +// - lost: some records missing from BOTH db and arc (true loss) +// Also records per-payment counts so we can size a restore. +// +// Read-only. Samples by default; pass --full to scan every user (slower). +// Usage: bun scripts/audit-missing-history.ts [--full] [--sample N] [--examples N] + +import { createClient } from "redis"; + +const full = process.argv.includes("--full"); +const sampleN = Number(process.argv[process.argv.indexOf("--sample") + 1]) || 4000; +const exN = Number(process.argv[process.argv.indexOf("--examples") + 1]) || 15; + +const db = createClient({ url: "redis://127.0.0.1:6379", socket: { reconnectStrategy: () => false } }); +const arc = createClient({ url: "redis://127.0.0.1:6380", socket: { reconnectStrategy: () => false } }); +await db.connect(); +await arc.connect(); + +// Resolve uid -> username lazily (only for examples) to keep memory low. +async function nameOf(uid: string) { + const u = await db.get(`user:${uid}`); + if (u && u[0] === "{") { try { return JSON.parse(u).username || uid.slice(0, 8); } catch {} } + return uid.slice(0, 8); +} + +let usersScanned = 0; +let usersWithPayments = 0; +const bucket = { clean: 0, recoverable: 0, lost: 0 }; +let totalRefs = 0, inDb = 0, inArcOnly = 0, missingBoth = 0; +const examples: { recoverable: string[]; lost: string[] } = { recoverable: [], lost: [] }; + +// Helper: does payment: exist in db / arc? (batched EXISTS) +async function classifyUser(uid: string): Promise<"clean" | "recoverable" | "lost" | null> { + const pids = await db.lRange(`${uid}:payments`, 0, -1); + if (!pids.length) return null; + usersWithPayments++; + + let anyArcOnly = false, anyMissing = false; + for (const pid of pids) { + totalRefs++; + const key = `payment:${pid}`; + const dbHas = await db.exists(key); + if (dbHas) { inDb++; continue; } + const arcHas = await arc.exists(key); + if (arcHas) { inArcOnly++; anyArcOnly = true; } + else { missingBoth++; anyMissing = true; } + } + if (anyMissing) return "lost"; + if (anyArcOnly) return "recoverable"; + return "clean"; +} + +const t0 = Date.now(); +let processed = 0; +for await (const key of db.scanIterator({ MATCH: "user:*", COUNT: 1000 })) { + // Only the canonical record key user: (skip user: pointers). + const id = (key as string).slice("user:".length); + if (!/^[0-9a-f]{8}-[0-9a-f]{4}-/.test(id)) continue; + usersScanned++; + if (!full && usersScanned > sampleN) break; + + const cls = await classifyUser(id); + if (!cls) continue; + bucket[cls]++; + if (cls === "recoverable" && examples.recoverable.length < exN) examples.recoverable.push(id); + if (cls === "lost" && examples.lost.length < exN) examples.lost.push(id); + processed++; +} +const secs = ((Date.now() - t0) / 1000).toFixed(1); + +const totalClassified = bucket.clean + bucket.recoverable + bucket.lost; +const pct = (n: number) => totalClassified ? ((n / totalClassified) * 100).toFixed(1) + "%" : "0%"; + +console.log("═".repeat(66)); +console.log(` MISSING-HISTORY AUDIT — ${full ? "FULL scan" : `sample of ${sampleN} users`}`); +console.log(` scanned ${usersScanned} user records, ${usersWithPayments} with payments, in ${secs}s`); +console.log("═".repeat(66)); +console.log(`\n Users with payment history: ${totalClassified}`); +console.log(` clean (all records present) ...... ${bucket.clean} (${pct(bucket.clean)})`); +console.log(` recoverable (in arc, not db) ..... ${bucket.recoverable} (${pct(bucket.recoverable)}) <- restorable`); +console.log(` lost (missing from db AND arc) ... ${bucket.lost} (${pct(bucket.lost)}) <- true loss`); +console.log(`\n Payment records referenced: ${totalRefs}`); +console.log(` in db ............ ${inDb} (${((inDb/totalRefs)*100||0).toFixed(1)}%)`); +console.log(` in arc only ...... ${inArcOnly} (${((inArcOnly/totalRefs)*100||0).toFixed(1)}%) <- restorable`); +console.log(` missing both ..... ${missingBoth} (${((missingBoth/totalRefs)*100||0).toFixed(1)}%) <- gone`); + +if (examples.recoverable.length) { + console.log(`\n Example RECOVERABLE users:`); + for (const uid of examples.recoverable) console.log(` ${await nameOf(uid)} (${uid})`); +} +if (examples.lost.length) { + console.log(`\n Example LOST-record users:`); + for (const uid of examples.lost) console.log(` ${await nameOf(uid)} (${uid})`); +} +if (!full) { + const est = Math.round((bucket.recoverable / Math.min(usersScanned, sampleN)) * usersScanned); + console.log(`\n (sample only — run --full for exact totals)`); +} +console.log("═".repeat(66)); +process.exit(0); diff --git a/scripts/ln-liquid-arb.ts b/scripts/ln-liquid-arb.ts new file mode 100644 index 00000000..02d0aa1e --- /dev/null +++ b/scripts/ln-liquid-arb.ts @@ -0,0 +1,126 @@ +#!/usr/bin/env bun +// Find users exhibiting the "lightning in, liquid out" pattern that drains the +// liquid hot wallet (forcing the liquid threshold down). For each user in the +// window, sum receives/sends per rail and surface those who take in meaningful +// lightning and push out meaningful liquid. +// +// Usage: bun scripts/ln-liquid-arb.ts [--days N] [--min SATS] [--top N] + +import { createClient } from "redis"; + +const argDays = Number(process.argv[process.argv.indexOf("--days") + 1]) || 14; +const MIN = Number(process.argv[process.argv.indexOf("--min") + 1]) || 500_000; +const TOP = Number(process.argv[process.argv.indexOf("--top") + 1]) || 30; +const cutoff = Date.now() - argDays * 24 * 60 * 60 * 1000; + +const db = createClient({ url: "redis://127.0.0.1:6379", socket: { reconnectStrategy: () => false } }); +await db.connect(); + +const fmt = (n: number) => n.toLocaleString("en-US"); + +// uid -> username +const uname = new Map(); +async function loadNames() { + let batch: string[] = []; + const drain = async () => { + if (!batch.length) return; + const vals = await db.mGet(batch); + for (const v of vals) { + if (!v || v[0] !== "{") continue; + try { const u = JSON.parse(v); if (u.id && u.username) uname.set(u.id, u.username); } catch {} + } + batch = []; + }; + for await (const k of db.scanIterator({ MATCH: "user:*", COUNT: 1000 })) { + batch.push(k as string); + if (batch.length >= 500) await drain(); + } + await drain(); +} +await loadNames(); +const nameOf = (uid: string) => uname.get(uid) || uid.slice(0, 8); + +// Per user per rail: { in, out } sats +type Rail = { in: number; out: number; inN: number; outN: number }; +const RAILS = ["lightning", "liquid", "bitcoin", "internal", "bolt12", "fund"]; +const stats = new Map>(); +const ensure = (uid: string) => { + let s = stats.get(uid); + if (!s) { + s = {}; + for (const r of RAILS) s[r] = { in: 0, out: 0, inN: 0, outN: 0 }; + stats.set(uid, s); + } + return s; +}; + +let scanned = 0; +let batch: string[] = []; +const drainP = async () => { + if (!batch.length) return; + const vals = await db.mGet(batch); + for (const v of vals) { + if (!v || v[0] !== "{") continue; + let p: any; try { p = JSON.parse(v); } catch { continue; } + if (!p.created || p.created < cutoff || typeof p.amount !== "number" || !p.uid) continue; + const rail = p.type; + if (!RAILS.includes(rail)) continue; + const s = ensure(p.uid)[rail]; + if (p.amount >= 0) { s.in += p.amount; s.inN++; } + else { s.out += Math.abs(p.amount); s.outN++; } + } + batch = []; +}; +for await (const k of db.scanIterator({ MATCH: "payment:*", COUNT: 2000 })) { + scanned++; + batch.push(k as string); + if (batch.length >= 1000) await drainP(); +} +await drainP(); + +// The arb pattern: significant lightning IN and significant liquid OUT. +const flagged = [...stats.entries()] + .map(([uid, s]) => ({ + uid, + lnIn: s.lightning.in, + lqOut: s.liquid.out, + bolt12In: s.bolt12.in, + arb: Math.min(s.lightning.in + s.bolt12.in, s.liquid.out), // overlap = the through-flow + s, + })) + .filter((x) => x.lqOut >= MIN && x.lnIn + x.bolt12In >= MIN) + .sort((a, b) => b.arb - a.arb) + .slice(0, TOP); + +console.log("═".repeat(72)); +console.log(` LIGHTNING-IN / LIQUID-OUT pattern — last ${argDays} days (min ${fmt(MIN)} sats each side)`); +console.log("═".repeat(72)); +console.log(` walked ${fmt(scanned)} payment keys\n`); + +if (!flagged.length) { + console.log(" No users matched the pattern at this threshold."); +} else { + console.log(" user ln_in bolt12_in liquid_out through-flow"); + for (const x of flagged) { + console.log( + ` ${nameOf(x.uid).padEnd(20)} ${fmt(x.lnIn).padStart(11)} ${fmt(x.bolt12In).padStart(11)} ${fmt(x.lqOut).padStart(13)} ${fmt(x.arb).padStart(13)}`, + ); + } + const totalThrough = flagged.reduce((s, x) => s + x.arb, 0); + console.log(`\n Combined through-flow (ln/bolt12 in -> liquid out): ${fmt(totalThrough)} sats`); +} + +// Also: who are the biggest NET liquid drainers regardless of funding source? +const netLiquid = [...stats.entries()] + .map(([uid, s]) => ({ uid, net: s.liquid.out - s.liquid.in, out: s.liquid.out })) + .filter((x) => x.net >= MIN) + .sort((a, b) => b.net - a.net) + .slice(0, 15); +console.log("\n" + "─".repeat(72)); +console.log(" TOP NET LIQUID DRAINERS (liquid_out - liquid_in, any funding source)"); +console.log("─".repeat(72)); +for (const x of netLiquid) { + console.log(` ${nameOf(x.uid).padEnd(20)} net out ${fmt(x.net).padStart(13)} (gross out ${fmt(x.out)})`); +} +console.log("═".repeat(72)); +process.exit(0); -- cgit v1.2.3