adjust ln minfeeregtest-data

9 files changed, 314 insertions, 26 deletions

diff --git a/index.ts b/index.ts
index 44d91d93..c191b069 100644
--- a/index.ts
+++ b/index.ts
@@ -29,7 +29,7 @@ try {
   catchUp();
   nwc();
   check();
-  startHealthCheck();
+  // startHealthCheck();
 } catch (e) {
   console.log(e);
 }
diff --git a/lib/app.ts b/lib/app.ts
index 7632520f..a1a31291 100644
--- a/lib/app.ts
+++ b/lib/app.ts
@@ -118,6 +118,7 @@ app.register(fastifyProxy, {
   prefix: "/ws",
   rewritePrefix: "/ws",
   websocket: true,
+  disableRequestLogging: true,
 });
 
 app.register(fastifySecureSession, {
diff --git a/lib/health.ts b/lib/health.ts
index c2e268c6..d9c38fa5 100644
--- a/lib/health.ts
+++ b/lib/health.ts
@@ -37,7 +37,7 @@ async function checkLightningHealth(): Promise<boolean> {
   const startTime = Date.now();
 
   try {
-    l("health check: testing lightning connection...");
+    // l("health check: testing lightning connection...");
     const info = await withTimeout(ln.getinfo(), RPC_TIMEOUT, "getinfo");
 
     if (!info?.id) {
@@ -71,7 +71,7 @@ async function checkLightningHealth(): Promise<boolean> {
     }
 
     const duration = Date.now() - startTime;
-    l(`health check: passed in ${duration}ms, node: ${info.id.slice(0, 16)}...`);
+    // l(`health check: passed in ${duration}ms, node: ${info.id.slice(0, 16)}...`);
 
     return true;
   } catch (e: any) {
@@ -139,9 +139,9 @@ export async function runHealthCheck() {
 }
 
 export function startHealthCheck() {
-  l(
-    `health check: starting (interval=${HEALTH_CHECK_INTERVAL / 1000}s, timeout=${RPC_TIMEOUT / 1000}s, max_failures=${MAX_CONSECUTIVE_FAILURES})`,
-  );
+  // l(
+  //   `health check: starting (interval=${HEALTH_CHECK_INTERVAL / 1000}s, timeout=${RPC_TIMEOUT / 1000}s, max_failures=${MAX_CONSECUTIVE_FAILURES})`,
+  // );
   // Run first check after a delay to let the app initialize
   checkTimer = setTimeout(runHealthCheck, 30000);
 }
diff --git a/lib/ln.ts b/lib/ln.ts
index 4dff06af..a1e4bd5c 100644
--- a/lib/ln.ts
+++ b/lib/ln.ts
@@ -123,7 +123,7 @@ export default ln;
 
 // Dedicated proxy for long-polling (waitanyinvoice) — uses its own socket
 // so it doesn't block the main connection
-export const lnListen = lightningProxy(config.lightning);
+export const lnListen = lightningProxy(config.lightningListen || config.lightning);
 
 export const lnb = lightningProxy(config.lightningb);
 export { LightningUnavailableError };
diff --git a/lib/nwc.ts b/lib/nwc.ts
index c888ee55..facf9dc8 100644
--- a/lib/nwc.ts
+++ b/lib/nwc.ts
@@ -185,14 +185,6 @@ const handle = (method, params, ev, app, user) =>
       );
 
       if (!created) {
-        warn(
-          "old nwc token",
-          pubkey,
-          user?.username,
-          spent,
-          amount,
-          max_amount,
-        );
         return error({
           code: "UNAUTHORIZED",
           message: `This NWC connection is no longer valid please create a new one at https://coinos.io/settings/nostr`,
@@ -334,7 +326,7 @@ const handle = (method, params, ev, app, user) =>
 
     async make_invoice() {
       const { amount, description, description_hash, expiry } = params;
-      l("nwc make_invoice", user.username);
+      // l("nwc make_invoice", user.username);
 
       const invoice = {
         amount: Math.round(amount / 1000),
diff --git a/lib/payments.ts b/lib/payments.ts
index 53b8dec1..151d4c99 100644
--- a/lib/payments.ts
+++ b/lib/payments.ts
@@ -309,7 +309,6 @@ export const completePayment = async (inv, p, user) => {
           };
         }
       } catch (e) {
-        console.log(e);
         withdrawal = { failed: true };
         warn(username, "autowithdraw failed", e.message);
       }
@@ -543,11 +542,12 @@ export const sendLightning = async ({
     payee = invoice_node_id;
   }
 
-  let minfee = 2;
+  const amt = amount_msat ? Math.round(amount_msat / 1000) : amount;
+  let minfee = Math.max(5, Math.round(amt * 0.005));
   const { channels } = await ln.listpeerchannels();
   if (channels.some((c) => c.peer_id === payee)) minfee = 0;
 
-  fee = Math.max(Number.parseInt(fee || 0), minfee);
+  fee = Math.max(Number.parseInt(fee) || minfee, minfee);
   if (fee < 0) fail("Fee cannot be negative");
 
   const { pays } = await ln.listpays(pr);
@@ -576,7 +576,8 @@ export const sendLightning = async ({
       // bolt11: pr.replace(/\s/g, "").toLowerCase(),
       amount_msat: amount_msat ? undefined : amount * 1000,
       maxfee: fee * 1000,
-      retry_for: 10,
+      retry_for: 30,
+      layers: ["prefer-kappa"],
     });
 
     try {
diff --git a/routes/invoices.ts b/routes/invoices.ts
index a18f4263..0cb73687 100644
--- a/routes/invoices.ts
+++ b/routes/invoices.ts
@@ -39,12 +39,6 @@ export default {
       const result = await generate({ invoice, user });
       res.send(result);
     } catch (e) {
-      err(
-        "problem generating invoice",
-        req.user?.username,
-        body.user?.username,
-        e.message,
-      );
       bail(res, e.message);
     }
   },
diff --git a/scripts/monitor-ln-socket.sh b/scripts/monitor-ln-socket.sh
new file mode 100755
index 00000000..66841f40
--- /dev/null
+++ b/scripts/monitor-ln-socket.sh
@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SOCKET_DIR="/home/adam/coinos-server/data/sockets"
+SOCKET_MAIN="$SOCKET_DIR/lightning-rpc"
+SOCKET_LISTEN="$SOCKET_DIR/lightning-rpc-listen"
+REMOTE_SOCKET="/home/adam/coinos-server/data/lightning/bitcoin/lightning-rpc"
+REMOTE_HOST="desk"
+
+log() {
+  echo "$(date '+%Y-%m-%d %H:%M:%S') [ln-socket-monitor] $*"
+}
+
+# Find SSH pids that are forwarding our sockets
+pid_main=$(pgrep -f "ssh.*-L ${SOCKET_MAIN}:${REMOTE_SOCKET} ${REMOTE_HOST}" 2>/dev/null || true)
+pid_listen=$(pgrep -f "ssh.*-L ${SOCKET_LISTEN}:${REMOTE_SOCKET} ${REMOTE_HOST}" 2>/dev/null || true)
+
+# Check if both processes are alive and both sockets exist
+if [[ -n "$pid_main" && -n "$pid_listen" && -S "$SOCKET_MAIN" && -S "$SOCKET_LISTEN" ]]; then
+  exit 0
+fi
+
+# Something is wrong - log what we found
+if [[ -z "$pid_main" ]]; then
+  log "main SSH forward is dead"
+elif [[ ! -S "$SOCKET_MAIN" ]]; then
+  log "main socket file missing"
+fi
+
+if [[ -z "$pid_listen" ]]; then
+  log "listen SSH forward is dead"
+elif [[ ! -S "$SOCKET_LISTEN" ]]; then
+  log "listen socket file missing"
+fi
+
+# Kill any remaining SSH forwards for our sockets
+for pid in $pid_main $pid_listen; do
+  if [[ -n "$pid" ]]; then
+    log "killing stale SSH forward (pid $pid)"
+    kill "$pid" 2>/dev/null || true
+  fi
+done
+
+# Brief wait for processes to exit
+sleep 1
+
+# Clean up socket files
+for sock in "$SOCKET_MAIN" "$SOCKET_LISTEN"; do
+  if [[ -e "$sock" ]]; then
+    log "removing stale socket $sock"
+    rm -f "$sock"
+  fi
+done
+
+# Ensure socket directory exists
+mkdir -p "$SOCKET_DIR"
+
+# Re-establish both SSH forwards
+log "starting main SSH forward"
+ssh -nNT -o ExitOnForwardFailure=yes \
+  -L "${SOCKET_MAIN}:${REMOTE_SOCKET}" "$REMOTE_HOST" &
+
+log "starting listen SSH forward"
+ssh -nNT -o ExitOnForwardFailure=yes \
+  -L "${SOCKET_LISTEN}:${REMOTE_SOCKET}" "$REMOTE_HOST" &
+
+# Wait briefly for sockets to appear
+for i in 1 2 3 4 5; do
+  if [[ -S "$SOCKET_MAIN" && -S "$SOCKET_LISTEN" ]]; then
+    break
+  fi
+  sleep 1
+done
+
+# Set permissions on sockets
+if [[ -S "$SOCKET_MAIN" ]]; then
+  chmod 777 "$SOCKET_MAIN"
+  log "set permissions on $SOCKET_MAIN"
+else
+  log "WARNING: $SOCKET_MAIN did not appear after 5s"
+fi
+
+if [[ -S "$SOCKET_LISTEN" ]]; then
+  chmod 777 "$SOCKET_LISTEN"
+  log "set permissions on $SOCKET_LISTEN"
+else
+  log "WARNING: $SOCKET_LISTEN did not appear after 5s"
+fi
+
+log "SSH forwards re-established"
diff --git a/scripts/payment-monitor.sh b/scripts/payment-monitor.sh
new file mode 100755
index 00000000..c50aad9e
--- /dev/null
+++ b/scripts/payment-monitor.sh
@@ -0,0 +1,210 @@
+#!/bin/bash
+# Payment failure monitor for coinos-server
+# Watches app container logs and compiles failure analytics
+# Run: ./scripts/payment-monitor.sh [hours_to_watch]
+# Output: periodic reports to stdout + detailed data to /tmp/payment-failures.jsonl
+
+WATCH_HOURS=${1:-24}
+DATA_FILE="/tmp/payment-failures.jsonl"
+REPORT_INTERVAL=1800  # report every 30 min
+NODE_ID=$(docker exec cl lightning-cli getinfo 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin)['id'])" 2>/dev/null || echo "unknown")
+
+echo "Payment failure monitor started (watching for ${WATCH_HOURS}h)"
+echo "Node: ${NODE_ID}"
+echo "Data file: ${DATA_FILE}"
+echo "Reports every ${REPORT_INTERVAL}s"
+echo "---"
+
+# Clear old data
+> "$DATA_FILE"
+
+docker logs app -f 2>&1 | timeout $((WATCH_HOURS * 3600)) python3 -u -c "
+import sys, json, time, re
+from collections import Counter, defaultdict
+from datetime import datetime
+
+data_file = '${DATA_FILE}'
+report_interval = ${REPORT_INTERVAL}
+last_report = time.time()
+
+# Tracking
+failures = []
+error_types = Counter()
+failed_destinations = Counter()
+failed_channels = Counter()  # channels that return temporary_channel_failure
+fee_insufficient_channels = Counter()
+failed_amounts = []  # (amount, error_type)
+successful = 0
+total_attempts = 0
+
+prev_line = None
+
+def parse_failure(line, error_line):
+    global successful, total_attempts
+    total_attempts += 1
+
+    try:
+        msg = json.loads(line)['msg']
+        err_msg = json.loads(error_line)['msg'] if error_line else ''
+    except:
+        return
+
+    # Parse: username payment failed amount balance hash
+    m = re.match(r'(\S+) payment failed (\d+) (\d+)\s+(.*)', msg)
+    if not m:
+        return
+
+    username, amount, balance, rest = m.group(1), int(m.group(2)), int(m.group(3)), m.group(4)
+
+    # Categorize error
+    if 'Insufficient funds' in err_msg or 'insuficientes' in err_msg.lower():
+        etype = 'insufficient_funds'
+    elif 'excessive cost' in err_msg:
+        etype = 'excessive_cost'
+    elif 'temporary_channel_failure' in err_msg:
+        etype = 'temporary_channel_failure'
+        # Extract failed channel IDs
+        for ch in re.findall(r'(\d+x\d+x\d+/\d+)', err_msg):
+            failed_channels[ch] += 1
+    elif 'fee_insufficient' in err_msg:
+        etype = 'fee_insufficient'
+        for ch in re.findall(r'(\d+x\d+x\d+/\d+)', err_msg):
+            fee_insufficient_channels[ch] += 1
+    elif 'Timed out' in err_msg:
+        etype = 'timed_out'
+        for ch in re.findall(r'(\d+x\d+x\d+/\d+)', err_msg):
+            if 'temporary_channel_failure' in err_msg:
+                failed_channels[ch] += 1
+            if 'fee_insufficient' in err_msg:
+                fee_insufficient_channels[ch] += 1
+    elif 'Cannot send to self' in err_msg:
+        etype = 'self_pay'
+    elif 'incorrect_or_unknown' in err_msg:
+        etype = 'unknown_invoice'
+    elif 'already underway' in err_msg:
+        etype = 'already_underway'
+    elif 'already been paid' in err_msg:
+        etype = 'already_paid'
+    elif 'user not provided' in err_msg:
+        etype = 'user_not_found'
+    elif err_msg:
+        etype = err_msg[:80]
+    else:
+        etype = 'unknown'
+
+    error_types[etype] += 1
+    failed_amounts.append((amount, etype))
+
+    # Try to get destination from invoice
+    if rest and rest.startswith('ln'):
+        bolt11_prefix = rest[:20]
+        failed_destinations[bolt11_prefix] += 1
+
+    record = {
+        'time': datetime.utcnow().isoformat(),
+        'user': username,
+        'amount': amount,
+        'balance': balance,
+        'error': etype,
+        'detail': err_msg[:200] if err_msg else '',
+    }
+
+    with open(data_file, 'a') as f:
+        f.write(json.dumps(record) + '\n')
+
+def parse_success(line):
+    global successful
+    successful += 1
+
+def report():
+    global last_report
+    now = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
+
+    print(f'\n====== PAYMENT FAILURE REPORT {now} UTC ======')
+    print(f'Total attempts: {total_attempts}  Successful: {successful}  Failed: {total_attempts}')
+    if total_attempts > 0:
+        print(f'Success rate: {successful/(successful+total_attempts)*100:.1f}%')
+
+    print(f'\n--- Error breakdown ---')
+    for etype, count in error_types.most_common(15):
+        print(f'  {count:>5}  {etype}')
+
+    if failed_channels:
+        print(f'\n--- Channels with liquidity failures (top 10) ---')
+        for ch, count in failed_channels.most_common(10):
+            print(f'  {count:>5}  {ch}')
+
+    if fee_insufficient_channels:
+        print(f'\n--- Channels with stale fee gossip (top 10) ---')
+        for ch, count in fee_insufficient_channels.most_common(10):
+            print(f'  {count:>5}  {ch}')
+
+    # Amount distribution of failures
+    if failed_amounts:
+        amounts_by_type = defaultdict(list)
+        for amt, etype in failed_amounts:
+            amounts_by_type[etype].append(amt)
+
+        print(f'\n--- Failed amount ranges by error ---')
+        for etype in ['insufficient_funds', 'excessive_cost', 'temporary_channel_failure', 'timed_out', 'fee_insufficient']:
+            amts = amounts_by_type.get(etype, [])
+            if amts:
+                print(f'  {etype}: min={min(amts)} max={max(amts)} avg={sum(amts)//len(amts)} count={len(amts)}')
+
+    print(f'\n--- Recommendations ---')
+    if error_types.get('insufficient_funds', 0) > 10:
+        print('  * HIGH insufficient_funds: Consider showing fee preview before payment attempt')
+    if error_types.get('excessive_cost', 0) > 5:
+        print('  * HIGH excessive_cost: Increase default maxfee or improve channel connectivity')
+    if error_types.get('fee_insufficient', 0) > 5:
+        print('  * STALE GOSSIP: Some channels advertise lower fees than they charge')
+        print('    Nodes to investigate:')
+        for ch, count in fee_insufficient_channels.most_common(5):
+            scid = ch.split('/')[0]
+            print(f'      {scid} ({count} failures)')
+    if failed_channels:
+        print('  * LIQUIDITY BOTTLENECKS:')
+        for ch, count in failed_channels.most_common(5):
+            if count >= 3:
+                scid = ch.split('/')[0]
+                print(f'      {scid} ({count} failures) - consider opening channel to bypass')
+    if error_types.get('self_pay', 0) > 5:
+        print('  * SELF-PAY: Users trying to pay own invoices - consider UI guard')
+
+    print('=' * 55)
+    sys.stdout.flush()
+    last_report = time.time()
+
+for line in sys.stdin:
+    line = line.strip()
+    if not line:
+        continue
+
+    try:
+        d = json.loads(line)
+        msg = d.get('msg', '')
+    except:
+        prev_line = line
+        continue
+
+    # Track successes
+    if 'sent lightning' in msg or 'sent liquid' in msg or 'sent bitcoin' in msg:
+        parse_success(line)
+
+    # Track failures - need the next line for error detail
+    if prev_line and 'payment failed' in prev_line:
+        try:
+            prev_d = json.loads(prev_line)
+            if 'payment failed' in prev_d.get('msg', ''):
+                parse_failure(prev_line, line)
+        except:
+            pass
+
+    prev_line = line
+
+    # Periodic report
+    if time.time() - last_report >= report_interval:
+        report()
+" 2>&1
+
+echo "Monitor finished after ${WATCH_HOURS}h"