diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/monitor-ln-socket.sh | 90 | ||||
| -rwxr-xr-x | scripts/payment-monitor.sh | 210 |
2 files changed, 300 insertions, 0 deletions
diff --git a/scripts/monitor-ln-socket.sh b/scripts/monitor-ln-socket.sh new file mode 100755 index 00000000..66841f40 --- /dev/null +++ b/scripts/monitor-ln-socket.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +set -euo pipefail + +SOCKET_DIR="/home/adam/coinos-server/data/sockets" +SOCKET_MAIN="$SOCKET_DIR/lightning-rpc" +SOCKET_LISTEN="$SOCKET_DIR/lightning-rpc-listen" +REMOTE_SOCKET="/home/adam/coinos-server/data/lightning/bitcoin/lightning-rpc" +REMOTE_HOST="desk" + +log() { + echo "$(date '+%Y-%m-%d %H:%M:%S') [ln-socket-monitor] $*" +} + +# Find SSH pids that are forwarding our sockets +pid_main=$(pgrep -f "ssh.*-L ${SOCKET_MAIN}:${REMOTE_SOCKET} ${REMOTE_HOST}" 2>/dev/null || true) +pid_listen=$(pgrep -f "ssh.*-L ${SOCKET_LISTEN}:${REMOTE_SOCKET} ${REMOTE_HOST}" 2>/dev/null || true) + +# Check if both processes are alive and both sockets exist +if [[ -n "$pid_main" && -n "$pid_listen" && -S "$SOCKET_MAIN" && -S "$SOCKET_LISTEN" ]]; then + exit 0 +fi + +# Something is wrong - log what we found +if [[ -z "$pid_main" ]]; then + log "main SSH forward is dead" +elif [[ ! -S "$SOCKET_MAIN" ]]; then + log "main socket file missing" +fi + +if [[ -z "$pid_listen" ]]; then + log "listen SSH forward is dead" +elif [[ ! -S "$SOCKET_LISTEN" ]]; then + log "listen socket file missing" +fi + +# Kill any remaining SSH forwards for our sockets +for pid in $pid_main $pid_listen; do + if [[ -n "$pid" ]]; then + log "killing stale SSH forward (pid $pid)" + kill "$pid" 2>/dev/null || true + fi +done + +# Brief wait for processes to exit +sleep 1 + +# Clean up socket files +for sock in "$SOCKET_MAIN" "$SOCKET_LISTEN"; do + if [[ -e "$sock" ]]; then + log "removing stale socket $sock" + rm -f "$sock" + fi +done + +# Ensure socket directory exists +mkdir -p "$SOCKET_DIR" + +# Re-establish both SSH forwards +log "starting main SSH forward" +ssh -nNT -o ExitOnForwardFailure=yes \ + -L "${SOCKET_MAIN}:${REMOTE_SOCKET}" "$REMOTE_HOST" & + +log "starting listen SSH forward" +ssh -nNT -o ExitOnForwardFailure=yes \ + -L "${SOCKET_LISTEN}:${REMOTE_SOCKET}" "$REMOTE_HOST" & + +# Wait briefly for sockets to appear +for i in 1 2 3 4 5; do + if [[ -S "$SOCKET_MAIN" && -S "$SOCKET_LISTEN" ]]; then + break + fi + sleep 1 +done + +# Set permissions on sockets +if [[ -S "$SOCKET_MAIN" ]]; then + chmod 777 "$SOCKET_MAIN" + log "set permissions on $SOCKET_MAIN" +else + log "WARNING: $SOCKET_MAIN did not appear after 5s" +fi + +if [[ -S "$SOCKET_LISTEN" ]]; then + chmod 777 "$SOCKET_LISTEN" + log "set permissions on $SOCKET_LISTEN" +else + log "WARNING: $SOCKET_LISTEN did not appear after 5s" +fi + +log "SSH forwards re-established" diff --git a/scripts/payment-monitor.sh b/scripts/payment-monitor.sh new file mode 100755 index 00000000..c50aad9e --- /dev/null +++ b/scripts/payment-monitor.sh @@ -0,0 +1,210 @@ +#!/bin/bash +# Payment failure monitor for coinos-server +# Watches app container logs and compiles failure analytics +# Run: ./scripts/payment-monitor.sh [hours_to_watch] +# Output: periodic reports to stdout + detailed data to /tmp/payment-failures.jsonl + +WATCH_HOURS=${1:-24} +DATA_FILE="/tmp/payment-failures.jsonl" +REPORT_INTERVAL=1800 # report every 30 min +NODE_ID=$(docker exec cl lightning-cli getinfo 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin)['id'])" 2>/dev/null || echo "unknown") + +echo "Payment failure monitor started (watching for ${WATCH_HOURS}h)" +echo "Node: ${NODE_ID}" +echo "Data file: ${DATA_FILE}" +echo "Reports every ${REPORT_INTERVAL}s" +echo "---" + +# Clear old data +> "$DATA_FILE" + +docker logs app -f 2>&1 | timeout $((WATCH_HOURS * 3600)) python3 -u -c " +import sys, json, time, re +from collections import Counter, defaultdict +from datetime import datetime + +data_file = '${DATA_FILE}' +report_interval = ${REPORT_INTERVAL} +last_report = time.time() + +# Tracking +failures = [] +error_types = Counter() +failed_destinations = Counter() +failed_channels = Counter() # channels that return temporary_channel_failure +fee_insufficient_channels = Counter() +failed_amounts = [] # (amount, error_type) +successful = 0 +total_attempts = 0 + +prev_line = None + +def parse_failure(line, error_line): + global successful, total_attempts + total_attempts += 1 + + try: + msg = json.loads(line)['msg'] + err_msg = json.loads(error_line)['msg'] if error_line else '' + except: + return + + # Parse: username payment failed amount balance hash + m = re.match(r'(\S+) payment failed (\d+) (\d+)\s+(.*)', msg) + if not m: + return + + username, amount, balance, rest = m.group(1), int(m.group(2)), int(m.group(3)), m.group(4) + + # Categorize error + if 'Insufficient funds' in err_msg or 'insuficientes' in err_msg.lower(): + etype = 'insufficient_funds' + elif 'excessive cost' in err_msg: + etype = 'excessive_cost' + elif 'temporary_channel_failure' in err_msg: + etype = 'temporary_channel_failure' + # Extract failed channel IDs + for ch in re.findall(r'(\d+x\d+x\d+/\d+)', err_msg): + failed_channels[ch] += 1 + elif 'fee_insufficient' in err_msg: + etype = 'fee_insufficient' + for ch in re.findall(r'(\d+x\d+x\d+/\d+)', err_msg): + fee_insufficient_channels[ch] += 1 + elif 'Timed out' in err_msg: + etype = 'timed_out' + for ch in re.findall(r'(\d+x\d+x\d+/\d+)', err_msg): + if 'temporary_channel_failure' in err_msg: + failed_channels[ch] += 1 + if 'fee_insufficient' in err_msg: + fee_insufficient_channels[ch] += 1 + elif 'Cannot send to self' in err_msg: + etype = 'self_pay' + elif 'incorrect_or_unknown' in err_msg: + etype = 'unknown_invoice' + elif 'already underway' in err_msg: + etype = 'already_underway' + elif 'already been paid' in err_msg: + etype = 'already_paid' + elif 'user not provided' in err_msg: + etype = 'user_not_found' + elif err_msg: + etype = err_msg[:80] + else: + etype = 'unknown' + + error_types[etype] += 1 + failed_amounts.append((amount, etype)) + + # Try to get destination from invoice + if rest and rest.startswith('ln'): + bolt11_prefix = rest[:20] + failed_destinations[bolt11_prefix] += 1 + + record = { + 'time': datetime.utcnow().isoformat(), + 'user': username, + 'amount': amount, + 'balance': balance, + 'error': etype, + 'detail': err_msg[:200] if err_msg else '', + } + + with open(data_file, 'a') as f: + f.write(json.dumps(record) + '\n') + +def parse_success(line): + global successful + successful += 1 + +def report(): + global last_report + now = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') + + print(f'\n====== PAYMENT FAILURE REPORT {now} UTC ======') + print(f'Total attempts: {total_attempts} Successful: {successful} Failed: {total_attempts}') + if total_attempts > 0: + print(f'Success rate: {successful/(successful+total_attempts)*100:.1f}%') + + print(f'\n--- Error breakdown ---') + for etype, count in error_types.most_common(15): + print(f' {count:>5} {etype}') + + if failed_channels: + print(f'\n--- Channels with liquidity failures (top 10) ---') + for ch, count in failed_channels.most_common(10): + print(f' {count:>5} {ch}') + + if fee_insufficient_channels: + print(f'\n--- Channels with stale fee gossip (top 10) ---') + for ch, count in fee_insufficient_channels.most_common(10): + print(f' {count:>5} {ch}') + + # Amount distribution of failures + if failed_amounts: + amounts_by_type = defaultdict(list) + for amt, etype in failed_amounts: + amounts_by_type[etype].append(amt) + + print(f'\n--- Failed amount ranges by error ---') + for etype in ['insufficient_funds', 'excessive_cost', 'temporary_channel_failure', 'timed_out', 'fee_insufficient']: + amts = amounts_by_type.get(etype, []) + if amts: + print(f' {etype}: min={min(amts)} max={max(amts)} avg={sum(amts)//len(amts)} count={len(amts)}') + + print(f'\n--- Recommendations ---') + if error_types.get('insufficient_funds', 0) > 10: + print(' * HIGH insufficient_funds: Consider showing fee preview before payment attempt') + if error_types.get('excessive_cost', 0) > 5: + print(' * HIGH excessive_cost: Increase default maxfee or improve channel connectivity') + if error_types.get('fee_insufficient', 0) > 5: + print(' * STALE GOSSIP: Some channels advertise lower fees than they charge') + print(' Nodes to investigate:') + for ch, count in fee_insufficient_channels.most_common(5): + scid = ch.split('/')[0] + print(f' {scid} ({count} failures)') + if failed_channels: + print(' * LIQUIDITY BOTTLENECKS:') + for ch, count in failed_channels.most_common(5): + if count >= 3: + scid = ch.split('/')[0] + print(f' {scid} ({count} failures) - consider opening channel to bypass') + if error_types.get('self_pay', 0) > 5: + print(' * SELF-PAY: Users trying to pay own invoices - consider UI guard') + + print('=' * 55) + sys.stdout.flush() + last_report = time.time() + +for line in sys.stdin: + line = line.strip() + if not line: + continue + + try: + d = json.loads(line) + msg = d.get('msg', '') + except: + prev_line = line + continue + + # Track successes + if 'sent lightning' in msg or 'sent liquid' in msg or 'sent bitcoin' in msg: + parse_success(line) + + # Track failures - need the next line for error detail + if prev_line and 'payment failed' in prev_line: + try: + prev_d = json.loads(prev_line) + if 'payment failed' in prev_d.get('msg', ''): + parse_failure(prev_line, line) + except: + pass + + prev_line = line + + # Periodic report + if time.time() - last_report >= report_interval: + report() +" 2>&1 + +echo "Monitor finished after ${WATCH_HOURS}h" |