sync vault

6 files changed, 269 insertions, 137 deletions

diff --git a/index.ts b/index.ts
index 784baa5d..f879b65f 100644
--- a/index.ts
+++ b/index.ts
@@ -119,6 +119,7 @@ app.post("/fetchinvoice", payments.fetchinvoice);
 app.get("/ark/address", payments.arkAddress);
 app.post("/ark/receive", auth, payments.arkReceive);
 app.post("/ark/sync", auth, payments.arkSync);
+app.post("/bitcoin/sync", auth, payments.bitcoinSync);
 
 app.get("/square/connect", auth, square.connect);
 app.get("/square/auth", auth, square.auth);
diff --git a/lib/bip353.ts b/lib/bip353.ts
index 4e37c46e..8b00635b 100644
--- a/lib/bip353.ts
+++ b/lib/bip353.ts
@@ -5,56 +5,58 @@ import { l, warn } from "$lib/logging";
 import { v4 } from "uuid";
 import { PaymentType } from "$lib/types";
 
-const { CF_EMAIL, CF_API_KEY, CF_ZONE_ID } = process.env;
 const cfApi = "https://api.cloudflare.com/client/v4";
-const cfHeaders = {
-  "X-Auth-Email": CF_EMAIL,
-  "X-Auth-Key": CF_API_KEY,
+const cfHeaders = () => ({
+  Authorization: `Bearer ${config.cloudflare?.apiToken}`,
   "Content-Type": "application/json",
-};
+});
 
 const createDnsRecord = async (username: string, bolt12: string) => {
-  if (!CF_API_KEY || !CF_ZONE_ID) return;
+  const { apiToken, zoneId } = config.cloudflare || {};
+  if (!apiToken || !zoneId) return;
 
   const name = `${username}.user._bitcoin-payment.${config.hostname}`;
   const content = `bitcoin:?lno=${bolt12}`;
 
+  const { zoneId } = config.cloudflare;
+
   const listRes = await fetch(
-    `${cfApi}/zones/${CF_ZONE_ID}/dns_records?type=TXT&name=${name}`,
-    { headers: cfHeaders },
+    `${cfApi}/zones/${zoneId}/dns_records?type=TXT&name=${name}`,
+    { headers: cfHeaders() },
   );
   const listData = (await listRes.json()) as any;
 
   if (listData.result?.length) {
     const recordId = listData.result[0].id;
-    await fetch(`${cfApi}/zones/${CF_ZONE_ID}/dns_records/${recordId}`, {
+    await fetch(`${cfApi}/zones/${zoneId}/dns_records/${recordId}`, {
       method: "PUT",
-      headers: cfHeaders,
+      headers: cfHeaders(),
       body: JSON.stringify({ type: "TXT", name, content, ttl: 3600 }),
     });
   } else {
-    await fetch(`${cfApi}/zones/${CF_ZONE_ID}/dns_records`, {
+    await fetch(`${cfApi}/zones/${zoneId}/dns_records`, {
       method: "POST",
-      headers: cfHeaders,
+      headers: cfHeaders(),
       body: JSON.stringify({ type: "TXT", name, content, ttl: 3600 }),
     });
   }
 };
 
 const deleteDnsRecord = async (username: string) => {
-  if (!CF_API_KEY || !CF_ZONE_ID) return;
+  const { apiToken, zoneId } = config.cloudflare || {};
+  if (!apiToken || !zoneId) return;
 
   const name = `${username}.user._bitcoin-payment.${config.hostname}`;
   const listRes = await fetch(
-    `${cfApi}/zones/${CF_ZONE_ID}/dns_records?type=TXT&name=${name}`,
-    { headers: cfHeaders },
+    `${cfApi}/zones/${zoneId}/dns_records?type=TXT&name=${name}`,
+    { headers: cfHeaders() },
   );
   const listData = (await listRes.json()) as any;
 
   for (const record of listData.result || []) {
-    await fetch(`${cfApi}/zones/${CF_ZONE_ID}/dns_records/${record.id}`, {
+    await fetch(`${cfApi}/zones/${zoneId}/dns_records/${record.id}`, {
       method: "DELETE",
-      headers: cfHeaders,
+      headers: cfHeaders(),
     });
   }
 };
diff --git a/lib/invoices.ts b/lib/invoices.ts
index c5194ce3..15de0687 100644
--- a/lib/invoices.ts
+++ b/lib/invoices.ts
@@ -100,7 +100,6 @@ export const generate = async ({ invoice, user }) => {
     account.nextIndex = nextIndex + 1;
     await s(`account:${aid}`, account);
     await s(`invoice:${hash}`, id);
-    await db.sAdd("watching", hash);
   } else if (type === PaymentType.lightning) {
     let r;
     if (bolt11) {
diff --git a/lib/payments.ts b/lib/payments.ts
index b3c63cfa..f2e4fcab 100644
--- a/lib/payments.ts
+++ b/lib/payments.ts
@@ -1325,153 +1325,214 @@ export const catchUp = async () => {
   }
 };
 
-export const importAccountHistory = async (account) => {
-  try {
-    const { id, uid, pubkey, fingerprint } = account;
-    const user = await getUser(uid);
+export const syncBitcoinVault = async (account, user) => {
+  const { id, uid, pubkey, fingerprint } = account;
 
-    let nextIndex = account.nextIndex || 0;
-    if (!account.importedAt) {
-      const lastUsed = await findLastUsedIndex(pubkey, fingerprint);
-      if (lastUsed > nextIndex) nextIndex = lastUsed;
-      account.nextIndex = nextIndex;
-    }
+  let nextIndex = account.nextIndex || 0;
+  const lastUsed = await findLastUsedIndex(pubkey, fingerprint);
+  if (lastUsed > nextIndex) {
+    nextIndex = lastUsed;
+    account.nextIndex = nextIndex;
+    await s(`account:${id}`, account);
+  }
 
-    const count = nextIndex + 1;
-    const externalAddrs = deriveAddresses(pubkey, fingerprint, count, false);
-    const internalAddrs = deriveAddresses(pubkey, fingerprint, count, true);
-    const allAddrs = [...externalAddrs, ...internalAddrs];
-    const addressSet = new Set(allAddrs);
+  const count = nextIndex + 1;
+  const externalAddrs = deriveAddresses(pubkey, fingerprint, count, false);
+  const internalAddrs = deriveAddresses(pubkey, fingerprint, count, true);
+  const allAddrs = [...externalAddrs, ...internalAddrs];
+  const addressSet = new Set(allAddrs);
 
-    const utxos = await getAddressUtxos(allAddrs);
-    const confirmedUtxos = utxos.filter((u) => u.status.confirmed && u.value >= 300);
-    const total = confirmedUtxos.reduce((sum, u) => sum + u.value, 0);
+  const utxos = await getAddressUtxos(allAddrs);
+  const confirmedUtxos = utxos.filter((u) => u.status.confirmed && u.value >= 300);
+  const pendingUtxos = utxos.filter((u) => !u.status.confirmed && u.value >= 300);
+  const confirmedTotal = confirmedUtxos.reduce((sum, u) => sum + u.value, 0);
+  const pendingTotal = pendingUtxos.reduce((sum, u) => sum + u.value, 0);
 
-    const rates = await g("rates");
-    const currency = account.currency || user.currency;
-    let rate = rates[currency];
-    if (!rate) await sleep(1000);
-    rate = rate || rates[currency];
+  const { rate, currency } = await getUserRate(user);
 
-    for (const u of confirmedUtxos) {
-      const existing = await getPayment(`${u.txid}:${u.vout}`);
-      if (existing?.aid === id) continue;
+  const newPayments = [];
 
-      if (existing) {
-        existing.aid = id;
-        await db
-          .multi()
-          .set(`payment:${existing.id}`, JSON.stringify(existing))
-          .lPush(`${id}:payments`, existing.id)
-          .exec();
-        continue;
+  // Process confirmed UTXOs (incoming)
+  for (const u of confirmedUtxos) {
+    const ref = `${u.txid}:${u.vout}`;
+    const existing = await getPayment(ref);
+    if (existing?.aid === id) {
+      if (!existing.confirmed) {
+        existing.confirmed = true;
+        await s(`payment:${existing.id}`, existing);
       }
+      continue;
+    }
 
-      const created = u.status?.block_time ? u.status.block_time * 1000 : Date.now();
-      const p = {
-        id: v4(),
-        aid: id,
-        amount: u.value,
-        fee: 0,
-        hash: u.address,
-        confirmed: true,
-        rate,
-        currency: user.currency,
-        type: PaymentType.bitcoin,
-        uid,
-        ref: `${u.txid}:${u.vout}`,
-        created,
-      };
-
+    if (existing) {
+      existing.aid = id;
+      existing.confirmed = true;
       await db
         .multi()
-        .set(`payment:${u.txid}:${u.vout}`, p.id)
-        .set(`payment:${p.id}`, JSON.stringify(p))
-        .lPush(`${id}:payments`, p.id)
-        .set(`${id}:payments:last`, p.created)
+        .set(`payment:${existing.id}`, JSON.stringify(existing))
+        .lPush(`${id}:payments`, existing.id)
         .exec();
+      continue;
     }
 
-    const txsById = new Map();
-    for (const address of allAddrs) {
-      const txs = await getAddressTxs(address);
-      for (const tx of txs as any) {
-        txsById.set(tx.txid, tx);
-      }
-    }
-
-    for (const tx of txsById.values()) {
-      if (!tx.status?.confirmed) continue;
-      const existing = await getPayment(tx.txid);
-      if (existing?.aid === id) continue;
-      if (existing) {
-        existing.aid = id;
-        await db
-          .multi()
-          .set(`payment:${existing.id}`, JSON.stringify(existing))
-          .lPush(`${id}:payments`, existing.id)
-          .exec();
-        continue;
-      }
+    const created = u.status?.block_time ? u.status.block_time * 1000 : Date.now();
+    const p = {
+      id: v4(),
+      aid: id,
+      amount: u.value,
+      fee: 0,
+      hash: u.address,
+      confirmed: true,
+      rate,
+      currency,
+      type: PaymentType.bitcoin,
+      uid,
+      ref,
+      created,
+    };
 
-      let inputsFromUs = 0;
-      let outputsToUs = 0;
-      let totalIn = 0;
-      let totalOut = 0;
+    await db
+      .multi()
+      .set(`payment:${ref}`, p.id)
+      .set(`payment:${p.id}`, JSON.stringify(p))
+      .lPush(`${id}:payments`, p.id)
+      .set(`${id}:payments:last`, p.created)
+      .exec();
 
-      for (const vin of tx.vin || []) {
-        const prev = vin.prevout;
-        if (!prev) continue;
-        const value = Number.parseInt(prev.value) || 0;
-        totalIn += value;
-        if (addressSet.has(prev.scriptpubkey_address)) inputsFromUs += value;
-      }
+    newPayments.push(p);
+  }
 
-      for (const vout of tx.vout || []) {
-        const value = Number.parseInt(vout.value) || 0;
-        totalOut += value;
-        if (addressSet.has(vout.scriptpubkey_address)) outputsToUs += value;
-      }
+  // Process pending (unconfirmed) UTXOs
+  for (const u of pendingUtxos) {
+    const ref = `${u.txid}:${u.vout}`;
+    const existing = await getPayment(ref);
+    if (existing?.aid === id) continue;
 
-      if (!inputsFromUs) continue;
+    if (existing) {
+      existing.aid = id;
+      await db
+        .multi()
+        .set(`payment:${existing.id}`, JSON.stringify(existing))
+        .lPush(`${id}:payments`, existing.id)
+        .exec();
+      continue;
+    }
 
-      const net = outputsToUs - inputsFromUs;
-      if (net >= 0) continue;
+    const p = {
+      id: v4(),
+      aid: id,
+      amount: u.value,
+      fee: 0,
+      hash: u.address,
+      confirmed: false,
+      rate,
+      currency,
+      type: PaymentType.bitcoin,
+      uid,
+      ref,
+      created: Date.now(),
+    };
 
-      const amount = totalOut - outputsToUs;
-      if (amount <= 0) continue;
+    await db
+      .multi()
+      .set(`payment:${ref}`, p.id)
+      .set(`payment:${p.id}`, JSON.stringify(p))
+      .lPush(`${id}:payments`, p.id)
+      .set(`${id}:payments:last`, p.created)
+      .exec();
 
-      const fee = Math.max(0, totalIn - totalOut);
-      const created = tx.status?.block_time ? tx.status.block_time * 1000 : Date.now();
+    newPayments.push(p);
+  }
 
-      const p = {
-        id: v4(),
-        aid: id,
-        amount: -amount,
-        fee,
-        hash: tx.txid,
-        confirmed: true,
-        rate,
-        currency: user.currency,
-        type: PaymentType.bitcoin,
-        uid,
-        created,
-      };
+  // Process outgoing transactions
+  const txsById = new Map();
+  for (const address of allAddrs) {
+    const txs = await getAddressTxs(address);
+    for (const tx of txs as any) {
+      txsById.set(tx.txid, tx);
+    }
+  }
 
+  for (const tx of txsById.values()) {
+    if (!tx.status?.confirmed) continue;
+    const existing = await getPayment(tx.txid);
+    if (existing?.aid === id) continue;
+    if (existing) {
+      existing.aid = id;
       await db
         .multi()
-        .set(`payment:${tx.txid}`, p.id)
-        .set(`payment:${p.id}`, JSON.stringify(p))
-        .lPush(`${id}:payments`, p.id)
-        .set(`${id}:payments:last`, p.created)
+        .set(`payment:${existing.id}`, JSON.stringify(existing))
+        .lPush(`${id}:payments`, existing.id)
         .exec();
+      continue;
     }
 
-    await tbSetBalance(id, total);
-    await tbSetPending(id, 0);
+    let inputsFromUs = 0;
+    let outputsToUs = 0;
+    let totalIn = 0;
+    let totalOut = 0;
+
+    for (const vin of tx.vin || []) {
+      const prev = vin.prevout;
+      if (!prev) continue;
+      const value = Number.parseInt(prev.value) || 0;
+      totalIn += value;
+      if (addressSet.has(prev.scriptpubkey_address)) inputsFromUs += value;
+    }
+
+    for (const vout of tx.vout || []) {
+      const value = Number.parseInt(vout.value) || 0;
+      totalOut += value;
+      if (addressSet.has(vout.scriptpubkey_address)) outputsToUs += value;
+    }
+
+    if (!inputsFromUs) continue;
+
+    const net = outputsToUs - inputsFromUs;
+    if (net >= 0) continue;
+
+    const amount = totalOut - outputsToUs;
+    if (amount <= 0) continue;
+
+    const fee = Math.max(0, totalIn - totalOut);
+    const created = tx.status?.block_time ? tx.status.block_time * 1000 : Date.now();
+
+    const p = {
+      id: v4(),
+      aid: id,
+      amount: -amount,
+      fee,
+      hash: tx.txid,
+      confirmed: true,
+      rate,
+      currency,
+      type: PaymentType.bitcoin,
+      uid,
+      created,
+    };
+
+    await db
+      .multi()
+      .set(`payment:${tx.txid}`, p.id)
+      .set(`payment:${p.id}`, JSON.stringify(p))
+      .lPush(`${id}:payments`, p.id)
+      .set(`${id}:payments:last`, p.created)
+      .exec();
+  }
+
+  return { confirmedTotal, pendingTotal, newPayments };
+};
+
+export const importAccountHistory = async (account) => {
+  try {
+    const user = await getUser(account.uid);
+    const { confirmedTotal } = await syncBitcoinVault(account, user);
+
+    await tbSetBalance(account.id, confirmedTotal);
+    await tbSetPending(account.id, 0);
 
     account.importedAt = account.importedAt || Date.now();
-    await s(`account:${id}`, account);
+    await s(`account:${account.id}`, account);
   } catch (e) {
     console.log(e);
     warn("problem importing account history", e.message, account);
diff --git a/routes/payments.ts b/routes/payments.ts
index 98581ebd..524f6f4d 100644
--- a/routes/payments.ts
+++ b/routes/payments.ts
@@ -23,6 +23,7 @@ import {
   sendInternal,
   sendLightning,
   sendOnchain,
+  syncBitcoinVault,
 } from "$lib/payments";
 import { emit } from "$lib/sockets";
 import {
@@ -1116,4 +1117,72 @@ export default {
       return bail(c, e.message);
     }
   },
+
+  async bitcoinSync(c) {
+    try {
+      const user = c.get("user");
+      const { aid } = await c.req.json();
+      const { id: uid } = user;
+
+      const pos = await db.lPos(`${uid}:accounts`, aid);
+      if (pos === null) return bail(c, "account not found");
+
+      const account = await g(`account:${aid}`);
+      if (!account?.pubkey || !account?.fingerprint) return bail(c, "not a bitcoin vault");
+
+      const lockKey = `btcsynclock:${aid}`;
+      const gotLock = await db.set(lockKey, "1", { NX: true, EX: 30 });
+      if (!gotLock) return c.json({ synced: 0, received: 0, payments: [] });
+
+      try {
+        const { pendingTotal, newPayments } = await syncBitcoinVault(account, user);
+
+        let received = 0;
+        const payments = [];
+
+        for (const p of newPayments) {
+          if (p.amount > 0) {
+            received += p.amount;
+            payments.push(p);
+
+            const invoiceIds = await db.lRange(`${aid}:invoices`, 0, 20);
+            for (const iid of invoiceIds) {
+              const inv = await getInvoice(iid);
+              if (!inv || inv.type !== "bitcoin") continue;
+              if (inv.received >= inv.amount && inv.amount > 0) continue;
+              if (inv.amount > 0 && p.amount < inv.amount) continue;
+              inv.received = (inv.received || 0) + p.amount;
+              p.iid = iid;
+              await s(`invoice:${iid}`, inv);
+              await s(`payment:${p.id}`, p);
+              emit(uid, "payment", p);
+              break;
+            }
+          }
+        }
+
+        // Compute balance from payments
+        const paymentIds = await db.lRange(`${aid}:payments`, 0, -1);
+        const paymentKeys = paymentIds.map((pid) => `payment:${pid}`);
+        const allPayments = await gfAll(paymentKeys);
+        let balance = 0;
+        for (const pay of allPayments) {
+          if (pay) balance += pay.amount;
+        }
+        balance = Math.max(balance, 0);
+
+        return c.json({
+          synced: newPayments.length,
+          received,
+          payments,
+          balance,
+          pending: pendingTotal,
+        });
+      } finally {
+        await db.del(lockKey);
+      }
+    } catch (e) {
+      return bail(c, e.message);
+    }
+  },
 };
diff --git a/routes/users.ts b/routes/users.ts
index 6948fb28..027d836e 100644
--- a/routes/users.ts
+++ b/routes/users.ts
@@ -894,7 +894,7 @@ export default {
           await importAccountHistory(account);
         }
 
-        if (account.type === PaymentType.ark) {
+        if (account.type === PaymentType.ark || (account.pubkey && account.fingerprint)) {
           const paymentIds = (await db.lRange(`${aid}:payments`, 0, -1)) as string[];
           const paymentKeys = paymentIds.map((pid) => `payment:${pid}`);
           const payments = await gfAll(paymentKeys);