request tuning

4 files changed, 115 insertions, 43 deletions

diff --git a/lib/db.ts b/lib/db.ts
index 9d09c997..fbd7a6d8 100644
--- a/lib/db.ts
+++ b/lib/db.ts
@@ -104,6 +104,30 @@ export const gf = async (k) => {
   }
 };
 
+// Batch get with fallback to archive
+export const gfAll = async (keys: string[]) => {
+  if (!keys.length) return [];
+  const values = await db.mGet(keys);
+  const missing: number[] = [];
+  for (let i = 0; i < values.length; i++) {
+    if (values[i] === null) missing.push(i);
+  }
+  if (missing.length) {
+    const archiveKeys = missing.map((i) => keys[i]);
+    const archiveValues = await archive.mGet(archiveKeys);
+    for (let j = 0; j < missing.length; j++) {
+      values[missing[j]] = archiveValues[j];
+    }
+  }
+  return values.map((v) => {
+    try {
+      return JSON.parse(v as string);
+    } catch {
+      return v;
+    }
+  });
+};
+
 // lRange with fallback to archive for missing items
 export const lRangeWithArchive = async (k, start, end) => {
   const items = await db.lRange(k, start, end);
diff --git a/routes/payments.ts b/routes/payments.ts
index 043d1c47..98581ebd 100644
--- a/routes/payments.ts
+++ b/routes/payments.ts
@@ -2,7 +2,7 @@ import config from "$config";
 import api from "$lib/api";
 import { getArkAddress, sendArk, verifyArkVtxo } from "$lib/ark";
 import { requirePin } from "$lib/auth";
-import { archive, db, g, gf, s } from "$lib/db";
+import { archive, db, g, gf, gfAll, s } from "$lib/db";
 import { getTx } from "$lib/esplora";
 import { generate } from "$lib/invoices";
 import { replay } from "$lib/lightning";
@@ -35,7 +35,7 @@ import {
   tbFundDebit,
 } from "$lib/tb";
 import { PaymentType } from "$lib/types";
-import { SATS, bail, fail, fields, getInvoice, getPayment, getUser, sats } from "$lib/utils";
+import { SATS, bail, fail, fields, getInvoice, getPayment, getUser, pick, sats } from "$lib/utils";
 import rpc from "@coinos/rpc";
 import got from "got";
 import { v4 } from "uuid";
@@ -197,24 +197,49 @@ export default {
       }
     }
 
-    payments = (
-      await Promise.all(
-        payments.map(async (pid) => {
-          const p = await gf(`payment:${pid}`);
-          if (!p) {
-            warn("user", id, "missing payment", pid);
-            await db.lRem(listKey, 0, pid);
-            return p;
-          }
-          if (received && p.amount < 0) return;
-          if (p.created < start || p.created > end) return;
-          if (p.type === PaymentType.internal) p.with = await getUser(p.ref, fields);
-          return p;
-        }),
-      )
-    )
-      .filter((p) => p)
-      .sort((a, b) => b.created - a.created);
+    const paymentKeys = payments.map((pid) => `payment:${pid}`);
+    const fetched = await gfAll(paymentKeys);
+
+    const missingIds: any[] = [];
+    const validPayments: any[] = [];
+
+    for (let i = 0; i < fetched.length; i++) {
+      const p = fetched[i];
+      if (!p) {
+        warn("user", id, "missing payment", payments[i]);
+        missingIds.push(payments[i]);
+        continue;
+      }
+      if (received && p.amount < 0) continue;
+      if (p.created < start || p.created > end) continue;
+      validPayments.push(p);
+    }
+
+    if (missingIds.length) {
+      for (const pid of missingIds) await db.lRem(listKey, 0, pid);
+    }
+
+    const internalRefs = [...new Set(
+      validPayments
+        .filter((p) => p.type === PaymentType.internal && p.ref)
+        .map((p) => p.ref),
+    )];
+
+    if (internalRefs.length) {
+      const userKeys = internalRefs.map((ref) => `user:${ref}`);
+      const users = await gfAll(userKeys);
+      const userMap = new Map<string, any>();
+      for (let i = 0; i < internalRefs.length; i++) {
+        let u = users[i];
+        if (typeof u === "string") u = await g(`user:${u}`);
+        if (u) userMap.set(internalRefs[i], fields ? pick(u, fields) : u);
+      }
+      for (const p of validPayments) {
+        if (p.type === PaymentType.internal && p.ref) p.with = userMap.get(p.ref);
+      }
+    }
+
+    payments = validPayments.sort((a, b) => b.created - a.created);
 
     const fn = (a, b) => ({
       ...a,
diff --git a/routes/rates.ts b/routes/rates.ts
index d4426a6e..6e14da82 100644
--- a/routes/rates.ts
+++ b/routes/rates.ts
@@ -4,15 +4,18 @@ import { rate } from "$lib/rates";
 export default {
   async fx(c) {
     const { fx } = await g("fx");
+    c.header("Cache-Control", "public, max-age=300");
     return c.json({ fx });
   },
 
   async last(c) {
+    c.header("Cache-Control", "public, max-age=30");
     return c.json(rate || (await g("rate")));
   },
 
   async index(c) {
     const { date, fx, ...rates } = await g("rates");
+    c.header("Cache-Control", "public, max-age=30");
     return c.json(rates);
   },
 };
diff --git a/routes/users.ts b/routes/users.ts
index bc899ac7..0a2c65c7 100644
--- a/routes/users.ts
+++ b/routes/users.ts
@@ -1,7 +1,7 @@
 import { writeFile } from "node:fs/promises";
 import config from "$config";
 import { requirePin } from "$lib/auth";
-import { db, g, ga, gf, s, scan } from "$lib/db";
+import { db, g, ga, gf, gfAll, s, scan } from "$lib/db";
 import { err, l, warn } from "$lib/logging";
 import { mail, templates } from "$lib/mail";
 import { getNostrUser, getProfile, serverPubkey2 } from "$lib/nostr";
@@ -623,15 +623,28 @@ export default {
     const pins = [...await db.sMembers(`${id}:pins`)];
     const trust = [...await db.sMembers(`${id}:trust`)];
 
-    for (const { ref } of (
-      await Promise.all(payments.reverse().map(async (id) => await gf(`payment:${id}`)))
-    ).filter((p) => p && p.type === PaymentType.internal && p.ref)) {
-      if (ref === id) continue;
-      const i = contacts.findIndex((c) => c && c.id === ref);
-      if (~i) contacts.splice(i, 1);
-      let u = await g(`user:${ref}`);
-      if (typeof u === "string") u = await g(`user:${ref}`);
-      if (u) contacts.unshift(pick(u, ["id", "picture", "username"]));
+    const paymentKeys = payments.reverse().map((pid) => `payment:${pid}`);
+    const fetched = await gfAll(paymentKeys);
+    const refs = fetched
+      .filter((p) => p && p.type === PaymentType.internal && p.ref && p.ref !== id)
+      .map((p) => p.ref);
+
+    const uniqueRefs = [...new Set(refs)];
+    if (uniqueRefs.length) {
+      const userKeys = uniqueRefs.map((ref) => `user:${ref}`);
+      const users = await gfAll(userKeys);
+      const userMap = new Map<string, any>();
+      for (let i = 0; i < uniqueRefs.length; i++) {
+        let u = users[i];
+        if (typeof u === "string") u = await g(`user:${u}`);
+        if (u) userMap.set(uniqueRefs[i], u);
+      }
+      for (const ref of refs) {
+        const i = contacts.findIndex((c) => c && c.id === ref);
+        if (~i) contacts.splice(i, 1);
+        const u = userMap.get(ref);
+        if (u) contacts.unshift(pick(u, ["id", "picture", "username"]));
+      }
     }
 
     await s(`${id}:contacts`, contacts);
@@ -849,28 +862,35 @@ export default {
     try {
       const user = c.get("user");
 
+      const accountIds = (await db.lRange(`${user.id}:accounts`, 0, -1)) as string[];
+      const accountKeys = accountIds.map((aid) => `account:${aid}`);
+      const accountData = await gfAll(accountKeys);
+
       const accounts = [];
-      for (const id of await db.lRange(`${user.id}:accounts`, 0, -1)) {
-        const aid = id as string;
-        const account = await g(`account:${aid}`);
-        if (account) {
-          if ((account.seed || user.seed) && account.pubkey && !account.importedAt) {
-            await importAccountHistory(account);
-          }
-          account.balance = await getBalance(aid);
-          account.pending = await getPending(aid);
-          accounts.push(account);
+      for (let i = 0; i < accountIds.length; i++) {
+        const aid = accountIds[i];
+        const account = accountData[i];
+        if (!account) continue;
+
+        if ((account.seed || user.seed) && account.pubkey && !account.importedAt) {
+          await importAccountHistory(account);
         }
 
         if (account.type === PaymentType.ark) {
-          const paymentIds = await db.lRange(`${id}:payments`, 0, -1);
+          const paymentIds = (await db.lRange(`${aid}:payments`, 0, -1)) as string[];
+          const paymentKeys = paymentIds.map((pid) => `payment:${pid}`);
+          const payments = await gfAll(paymentKeys);
           let sum = 0;
-          for (const pid of paymentIds) {
-            const pay = await g(`payment:${pid}`);
+          for (const pay of payments) {
             if (pay) sum += pay.amount;
           }
           account.balance = Math.max(sum, 0);
+        } else {
+          account.balance = await getBalance(aid);
         }
+
+        account.pending = await getPending(aid);
+        accounts.push(account);
       }
 
       return c.json(accounts.reverse());