use strfry scan

10 files changed, 118 insertions, 173 deletions

diff --git a/bun.lockb b/bun.lockb
index db0e8d09..db190fa1 100755
--- a/bun.lockb
+++ b/bun.lockb
diff --git a/index.ts b/index.ts
index 512d4af9..8d12705f 100644
--- a/index.ts
+++ b/index.ts
@@ -1,5 +1,3 @@
-Error.stackTraceLimit = Infinity;
-
 import app from "$lib/app";
 import { admin, auth, optional } from "$lib/auth";
 
diff --git a/lib/nostr.ts b/lib/nostr.ts
index f3befa03..5ef8af3d 100644
--- a/lib/nostr.ts
+++ b/lib/nostr.ts
@@ -1,6 +1,7 @@
 import config from "$config";
 import { g, s } from "$lib/db";
 import { l } from "$lib/logging";
+import { count, scan } from "$lib/strfry";
 import { fail } from "$lib/utils";
 import { finalizeEvent, getPublicKey, nip19 } from "nostr-tools";
 import { AbstractSimplePool } from "nostr-tools/abstract-pool";
@@ -18,7 +19,7 @@ export const pool = new AbstractSimplePool({ verifyEvent: alwaysTrue });
 const opts = { maxWait: 2000 };
 
 export const anon = (pubkey) => ({
-  name: pubkey.substr(0, 6),
+  username: pubkey.substr(0, 6),
   pubkey,
   anon: true,
   follows: [],
@@ -101,15 +102,16 @@ export async function handleZap(invoice) {
 export const getRelays = async (pubkey): Promise<any> => {
   const { relays } = config;
   const filter = { authors: [pubkey], kinds: [10002] };
-  const event = await pool.get([config.nostr], filter, opts);
+  const result = await scan(filter);
 
   let read = relays;
   let write = relays;
 
-  if (event) {
+  if (result.length) {
+    const { tags } = result[0];
     read = [];
     write = [];
-    for (const r of event.tags) {
+    for (const r of tags) {
       if (r[0] !== "r") continue;
       if (!r[2] || r[2] === "write") write.push(r[1]);
       if (!r[2] || r[2] === "read") read.push(r[1]);
@@ -157,44 +159,15 @@ export const getProfile = async (pubkey, relays = [config.nostr]) => {
 };
 
 export const getCount = async (pubkey) => {
-  let follows = await g(`${pubkey}:follows:n`);
-  let followers = await g(`${pubkey}:followers:n`);
-
-  if (follows === null) {
-    const filter: any = { kinds: [3], authors: [pubkey] };
-    let ev = await pool.get([config.nostr], filter, opts);
-
-    if (!ev) {
-      let { relays } = config;
-      ({ write: relays } = await getRelays(pubkey));
-      ev = await pool.get(relays, filter, opts);
-      send(ev);
-    }
-
-    if (ev) {
-      follows = ev.tags
-        .map((t) => t[0] === "p" && t[1])
-        .filter((p) => p && p.length === 64).length;
-    }
-
-    follows ||= 0;
-    await s(`${pubkey}:follows:n`, follows);
-  }
-
-  if (followers === null) {
-    const filter: any = { cache: ["user_infos", { pubkeys: [pubkey] }] };
-    const infos = await pool.querySync([config.cache], filter, opts);
-
-    const f = infos.find((e) => e.kind === 10000133);
-    let counts = {};
-    if (f?.content) {
-      counts = JSON.parse(f.content);
-      followers = counts[pubkey];
-    }
+  try {
+    const result = await scan({ authors: [pubkey], kinds: [3] });
+    const follows = result.length
+      ? result[0].tags.filter((t) => t[0] === "p").length
+      : 0;
 
-    followers ||= 0;
-    await s(`${pubkey}:followers:n`, followers);
+    const [followers] = await count({ "#p": [pubkey], kinds: [3] });
+    return { follows, followers };
+  } catch (e) {
+    console.log(e);
   }
-
-  return { follows, followers };
 };
diff --git a/lib/strfry.ts b/lib/strfry.ts
index 9caa539e..231e3533 100644
--- a/lib/strfry.ts
+++ b/lib/strfry.ts
@@ -1,5 +1,5 @@
-import { createServer, connect } from "net";
 import { randomUUID } from "crypto";
+import { connect, createServer } from "net";
 
 const exec = async (cmd) =>
   new Promise((resolve, reject) => {
@@ -13,7 +13,6 @@ const exec = async (cmd) =>
       const results = [];
       socket.on("data", (data) => {
         resultBuffer += data.toString();
-
         const parts = resultBuffer.split("\n");
 
         for (let i = 0; i < parts.length - 1; i++) {
@@ -44,16 +43,17 @@ const exec = async (cmd) =>
         controlClient.end();
       });
 
-      controlClient.on("error", (e) =>
-        reject(`Control socket error: ${e.message}`),
-      );
+      controlClient.on("error", (e) => {
+        reject(`Control socket error: ${e.message}`);
+      });
     });
 
-    resultServer.on("error", (e) =>
-      reject(`Result server error: ${e.message}`),
-    );
+    resultServer.on("error", (e) => {
+      reject(`Result server error: ${e.message}`);
+    });
   });
 
+export const count = (f) => exec(`scan --count '${JSON.stringify(f)}'\n`);
 export const scan = (f) => exec(`scan '${JSON.stringify(f)}'\n`);
 export const sync = (r, f) =>
   exec(`sync ${r} --dir down --filter '${JSON.stringify(f)}'\n`);
diff --git a/lib/utils.ts b/lib/utils.ts
index 7dc6b9ef..9cb6a690 100644
--- a/lib/utils.ts
+++ b/lib/utils.ts
@@ -2,6 +2,7 @@ import config from "$config";
 import { g, s } from "$lib/db";
 import locales from "$lib/locales/index";
 import migrate from "$lib/migrate";
+import whitelist from "$lib/whitelist";
 import { bytesToHex, randomBytes } from "@noble/hashes/utils";
 import { getPublicKey } from "nostr";
 
@@ -68,7 +69,7 @@ export const fields = [
   "banner",
 ];
 
-export const getUser = async (username) => {
+export const getUser = async (username, fields = whitelist) => {
   let update;
   if (username === "undefined") fail("invalid user");
   const user = await migrate(username);
@@ -206,3 +207,22 @@ export const fmt = (sats) =>
   )}`;
 
 export const link = (id) => `${URL}/payment/${id}`;
+
+export const publicFields = [
+  "about",
+  "anon",
+  "banner",
+  "banner",
+  "currency",
+  "display",
+  "id",
+  "hidepay",
+  "lud16",
+  "memoPrompt",
+  "npub",
+  "picture",
+  "prompt",
+  "pubkey",
+  "username",
+  "website",
+];
diff --git a/package.json b/package.json
index 115aa6e7..4a726347 100644
--- a/package.json
+++ b/package.json
@@ -44,6 +44,7 @@
     "readable-stream-clone": "^0.0.7",
     "redis": "^4.6.10",
     "sharp": "^0.33.5",
+    "strfry-rpc": "^0.0.1",
     "uuid": "8",
     "web-push": "^3.6.7",
     "ws": "^7.4.6"
diff --git a/routes/nostr.ts b/routes/nostr.ts
index b909f1ab..6c252927 100644
--- a/routes/nostr.ts
+++ b/routes/nostr.ts
@@ -9,7 +9,8 @@ import {
   send,
   serverPubkey,
 } from "$lib/nostr";
-import { bail, fail, getUser } from "$lib/utils";
+import { count, scan } from "$lib/strfry";
+import { bail, fail, getUser, publicFields } from "$lib/utils";
 import { decode } from "nostr-tools/nip19";
 import type { ProfilePointer } from "nostr-tools/nip19";
 
@@ -59,65 +60,41 @@ export default {
   async follows(req, res) {
     const {
       params: { pubkey },
-      query: { pubkeysOnly, nocache },
+      query: { limit = 20, offset = 0, pubkeysOnly },
     } = req;
     try {
-      let created_at;
-      let follows = [];
-
-      let pubkeys = await g(`${pubkey}:pubkeys`);
-      if (!pubkeys || !pubkeysOnly || nocache) {
-        let filter = { kinds: [3], authors: [pubkey] };
-        let ev = await pool.get([config.nostr], filter, opts);
-
-        if (!ev) {
-          let { relays } = config;
-          ({ write: relays } = await getRelays(pubkey));
-
-          filter = { kinds: [3], authors: [pubkey] };
-          ev = await pool.get(relays, filter, opts);
-          send(ev);
-        }
+      const events = await scan({ authors: [pubkey], kinds: [3] });
+      if (!events.length) return res.send([]);
 
-        let tags = [];
+      const event = events[0];
+      const pubkeys = event.tags
+        .filter((tag) => tag[0] === "p")
+        .map((tag) => tag[1])
+        .slice(offset, limit);
 
-        if (ev) ({ created_at, tags } = ev);
-        pubkeys = tags
-          .map((t) => t[0] === "p" && t[1])
-          .filter((p) => p && p.length === 64);
-
-        await s(`${pubkey}:pubkeys`, pubkeys);
-      }
-      if (!pubkeys.length || pubkeysOnly) return res.send(pubkeys);
-
-      const cache = await g(`${pubkey}:follows`);
-      if (cache && cache.t >= created_at) ({ follows } = cache);
+      let follows = [];
+      if (pubkeysOnly) follows = pubkeys;
       else {
-        const filter: any = { cache: ["user_infos", { pubkeys }] };
-        const infos = await pool.querySync([config.cache], filter, opts);
-        const profiles = infos.filter((e) => e.kind === 0);
-
-        const f = infos.find((e) => e.kind === 10000133);
-        let counts = {};
-        if (f) counts = JSON.parse(f.content);
-
+        const profiles = await scan({ authors: pubkeys, kinds: [0] });
         for (const p of profiles) {
           const { content, pubkey } = p;
-          const user = JSON.parse(content);
-          user.count = counts[pubkey];
+          let user = await getUser(pubkey, publicFields);
+          user = { ...user, ...JSON.parse(content) };
           user.pubkey = pubkey;
           follows.push(user);
         }
 
-        const followKeys = follows.map((f) => f.pubkey);
-        const missing = pubkeys
-          .filter((p) => !followKeys.includes(p))
-          .map((pubkey) => anon(pubkey));
+        const keys = follows.map((f) => f.pubkey);
+        const missing = await Promise.all(
+          pubkeys
+            .filter((p) => !keys.includes(p))
+            .map(
+              async (pubkey) =>
+                (await getUser(pubkey, publicFields)) || anon(pubkey),
+            ),
+        );
 
-        follows.sort((a: any, b: any) => b.count - a.count);
         follows.push(...missing);
-
-        s(`${pubkey}:follows`, { follows, t: created_at });
       }
 
       res.send(follows);
@@ -130,38 +107,36 @@ export default {
   async followers(req, res) {
     const {
       params: { pubkey },
+      query: { limit = 20, offset = 0 },
     } = req;
     try {
-      let filter: any = { kinds: [3], "#p": [pubkey] };
-      const ev = await pool.get([config.nostr], filter, opts);
-      let created_at;
-      if (ev) ({ created_at } = ev);
+      const filter = { kinds: [3], "#p": [pubkey] };
+      const events = await scan({ ...filter, limit });
+      if (!events.length) return res.send([]);
 
-      let followers = [];
-      const cache = await g(`${pubkey}:followers`);
-      if (cache && cache.t >= created_at) ({ followers } = cache);
-      else {
-        filter = { cache: ["user_followers", { pubkey }] };
-        const data = await pool.querySync([config.cache], filter, opts);
-        for (const ev of data.filter((f) => f.kind === 0)) {
-          const { content, pubkey } = ev;
-          const user = JSON.parse(content);
-          user.pubkey = pubkey;
-          followers.push(user);
-        }
+      const pubkeys = events.map((e) => e.pubkey).slice(offset, limit);
 
-        const pubkeys = followers.map((f) => f.pubkey);
-        filter = { cache: ["user_infos", { pubkeys }] };
-        const infos = await pool.querySync([config.cache], filter, opts);
+      const followers = [];
+      const profiles = await scan({ authors: pubkeys, kinds: [0] });
+      for (const p of profiles) {
+        const { content, pubkey } = p;
+        let user = await getUser(pubkey, publicFields);
+        user = { ...user, ...JSON.parse(content) };
+        user.pubkey = pubkey;
+        followers.push(user);
+      }
 
-        let counts = {};
-        const f = infos.find((e) => e.kind === 10000133);
-        if (f) counts = JSON.parse(f.content);
-        followers.map((f) => (f.count = counts[f.pubkey]));
-        followers.sort((a: any, b: any) => b.count - a.count);
+      const keys = followers.map((f) => f.pubkey);
+      const missing = await Promise.all(
+        pubkeys
+          .filter((p) => !keys.includes(p))
+          .map(
+            async (pubkey) =>
+              (await getUser(pubkey, publicFields)) || anon(pubkey),
+          ),
+      );
 
-        await s(`${pubkey}:followers`, { followers, t: created_at });
-      }
+      followers.push(...missing);
 
       res.send(followers);
     } catch (e) {
diff --git a/routes/users.ts b/routes/users.ts
index af5794d5..dc3636e8 100644
--- a/routes/users.ts
+++ b/routes/users.ts
@@ -10,7 +10,7 @@ import { reconcile, types } from "$lib/payments";
 import register from "$lib/register";
 import { emit } from "$lib/sockets";
 import upload from "$lib/upload";
-import { bail, fail, getUser, pick } from "$lib/utils";
+import { bail, fail, publicFields, getUser, pick } from "$lib/utils";
 import whitelist from "$lib/whitelist";
 import rpc from "@coinos/rpc";
 import { bytesToHex, randomBytes } from "@noble/hashes/utils";
@@ -114,29 +114,10 @@ export default {
 
       if (!user) return res.code(500).send("User not found");
 
-      const whitelist = [
-        "about",
-        "anon",
-        "banner",
-        "banner",
-        "currency",
-        "display",
-        "id",
-        "hidepay",
-        "lud16",
-        "memoPrompt",
-        "npub",
-        "picture",
-        "prompt",
-        "pubkey",
-        "username",
-        "website",
-      ];
-
       if (user.pubkey) user.npub = nip19.npubEncode(user.pubkey);
       user.prompt = !!user.prompt;
 
-      res.send(pick(user, whitelist));
+      res.send(pick(user, publicFields));
     } catch (e) {
       err("problem getting user", e.message);
       res.code(500).send(e.message);
@@ -213,9 +194,17 @@ export default {
       l("updating user", user.username);
 
       const { confirm, password, pin, newpin, username } = body;
+      let exists;
 
       let { pubkey } = body;
       if (pubkey) {
+        exists = await getUser(pubkey);
+        if (exists && ![username].includes(exists.username)) {
+          warn("key in use", pubkey, exists.username);
+          if (exists.anon) await db.del(`user:${pubkey}`);
+          else fail("Key in use by another account");
+        }
+
         const { challenge } = body;
         const event = JSON.parse(body.event);
         const c = await g(`challenge:${challenge}`);
@@ -241,32 +230,21 @@ export default {
         user.pin = newpin;
       if (user.pin === "delete") user.pin = undefined;
 
-      let exists;
-      if (username) exists = await getUser(username);
-
-      let token;
-      let oldname;
-      if (
-        username &&
-        user.username.toLowerCase() !== username.toLowerCase() &&
-        exists
-      ) {
-        err("username taken", username, user.username, exists.username);
-        fail("Username taken");
-      } else if (username) {
-        if (user.username.toLowerCase() !== username.toLowerCase())
-          l("changing username", user.username, username);
+      if (username) {
+        let oldname;
+        exists = await getUser(username);
 
-        await db.del(`user:${user.username}`);
-        oldname = user.username;
-        user.username = username;
-      }
+        if (user.username.toLowerCase() !== username.toLowerCase() && exists) {
+          err("username taken", username, user.username, exists.username);
+          fail("Username taken");
+        } else if (username) {
+          if (user.username.toLowerCase() !== username.toLowerCase())
+            l("changing username", user.username, username);
 
-      if (pubkey) exists = await getUser(pubkey);
-      if (exists && ![oldname, username].includes(exists.username)) {
-        warn("key in use", pubkey, exists.username);
-        if (exists.anon) await db.del(`user:${pubkey}`);
-        else fail("Key in use by another account");
+          await db.del(`user:${user.username}`);
+          oldname = user.username;
+          user.username = username;
+        }
       }
 
       const attributes = [
@@ -319,7 +297,7 @@ export default {
       if (user.nip5) await db.sAdd("nip5", `${user.username}:${user.pubkey}`);
 
       emit(user.id, "user", user);
-      res.send({ user, token });
+      res.send({ user });
     } catch (e) {
       warn("failed to update", user.username, e.message);
       bail(res, e.message);
diff --git a/server b/server
new file mode 100755
index 00000000..76b9e336
--- /dev/null
+++ b/server
diff --git a/server.c b/server.c
index cd49f18b..8522785b 100644
--- a/server.c
+++ b/server.c
@@ -6,7 +6,7 @@
 #include <unistd.h>
 
 #define SOCKET_PATH "/sockets/ctrl"
-#define BUFFER_SIZE 1024
+#define BUFFER_SIZE 262144
 
 void handle_client(const char *result_socket, const char *command);