5 files changed, 216 insertions, 0 deletions
diff --git a/feed/README.md b/feed/README.md
new file mode 100644
index 0000000..550445e
--- /dev/null
+++ b/feed/README.md
@@ -0,0 +1,66 @@
+# Shared public feed cache
+
+The default Popular feed fetches `/api/feed` before querying Nostr relays.
+The service collects the same popular-post candidates as the browser, with
+signed author profiles. Language, replies, spam and mute filtering remain in
+the client. Custom relay selections and pagination use the existing relay path.
+The client verifies signatures from the HTTP snapshot too.
+
+Run from the repository with dependencies installed:
+
+```sh
+bun run feed:serve
+```
+
+The service binds to `127.0.0.1:8792` by default (`HOST` and `PORT` override).
+Run it under your existing process supervisor, then add a same-origin proxy
+alongside the static site's routes. For nginx:
+
+```nginx
+location = /api/feed {
+    proxy_pass http://127.0.0.1:8792;
+}
+```
+
+Rebuild and deploy the frontend with `bun run build`. `bun run dev` includes
+the cache endpoint automatically. Static or standalone installations without
+the endpoint fall back to relays; no separate cache host is hardcoded.
+
+One background job queries a fixed set of four public relays, with a one-minute
+snapshot lifetime. Visitors receive the previous snapshot during refreshes.
+Concurrent requests cannot launch duplicate jobs. Failed/empty refreshes retain
+the previous result and retry after at least 15 seconds. A snapshot older than
+24 hours is not served. Cold starts without a saved snapshot return 503 promptly
+so browsers can fall back while the service warms.
+
+The latest snapshot is atomically saved to `db/public-feed.json`; set
+`FEED_CACHE_FILE` to change the path. Give the service user write access to its
+parent directory. The snapshot holds only public posts/profiles, at most 240 of
+each, with individual events capped at 32 KB. It is shared across languages and
+users; the server accepts no user-selected relays or personalized query inputs.
+Browsers cache HTTP responses for 15 seconds; the service worker bypasses this
+endpoint so it cannot freeze an old feed indefinitely.
+
+This removes repeated initial reaction scans and profile discovery. Images,
+link previews, quoted posts, live updates and older pages can still require
+client requests. A language with no usable cached posts falls back to relays.
+
+Validation: `bun tools/public-feed-cache-test.js`.
+
+## Current production deployment
+
+`v3.coinos.io` is served by the `bitcoin-wallet` nginx container on SSH host
+`cs`. The `public-feed` container runs the bundled `feed/server.js` on the
+existing Docker `net` network, with `unless-stopped` restart policy. No host
+port is published. Its bundle is `/home/adam/public-feed/server.js`, and its
+persistent snapshot is `/home/adam/public-feed/data/public-feed.json`.
+The nginx config is `/home/adam/bitcoin-wallet-nginx.conf`; `/api/feed` proxies
+to `public-feed:8792` using Docker DNS with periodic resolution.
+
+To update the service, bundle with
+`bun build feed/server.js --target=bun --minify --outfile=/tmp/coinos-public-feed-server.js`,
+copy that file over the existing remote bundle, then `ssh cs 'docker restart public-feed'`.
+The frontend is built locally and copied to `cs:bitcoin-wallet/`, retaining
+older hashed assets for existing browser sessions. Publish assets before
+`index.html` and `sw.js`. Pre-cache-deployment nginx, HTML and service-worker
+backups are in `/home/adam/public-feed/backups/` on `cs`.
diff --git a/feed/cache.js b/feed/cache.js
new file mode 100644
index 0000000..4edc017
--- /dev/null
+++ b/feed/cache.js
@@ -0,0 +1,59 @@
+import { PUBLIC_FEED_RELAYS, popularCandidates } from '../src/popular-feed.js';
+import { isMachinePost } from '../src/lang-guess.js';
+
+// One snapshot for all languages; language/mute filtering stays in the browser.
+export async function buildSnapshot(query, now = Date.now) {
+  const notes = new Map();
+  let until;
+  for (let round = 0; round < 3; round++) {
+    const reactions = await query({ kinds: [6, 7, 9735], limit: 600, ...(until ? { until } : {}) });
+    if (!reactions.length) break;
+    until = Math.min(...reactions.map((e) => e.created_at)) - 1;
+    const candidates = popularCandidates(reactions);
+    const batches = [];
+    for (let i = 0; i < candidates.ids.length; i += 40) batches.push(query({ ids: candidates.ids.slice(i, i + 40) }));
+    for (const e of (await Promise.all(batches)).flat()) {
+      if (candidates.qualifies(e) && !isMachinePost(e.content) && JSON.stringify(e).length <= 32_000) notes.set(e.id, e);
+    }
+  }
+  if (!notes.size) throw new Error('No public posts returned; retaining previous snapshot');
+  const posts = [...notes.values()].sort((a, b) => b.created_at - a.created_at);
+  const authors = [...new Set(posts.map((e) => e.pubkey))];
+  const profiles = new Map();
+  for (let i = 0; i < authors.length; i += 100) {
+    for (const e of await query({ kinds: [0], authors: authors.slice(i, i + 100) })) {
+      if (e.kind === 0 && authors.includes(e.pubkey) && JSON.stringify(e).length <= 32_000
+        && (!profiles.has(e.pubkey) || profiles.get(e.pubkey).created_at < e.created_at)) profiles.set(e.pubkey, e);
+    }
+  }
+  return { version: 1, generatedAt: now(), notes: posts, profiles: [...profiles.values()], popUntil: until + 1 };
+}
+
+// Single flight, stale-while-refresh, and a retry delay when relays are down.
+export function createSnapshotCache({ build, initial = null, save = async () => {}, now = Date.now, ttl = 60_000, maxAge = 86400_000, onError = console.error }) {
+  let snapshot = initial, pending = null, retryAt = 0;
+  function refresh() {
+    if (pending) return pending;
+    if (now() < retryAt) return Promise.resolve();
+    pending = Promise.resolve().then(build).then(async (next) => {
+      snapshot = next;
+      await save(next);
+    }).catch(onError).finally(() => { pending = null; retryAt = now() + 15_000; });
+    return pending;
+  }
+  function read() {
+    if (!snapshot || now() - snapshot.generatedAt >= ttl) void refresh();
+    return snapshot && now() - snapshot.generatedAt <= maxAge ? snapshot : null;
+  }
+  return { read, refresh };
+}
+
+export function snapshotResponse(cache, req) {
+  if (!['GET', 'HEAD'].includes(req.method)) return new Response(null, { status: 405, headers: { Allow: 'GET, HEAD' } });
+  const snapshot = cache.read();
+  if (!snapshot) return new Response(null, { status: 503, headers: { 'Retry-After': '15', 'Cache-Control': 'no-store' } });
+  return new Response(req.method === 'HEAD' ? null : JSON.stringify(snapshot), {
+    headers: { 'Content-Type': 'application/json', 'Cache-Control': 'public, max-age=15, stale-while-revalidate=60' },
+  });
+}
+export { PUBLIC_FEED_RELAYS };
diff --git a/feed/server.js b/feed/server.js
new file mode 100644
index 0000000..2cf26a1
--- /dev/null
+++ b/feed/server.js
@@ -0,0 +1,12 @@
+import { startFeedCache } from './service.js';
+import { snapshotResponse } from './cache.js';
+const cache = await startFeedCache();
+const server = Bun.serve({
+  hostname: process.env.HOST || '127.0.0.1',
+  port: Number(process.env.PORT || 8792),
+  fetch(req) {
+    if (new URL(req.url).pathname !== '/api/feed') return new Response('Not found', { status: 404 });
+    return snapshotResponse(cache, req);
+  },
+});
+console.log(`Public feed cache listening on ${server.url}`);
diff --git a/feed/service.js b/feed/service.js
new file mode 100644
index 0000000..304b40f
--- /dev/null
+++ b/feed/service.js
@@ -0,0 +1,29 @@
+import { SimplePool } from 'nostr-tools/pool';
+import { readFile, mkdir, writeFile, rename } from 'node:fs/promises';
+import { dirname } from 'node:path';
+import { buildSnapshot, createSnapshotCache, PUBLIC_FEED_RELAYS } from './cache.js';
+
+export async function startFeedCache() {
+  const file = process.env.FEED_CACHE_FILE || new URL('../db/public-feed.json', import.meta.url).pathname;
+  let initial = null;
+  try {
+    const data = JSON.parse(await readFile(file, 'utf8'));
+    if (data.version === 1 && Number.isFinite(data.generatedAt) && Array.isArray(data.notes) && data.notes.length && Array.isArray(data.profiles)) initial = data;
+  } catch {}
+  // SimplePool verifies signatures and matches filters before accepting events.
+  const pool = new SimplePool();
+  const cache = createSnapshotCache({
+    initial,
+    build: () => buildSnapshot((filter) => pool.querySync(PUBLIC_FEED_RELAYS, filter, { maxWait: 4500 })),
+    save: async (data) => {
+      await mkdir(dirname(file), { recursive: true });
+      await writeFile(file + '.tmp', JSON.stringify(data));
+      await rename(file + '.tmp', file);
+    },
+    onError: (e) => console.warn('[public feed]', e.message),
+  });
+  cache.read(); // warm before the first visitor, reusing the persisted snapshot
+  const timer = setInterval(() => cache.read(), 15_000);
+  timer.unref();
+  return cache;
+}
diff --git a/tools/public-feed-cache-test.js b/tools/public-feed-cache-test.js
new file mode 100644
index 0000000..77f833a
--- /dev/null
+++ b/tools/public-feed-cache-test.js
@@ -0,0 +1,50 @@
+import assert from 'node:assert/strict';
+import { buildSnapshot, createSnapshotCache, snapshotResponse } from '../feed/cache.js';
+import { popularCandidates } from '../src/popular-feed.js';
+const id = 'a'.repeat(64), author = 'b'.repeat(64);
+const reaction = (pk, kind = 7, tags = []) => ({ kind, pubkey: pk, created_at: 100, tags: [['e', id], ...tags] });
+const post = { kind: 1, id, pubkey: author, created_at: 90, content: 'This is a post about the things we build together.', tags: [] };
+assert.equal(popularCandidates([reaction('c'), reaction('c')]).ids.length, 0);
+assert.equal(popularCandidates([reaction(author), reaction('c')]).qualifies(post), false);
+assert.equal(popularCandidates([reaction('c'), reaction('d')]).qualifies(post), true);
+assert.equal(popularCandidates([reaction('zap-service', 9735, [['P', 'c']]), reaction('c')]).ids.length, 0);
+const snapshot = await buildSnapshot(async (filter) => {
+  if (filter.ids) return [post, post];
+  if (filter.kinds[0] === 0) return [{ kind: 0, pubkey: author, created_at: 1, content: '{}' }, { kind: 0, pubkey: author, created_at: 2, content: '{}' }];
+  return [reaction('c'), reaction('d')];
+}, () => 1000);
+assert.equal(snapshot.notes.length, 1);
+assert.equal(snapshot.profiles[0].created_at, 2);
+assert.equal(snapshot.popUntil, 100);
+await assert.rejects(buildSnapshot(async () => []), /No public posts/);
+let time = 1000, builds = 0, finish;
+const cache = createSnapshotCache({ now: () => time, build: () => { builds++; return new Promise((resolve) => { finish = resolve; }); } });
+const request = new Request('http://localhost/api/feed');
+assert.equal(snapshotResponse(cache, request).status, 503);
+for (let i = 0; i < 100; i++) assert.equal(cache.read(), null);
+await Promise.resolve();
+assert.equal(builds, 1, 'cold visitors share one build');
+finish(snapshot);
+await cache.refresh();
+assert.equal(cache.read(), snapshot);
+assert.equal(snapshotResponse(cache, request).status, 200);
+assert.equal(snapshotResponse(cache, new Request(request, { method: 'POST' })).status, 405);
+assert.equal(await snapshotResponse(cache, new Request(request, { method: 'HEAD' })).text(), '');
+time += 61_000;
+for (let i = 0; i < 100; i++) assert.equal(cache.read(), snapshot);
+await Promise.resolve();
+assert.equal(builds, 2, 'stale visitors share one background build');
+finish({ ...snapshot, generatedAt: time });
+await cache.refresh();
+assert.equal(cache.read().generatedAt, time);
+let failures = 0;
+const failing = createSnapshotCache({ initial: snapshot, now: () => time, build: async () => { failures++; throw Error('offline'); }, onError: () => {} });
+assert.equal(failing.read(), snapshot);
+await failing.refresh();
+assert.equal(failing.read(), snapshot);
+await failing.refresh();
+assert.equal(failures, 1, 'failed refreshes are throttled and preserve stale data');
+time += 86400_000;
+assert.equal(failing.read(), null, 'expired snapshots are not served');
+await failing.refresh();
+console.log('✓ public feed selection, profiles, single flight, stale refresh, failure recovery, expiry and HTTP behavior');