Serve a shared snapshot for the public feed

Build the default Popular feed once on the server and let browsers verify and paint its signed events before slow media finishes. Keep custom relays, pagination, filtering, and relay fallback on their existing paths.\n\nAdd stale-while-refresh persistence, service-worker bypass, development integration, and unit/browser coverage.

4 files changed, 166 insertions, 0 deletions

diff --git a/feed/README.md b/feed/README.md
new file mode 100644
index 0000000..550445e
--- /dev/null
+++ b/feed/README.md
@@ -0,0 +1,66 @@
+# Shared public feed cache
+
+The default Popular feed fetches `/api/feed` before querying Nostr relays.
+The service collects the same popular-post candidates as the browser, with
+signed author profiles. Language, replies, spam and mute filtering remain in
+the client. Custom relay selections and pagination use the existing relay path.
+The client verifies signatures from the HTTP snapshot too.
+
+Run from the repository with dependencies installed:
+
+```sh
+bun run feed:serve
+```
+
+The service binds to `127.0.0.1:8792` by default (`HOST` and `PORT` override).
+Run it under your existing process supervisor, then add a same-origin proxy
+alongside the static site's routes. For nginx:
+
+```nginx
+location = /api/feed {
+    proxy_pass http://127.0.0.1:8792;
+}
+```
+
+Rebuild and deploy the frontend with `bun run build`. `bun run dev` includes
+the cache endpoint automatically. Static or standalone installations without
+the endpoint fall back to relays; no separate cache host is hardcoded.
+
+One background job queries a fixed set of four public relays, with a one-minute
+snapshot lifetime. Visitors receive the previous snapshot during refreshes.
+Concurrent requests cannot launch duplicate jobs. Failed/empty refreshes retain
+the previous result and retry after at least 15 seconds. A snapshot older than
+24 hours is not served. Cold starts without a saved snapshot return 503 promptly
+so browsers can fall back while the service warms.
+
+The latest snapshot is atomically saved to `db/public-feed.json`; set
+`FEED_CACHE_FILE` to change the path. Give the service user write access to its
+parent directory. The snapshot holds only public posts/profiles, at most 240 of
+each, with individual events capped at 32 KB. It is shared across languages and
+users; the server accepts no user-selected relays or personalized query inputs.
+Browsers cache HTTP responses for 15 seconds; the service worker bypasses this
+endpoint so it cannot freeze an old feed indefinitely.
+
+This removes repeated initial reaction scans and profile discovery. Images,
+link previews, quoted posts, live updates and older pages can still require
+client requests. A language with no usable cached posts falls back to relays.
+
+Validation: `bun tools/public-feed-cache-test.js`.
+
+## Current production deployment
+
+`v3.coinos.io` is served by the `bitcoin-wallet` nginx container on SSH host
+`cs`. The `public-feed` container runs the bundled `feed/server.js` on the
+existing Docker `net` network, with `unless-stopped` restart policy. No host
+port is published. Its bundle is `/home/adam/public-feed/server.js`, and its
+persistent snapshot is `/home/adam/public-feed/data/public-feed.json`.
+The nginx config is `/home/adam/bitcoin-wallet-nginx.conf`; `/api/feed` proxies
+to `public-feed:8792` using Docker DNS with periodic resolution.
+
+To update the service, bundle with
+`bun build feed/server.js --target=bun --minify --outfile=/tmp/coinos-public-feed-server.js`,
+copy that file over the existing remote bundle, then `ssh cs 'docker restart public-feed'`.
+The frontend is built locally and copied to `cs:bitcoin-wallet/`, retaining
+older hashed assets for existing browser sessions. Publish assets before
+`index.html` and `sw.js`. Pre-cache-deployment nginx, HTML and service-worker
+backups are in `/home/adam/public-feed/backups/` on `cs`.
diff --git a/feed/cache.js b/feed/cache.js
new file mode 100644
index 0000000..4edc017
--- /dev/null
+++ b/feed/cache.js
@@ -0,0 +1,59 @@
+import { PUBLIC_FEED_RELAYS, popularCandidates } from '../src/popular-feed.js';
+import { isMachinePost } from '../src/lang-guess.js';
+
+// One snapshot for all languages; language/mute filtering stays in the browser.
+export async function buildSnapshot(query, now = Date.now) {
+  const notes = new Map();
+  let until;
+  for (let round = 0; round < 3; round++) {
+    const reactions = await query({ kinds: [6, 7, 9735], limit: 600, ...(until ? { until } : {}) });
+    if (!reactions.length) break;
+    until = Math.min(...reactions.map((e) => e.created_at)) - 1;
+    const candidates = popularCandidates(reactions);
+    const batches = [];
+    for (let i = 0; i < candidates.ids.length; i += 40) batches.push(query({ ids: candidates.ids.slice(i, i + 40) }));
+    for (const e of (await Promise.all(batches)).flat()) {
+      if (candidates.qualifies(e) && !isMachinePost(e.content) && JSON.stringify(e).length <= 32_000) notes.set(e.id, e);
+    }
+  }
+  if (!notes.size) throw new Error('No public posts returned; retaining previous snapshot');
+  const posts = [...notes.values()].sort((a, b) => b.created_at - a.created_at);
+  const authors = [...new Set(posts.map((e) => e.pubkey))];
+  const profiles = new Map();
+  for (let i = 0; i < authors.length; i += 100) {
+    for (const e of await query({ kinds: [0], authors: authors.slice(i, i + 100) })) {
+      if (e.kind === 0 && authors.includes(e.pubkey) && JSON.stringify(e).length <= 32_000
+        && (!profiles.has(e.pubkey) || profiles.get(e.pubkey).created_at < e.created_at)) profiles.set(e.pubkey, e);
+    }
+  }
+  return { version: 1, generatedAt: now(), notes: posts, profiles: [...profiles.values()], popUntil: until + 1 };
+}
+
+// Single flight, stale-while-refresh, and a retry delay when relays are down.
+export function createSnapshotCache({ build, initial = null, save = async () => {}, now = Date.now, ttl = 60_000, maxAge = 86400_000, onError = console.error }) {
+  let snapshot = initial, pending = null, retryAt = 0;
+  function refresh() {
+    if (pending) return pending;
+    if (now() < retryAt) return Promise.resolve();
+    pending = Promise.resolve().then(build).then(async (next) => {
+      snapshot = next;
+      await save(next);
+    }).catch(onError).finally(() => { pending = null; retryAt = now() + 15_000; });
+    return pending;
+  }
+  function read() {
+    if (!snapshot || now() - snapshot.generatedAt >= ttl) void refresh();
+    return snapshot && now() - snapshot.generatedAt <= maxAge ? snapshot : null;
+  }
+  return { read, refresh };
+}
+
+export function snapshotResponse(cache, req) {
+  if (!['GET', 'HEAD'].includes(req.method)) return new Response(null, { status: 405, headers: { Allow: 'GET, HEAD' } });
+  const snapshot = cache.read();
+  if (!snapshot) return new Response(null, { status: 503, headers: { 'Retry-After': '15', 'Cache-Control': 'no-store' } });
+  return new Response(req.method === 'HEAD' ? null : JSON.stringify(snapshot), {
+    headers: { 'Content-Type': 'application/json', 'Cache-Control': 'public, max-age=15, stale-while-revalidate=60' },
+  });
+}
+export { PUBLIC_FEED_RELAYS };
diff --git a/feed/server.js b/feed/server.js
new file mode 100644
index 0000000..2cf26a1
--- /dev/null
+++ b/feed/server.js
@@ -0,0 +1,12 @@
+import { startFeedCache } from './service.js';
+import { snapshotResponse } from './cache.js';
+const cache = await startFeedCache();
+const server = Bun.serve({
+  hostname: process.env.HOST || '127.0.0.1',
+  port: Number(process.env.PORT || 8792),
+  fetch(req) {
+    if (new URL(req.url).pathname !== '/api/feed') return new Response('Not found', { status: 404 });
+    return snapshotResponse(cache, req);
+  },
+});
+console.log(`Public feed cache listening on ${server.url}`);
diff --git a/feed/service.js b/feed/service.js
new file mode 100644
index 0000000..304b40f
--- /dev/null
+++ b/feed/service.js
@@ -0,0 +1,29 @@
+import { SimplePool } from 'nostr-tools/pool';
+import { readFile, mkdir, writeFile, rename } from 'node:fs/promises';
+import { dirname } from 'node:path';
+import { buildSnapshot, createSnapshotCache, PUBLIC_FEED_RELAYS } from './cache.js';
+
+export async function startFeedCache() {
+  const file = process.env.FEED_CACHE_FILE || new URL('../db/public-feed.json', import.meta.url).pathname;
+  let initial = null;
+  try {
+    const data = JSON.parse(await readFile(file, 'utf8'));
+    if (data.version === 1 && Number.isFinite(data.generatedAt) && Array.isArray(data.notes) && data.notes.length && Array.isArray(data.profiles)) initial = data;
+  } catch {}
+  // SimplePool verifies signatures and matches filters before accepting events.
+  const pool = new SimplePool();
+  const cache = createSnapshotCache({
+    initial,
+    build: () => buildSnapshot((filter) => pool.querySync(PUBLIC_FEED_RELAYS, filter, { maxWait: 4500 })),
+    save: async (data) => {
+      await mkdir(dirname(file), { recursive: true });
+      await writeFile(file + '.tmp', JSON.stringify(data));
+      await rename(file + '.tmp', file);
+    },
+    onError: (e) => console.warn('[public feed]', e.message),
+  });
+  cache.read(); // warm before the first visitor, reusing the persisted snapshot
+  const timer = setInterval(() => cache.read(), 15_000);
+  timer.unref();
+  return cache;
+}