Rebuildable caches move from localStorage to IndexedDB (cache-store.js)

localStorage is ~5M characters per site whatever the device, and a phone
had filled it. Profiles, faceless list, reports, follow packs, profile
pages, relay lists, the moderation baseline, community directory, link
previews, video sizes/probes and search profiles (~3M on that phone) now
live in IndexedDB, which shares the site's disk allowance (GBs).

cacheStorage keeps localStorage's string API: reads from memory,
coalesced write-behind, flushed on hide. Copies migrate on open and are
removed once written. Boot waits for it (max 3s) beside the Ark store;
caches read at construction re-merge on ready; an early write can't
replace the stored copy. A copy an older build writes to localStorage
later moves in on first ask. Wallet wipes reach it; private mode stays on
localStorage. Read markers, drafts, follows, note threads and sync
outboxes stay in localStorage on purpose (cross-tab consistency, early
server-rendered seeding). Also asks navigator.storage.persist().

Co-Authored-By: Claude Opus 5.5 <[email protected]>

7 files changed, 155 insertions, 17 deletions

diff --git a/tools/avatar-thumb-test.js b/tools/avatar-thumb-test.js
index f70f6b3..e7ff843 100644
--- a/tools/avatar-thumb-test.js
+++ b/tools/avatar-thumb-test.js
@@ -102,8 +102,8 @@ try {
   await sleep(2500);
 
   const row = await page.evaluate(() => {
-    const rows = Object.keys(localStorage).filter((k) => /^btc-wallet-cache:[0-9a-f]+:profiles$/.test(k))
-      .flatMap((k) => Object.values(JSON.parse(localStorage.getItem(k) || '{}')));
+    const rows = window.coinosCache.keys().filter((k) => /^btc-wallet-cache:[0-9a-f]+:profiles$/.test(k))
+      .flatMap((k) => Object.values(JSON.parse(window.coinosCache.getItem(k) || '{}')));
     const r = rows.find((x) => x && x.thumb) || rows[0] || {};
     return { hasThumb: typeof r.thumb === 'string', kind: (r.thumb || '').slice(0, 20), len: (r.thumb || '').length, thumbFor: r.thumbFor === r.picture };
   });
@@ -123,8 +123,8 @@ try {
     document.body.appendChild(el);
     const css = parseFloat(getComputedStyle(el).width);
     el.remove();
-    const rows = Object.keys(localStorage).filter((k) => /^btc-wallet-cache:[0-9a-f]+:profiles$/.test(k))
-      .flatMap((k) => Object.values(JSON.parse(localStorage.getItem(k) || '{}')));
+    const rows = window.coinosCache.keys().filter((k) => /^btc-wallet-cache:[0-9a-f]+:profiles$/.test(k))
+      .flatMap((k) => Object.values(JSON.parse(window.coinosCache.getItem(k) || '{}')));
     const r = rows.find((x) => x && (x.thumb || '').length) || {};
     return new Promise((done) => {
       const i = new Image();
@@ -186,8 +186,9 @@ try {
 
   // And someone we have never cached at all — the case that used to sit as a
   // blank circle until a relay answered — is drawn from their pubkey alone.
-  await page.evaluate(() => {
-    for (const k of Object.keys(localStorage).filter((x) => /^btc-wallet-cache:[0-9a-f]+:profiles$/.test(x))) localStorage.removeItem(k);
+  await page.evaluate(async () => {
+    for (const k of window.coinosCache.keys().filter((x) => /^btc-wallet-cache:[0-9a-f]+:profiles$/.test(x))) window.coinosCache.removeItem(k);
+    await window.coinosCache.flush();
   });
   await page.reload({ waitUntil: 'domcontentloaded' });
   await waitText('receive', 20000);
@@ -210,8 +211,8 @@ try {
   await waitText('receive', 20000);
   await sleep(3000);
   const redirThumb = await page.evaluate(() => {
-    const rows = Object.keys(localStorage).filter((k) => /^btc-wallet-cache:[0-9a-f]+:profiles$/.test(k))
-      .flatMap((k) => Object.values(JSON.parse(localStorage.getItem(k) || '{}')));
+    const rows = window.coinosCache.keys().filter((k) => /^btc-wallet-cache:[0-9a-f]+:profiles$/.test(k))
+      .flatMap((k) => Object.values(JSON.parse(window.coinosCache.getItem(k) || '{}')));
     // The seed above wrote this profile into EVERY wallet's cache, but only
     // the one actually on screen paints the avatar and so makes the
     // thumbnail. Asking the first matching row was a coin toss on
@@ -235,9 +236,9 @@ try {
   await page.reload({ waitUntil: 'domcontentloaded' });
   await waitText('receive', 20000);
   await sleep(1500);
-  const failure = await page.evaluate(() => Object.keys(localStorage)
+  const failure = await page.evaluate(() => window.coinosCache.keys()
     .filter((k) => /^btc-wallet-cache:[0-9a-f]+:profiles$/.test(k))
-    .flatMap((k) => Object.values(JSON.parse(localStorage.getItem(k) || '{}')))
+    .flatMap((k) => Object.values(JSON.parse(window.coinosCache.getItem(k) || '{}')))
     .find((r) => r.thumbFailVersion === 1));
   check('a fresh failure starts a new backoff', failedCorsHits === 1 && failure?.thumbFails === 1);
   failedCorsHits = 0;
diff --git a/tools/cache-store-test.js b/tools/cache-store-test.js
new file mode 100644
index 0000000..20ad9e1
--- /dev/null
+++ b/tools/cache-store-test.js
@@ -0,0 +1,137 @@
+// The IndexedDB cache store (src/cache-store.js): localStorage copies move in
+// and leave, an early write can't replace the full copy, re-reads fire once
+// it opens, writes land behind, wipes reach it, and a refused IndexedDB
+// leaves everything in localStorage as before.
+// Run: bun tools/cache-store-test.js
+import { CacheStore, isCacheKey } from '../src/cache-store.js';
+
+let fails = 0;
+const check = (name, ok, detail = '') => {
+  console.log(` ${ok ? '✓' : '✗'} ${name}${detail ? ' — ' + detail : ''}`);
+  if (!ok) fails++;
+};
+const fakeLs = (init = {}) => {
+  const m = new Map(Object.entries(init));
+  return {
+    get length() { return m.size; },
+    key: (i) => [...m.keys()][i] ?? null,
+    getItem: (k) => (m.has(k) ? m.get(k) : null),
+    setItem: (k, v) => { m.set(k, String(v)); },
+    removeItem: (k) => { m.delete(k); },
+    dump: () => m,
+  };
+};
+const memBackend = (init = []) => {
+  const m = new Map(init);
+  return { writes: 0, async list() { return [...m]; }, async write(puts, dels) { this.writes++; for (const [k, v] of puts) m.set(k, v); for (const k of dels) m.delete(k); }, dump: () => m };
+};
+const W = 'btc-wallet-cache:abc123';
+
+console.log('[which keys]');
+check('a wallet\'s profile cache is one', isCacheKey(W + ':profiles'));
+check('link previews are one', isCacheKey('coinos-link-previews'));
+check('the messages state is not (read markers, drafts)', !isCacheKey(W + ':messages'));
+check('the follow list is not', !isCacheKey(W + ':follows'));
+check('note threads stay (server-rendered pages seed them early)', !isCacheKey('btc-wallet-note-threads'));
+
+console.log('\n[first open: localStorage moves in]');
+{
+  const ls = fakeLs({ [W + ':profiles']: '{"a":1}', 'coinos-link-previews': '[["u",1]]', [W + ':messages']: '{"read":{}}' });
+  const be = memBackend();
+  const s = new CacheStore({ backend: be, ls: () => ls, listen: false });
+  check('before opening, reads come from localStorage', s.getItem(W + ':profiles') === '{"a":1}');
+  let hooked = 0;
+  s.onReady(() => hooked++);
+  await s.open();
+  check('the ready hook ran once', hooked === 1);
+  check('reads now come from memory', s.getItem(W + ':profiles') === '{"a":1}');
+  await s.flush(); await new Promise((r) => setTimeout(r, 0));
+  check('the copies are in IndexedDB', be.dump().get(W + ':profiles') === '{"a":1}' && be.dump().get('coinos-link-previews') === '[["u",1]]');
+  check('...and gone from localStorage', ls.getItem(W + ':profiles') === null && ls.getItem('coinos-link-previews') === null);
+  check('state that isn\'t a cache stays put', ls.getItem(W + ':messages') === '{"read":{}}');
+  check('a non-cache key still reads from localStorage', s.getItem(W + ':messages') === '{"read":{}}');
+}
+
+console.log('\n[a later boot: an early write must not replace the full copy]');
+{
+  const ls = fakeLs();
+  const be = memBackend([[W + ':profiles', '{"full":true}']]);
+  const s = new CacheStore({ backend: be, ls: () => ls, listen: false });
+  check('an early read finds nothing (migrated)', s.getItem(W + ':profiles') === null);
+  s.setItem(W + ':profiles', '{"early":true}');
+  check('the early write didn\'t land in localStorage', ls.getItem(W + ':profiles') === null);
+  await s.open();
+  check('the full copy survived', s.getItem(W + ':profiles') === '{"full":true}');
+}
+
+console.log('\n[an older build in another tab wrote localStorage again]');
+{
+  const ls = fakeLs({ [W + ':reports']: '{"newer":1}' });
+  const be = memBackend([[W + ':reports', '{"older":1}']]);
+  const s = new CacheStore({ backend: be, ls: () => ls, listen: false });
+  s.setItem(W + ':reports', '{"early":1}');
+  check('an early write to a key still in localStorage goes there', ls.getItem(W + ':reports') === '{"early":1}');
+  await s.open();
+  check('localStorage\'s copy wins on open', s.getItem(W + ':reports') === '{"early":1}');
+}
+
+console.log('\n[a copy written to localStorage after opening]');
+{
+  const ls = fakeLs();
+  const be = memBackend();
+  const s = new CacheStore({ backend: be, ls: () => ls, listen: false });
+  await s.open();
+  ls.setItem(W + ':profPages', '{"late":1}');
+  check('is found on first ask', s.getItem(W + ':profPages') === '{"late":1}');
+  await s.flush(); await new Promise((r) => setTimeout(r, 0)); await s.flush(); await new Promise((r) => setTimeout(r, 0));
+  check('...and moves in', be.dump().get(W + ':profPages') === '{"late":1}' && ls.getItem(W + ':profPages') === null);
+}
+
+console.log('\n[writes land behind, deletes and wipes reach it]');
+{
+  const ls = fakeLs();
+  const be = memBackend([[W + ':profiles', 'x'], [W + ':reports', 'y'], ['btc-wallet-cache:other:profiles', 'z']]);
+  const s = new CacheStore({ backend: be, ls: () => ls, listen: false });
+  await s.open();
+  s.setItem(W + ':profiles', 'x2');
+  s.setItem(W + ':profiles', 'x3');
+  check('nothing written yet (coalesced)', be.dump().get(W + ':profiles') === 'x');
+  await s.flush();
+  check('one write with the latest value', be.dump().get(W + ':profiles') === 'x3' && be.writes === 1);
+  s.removePrefix(W);
+  await s.flush();
+  check('a wallet wipe removes its caches', !be.dump().has(W + ':profiles') && !be.dump().has(W + ':reports'));
+  check('...and only its own', be.dump().get('btc-wallet-cache:other:profiles') === 'z');
+  check('a non-cache key written through it goes to localStorage', (s.setItem(W + ':messages', 'm'), ls.getItem(W + ':messages') === 'm'));
+}
+
+console.log('\n[a failed write is retried]');
+{
+  const ls = fakeLs();
+  let fail = true;
+  const m = new Map();
+  const be = { async list() { return []; }, async write(puts) { if (fail) throw new Error('quota'); for (const [k, v] of puts) m.set(k, v); } };
+  const s = new CacheStore({ backend: be, ls: () => ls, listen: false });
+  await s.open();
+  s.setItem(W + ':profiles', 'p');
+  await s.flush();
+  check('kept in memory after the failure', s.getItem(W + ':profiles') === 'p');
+  fail = false;
+  await s.flush();
+  check('written on the next flush', m.get(W + ':profiles') === 'p');
+}
+
+console.log('\n[IndexedDB refused]');
+{
+  const ls = fakeLs({ [W + ':profiles']: 'kept' });
+  const s = new CacheStore({ backend: { async list() { throw new Error('private mode'); } }, ls: () => ls, listen: false });
+  let hooked = 0; s.onReady(() => hooked++);
+  await s.open();
+  check('still opens (hooks run)', hooked === 1);
+  check('reads localStorage', s.getItem(W + ':profiles') === 'kept');
+  s.setItem(W + ':profiles', 'new');
+  check('writes localStorage', ls.getItem(W + ':profiles') === 'new');
+}
+
+console.log(fails ? `\n❌ ${fails} failed` : '\n✅ cache store holds');
+process.exit(fails ? 1 : 0);
diff --git a/tools/feed-media-cache-test.js b/tools/feed-media-cache-test.js
index f762375..fa90a15 100644
--- a/tools/feed-media-cache-test.js
+++ b/tools/feed-media-cache-test.js
@@ -148,8 +148,8 @@ try {
     && steps.every((u, i) => !i || u < steps[i - 1]), JSON.stringify(steps));
   check('tile frames are kept as pictures (all but the no-CORS host\'s)', await waitFor(async () => (await (await caches.open('coinos-video-thumbs-v1')).keys()).length === 5, 12000),
     String(await page.evaluate(async () => (await (await caches.open('coinos-video-thumbs-v1')).keys()).map((r) => decodeURIComponent(r.url).split('/').pop()).join(','))));
-  await waitFor(() => /h4\.mp4[^\]]*probed|probed[^\]]*h4/.test(localStorage.getItem('coinos-video-info') || '') && /n5\.mp4[^\]]*failedAt/.test(localStorage.getItem('coinos-video-info') || ''), 8000); // the write-behind
-  const info = await page.evaluate(() => Object.fromEntries(JSON.parse(localStorage.getItem('coinos-video-info') || '[]').map(([u, d]) => [u.split('/').pop(), d])));
+  await waitFor(() => /h4\.mp4[^\]]*probed|probed[^\]]*h4/.test((window.coinosCache || localStorage).getItem('coinos-video-info') || '') && /n5\.mp4[^\]]*failedAt/.test((window.coinosCache || localStorage).getItem('coinos-video-info') || ''), 8000); // the write-behind
+  const info = await page.evaluate(() => Object.fromEntries(JSON.parse((window.coinosCache || localStorage).getItem('coinos-video-info') || '[]').map(([u, d]) => [u.split('/').pop(), d])));
   check('a host hiding Content-Range gives its size to a HEAD', info['h4.mp4']?.probed && info['h4.mp4'].size > 1000 && info['h4.mp4'].size !== 12000, JSON.stringify(info['h4.mp4']));
   check('a clip the browser cached from a plain load is still measured, and its frame kept', info['r6.mp4']?.probed === true
     && await page.evaluate(async () => (await (await caches.open('coinos-video-thumbs-v1')).keys()).some((r) => /r6\.mp4/.test(decodeURIComponent(r.url)))), JSON.stringify(info['r6.mp4']));
diff --git a/tools/feed-media-views-test.js b/tools/feed-media-views-test.js
index 52fa810..ccfd636 100644
--- a/tools/feed-media-views-test.js
+++ b/tools/feed-media-views-test.js
@@ -162,7 +162,7 @@ try {
   check('it waits under a cover with its size', await waitFor(() => [...document.querySelectorAll('.note-video-box .vid-heavy')].some((b) => /50 MB/.test(b.textContent)), 8000));
   check('feed clips fetch nothing until they play', await page.evaluate(() => [...document.querySelectorAll('.note-video-box video')].every((v) => v.preload === 'none' || !v.paused)));
   console.log('\n[Videos: flip through]');
-  const info = () => page.evaluate(() => Object.fromEntries(JSON.parse(localStorage.getItem('coinos-video-info') || '[]').map(([u, d]) => [u.split('/').pop(), d])));
+  const info = () => page.evaluate(() => Object.fromEntries(JSON.parse((window.coinosCache || localStorage).getItem('coinos-video-info') || '[]').map(([u, d]) => [u.split('/').pop(), d])));
   const srcs = () => page.evaluate(() => [...document.querySelectorAll('.media-pager video')].map((v) => ({ src: !!v.getAttribute('src'), pre: v.preload })));
   await mode('videos');
   check('Videos shows a grid of the vertical clips, not the pager yet', await waitFor(() => document.querySelectorAll('.media-grid .media-tile.video').length === 3, 8000)
@@ -181,7 +181,7 @@ try {
   let ss = await srcs();
   check('only the clip on screen loads in full; the next gets its opening; the rest nothing',
     ss[0].src && ss[0].pre === 'auto' && ss[1].src && ss[1].pre === 'metadata' && !ss[2].src, JSON.stringify(ss));
-  await waitFor(() => Object.keys(JSON.parse(localStorage.getItem('coinos-video-info') || '[]')).length >= 3, 6000);
+  await waitFor(() => Object.keys(JSON.parse((window.coinosCache || localStorage).getItem('coinos-video-info') || '[]')).length >= 3, 6000);
   await sleep(1500);
   const vi = await info();
   check('a probe reads the size, ranges, layout and picture size from 64 KB', vi['a.mp4']?.probed && vi['a.mp4'].ranges && vi['a.mp4'].faststart
diff --git a/tools/feed-video-aspect-test.js b/tools/feed-video-aspect-test.js
index d3228db..65cf9bb 100644
--- a/tools/feed-video-aspect-test.js
+++ b/tools/feed-video-aspect-test.js
@@ -123,7 +123,7 @@ try {
   check('a small photo is its own original', imgs.small && imgs.small.src === 'http:' && imgs.small.nw === 800, JSON.stringify(imgs.small));
   check('the copy is kept on disk for the next visit', await page.evaluate(async () => !!(await (await caches.open('coinos-media-thumbs-v1')).match(location.origin + '/i/big.jpg'))));
   await sleep(1500);
-  check('shapes are remembered on the device', await page.evaluate(() => (JSON.parse(localStorage.getItem('coinos-video-dims') || '[]')).length >= 3));
+  check('shapes are remembered on the device', await page.evaluate(() => (JSON.parse((window.coinosCache || localStorage).getItem('coinos-video-dims') || '[]')).length >= 3));
   check('no page errors', errs.length === 0, errs.join('; ').slice(0, 300));
 } catch (e) {
   console.log('✗ threw', e.stack); ok = false;
diff --git a/tools/link-preview-test.js b/tools/link-preview-test.js
index 673cb20..d90dbd2 100644
--- a/tools/link-preview-test.js
+++ b/tools/link-preview-test.js
@@ -116,7 +116,7 @@ try {
   check('a picture link is still a picture, not a card', !!pic && pic.img && !pic.card, JSON.stringify(pic));
   check('the registrar was asked once per link', previewHits.filter((u) => u === 'https://news.example/story/1').length === 1 && !previewHits.some((u) => u.endsWith('.png')), JSON.stringify(previewHits));
   let kept = 0; // written behind, a second after the card was learned
-  for (let i = 0; i < 12 && !kept; i++) { kept = await page.evaluate(() => JSON.parse(localStorage.getItem('coinos-link-previews') || '[]').length); if (!kept) await sleep(300); }
+  for (let i = 0; i < 12 && !kept; i++) { kept = await page.evaluate(() => JSON.parse((window.coinosCache || localStorage).getItem('coinos-link-previews') || '[]').length); if (!kept) await sleep(300); }
   check('the card is remembered for next time', kept === 1, kept + ' kept');
   check('no page errors', errs.length === 0, errs.slice(0, 2).join(' | '));
 } finally { await browser.close(); server.stop(true); }
diff --git a/tools/notif-faces-test.js b/tools/notif-faces-test.js
index 18a2f00..3093f51 100644
--- a/tools/notif-faces-test.js
+++ b/tools/notif-faces-test.js
@@ -87,7 +87,7 @@ try {
     [...document.querySelectorAll('.app-nav-button.unread')].some((n) => /Notifications/i.test(n.getAttribute('aria-label') || ''))));
   check('the actors\' profiles were asked for', (await page.evaluate(() => window.__profileReqs)) > 0);
   check('...and their pictures fetched', faceHits.length >= 5, faceHits.length + ' of 5');
-  const profs = await page.evaluate((k) => Object.keys(JSON.parse(localStorage.getItem(k + ':profiles') || '{}')).length, base);
+  const profs = await page.evaluate((k) => Object.keys(JSON.parse((window.coinosCache || localStorage).getItem(k + ':profiles') || '{}')).length, base);
   check('...and remembered in the profile cache', profs >= 5, profs + ' profiles');
 
   console.log('\n[opening the page]');