Ark: persist vtxo bytes as base64 (a third smaller than hex); hex still read, hex still on the wire.

The live coins' signed bytes are most of a deep wallet's state — 560KB of
hex, which localStorage then holds as UTF-16. vtxoBytesToStr/FromStr in
proto.js persist base64 and read either form, _addVtxo writes base64, and
pruneArkState converts hex to base64 on every save, so an existing state
migrates itself the first time it saves. Measured on a live deep wallet:
904KB → 665KB (-26%), all 41 coins decoding identically.

The sync wire (slimArkForSync) keeps sending hex for now: a device still
on an older build would otherwise fail to decode every coin merged from a
newer one. Flip it once pre-2026-09-10 builds are gone.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
Claude-Session: https://claude.ai/code/session_01Nw1Dijik1tLVKRM1WkU7xE

4 files changed, 61 insertions, 26 deletions

diff --git a/src/ark/manager.js b/src/ark/manager.js
index d40a65b..c0e972b 100644
--- a/src/ark/manager.js
+++ b/src/ark/manager.js
@@ -21,8 +21,7 @@ import {
   getArkInfo, handshake, encodeAddress, decodeAddress, blindMailboxId,
   readMailbox, decodeVtxo, arkIdFromServerPubkey, GrpcError,
   grpcStream, decodeMailboxMessage, mailboxRequestBytes,
-  getVtxoStatus, VTXO_STATE_SPENT,
-} from './proto.js';
+  getVtxoStatus, VTXO_STATE_SPENT, vtxoBytesToStr, vtxoBytesFromStr, vtxoBytesNormalize } from './proto.js';
 import {
   buildArkoorSend, cosignWithServer, cosignPackageWithServer, buildAllSignedVtxos,
   registerVtxoTransactions, postArkoorMessage, txid,
@@ -101,9 +100,12 @@ export function pruneArkState(state, now = Date.now()) {
     for (const id of a.htlcVtxoIds || []) name(id, ts);
   }
   for (const v of state.vtxos || []) {
-    if (v.state !== 'spent' || !v.bytes || inFlight.has(v.id)) continue;
-    const ts = lastNamed.get(v.id);
-    if (ts && now - ts > SPENT_BYTES_AFTER) delete v.bytes;
+    if (!v.bytes) continue;
+    if (v.state === 'spent' && !inFlight.has(v.id)) {
+      const ts = lastNamed.get(v.id);
+      if (ts && now - ts > SPENT_BYTES_AFTER) { delete v.bytes; continue; }
+    }
+    v.bytes = vtxoBytesNormalize(v.bytes); // hex from older builds → base64
   }
   for (const a of actions) {
     if (a.step !== 'done') continue;
@@ -198,7 +200,7 @@ export class ArkManager {
     // spendable here and must not be shown.
     const coinOwner = (() => {
       const v = (this.state.vtxos || []).find((x) => x.bytes);
-      try { return v ? decodeVtxo(hex.decode(v.bytes)).serverPubkey : null; } catch { return null; }
+      try { return v ? decodeVtxo(vtxoBytesFromStr(v.bytes)).serverPubkey : null; } catch { return null; }
     })();
     if ((this.state.serverPubkey && this.state.serverPubkey !== this.info.serverPubkey)
         || (coinOwner && coinOwner !== this.info.serverPubkey)) {
@@ -249,7 +251,7 @@ export class ArkManager {
   _addVtxo(decoded, bytes, keyIndex, state = 'spendable') {
     if (this._vtxo(decoded.id)) return false;
     this.state.vtxos.push({
-      id: decoded.id, bytes: hex.encode(bytes), keyIndex,
+      id: decoded.id, bytes: vtxoBytesToStr(bytes), keyIndex,
       amountSat: decoded.amountSat, expiryHeight: decoded.expiryHeight, state,
     });
     return true;
@@ -259,7 +261,7 @@ export class ArkManager {
     // fee estimates decode every coin on every render without this.
     const hit = this._decodeCache && this._decodeCache.get(v.id);
     if (hit) return hit;
-    const d = decodeVtxo(hex.decode(v.bytes));
+    const d = decodeVtxo(vtxoBytesFromStr(v.bytes));
     (this._decodeCache ||= new Map()).set(v.id, d);
     if (this._decodeCache.size > 200) this._decodeCache.delete(this._decodeCache.keys().next().value);
     return d;
@@ -667,7 +669,7 @@ export class ArkManager {
       const keys = inputRecs.map((v) => this._keyForVtxo(v));
       const spk = hex.decode(action.spkHex);
       const inputIdRaws = decoded.map((d) => d.point.raw);
-      await registerVtxoTransactions(this.arkUrl, inputRecs.map((v) => hex.decode(v.bytes)));
+      await registerVtxoTransactions(this.arkUrl, inputRecs.map((v) => vtxoBytesFromStr(v.bytes)));
       const attestations = keys.map((k) =>
         offboardAttestation({ netSat: action.netSat, spk, inputIdRaws }, k.privkey));
       const { txBytes, serverNonces } = await prepareOffboard(this.arkUrl, {
@@ -1772,7 +1774,7 @@ export class ArkManager {
     const inputRecs = action.inputIds.map((id) => this._vtxo(id));
 
     if (action.step === 'created') {
-      const inputBytes = inputRecs.map((v) => hex.decode(v.bytes));
+      const inputBytes = inputRecs.map((v) => vtxoBytesFromStr(v.bytes));
       await registerVtxoTransactions(this.arkUrl, inputBytes);
       let unlockHash;
       try {
diff --git a/src/ark/proto.js b/src/ark/proto.js
index b8c091d..829eba4 100644
--- a/src/ark/proto.js
+++ b/src/ark/proto.js
@@ -7,7 +7,17 @@
 // Wire formats mirrored from github.com/ark-bitcoin/bark:
 //   lib/src/address.rs, lib/src/mailbox.rs, lib/src/encode.rs, lib/src/vtxo/mod.rs
 
-import { bech32m, hex } from '@scure/base';
+import { bech32m, hex, base64 } from '@scure/base';
+
+// Persisted vtxo bytes are base64: a third smaller than hex, and half the
+// UTF-16 cost in localStorage (a deep coin set was 560KB of hex). Readers
+// accept the hex that older builds wrote — states are never migrated in
+// place, they just re-save in the new form.
+const isHexStr = (s) => s.length % 2 === 0 && /^[0-9a-f]*$/.test(s);
+export const vtxoBytesToStr = (bytes) => base64.encode(bytes);
+export const vtxoBytesFromStr = (s) => (isHexStr(s) ? hex.decode(s) : base64.decode(s));
+export const vtxoBytesToHex = (s) => (isHexStr(s) ? s : hex.encode(base64.decode(s)));
+export const vtxoBytesNormalize = (s) => (isHexStr(s) ? base64.encode(hex.decode(s)) : s);
 import { sha256 } from '@noble/hashes/sha256';
 import { schnorr, secp256k1 } from '@noble/curves/secp256k1';
 
diff --git a/src/features/ark.js b/src/features/ark.js
index a5264ad..1dcfa8c 100644
--- a/src/features/ark.js
+++ b/src/features/ark.js
@@ -10,7 +10,7 @@ import { ArkManager } from '../ark/manager.js';
 import { loadBg, saveBg, buildBg, disarmSiblingRecords } from '../nwc-bg.js';
 import { boardFee, p2trAddress } from '../ark/board.js';
 import { maybeBolt11, maybeLnInvoice, lnSendFee } from '../ark/lightning.js';
-import { decodeVtxo, getVtxoStatus, VTXO_STATE_SPENT, concatBytes } from '../ark/proto.js';
+import { decodeVtxo, getVtxoStatus, VTXO_STATE_SPENT, concatBytes, vtxoBytesFromStr, vtxoBytesToHex } from '../ark/proto.js';
 import { signedExitTxs, exitTxVsizes, buildBumpChild, buildExitClaim, submitPackage } from '../ark/exit.js';
 import { utxoId } from '../wallet.js';
 import {
@@ -65,7 +65,7 @@ export function installArkWallet(wallet) {
     _arkVtxoOwner(st) {
       const v = (st?.vtxos || []).find((x) => x.bytes);
       if (!v) return null;
-      try { return decodeVtxo(hex.decode(v.bytes)).serverPubkey; } catch { return null; }
+      try { return decodeVtxo(vtxoBytesFromStr(v.bytes)).serverPubkey; } catch { return null; }
     },
     // Park state under the server that actually owns it, until the URL for
     // that server is selected and can adopt it.
@@ -116,7 +116,7 @@ export function installArkWallet(wallet) {
         let owner = st.serverPubkey || null;
         if (!owner) {
           const v = (st.vtxos || []).find((x) => x.bytes);
-          if (v) { try { owner = decodeVtxo(hex.decode(v.bytes)).serverPubkey; } catch {} }
+          if (v) { try { owner = decodeVtxo(vtxoBytesFromStr(v.bytes)).serverPubkey; } catch {} }
         }
         if (owner && owner !== serverPubkey) return null; // belongs to another ASP
         if (!owner) return null;                          // can't prove ownership: leave it
@@ -147,7 +147,11 @@ export function slimArkForSync(s) {
     // Spent stubs and done actions are pure history and grow forever — cap
     // them (newest kept) so the sync event's size is bounded for life.
     vtxos: (() => {
-      const live = (s.vtxos || []).filter((v) => v.state !== 'spent');
+      // On the wire the bytes stay HEX for now: a device still on a build
+      // that only reads hex would otherwise fail to decode every coin merged
+      // from this one. Flip to base64 once builds before 2026-09-10 are gone.
+      const live = (s.vtxos || []).filter((v) => v.state !== 'spent')
+        .map((v) => (v.bytes ? { ...v, bytes: vtxoBytesToHex(v.bytes) } : v));
       const stubs = (s.vtxos || []).filter((v) => v.state === 'spent').slice(-200)
         .map((v) => ({ id: v.id, amountSat: v.amountSat, state: 'spent', keyIndex: v.keyIndex, expiryHeight: v.expiryHeight }));
       return [...live, ...stubs];
diff --git a/tools/ark-prune-test.js b/tools/ark-prune-test.js
index e12b02e..fcf0166 100644
--- a/tools/ark-prune-test.js
+++ b/tools/ark-prune-test.js
@@ -8,12 +8,12 @@ const DAY = 86400_000;
 const now = 1_800_000_000_000;
 const state = {
   vtxos: [
-    { id: 'live', state: 'spendable', bytes: 'aa' },
-    { id: 'oldspent', state: 'spent', bytes: 'bb' },        // spent 40d ago → bytes go
-    { id: 'freshspent', state: 'spent', bytes: 'cc' },      // spent yesterday → kept
-    { id: 'inflight', state: 'spent', bytes: 'dd' },        // named by a live action → kept
-    { id: 'unnamed', state: 'spent', bytes: 'ee' },         // no date known → kept
-    { id: 'pending', state: 'pending', bytes: 'ff' },
+    { id: 'live', state: 'spendable', bytes: 'LIVE' },
+    { id: 'oldspent', state: 'spent', bytes: 'OLD=' },        // spent 40d ago → bytes go
+    { id: 'freshspent', state: 'spent', bytes: 'NEW=' },      // spent yesterday → kept
+    { id: 'inflight', state: 'spent', bytes: 'FLY=' },        // named by a live action → kept
+    { id: 'unnamed', state: 'spent', bytes: 'UNK=' },         // no date known → kept
+    { id: 'pending', state: 'pending', bytes: 'PND=' },
   ],
   actions: [
     { id: `send-${now - 40 * DAY}`, type: 'send', step: 'done', inputIds: ['oldspent'], destBytesList: ['hex'], changeBytesList: ['hex'], amountSat: 5 },
@@ -27,12 +27,12 @@ const state = {
 const before = JSON.stringify(state).length;
 pruneArkState(state, now);
 const v = (id) => state.vtxos.find((x) => x.id === id);
-assert.equal(v('live').bytes, 'aa');
-assert.equal(v('pending').bytes, 'ff');
+assert.equal(v('live').bytes, 'LIVE');
+assert.equal(v('pending').bytes, 'PND=');
 assert.equal(v('oldspent').bytes, undefined, 'old spend sheds its bytes');
-assert.equal(v('freshspent').bytes, 'cc', 'a recent spend keeps them');
-assert.equal(v('inflight').bytes, 'dd', 'an in-flight action pins them');
-assert.equal(v('unnamed').bytes, 'ee', 'no known spend date → keep');
+assert.equal(v('freshspent').bytes, 'NEW=', 'a recent spend keeps them');
+assert.equal(v('inflight').bytes, 'FLY=', 'an in-flight action pins them');
+assert.equal(v('unnamed').bytes, 'UNK=', 'no known spend date → keep');
 const a = (i) => state.actions[i];
 assert.ok(!('destBytesList' in a(0)) && !('changeBytesList' in a(0)) && a(0).amountSat === 5 && a(0).inputIds[0] === 'oldspent', 'old done action loses hex, keeps ids');
 assert.equal(a(1).htlcBytesList[0], 'hex', 'a day-old done action still has its hex');
@@ -42,3 +42,22 @@ assert.equal(a(4).htlcBytesList[0], 'hex', 'a failed ln-pay keeps its HTLC bytes
 assert.ok(JSON.stringify(state).length < before);
 pruneArkState(state, now);
 console.log('✓ ark state pruning is selective and idempotent');
+
+// ---- bytes encoding: base64 persisted, hex still readable, hex on the wire ----
+import { vtxoBytesToStr, vtxoBytesFromStr, vtxoBytesToHex, vtxoBytesNormalize } from '../src/ark/proto.js';
+import { slimArkForSync } from '../src/features/ark.js';
+const raw = new Uint8Array(300).map((_, i) => (i * 37) & 255);
+const hexStr = [...raw].map((b) => b.toString(16).padStart(2, '0')).join('');
+const b64 = vtxoBytesToStr(raw);
+assert.ok(b64.length < hexStr.length * 0.7);
+assert.deepEqual(vtxoBytesFromStr(b64), raw);
+assert.deepEqual(vtxoBytesFromStr(hexStr), raw, 'hex from older builds still decodes');
+assert.equal(vtxoBytesNormalize(hexStr), b64);
+assert.equal(vtxoBytesNormalize(b64), b64);
+assert.equal(vtxoBytesToHex(b64), hexStr);
+const st2 = { vtxos: [{ id: 'a', state: 'spendable', bytes: hexStr }, { id: 'b', state: 'spent', bytes: hexStr }], actions: [], movements: [] };
+pruneArkState(st2, now);
+assert.equal(st2.vtxos[0].bytes, b64, 'a save converts persisted hex to base64');
+assert.equal(st2.vtxos[1].bytes, b64);
+assert.equal(slimArkForSync(st2).vtxos[0].bytes, hexStr, 'the sync wire stays hex for older devices');
+console.log('✓ vtxo bytes persist as base64, read hex, ship hex');