playlists: the store, and putting it on nodes

playlists.js is IndexedDB, WebCrypto and a transport, and node has no
IndexedDB — so it is driven in Chrome against a node stubbed to record
what it was handed, which is also how what leaves the browser is
checked to be sealed.

Sync asks the node what it holds (user_blob_list) rather than comparing
against the merged watermark, which says nothing about that node: the
first version pushed every body on every sync. A tombstoned playlist's
body is deleted as each node is reached, or the quota fills with graves.

The database version and its stores stay in hub-client.js — two modules
opening one database at versions of their own is a VersionError thrown
at whichever runs second.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

1 files changed, 484 insertions, 0 deletions

diff --git a/packages/meshbay-hub/src/meshbay_hub/static/playlists.js b/packages/meshbay-hub/src/meshbay_hub/static/playlists.js
new file mode 100644
index 0000000..11525eb
--- /dev/null
+++ b/packages/meshbay-hub/src/meshbay_hub/static/playlists.js
@@ -0,0 +1,484 @@
+import {
+  MANIFEST_KIND, bodyKind, FAVORITES_ID,
+  emptyManifest, emptyBody, mergeManifests, mergeBodies,
+  toStored, fromStored, livePlaylists, repairTracks, indexTracks,
+} from './playlist-merge.js';
+import { derivePlaylistKey, seal, open } from './playlist-crypto.js';
+import {
+  session, _loadBundleKey, openDB, IDB_PLAYLISTS,
+} from './hub-client.js';
+
+/**
+ * Playlists: the store, and putting it on nodes.
+ *
+ * `docs/playlists.md` §7 and §14. The rules live in `playlist-merge.js` and the
+ * sealing in `playlist-crypto.js`, both of which are pure and both of which are
+ * executed by their own tests. What is here is the part that cannot be: reading
+ * and writing IndexedDB, and talking to nodes.
+ *
+ * **The client is the authority.** The merged state is what is in this
+ * browser's IndexedDB, and every merge takes the maximum across the local copy
+ * *plus* whatever each node answers. A node serving a stale copy, or a fresh
+ * one it never received, can only lose the tie — it can never lower the merged
+ * state, because the local copy is one of the inputs. That is what makes an
+ * offline node a non-event rather than a hazard.
+ *
+ * **Nothing here dials.** Sync rides connections that were being opened anyway:
+ * opening a group, the Search page's sweep, the music pool connecting to play a
+ * track. A sweep of every group costs ten seconds per unreachable one, and
+ * doing that at sign-in for a feature nobody has asked to use yet is exactly
+ * what §7 refuses.
+ */
+
+// ── local identity ───────────────────────────────────────────────────────────
+
+const DEVICE_KEY = 'meshbay_playlist_device';
+
+/**
+ * A stable id for *this browser*, for breaking merge ties and nothing else.
+ *
+ * Not an identity and not a secret: it never leaves the sealed blob, and two
+ * devices needing different values is the whole requirement. Random rather than
+ * derived from the account, because two browsers of one account must not
+ * collide — which is the one case the tie-break exists for.
+ */
+function deviceId() {
+  try {
+    let id = localStorage.getItem(DEVICE_KEY);
+    if (!id) {
+      id = Array.from(crypto.getRandomValues(new Uint8Array(8)),
+                      (b) => b.toString(16).padStart(2, '0')).join('');
+      localStorage.setItem(DEVICE_KEY, id);
+    }
+    return id;
+  } catch {
+    // A private window with storage refused. A per-session id still breaks
+    // ties correctly; it just stops being the *same* device next time, which
+    // costs nothing but a different arbitrary winner on a tie.
+    return 'ephemeral';
+  }
+}
+
+// ── the key ──────────────────────────────────────────────────────────────────
+
+let _key = null;
+let _keyFor = null;
+
+/**
+ * The playlist key, derived once per sign-in from the HKDF handle that rides
+ * alongside the bundle key (`keyderive.js`'s deriveBundleKeys).
+ *
+ * `v2hkdf` is absent when this browser's session predates it — a stored bundle
+ * key from before the change, loaded out of IndexedDB. There is nothing to do
+ * about that here and nothing to fall back to: the passphrase is not in memory
+ * to re-derive from. Playlists stay local until the next sign-in, which is a
+ * degradation rather than a failure and is reported as one.
+ */
+async function playlistKey(userId) {
+  if (_key && _keyFor === userId) return _key;
+  let bundleKey = session.bundleKey;
+  if (!bundleKey) {
+    bundleKey = await _loadBundleKey();
+    if (bundleKey) session.bundleKey = bundleKey;
+  }
+  if (!bundleKey || !bundleKey.v2hkdf) return null;
+  _key = await derivePlaylistKey(bundleKey.v2hkdf);
+  _keyFor = userId;
+  return _key;
+}
+
+/** Sign-out: the next account on this browser must not inherit this one's key. */
+function forgetPlaylistKey() {
+  _key = null;
+  _keyFor = null;
+  _state = null;
+}
+
+// ── local storage ────────────────────────────────────────────────────────────
+//
+// Plaintext in IndexedDB, exactly as the cached group indexes already are
+// (`hub-client.js`'s `group_indexes`). What is sealed is the copy that leaves
+// this browser; a second encryption layer over the local cache would protect
+// against nothing the index cache is not already exposed to, and would need a
+// key held in the same place.
+//
+// Keyed by `userId|kind`, not by `kind`. Two accounts on one browser is
+// ordinary, and a sign-out that fails to run is ordinary too — so the
+// separation is in the key rather than in a cleanup path that has to happen.
+
+// The database, its version and its stores are `hub-client.js`'s: two modules
+// opening one database at versions of their own is a `VersionError` thrown at
+// whichever runs second, and the store this uses was added there at version 2
+// (docs/playlists.md §14.2).
+
+async function _idbGet(userId, kind) {
+  try {
+    const db = await openDB();
+    const tx = db.transaction(IDB_PLAYLISTS, 'readonly');
+    const req = tx.objectStore(IDB_PLAYLISTS).get(`${userId}|${kind}`);
+    const value = await new Promise((res, rej) => {
+      req.onsuccess = () => res(req.result);
+      req.onerror = () => rej(req.error);
+    });
+    db.close();
+    return value || null;
+  } catch { return null; }
+}
+
+async function _idbPut(userId, kind, value) {
+  try {
+    const db = await openDB();
+    const tx = db.transaction(IDB_PLAYLISTS, 'readwrite');
+    tx.objectStore(IDB_PLAYLISTS).put(value, `${userId}|${kind}`);
+    await new Promise((res, rej) => { tx.oncomplete = res; tx.onerror = rej; });
+    db.close();
+  } catch { /* a private window, or storage refused — the session still works */ }
+}
+
+// ── in-memory state ──────────────────────────────────────────────────────────
+
+let _state = null;   // { userId, manifest, bodies: Map(id -> body) }
+
+async function ensureLoaded(userId) {
+  if (_state && _state.userId === userId) return _state;
+  const manifest = (await _idbGet(userId, MANIFEST_KIND)) || emptyManifest();
+  _state = { userId, manifest, bodies: new Map() };
+  return _state;
+}
+
+async function _body(userId, id) {
+  const st = await ensureLoaded(userId);
+  if (st.bodies.has(id)) return st.bodies.get(id);
+  const body = (await _idbGet(userId, bodyKind(id))) || emptyBody(id);
+  st.bodies.set(id, body);
+  return body;
+}
+
+async function _saveManifest(st) {
+  // Its own counter, separate from the per-playlist revisions: it is what the
+  // node's `rev` column carries, and what a listing compares without opening
+  // anything.
+  st.manifest.rev = (st.manifest.rev || 0) + 1;
+  await _idbPut(st.userId, MANIFEST_KIND, st.manifest);
+}
+
+async function _saveBody(st, body) {
+  st.bodies.set(body.id, body);
+  await _idbPut(st.userId, bodyKind(body.id), body);
+}
+
+/** The manifest entry for `id`, materialised if this is Favourites. */
+function _entry(st, id, name) {
+  let e = st.manifest.playlists[id];
+  if (!e) {
+    e = {
+      name: name || '', rev: 1, body_rev: 0, device: deviceId(),
+      updated_at: Math.floor(Date.now() / 1000), deleted: false, count: 0,
+    };
+    st.manifest.playlists[id] = e;
+  }
+  return e;
+}
+
+// ── reading ──────────────────────────────────────────────────────────────────
+
+/**
+ * Every playlist a reader should see, Favourites first.
+ *
+ * Drawn from the manifest alone — names, counts, a few kilobytes — so the
+ * "add to playlist" menu opens instantly with every node offline and no body
+ * fetched. That is what the manifest is for.
+ */
+async function listPlaylists(userId) {
+  const st = await ensureLoaded(userId);
+  return livePlaylists(st.manifest);
+}
+
+/**
+ * One playlist's tracks, in the shape the player consumes.
+ *
+ * `cachedIndexes` is optional and is `[{ groupId, entries }]`: given it, an
+ * entry whose hash is no longer in its group's index but whose path is gets
+ * its id rewritten, and the other way round. Repaired in memory and written
+ * back, once, so the next read costs nothing.
+ */
+async function getPlaylistTracks(userId, id, cachedIndexes) {
+  const body = await _body(userId, id);
+  let tracks = body.tracks || [];
+  if (cachedIndexes && cachedIndexes.length) {
+    let changed = 0;
+    for (const { groupId, entries } of cachedIndexes) {
+      const res = repairTracks(tracks, groupId, indexTracks(entries));
+      tracks = res.tracks;
+      changed += res.repaired;
+    }
+    if (changed) {
+      const st = await ensureLoaded(userId);
+      await _saveBody(st, { ...body, tracks });
+    }
+  }
+  return tracks.map(fromStored);
+}
+
+// ── writing ──────────────────────────────────────────────────────────────────
+
+function _newId() {
+  return (crypto.randomUUID && crypto.randomUUID())
+    || Array.from(crypto.getRandomValues(new Uint8Array(16)),
+                  (b) => b.toString(16).padStart(2, '0')).join('');
+}
+
+/**
+ * Create a playlist under a name that is not already taken.
+ *
+ * Case- and accent-folded, because "Soirée" and "soiree" being two playlists
+ * is not something anyone wants and is very easy to do by accident.
+ */
+async function createPlaylist(userId, name) {
+  const st = await ensureLoaded(userId);
+  const wanted = String(name || '').trim();
+  if (!wanted) throw new Error('empty name');
+  const fold = (s) => s.normalize('NFD').replace(/[̀-ͯ]/g, '').toLowerCase();
+  for (const p of livePlaylists(st.manifest)) {
+    if (fold(p.name) === fold(wanted)) throw new Error('duplicate name');
+  }
+  const id = _newId();
+  _entry(st, id, wanted);
+  await _saveManifest(st);
+  await _saveBody(st, emptyBody(id));
+  return id;
+}
+
+async function renamePlaylist(userId, id, name) {
+  const st = await ensureLoaded(userId);
+  const e = _entry(st, id);
+  e.name = String(name || '').trim();
+  e.rev += 1;
+  e.device = deviceId();
+  e.updated_at = Math.floor(Date.now() / 1000);
+  await _saveManifest(st);
+}
+
+/**
+ * Tombstone a playlist. Favourites is refused outright.
+ *
+ * Refusing rather than special-casing is what keeps the tombstone rule free of
+ * an exception: a tombstone on the reserved id would have to be ignored by
+ * every reader, and a tombstone rule with an exception is how the resurrection
+ * defect ships.
+ */
+async function deletePlaylist(userId, id) {
+  if (id === FAVORITES_ID) throw new Error('favorites cannot be deleted');
+  const st = await ensureLoaded(userId);
+  const e = _entry(st, id);
+  e.deleted = true;
+  e.rev += 1;
+  e.device = deviceId();
+  e.updated_at = Math.floor(Date.now() / 1000);
+  await _saveManifest(st);
+  st.bodies.delete(id);
+  await _idbPut(st.userId, bodyKind(id), null);
+}
+
+/**
+ * Append entries to a playlist, creating it if this is Favourites' first use.
+ *
+ * Duplicates are allowed in an ordinary playlist — real ones have them — and
+ * refused in Favourites, where the gesture is a toggle rather than an append.
+ * Returns how many were actually added, so the caller can say so.
+ */
+async function addTracks(userId, id, entries, groupId, name) {
+  const st = await ensureLoaded(userId);
+  const e = _entry(st, id, id === FAVORITES_ID ? (name || 'Favorites') : name);
+  const body = await _body(userId, id);
+  const existing = new Set(body.tracks.map((tr) => tr.id));
+  const incoming = entries
+    .map((entry) => toStored(entry, groupId))
+    .filter((tr) => (id === FAVORITES_ID ? !existing.has(tr.id) : true));
+  if (!incoming.length) return 0;
+
+  const next = {
+    ...body,
+    rev: (body.rev || 0) + 1,
+    device: deviceId(),
+    tracks: [...body.tracks, ...incoming],
+  };
+  await _saveBody(st, next);
+  e.body_rev = next.rev;
+  e.count = next.tracks.length;
+  e.updated_at = Math.floor(Date.now() / 1000);
+  await _saveManifest(st);
+  return incoming.length;
+}
+
+async function removeTrackAt(userId, id, at) {
+  const st = await ensureLoaded(userId);
+  const body = await _body(userId, id);
+  if (at < 0 || at >= body.tracks.length) return false;
+  const next = {
+    ...body,
+    rev: (body.rev || 0) + 1,
+    device: deviceId(),
+    tracks: [...body.tracks.slice(0, at), ...body.tracks.slice(at + 1)],
+  };
+  await _saveBody(st, next);
+  const e = _entry(st, id);
+  e.body_rev = next.rev;
+  e.count = next.tracks.length;
+  e.updated_at = Math.floor(Date.now() / 1000);
+  await _saveManifest(st);
+  return true;
+}
+
+/**
+ * The current queue as a new playlist.
+ *
+ * The caller passes the tracks in **play order** — what the queue panel is
+ * showing — rather than in the order they were added, because that is what
+ * "save what I am listening to" means, shuffle included.
+ */
+async function saveQueueAsPlaylist(userId, name, entries) {
+  const id = await createPlaylist(userId, name);
+  await addTracks(userId, id, entries, null);
+  return id;
+}
+
+// ── sync ─────────────────────────────────────────────────────────────────────
+
+/**
+ * Reconcile with one node, over a connection that was open anyway.
+ *
+ * Manifest first, always; bodies only when the merged manifest says this
+ * browser's copy is behind, or when the node's is. A reader with forty
+ * playlists who only ever plays two never transfers the other thirty-eight.
+ *
+ * Every failure is swallowed and reported in the return value rather than
+ * thrown: this rides someone else's connection, and a playlist that could not
+ * be reconciled must never break whatever that connection was opened for.
+ */
+async function syncWith(transport, userId) {
+  const result = { ok: false, pulled: 0, pushed: 0, reason: null };
+  if (!transport || !transport.connected) {
+    result.reason = 'offline';
+    return result;
+  }
+  const key = await playlistKey(userId);
+  if (!key) {
+    // No HKDF handle: a session from before it existed. Nothing to fall back
+    // to, and silently doing nothing would be the worse answer.
+    result.reason = 'no_key';
+    return result;
+  }
+  const st = await ensureLoaded(userId);
+
+  // What this node holds, and at what revision — kinds and numbers, no
+  // payloads, one message. Without it there is no way to tell "the node is
+  // behind" from "the node is up to date", and the first version of this
+  // pushed every body on every sync because it could not tell the difference.
+  const have = new Map();
+  try {
+    for (const b of await transport.listUserBlobs()) have.set(b.kind, b.rev);
+  } catch (err) {
+    // A node too old to know these messages at all. Nothing to reconcile with,
+    // and nothing lost: the local copy is the authority and the next node this
+    // browser reaches will take the writes.
+    result.reason = err.message || 'unsupported';
+    return result;
+  }
+
+  let theirs = null;
+  if (have.has(MANIFEST_KIND)) {
+    try {
+      const row = await transport.fetchUserBlob(MANIFEST_KIND);
+      if (row && row.blob_enc) {
+        theirs = await open(row.blob_enc, MANIFEST_KIND, userId, key);
+      }
+    } catch (err) {
+      result.reason = err.message || 'fetch failed';
+      return result;
+    }
+  }
+
+  const merged = mergeManifests(st.manifest, theirs || emptyManifest());
+  merged.rev = Math.max(st.manifest.rev || 0, (theirs && theirs.rev) || 0);
+  const grew = JSON.stringify(merged.playlists) !== JSON.stringify(st.manifest.playlists);
+  if (grew) {
+    st.manifest = merged;
+    await _idbPut(userId, MANIFEST_KIND, merged);
+    result.pulled += 1;
+  }
+
+  // Bodies, compared against what *this node* holds rather than against the
+  // merged watermark: the watermark is the highest revision anybody has seen,
+  // which says nothing about whether this particular node has it.
+  //
+  // A body is fetched only when this node is ahead, and pushed only when it is
+  // behind. A reader with forty playlists who only ever plays two never
+  // transfers the other thirty-eight, and a sync with nothing to do writes
+  // nothing at all.
+  for (const p of livePlaylists(merged)) {
+    const local = await _body(userId, p.id);
+    const kind = bodyKind(p.id);
+    const nodeRev = have.has(kind) ? (have.get(kind) || 0) : -1;
+    const localRev = local.rev || 0;
+
+    if (nodeRev > localRev) {
+      try {
+        const row = await transport.fetchUserBlob(kind);
+        if (row && row.blob_enc) {
+          const remote = await open(row.blob_enc, kind, userId, key);
+          const best = mergeBodies(local, remote);
+          if (best && (best.rev || 0) > localRev) {
+            await _saveBody(st, best);
+            result.pulled += 1;
+          }
+        }
+      } catch { /* one body failing must not stop the rest */ }
+    } else if (localRev > nodeRev && localRev > 0) {
+      try {
+        await transport.storeUserBlob(
+          kind, localRev, await seal(local, kind, userId, key));
+        result.pushed += 1;
+      } catch { /* a cap, or a node that went away mid-sweep */ }
+    }
+  }
+
+  // A tombstoned playlist's body is dropped from every node as it is reached.
+  // The tombstone in the manifest is what has to survive, not the tracks —
+  // and without this the body of every playlist ever deleted stays on every
+  // node for ever, filling the account's quota with graves.
+  for (const [id, e] of Object.entries(merged.playlists)) {
+    if (!e.deleted) continue;
+    const kind = bodyKind(id);
+    if (!have.has(kind)) continue;
+    try { await transport.deleteUserBlob(kind); } catch { /* next time round */ }
+  }
+
+  // The manifest goes last, so a node never advertises a body it has not been
+  // given: a reader on a third device would fetch a watermark, ask for the
+  // body behind it and be told there is none.
+  const mine = JSON.stringify(st.manifest.playlists);
+  if (!theirs || mine !== JSON.stringify(theirs.playlists)) {
+    try {
+      await transport.storeUserBlob(
+        MANIFEST_KIND, st.manifest.rev || 1,
+        await seal(st.manifest, MANIFEST_KIND, userId, key));
+      result.pushed += 1;
+    } catch (err) {
+      result.reason = err.message || 'store failed';
+      return result;
+    }
+  }
+
+  result.ok = true;
+  return result;
+}
+
+export {
+  FAVORITES_ID,
+  deviceId, playlistKey, forgetPlaylistKey, ensureLoaded,
+  listPlaylists, getPlaylistTracks,
+  createPlaylist, renamePlaylist, deletePlaylist,
+  addTracks, removeTrackAt, saveQueueAsPlaylist,
+  syncWith,
+};