feat(android): send a manifest of what each backup run sent

Photos, videos and files record, per file, their path on the node, their path
and album on the phone, dates, size and SHA-256, uploaded as
meshbay-manifest/manifest-<date>.jsonl once the run is done.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

7 files changed, 226 insertions, 33 deletions

diff --git a/packages/meshbay-android/app/src/main/assets/bridge/meshbay-bridge.js b/packages/meshbay-android/app/src/main/assets/bridge/meshbay-bridge.js
index 32cc4ab..d8237ec 100644
--- a/packages/meshbay-android/app/src/main/assets/bridge/meshbay-bridge.js
+++ b/packages/meshbay-android/app/src/main/assets/bridge/meshbay-bridge.js
@@ -213,6 +213,8 @@
         estimate: (settings) => call('photosync:estimate', settings),
         plan: () => call('photosync:plan'),
         sent: (token, dir, name) => call('photosync:sent', token, dir, name),
+        manifest: () => call('photosync:manifest'),
+        manifestSent: (token) => call('photosync:manifest-sent', token),
         completed: () => call('photosync:completed'),
         failed: (code, text) => call('photosync:failed', code, text),
         keepAlive: (on, text) => call('photosync:keep-alive', on === true, text || ''),
@@ -259,6 +261,8 @@
         configure: (settings) => call('drivesync:configure', settings || null),
         plan: () => call('drivesync:plan'),
         sent: (token, dir, name) => call('drivesync:sent', token, dir, name),
+        manifest: () => call('drivesync:manifest'),
+        manifestSent: (token) => call('drivesync:manifest-sent', token),
         completed: () => call('drivesync:completed'),
         failed: (code, text) => call('drivesync:failed', code, text),
         keepAlive: (on, text) => call('drivesync:keep-alive', on === true, text || ''),
diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveChannels.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveChannels.kt
index 4328774..3f87359 100644
--- a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveChannels.kt
+++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveChannels.kt
@@ -9,9 +9,10 @@ import android.webkit.WebResourceResponse
 import org.json.JSONArray
 import org.json.JSONObject
 import org.meshbay.client.bridge.Refused
+import org.meshbay.client.phonesync.Destination
 import org.meshbay.client.phonesync.DestinationChannels
+import org.meshbay.client.phonesync.ManifestLog
 import org.meshbay.client.phonesync.DocPlan
-import org.meshbay.client.photos.ByteRange
 import org.meshbay.client.photos.PhotoChannels
 import org.meshbay.client.shell.Pickers
 import java.io.File
@@ -61,6 +62,8 @@ class DriveChannels(
         "drivesync:configure" -> { configure(args.optJSONObject(0)); status() }
         "drivesync:plan" -> plan()
         "drivesync:sent" -> { sent(args.optString(0, ""), args.optString(1, ""), args.optString(2, "")); true }
+        "drivesync:manifest" -> manifest()
+        "drivesync:manifest-sent" -> { manifestSent(args.optString(0, "")); true }
         "drivesync:completed" -> { completed(); status() }
         "drivesync:failed" -> failed(args.optString(0, ""), args.optString(1, ""))
         "drivesync:keep-alive" -> { onKeepAlive(args.optBoolean(0, false), args.optString(1, "").take(200)); true }
@@ -91,10 +94,15 @@ class DriveChannels(
         }
     }
 
-    private fun base(d: org.meshbay.client.phonesync.Destination) = DocPlan.dirFor(d, SUFFIX)
+    private fun base(d: Destination) = DocPlan.dirFor(d, SUFFIX)
 
-    private fun ledger(d: org.meshbay.client.phonesync.Destination) =
-        DriveLedger(File(dir, PhotoChannels.hex(PhotoChannels.sha256Of(d.ledgerKey.toByteArray())).take(32) + ".jsonl"))
+    private fun keyOf(d: Destination) = PhotoChannels.hex(PhotoChannels.sha256Of(d.ledgerKey.toByteArray())).take(32)
+
+    private fun ledger(d: Destination) = DriveLedger(File(dir, keyOf(d) + ".jsonl"))
+
+    private fun manifestLog(d: Destination) = ManifestLog(File(dir, keyOf(d) + ".manifest"))
+
+    @Volatile private var manifestToken: Pair<String, File>? = null
 
     fun status(): JSONObject {
         val d = destinations.get()
@@ -175,7 +183,7 @@ class DriveChannels(
             out.put(JSONObject().put("token", token).put("name", p.name).put("dir", p.dir)
                 .put("size", p.file.size).put("edited", p.edited))
         }
-        return JSONObject().put("items", out)
+        return JSONObject().put("items", out).put("manifest", manifestLog(d).waiting())
     }
 
     /** The node took it (or already had it): into the ledger, hashed from the phone now. */
@@ -184,23 +192,45 @@ class DriveChannels(
         val d = destinations.get()?.takeIf { it.ledgerKey == issued.key } ?: throw Refused("Refused: the backup changed")
         val f = issued.pending.file
         val sha = hashOf(f) ?: throw Refused("Refused: the file is gone")
-        ledger(d).record(DriveLedger.Entry(f.docId, f.modified, f.size, sha, dir.take(1024), name.take(256),
-                                           System.currentTimeMillis()))
+        val now = System.currentTimeMillis()
+        ledger(d).record(DriveLedger.Entry(f.docId, f.modified, f.size, sha, dir.take(1024), name.take(256), now))
+        manifestLog(d).append(JSONObject()
+            .put("kind", "file")
+            .put("node", "${dir.take(1024)}/${name.take(256)}")
+            .put("source", DrivePlan.pathOf(f.docId)).put("name", f.name)
+            .put("folder", folders().firstOrNull { it.uri == f.tree }?.name ?: "")
+            .put("modified", f.modified).put("size", f.size).put("sha256", sha).put("mime", f.mime)
+            .put("edited", issued.pending.edited).put("sentAt", now))
         tokens.remove(token)
     }
 
+    /** The manifest of what was sent and not yet described on the node (ManifestLog), or `item: null`. */
+    private fun manifest(): JSONObject {
+        val d = destinations.get()?.takeIf { prefs.getBoolean(ON, false) } ?: throw Refused("Refused: files backup is off")
+        val file = manifestLog(d).issue() ?: return JSONObject().put("item", JSONObject.NULL)
+        val token = PhotoChannels.hex(ByteArray(16).also { random.nextBytes(it) })
+        manifestToken = token to file
+        return JSONObject().put("item", JSONObject().put("token", token)
+            .put("name", ManifestLog.nameFor(System.currentTimeMillis(), TimeZone.getDefault()))
+            .put("dir", base(d) + "/" + ManifestLog.DIR).put("size", file.length()))
+    }
+
+    private fun manifestSent(token: String) {
+        if (manifestToken?.first != token) throw Refused("Refused: unknown manifest")
+        val d = destinations.get() ?: throw Refused("Refused: the backup changed")
+        manifestLog(d).confirm()
+        manifestToken = null
+    }
+
     /** A range of an issued file, for `/drivesync/<token>` on the packaged origin. */
     fun serve(path: String, range: String?): WebResourceResponse? {
+        manifestToken?.takeIf { it.first == path.removePrefix(PATH) }?.let { (_, file) ->
+            return PhotoChannels.serveRange(file.inputStream(), file.length(), "application/x-ndjson", range)
+        }
         val issued = tokens[path.removePrefix(PATH)] ?: return null
         val f = issued.pending.file
-        val r = ByteRange.parse(range, f.size) ?: return WebResourceResponse("text/plain", null, 416,
-            "Range Not Satisfiable", mapOf("Content-Range" to "bytes */${f.size}"), "".byteInputStream())
         val raw = try { source.open(f) } catch (e: Exception) { null } ?: return null
-        val headers = mapOf("Cache-Control" to "no-store", "X-Content-Type-Options" to "nosniff",
-                            "Content-Range" to "bytes ${r.first}-${r.last}/${f.size}",
-                            "Content-Length" to (r.last - r.first + 1).toString())
-        return WebResourceResponse(f.mime.ifEmpty { "application/octet-stream" }, null, 206, "Partial Content",
-                                   headers, ByteRange.slice(raw, r))
+        return PhotoChannels.serveRange(raw, f.size, f.mime.ifEmpty { "application/octet-stream" }, range)
     }
 
     private fun hashOf(f: DriveFile): String? = try {
diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/phonesync/ManifestLog.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/phonesync/ManifestLog.kt
new file mode 100644
index 0000000..0d7142c
--- /dev/null
+++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/phonesync/ManifestLog.kt
@@ -0,0 +1,56 @@
+package org.meshbay.client.phonesync
+
+import org.json.JSONObject
+import java.io.File
+import java.text.SimpleDateFormat
+import java.util.Date
+import java.util.Locale
+import java.util.TimeZone
+
+/**
+ * What a backup sent, as the phone knew it, for a restore or a merge later
+ * (docs/MESHBAY_DESIGN.md §9.12): one JSON line per file the node took, with
+ * where it came from on the phone, its album or folder, its dates, size and
+ * SHA-256. The node has the files; only the phone knew these.
+ *
+ * Lines wait here until a manifest carrying them is on the node: `issue()`
+ * gathers them into the file to send, `confirm()` forgets them once the node
+ * has it. A manifest that did not arrive is sent again, with whatever was
+ * added since, at the next run: nothing is lost to an interrupted one.
+ */
+class ManifestLog(private val pending: File) {
+    private val sending = File(pending.path + ".sending")
+
+    fun append(line: JSONObject) {
+        pending.parentFile?.mkdirs()
+        pending.appendText(line.toString() + "\n")
+    }
+
+    /** True when there are lines no manifest on the node holds yet. */
+    fun waiting(): Boolean = (sending.exists() && sending.length() > 0) || (pending.exists() && pending.length() > 0)
+
+    /** The file to send now, holding every waiting line, or null. */
+    fun issue(): File? {
+        if (pending.exists() && pending.length() > 0) {
+            if (sending.exists()) { sending.appendText(pending.readText()); pending.delete() }
+            else if (!pending.renameTo(sending)) return null
+        }
+        return sending.takeIf { it.exists() && it.length() > 0 }
+    }
+
+    /** The node has it: those lines are done. */
+    fun confirm() { sending.delete() }
+
+    /** Everything forgotten: the destination changed, or the backup was turned off. */
+    fun clear() { pending.delete(); sending.delete() }
+
+    companion object {
+        /** The folder manifests go into, under a kind's own folder. */
+        const val DIR = "meshbay-manifest"
+
+        /** `manifest-2026-10-10-143205.jsonl`, in the phone's time: never one name twice. */
+        fun nameFor(now: Long, zone: TimeZone): String =
+            "manifest-" + SimpleDateFormat("yyyy-MM-dd-HHmmss", Locale.ROOT).apply { timeZone = zone }.format(Date(now)) +
+                ".jsonl"
+    }
+}
diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoChannels.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoChannels.kt
index 422103b..2e7c519 100644
--- a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoChannels.kt
+++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoChannels.kt
@@ -22,6 +22,7 @@ import org.meshbay.client.bridge.Refused
 import org.meshbay.client.notify.Notifier
 import org.meshbay.client.phonesync.Destination
 import org.meshbay.client.phonesync.DestinationChannels
+import org.meshbay.client.phonesync.ManifestLog
 import java.io.File
 import java.io.FilterInputStream
 import java.io.InputStream
@@ -81,6 +82,8 @@ class PhotoChannels(
         "photosync:estimate" -> { requirePermission(); estimate(args.optJSONObject(0) ?: throw Refused("Refused: no settings")) }
         "photosync:plan" -> { requirePermission(); plan() }
         "photosync:sent" -> { sent(args.optString(0, ""), args.optString(1, ""), args.optString(2, "")); true }
+        "photosync:manifest" -> manifest()
+        "photosync:manifest-sent" -> { manifestSent(args.optString(0, "")); true }
         "photosync:completed" -> { completed(); status() }
         "photosync:failed" -> failed(args.optString(0, ""), args.optString(1, ""))
         "photosync:keep-alive" -> { onKeepAlive(args.optBoolean(0, false), args.optString(1, "").take(200)); true }
@@ -97,7 +100,13 @@ class PhotoChannels(
         return (destinations.get() ?: return null) to c
     }
 
-    private fun ledger(d: Destination) = PhotoLedger(File(dir, hex(sha256Of(d.ledgerKey.toByteArray())).take(32) + ".jsonl"))
+    private fun keyOf(d: Destination) = hex(sha256Of(d.ledgerKey.toByteArray())).take(32)
+
+    private fun ledger(d: Destination) = PhotoLedger(File(dir, keyOf(d) + ".jsonl"))
+
+    private fun manifestLog(d: Destination) = ManifestLog(File(dir, keyOf(d) + ".manifest"))
+
+    @Volatile private var manifestToken: Pair<String, File>? = null
 
     fun status(): JSONObject {
         val c = config()
@@ -191,7 +200,7 @@ class PhotoChannels(
                 // folder for what is already there — an edit under its own name too.
                 .put("alsoKnownAs", PhotoPlan.editedName(p.photo, java.util.TimeZone.getDefault())))
         }
-        return JSONObject().put("items", out)
+        return JSONObject().put("items", out).put("manifest", manifestLog(d).waiting())
     }
 
     /** The node took it (or already had it): into the ledger, under the name its ack gave. */
@@ -200,12 +209,41 @@ class PhotoChannels(
         val d = active()?.first?.takeIf { it.ledgerKey == issued.key } ?: throw Refused("Refused: the backup changed")
         val p = issued.pending.photo
         val sha = issued.sha256 ?: hashOf(p) ?: throw Refused("Refused: the photo is gone")
+        val now = System.currentTimeMillis()
         ledger(d).record(PhotoLedger.Entry(p.mediaId, p.modified, p.size, sha,
-                                           dir.take(1024), name.take(256), System.currentTimeMillis()))
+                                           dir.take(1024), name.take(256), now))
+        manifestLog(d).append(JSONObject()
+            .put("kind", if (p.video) "video" else "photo")
+            .put("node", "${dir.take(1024)}/${name.take(256)}")
+            .put("source", p.relPath).put("album", p.album)
+            .put("taken", PhotoPlan.whenTaken(p)).put("modified", p.modified * 1000)
+            .put("size", p.size).put("sha256", sha).put("mime", p.mime)
+            .put("edited", issued.pending.edited).put("sentAt", now))
         tokens.remove(token)
     }
 
     /**
+     * The manifest of what was sent and not yet described on the node, for
+     * `<base>/meshbay-manifest/` (ManifestLog), or `item: null`.
+     */
+    private fun manifest(): JSONObject {
+        val (d, _) = active() ?: throw Refused("Refused: photo backup is off")
+        val file = manifestLog(d).issue() ?: return JSONObject().put("item", JSONObject.NULL)
+        val token = hex(ByteArray(16).also { random.nextBytes(it) })
+        manifestToken = token to file
+        return JSONObject().put("item", JSONObject().put("token", token)
+            .put("name", ManifestLog.nameFor(System.currentTimeMillis(), java.util.TimeZone.getDefault()))
+            .put("dir", PhotoPlan.baseFor(d) + "/" + ManifestLog.DIR).put("size", file.length()))
+    }
+
+    private fun manifestSent(token: String) {
+        if (manifestToken?.first != token) throw Refused("Refused: unknown manifest")
+        val d = destinations.get() ?: throw Refused("Refused: the backup changed")
+        manifestLog(d).confirm()
+        manifestToken = null
+    }
+
+    /**
      * The bytes of an issued photo or video, for `/photosync/<token>` on the
      * packaged origin. Asked a range at a time (ByteRange), as the upload
      * reads them, so a video never sits whole in the page; the ledger's hash
@@ -213,21 +251,13 @@ class PhotoChannels(
      * as it goes out.
      */
     fun serve(path: String, range: String? = null): WebResourceResponse? {
+        manifestToken?.takeIf { it.first == path.removePrefix(PATH) }?.let { (_, file) ->
+            return serveRange(file.inputStream(), file.length(), "application/x-ndjson", range)
+        }
         val issued = tokens[path.removePrefix(PATH)] ?: return null
         val raw = try { source.open(issued.pending.photo) } catch (e: Exception) { null } ?: return null
         val mime = issued.pending.photo.mime.ifEmpty { "application/octet-stream" }
-        if (range != null) {
-            val size = issued.pending.photo.size
-            val r = ByteRange.parse(range, size) ?: run {
-                raw.close()
-                return WebResourceResponse("text/plain", null, 416, "Range Not Satisfiable",
-                                           mapOf("Content-Range" to "bytes */$size"), "".byteInputStream())
-            }
-            val headers = mapOf("Cache-Control" to "no-store", "X-Content-Type-Options" to "nosniff",
-                                "Content-Range" to "bytes ${r.first}-${r.last}/$size",
-                                "Content-Length" to (r.last - r.first + 1).toString())
-            return WebResourceResponse(mime, null, 206, "Partial Content", headers, ByteRange.slice(raw, r))
-        }
+        if (range != null) return serveRange(raw, issued.pending.photo.size, mime, range)
         val digest = MessageDigest.getInstance("SHA-256")
         val stream = object : FilterInputStream(raw) {
             private var done = false
@@ -337,6 +367,19 @@ class PhotoChannels(
                 .setContentIntent(pending).setAutoCancel(true).build())
         }
 
+        /** `range` of `stream`, `size` bytes long, as a 206 (or a 416 for a range that cannot be served). */
+        fun serveRange(stream: InputStream, size: Long, mime: String, range: String?): WebResourceResponse {
+            val r = ByteRange.parse(range, size) ?: run {
+                stream.close()
+                return WebResourceResponse("text/plain", null, 416, "Range Not Satisfiable",
+                                           mapOf("Content-Range" to "bytes */$size"), "".byteInputStream())
+            }
+            val headers = mapOf("Cache-Control" to "no-store", "X-Content-Type-Options" to "nosniff",
+                                "Content-Range" to "bytes ${r.first}-${r.last}/$size",
+                                "Content-Length" to (r.last - r.first + 1).toString())
+            return WebResourceResponse(mime, null, 206, "Partial Content", headers, ByteRange.slice(stream, r))
+        }
+
         fun digestOf(s: InputStream): ByteArray {
             val d = MessageDigest.getInstance("SHA-256")
             val buf = ByteArray(64 * 1024)
diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoPlan.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoPlan.kt
index a8bf3b5..2cb85e6 100644
--- a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoPlan.kt
+++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoPlan.kt
@@ -23,6 +23,9 @@ data class Photo(
     val mime: String,
     /** From MediaStore's video collection rather than its images. */
     val video: Boolean = false,
+    /** Where it is on the phone, `DCIM/Camera/PXL_….jpg`, and its album's name, for the manifest. */
+    val relPath: String = "",
+    val album: String = "",
 )
 
 /**
diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoSource.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoSource.kt
index ce21965..b3b2834 100644
--- a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoSource.kt
+++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoSource.kt
@@ -112,13 +112,18 @@ class PhotoSource(private val context: Context) {
             val mime = c.getColumnIndexOrThrow(cols[8]); val where2 = c.getColumnIndexOrThrow(cols[9])
             while (c.moveToNext()) {
                 val bucketId = c.getString(bucket) ?: continue
+                val path = (c.getString(where2) ?: "").replace('\\', '/')
+                val display = c.getString(name) ?: ""
+                val album = c.getString(bucketName) ?: ""
+                // Where it was on the phone, from the top of its volume: what a
+                // restore puts it back to (the manifest, §9.12).
+                val relPath = if (Build.VERSION.SDK_INT >= 29) path.trim('/') + "/" + display else volumeRelative(path)
                 val photo = Photo(
-                    c.getLong(id), c.getString(name) ?: "", c.getLong(size),
+                    c.getLong(id), display, c.getLong(size),
                     if (c.isNull(taken)) 0 else c.getLong(taken), c.getLong(added), c.getLong(modified),
-                    bucketId, c.getString(mime) ?: "", video,
+                    bucketId, c.getString(mime) ?: "", video, relPath, album,
                 )
-                val path = (c.getString(where2) ?: "").replace('\\', '/')
-                each(photo, c.getString(bucketName) ?: "", isCamera(path))
+                each(photo, album, isCamera(path))
             }
         }
     }
diff --git a/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/ManifestLogTest.kt b/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/ManifestLogTest.kt
new file mode 100644
index 0000000..bb3a67f
--- /dev/null
+++ b/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/ManifestLogTest.kt
@@ -0,0 +1,52 @@
+package org.meshbay.client
+
+import org.json.JSONObject
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertFalse
+import org.junit.Assert.assertNull
+import org.junit.Assert.assertTrue
+import org.junit.Rule
+import org.junit.Test
+import org.junit.rules.TemporaryFolder
+import org.meshbay.client.phonesync.ManifestLog
+import java.io.File
+import java.util.TimeZone
+
+class ManifestLogTest {
+    @get:Rule val tmp = TemporaryFolder()
+
+    private fun line(n: Int) = JSONObject().put("node", "d/$n.jpg").put("source", "DCIM/Camera/$n.jpg")
+
+    private fun nodes(f: File) = f.readLines().filter { it.isNotBlank() }.map { JSONObject(it).getString("node") }
+
+    @Test fun `nothing waiting means nothing to send`() {
+        val log = ManifestLog(File(tmp.root, "m"))
+        assertFalse(log.waiting())
+        assertNull(log.issue())
+    }
+
+    @Test fun `what was sent is described once the node has the manifest, and only then forgotten`() {
+        val log = ManifestLog(File(tmp.root, "m"))
+        log.append(line(1)); log.append(line(2))
+        assertTrue(log.waiting())
+        assertEquals(listOf("d/1.jpg", "d/2.jpg"), nodes(log.issue()!!))
+        log.confirm()
+        assertFalse(log.waiting())
+        assertNull(log.issue())
+    }
+
+    @Test fun `a manifest that did not arrive goes again, with what came since`() {
+        val log = ManifestLog(File(tmp.root, "m"))
+        log.append(line(1))
+        log.issue()                      // sent, never confirmed: the run was cut short
+        log.append(line(2))
+        val again = ManifestLog(File(tmp.root, "m"))   // a restart in between
+        assertTrue(again.waiting())
+        assertEquals(listOf("d/1.jpg", "d/2.jpg"), nodes(again.issue()!!))
+    }
+
+    @Test fun `manifests are named for when they were written`() {
+        assertEquals("manifest-2026-10-09-120000.jsonl",
+                     ManifestLog.nameFor(1791547200000L, TimeZone.getTimeZone("UTC")))
+    }
+}