diff options
| author | Christophe Besson <cbesson@gmail.com> | 2026-10-10 16:15:48 +0200 |
|---|---|---|
| committer | Christophe Besson <cbesson@gmail.com> | 2026-10-10 16:15:48 +0200 |
| commit | 24547e4efd3a45b36b3bea8c29cfc687230a6d3a (patch) | |
| tree | 05aed558b665841230b0a1b4d2f987091c573ab8 | |
| parent | 1c97ffb91bb4928022e995709d7e55b19cfd18eb (diff) | |
| download | meshbay-24547e4efd3a45b36b3bea8c29cfc687230a6d3a.tar.gz | |
feat(android): send a manifest of what each backup run sent
Photos, videos and files record, per file, their path on the node, their path
and album on the phone, dates, size and SHA-256, uploaded as
meshbay-manifest/manifest-<date>.jsonl once the run is done.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
13 files changed, 315 insertions, 40 deletions
diff --git a/docs/MESHBAY_DESIGN.md b/docs/MESHBAY_DESIGN.md index bc6e20a..6025875 100644 --- a/docs/MESHBAY_DESIGN.md +++ b/docs/MESHBAY_DESIGN.md @@ -3435,9 +3435,24 @@ answered 206 by `ByteRange.kt`), so a video of gigabytes costs the WebView one chunk of memory; the SHA-256 the ledger keeps is then read from the phone once the node has the file. The Photos tab shows them in their albums (§9.9). -**Location.** The manifest does not ask for `ACCESS_MEDIA_LOCATION`, so a -photo or video read through MediaStore has its location **redacted by the -platform** (Android 10+): a photo's EXIF, and in an MP4 or MOV the location +**A manifest, for a restore or a merge later.** The node keeps the files; only +the phone knew where each came from. Every file the node takes adds a line to +a log the phone keeps (`ManifestLog.kt`): its path on the node, its path and +album on the phone (`DCIM/Camera/PXL_….jpg`, `Camera`), when it was taken and +last modified, its size, MIME type and the SHA-256 of the bytes sent. A run +that sent anything ends by uploading the waiting lines as +`<kind folder>/meshbay-manifest/manifest-<date>.jsonl`; the phone forgets them +only once the node has that file, so a run cut short sends them with the next +one. Restoring puts each photo back into its album and each file back under +its own name and date; merging onto a phone that already has some of them +compares hashes without downloading anything. Photos, videos and files have +one; contacts, calendars and messages are each one file that describes +itself. What the manifest cannot bring back is a photo's location, removed +before it left the phone. + +**Location.** The application's Android manifest does not ask for +`ACCESS_MEDIA_LOCATION`, so a photo or video read through MediaStore has its +location **redacted by the platform** (Android 10+): a photo's EXIF, and in an MP4 or MOV the location boxes MediaProvider finds (`IsoInterface`). A camera roll going to a group does not say where its owner lives, and nobody had to do anything for it. On Android 8 and 9 nothing redacts it. diff --git a/docs/USERGUIDE.md b/docs/USERGUIDE.md index 207e57e..dc57f27 100644 --- a/docs/USERGUIDE.md +++ b/docs/USERGUIDE.md @@ -479,6 +479,11 @@ backup at all. earlier copy, with `-modified-` and the date in its name; nothing is ever replaced, and a file you delete on the phone stays in the group. +Beside your photos and files, a `meshbay-manifest` folder holds a small file +per backup run that records where each one came from on the phone (its +album or folder, its original name and dates). It is what will let a restore +put everything back where it was. Leave it in place. + If a backup cannot go on (the node's disk is full, the folder no longer accepts files, somebody else joined the group), it stops, says why once in a notification and on the Android Sync page, and tries again the next day. diff --git a/packages/meshbay-android/app/src/main/assets/bridge/meshbay-bridge.js b/packages/meshbay-android/app/src/main/assets/bridge/meshbay-bridge.js index 32cc4ab..d8237ec 100644 --- a/packages/meshbay-android/app/src/main/assets/bridge/meshbay-bridge.js +++ b/packages/meshbay-android/app/src/main/assets/bridge/meshbay-bridge.js @@ -213,6 +213,8 @@ estimate: (settings) => call('photosync:estimate', settings), plan: () => call('photosync:plan'), sent: (token, dir, name) => call('photosync:sent', token, dir, name), + manifest: () => call('photosync:manifest'), + manifestSent: (token) => call('photosync:manifest-sent', token), completed: () => call('photosync:completed'), failed: (code, text) => call('photosync:failed', code, text), keepAlive: (on, text) => call('photosync:keep-alive', on === true, text || ''), @@ -259,6 +261,8 @@ configure: (settings) => call('drivesync:configure', settings || null), plan: () => call('drivesync:plan'), sent: (token, dir, name) => call('drivesync:sent', token, dir, name), + manifest: () => call('drivesync:manifest'), + manifestSent: (token) => call('drivesync:manifest-sent', token), completed: () => call('drivesync:completed'), failed: (code, text) => call('drivesync:failed', code, text), keepAlive: (on, text) => call('drivesync:keep-alive', on === true, text || ''), diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveChannels.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveChannels.kt index 4328774..3f87359 100644 --- a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveChannels.kt +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveChannels.kt @@ -9,9 +9,10 @@ import android.webkit.WebResourceResponse import org.json.JSONArray import org.json.JSONObject import org.meshbay.client.bridge.Refused +import org.meshbay.client.phonesync.Destination import org.meshbay.client.phonesync.DestinationChannels +import org.meshbay.client.phonesync.ManifestLog import org.meshbay.client.phonesync.DocPlan -import org.meshbay.client.photos.ByteRange import org.meshbay.client.photos.PhotoChannels import org.meshbay.client.shell.Pickers import java.io.File @@ -61,6 +62,8 @@ class DriveChannels( "drivesync:configure" -> { configure(args.optJSONObject(0)); status() } "drivesync:plan" -> plan() "drivesync:sent" -> { sent(args.optString(0, ""), args.optString(1, ""), args.optString(2, "")); true } + "drivesync:manifest" -> manifest() + "drivesync:manifest-sent" -> { manifestSent(args.optString(0, "")); true } "drivesync:completed" -> { completed(); status() } "drivesync:failed" -> failed(args.optString(0, ""), args.optString(1, "")) "drivesync:keep-alive" -> { onKeepAlive(args.optBoolean(0, false), args.optString(1, "").take(200)); true } @@ -91,10 +94,15 @@ class DriveChannels( } } - private fun base(d: org.meshbay.client.phonesync.Destination) = DocPlan.dirFor(d, SUFFIX) + private fun base(d: Destination) = DocPlan.dirFor(d, SUFFIX) - private fun ledger(d: org.meshbay.client.phonesync.Destination) = - DriveLedger(File(dir, PhotoChannels.hex(PhotoChannels.sha256Of(d.ledgerKey.toByteArray())).take(32) + ".jsonl")) + private fun keyOf(d: Destination) = PhotoChannels.hex(PhotoChannels.sha256Of(d.ledgerKey.toByteArray())).take(32) + + private fun ledger(d: Destination) = DriveLedger(File(dir, keyOf(d) + ".jsonl")) + + private fun manifestLog(d: Destination) = ManifestLog(File(dir, keyOf(d) + ".manifest")) + + @Volatile private var manifestToken: Pair<String, File>? = null fun status(): JSONObject { val d = destinations.get() @@ -175,7 +183,7 @@ class DriveChannels( out.put(JSONObject().put("token", token).put("name", p.name).put("dir", p.dir) .put("size", p.file.size).put("edited", p.edited)) } - return JSONObject().put("items", out) + return JSONObject().put("items", out).put("manifest", manifestLog(d).waiting()) } /** The node took it (or already had it): into the ledger, hashed from the phone now. */ @@ -184,23 +192,45 @@ class DriveChannels( val d = destinations.get()?.takeIf { it.ledgerKey == issued.key } ?: throw Refused("Refused: the backup changed") val f = issued.pending.file val sha = hashOf(f) ?: throw Refused("Refused: the file is gone") - ledger(d).record(DriveLedger.Entry(f.docId, f.modified, f.size, sha, dir.take(1024), name.take(256), - System.currentTimeMillis())) + val now = System.currentTimeMillis() + ledger(d).record(DriveLedger.Entry(f.docId, f.modified, f.size, sha, dir.take(1024), name.take(256), now)) + manifestLog(d).append(JSONObject() + .put("kind", "file") + .put("node", "${dir.take(1024)}/${name.take(256)}") + .put("source", DrivePlan.pathOf(f.docId)).put("name", f.name) + .put("folder", folders().firstOrNull { it.uri == f.tree }?.name ?: "") + .put("modified", f.modified).put("size", f.size).put("sha256", sha).put("mime", f.mime) + .put("edited", issued.pending.edited).put("sentAt", now)) tokens.remove(token) } + /** The manifest of what was sent and not yet described on the node (ManifestLog), or `item: null`. */ + private fun manifest(): JSONObject { + val d = destinations.get()?.takeIf { prefs.getBoolean(ON, false) } ?: throw Refused("Refused: files backup is off") + val file = manifestLog(d).issue() ?: return JSONObject().put("item", JSONObject.NULL) + val token = PhotoChannels.hex(ByteArray(16).also { random.nextBytes(it) }) + manifestToken = token to file + return JSONObject().put("item", JSONObject().put("token", token) + .put("name", ManifestLog.nameFor(System.currentTimeMillis(), TimeZone.getDefault())) + .put("dir", base(d) + "/" + ManifestLog.DIR).put("size", file.length())) + } + + private fun manifestSent(token: String) { + if (manifestToken?.first != token) throw Refused("Refused: unknown manifest") + val d = destinations.get() ?: throw Refused("Refused: the backup changed") + manifestLog(d).confirm() + manifestToken = null + } + /** A range of an issued file, for `/drivesync/<token>` on the packaged origin. */ fun serve(path: String, range: String?): WebResourceResponse? { + manifestToken?.takeIf { it.first == path.removePrefix(PATH) }?.let { (_, file) -> + return PhotoChannels.serveRange(file.inputStream(), file.length(), "application/x-ndjson", range) + } val issued = tokens[path.removePrefix(PATH)] ?: return null val f = issued.pending.file - val r = ByteRange.parse(range, f.size) ?: return WebResourceResponse("text/plain", null, 416, - "Range Not Satisfiable", mapOf("Content-Range" to "bytes */${f.size}"), "".byteInputStream()) val raw = try { source.open(f) } catch (e: Exception) { null } ?: return null - val headers = mapOf("Cache-Control" to "no-store", "X-Content-Type-Options" to "nosniff", - "Content-Range" to "bytes ${r.first}-${r.last}/${f.size}", - "Content-Length" to (r.last - r.first + 1).toString()) - return WebResourceResponse(f.mime.ifEmpty { "application/octet-stream" }, null, 206, "Partial Content", - headers, ByteRange.slice(raw, r)) + return PhotoChannels.serveRange(raw, f.size, f.mime.ifEmpty { "application/octet-stream" }, range) } private fun hashOf(f: DriveFile): String? = try { diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/phonesync/ManifestLog.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/phonesync/ManifestLog.kt new file mode 100644 index 0000000..0d7142c --- /dev/null +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/phonesync/ManifestLog.kt @@ -0,0 +1,56 @@ +package org.meshbay.client.phonesync + +import org.json.JSONObject +import java.io.File +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Locale +import java.util.TimeZone + +/** + * What a backup sent, as the phone knew it, for a restore or a merge later + * (docs/MESHBAY_DESIGN.md §9.12): one JSON line per file the node took, with + * where it came from on the phone, its album or folder, its dates, size and + * SHA-256. The node has the files; only the phone knew these. + * + * Lines wait here until a manifest carrying them is on the node: `issue()` + * gathers them into the file to send, `confirm()` forgets them once the node + * has it. A manifest that did not arrive is sent again, with whatever was + * added since, at the next run: nothing is lost to an interrupted one. + */ +class ManifestLog(private val pending: File) { + private val sending = File(pending.path + ".sending") + + fun append(line: JSONObject) { + pending.parentFile?.mkdirs() + pending.appendText(line.toString() + "\n") + } + + /** True when there are lines no manifest on the node holds yet. */ + fun waiting(): Boolean = (sending.exists() && sending.length() > 0) || (pending.exists() && pending.length() > 0) + + /** The file to send now, holding every waiting line, or null. */ + fun issue(): File? { + if (pending.exists() && pending.length() > 0) { + if (sending.exists()) { sending.appendText(pending.readText()); pending.delete() } + else if (!pending.renameTo(sending)) return null + } + return sending.takeIf { it.exists() && it.length() > 0 } + } + + /** The node has it: those lines are done. */ + fun confirm() { sending.delete() } + + /** Everything forgotten: the destination changed, or the backup was turned off. */ + fun clear() { pending.delete(); sending.delete() } + + companion object { + /** The folder manifests go into, under a kind's own folder. */ + const val DIR = "meshbay-manifest" + + /** `manifest-2026-10-10-143205.jsonl`, in the phone's time: never one name twice. */ + fun nameFor(now: Long, zone: TimeZone): String = + "manifest-" + SimpleDateFormat("yyyy-MM-dd-HHmmss", Locale.ROOT).apply { timeZone = zone }.format(Date(now)) + + ".jsonl" + } +} diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoChannels.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoChannels.kt index 422103b..2e7c519 100644 --- a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoChannels.kt +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoChannels.kt @@ -22,6 +22,7 @@ import org.meshbay.client.bridge.Refused import org.meshbay.client.notify.Notifier import org.meshbay.client.phonesync.Destination import org.meshbay.client.phonesync.DestinationChannels +import org.meshbay.client.phonesync.ManifestLog import java.io.File import java.io.FilterInputStream import java.io.InputStream @@ -81,6 +82,8 @@ class PhotoChannels( "photosync:estimate" -> { requirePermission(); estimate(args.optJSONObject(0) ?: throw Refused("Refused: no settings")) } "photosync:plan" -> { requirePermission(); plan() } "photosync:sent" -> { sent(args.optString(0, ""), args.optString(1, ""), args.optString(2, "")); true } + "photosync:manifest" -> manifest() + "photosync:manifest-sent" -> { manifestSent(args.optString(0, "")); true } "photosync:completed" -> { completed(); status() } "photosync:failed" -> failed(args.optString(0, ""), args.optString(1, "")) "photosync:keep-alive" -> { onKeepAlive(args.optBoolean(0, false), args.optString(1, "").take(200)); true } @@ -97,7 +100,13 @@ class PhotoChannels( return (destinations.get() ?: return null) to c } - private fun ledger(d: Destination) = PhotoLedger(File(dir, hex(sha256Of(d.ledgerKey.toByteArray())).take(32) + ".jsonl")) + private fun keyOf(d: Destination) = hex(sha256Of(d.ledgerKey.toByteArray())).take(32) + + private fun ledger(d: Destination) = PhotoLedger(File(dir, keyOf(d) + ".jsonl")) + + private fun manifestLog(d: Destination) = ManifestLog(File(dir, keyOf(d) + ".manifest")) + + @Volatile private var manifestToken: Pair<String, File>? = null fun status(): JSONObject { val c = config() @@ -191,7 +200,7 @@ class PhotoChannels( // folder for what is already there — an edit under its own name too. .put("alsoKnownAs", PhotoPlan.editedName(p.photo, java.util.TimeZone.getDefault()))) } - return JSONObject().put("items", out) + return JSONObject().put("items", out).put("manifest", manifestLog(d).waiting()) } /** The node took it (or already had it): into the ledger, under the name its ack gave. */ @@ -200,12 +209,41 @@ class PhotoChannels( val d = active()?.first?.takeIf { it.ledgerKey == issued.key } ?: throw Refused("Refused: the backup changed") val p = issued.pending.photo val sha = issued.sha256 ?: hashOf(p) ?: throw Refused("Refused: the photo is gone") + val now = System.currentTimeMillis() ledger(d).record(PhotoLedger.Entry(p.mediaId, p.modified, p.size, sha, - dir.take(1024), name.take(256), System.currentTimeMillis())) + dir.take(1024), name.take(256), now)) + manifestLog(d).append(JSONObject() + .put("kind", if (p.video) "video" else "photo") + .put("node", "${dir.take(1024)}/${name.take(256)}") + .put("source", p.relPath).put("album", p.album) + .put("taken", PhotoPlan.whenTaken(p)).put("modified", p.modified * 1000) + .put("size", p.size).put("sha256", sha).put("mime", p.mime) + .put("edited", issued.pending.edited).put("sentAt", now)) tokens.remove(token) } /** + * The manifest of what was sent and not yet described on the node, for + * `<base>/meshbay-manifest/` (ManifestLog), or `item: null`. + */ + private fun manifest(): JSONObject { + val (d, _) = active() ?: throw Refused("Refused: photo backup is off") + val file = manifestLog(d).issue() ?: return JSONObject().put("item", JSONObject.NULL) + val token = hex(ByteArray(16).also { random.nextBytes(it) }) + manifestToken = token to file + return JSONObject().put("item", JSONObject().put("token", token) + .put("name", ManifestLog.nameFor(System.currentTimeMillis(), java.util.TimeZone.getDefault())) + .put("dir", PhotoPlan.baseFor(d) + "/" + ManifestLog.DIR).put("size", file.length())) + } + + private fun manifestSent(token: String) { + if (manifestToken?.first != token) throw Refused("Refused: unknown manifest") + val d = destinations.get() ?: throw Refused("Refused: the backup changed") + manifestLog(d).confirm() + manifestToken = null + } + + /** * The bytes of an issued photo or video, for `/photosync/<token>` on the * packaged origin. Asked a range at a time (ByteRange), as the upload * reads them, so a video never sits whole in the page; the ledger's hash @@ -213,21 +251,13 @@ class PhotoChannels( * as it goes out. */ fun serve(path: String, range: String? = null): WebResourceResponse? { + manifestToken?.takeIf { it.first == path.removePrefix(PATH) }?.let { (_, file) -> + return serveRange(file.inputStream(), file.length(), "application/x-ndjson", range) + } val issued = tokens[path.removePrefix(PATH)] ?: return null val raw = try { source.open(issued.pending.photo) } catch (e: Exception) { null } ?: return null val mime = issued.pending.photo.mime.ifEmpty { "application/octet-stream" } - if (range != null) { - val size = issued.pending.photo.size - val r = ByteRange.parse(range, size) ?: run { - raw.close() - return WebResourceResponse("text/plain", null, 416, "Range Not Satisfiable", - mapOf("Content-Range" to "bytes */$size"), "".byteInputStream()) - } - val headers = mapOf("Cache-Control" to "no-store", "X-Content-Type-Options" to "nosniff", - "Content-Range" to "bytes ${r.first}-${r.last}/$size", - "Content-Length" to (r.last - r.first + 1).toString()) - return WebResourceResponse(mime, null, 206, "Partial Content", headers, ByteRange.slice(raw, r)) - } + if (range != null) return serveRange(raw, issued.pending.photo.size, mime, range) val digest = MessageDigest.getInstance("SHA-256") val stream = object : FilterInputStream(raw) { private var done = false @@ -337,6 +367,19 @@ class PhotoChannels( .setContentIntent(pending).setAutoCancel(true).build()) } + /** `range` of `stream`, `size` bytes long, as a 206 (or a 416 for a range that cannot be served). */ + fun serveRange(stream: InputStream, size: Long, mime: String, range: String?): WebResourceResponse { + val r = ByteRange.parse(range, size) ?: run { + stream.close() + return WebResourceResponse("text/plain", null, 416, "Range Not Satisfiable", + mapOf("Content-Range" to "bytes */$size"), "".byteInputStream()) + } + val headers = mapOf("Cache-Control" to "no-store", "X-Content-Type-Options" to "nosniff", + "Content-Range" to "bytes ${r.first}-${r.last}/$size", + "Content-Length" to (r.last - r.first + 1).toString()) + return WebResourceResponse(mime, null, 206, "Partial Content", headers, ByteRange.slice(stream, r)) + } + fun digestOf(s: InputStream): ByteArray { val d = MessageDigest.getInstance("SHA-256") val buf = ByteArray(64 * 1024) diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoPlan.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoPlan.kt index a8bf3b5..2cb85e6 100644 --- a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoPlan.kt +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoPlan.kt @@ -23,6 +23,9 @@ data class Photo( val mime: String, /** From MediaStore's video collection rather than its images. */ val video: Boolean = false, + /** Where it is on the phone, `DCIM/Camera/PXL_….jpg`, and its album's name, for the manifest. */ + val relPath: String = "", + val album: String = "", ) /** diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoSource.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoSource.kt index ce21965..b3b2834 100644 --- a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoSource.kt +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoSource.kt @@ -112,13 +112,18 @@ class PhotoSource(private val context: Context) { val mime = c.getColumnIndexOrThrow(cols[8]); val where2 = c.getColumnIndexOrThrow(cols[9]) while (c.moveToNext()) { val bucketId = c.getString(bucket) ?: continue + val path = (c.getString(where2) ?: "").replace('\\', '/') + val display = c.getString(name) ?: "" + val album = c.getString(bucketName) ?: "" + // Where it was on the phone, from the top of its volume: what a + // restore puts it back to (the manifest, §9.12). + val relPath = if (Build.VERSION.SDK_INT >= 29) path.trim('/') + "/" + display else volumeRelative(path) val photo = Photo( - c.getLong(id), c.getString(name) ?: "", c.getLong(size), + c.getLong(id), display, c.getLong(size), if (c.isNull(taken)) 0 else c.getLong(taken), c.getLong(added), c.getLong(modified), - bucketId, c.getString(mime) ?: "", video, + bucketId, c.getString(mime) ?: "", video, relPath, album, ) - val path = (c.getString(where2) ?: "").replace('\\', '/') - each(photo, c.getString(bucketName) ?: "", isCamera(path)) + each(photo, album, isCamera(path)) } } } diff --git a/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/ManifestLogTest.kt b/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/ManifestLogTest.kt new file mode 100644 index 0000000..bb3a67f --- /dev/null +++ b/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/ManifestLogTest.kt @@ -0,0 +1,52 @@ +package org.meshbay.client + +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.meshbay.client.phonesync.ManifestLog +import java.io.File +import java.util.TimeZone + +class ManifestLogTest { + @get:Rule val tmp = TemporaryFolder() + + private fun line(n: Int) = JSONObject().put("node", "d/$n.jpg").put("source", "DCIM/Camera/$n.jpg") + + private fun nodes(f: File) = f.readLines().filter { it.isNotBlank() }.map { JSONObject(it).getString("node") } + + @Test fun `nothing waiting means nothing to send`() { + val log = ManifestLog(File(tmp.root, "m")) + assertFalse(log.waiting()) + assertNull(log.issue()) + } + + @Test fun `what was sent is described once the node has the manifest, and only then forgotten`() { + val log = ManifestLog(File(tmp.root, "m")) + log.append(line(1)); log.append(line(2)) + assertTrue(log.waiting()) + assertEquals(listOf("d/1.jpg", "d/2.jpg"), nodes(log.issue()!!)) + log.confirm() + assertFalse(log.waiting()) + assertNull(log.issue()) + } + + @Test fun `a manifest that did not arrive goes again, with what came since`() { + val log = ManifestLog(File(tmp.root, "m")) + log.append(line(1)) + log.issue() // sent, never confirmed: the run was cut short + log.append(line(2)) + val again = ManifestLog(File(tmp.root, "m")) // a restart in between + assertTrue(again.waiting()) + assertEquals(listOf("d/1.jpg", "d/2.jpg"), nodes(again.issue()!!)) + } + + @Test fun `manifests are named for when they were written`() { + assertEquals("manifest-2026-10-09-120000.jsonl", + ManifestLog.nameFor(1791547200000L, TimeZone.getTimeZone("UTC"))) + } +} diff --git a/packages/meshbay-hub/src/meshbay_hub/static/photo-sync.js b/packages/meshbay-hub/src/meshbay_hub/static/photo-sync.js index 1439c20..a9d4517 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/photo-sync.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/photo-sync.js @@ -172,8 +172,8 @@ export function createBackup({ if (refused === 'offline') { setState({ phase: 'offline' }); return; } if (refused) { await lasting(refused); return; } - const { items } = await sync.plan(); - if (!items.length) { + const { items, manifest } = await sync.plan(); + if (!items.length && !manifest) { await sync.completed(); setState({ phase: 'up_to_date', done: 0, total: 0 }); return; @@ -246,10 +246,29 @@ export function createBackup({ done += 1; await sync.keepAlive(true, report()); } + if (!await sendManifest(transport, dest, dirs)) return; await sync.completed(); setState({ phase: 'up_to_date', done, total: items.length }); } + // What the phone knew of what was sent (its path and album, its dates and + // hash), for a restore or a merge later: one file per run that sent + // anything, beside the files, kept by the phone until the node has it. + // False when a lasting refusal stopped it, which is then said. + async function sendManifest(transport, dest, dirs) { + const { item } = await sync.manifest(); + if (!item) return true; + try { + await makeDirs(transport, dest.folder, item.dir, dirs); + await upload(transport, item, sync.file); + } catch (e) { + if (e.code && failures[e.code]) { await lasting(e.code); return false; } + throw e; + } + await sync.manifestSent(item.token); + return true; + } + async function lasting(code) { setState({ phase: 'failed', reason: code }); await sync.failed(code, say(code)).catch(() => {}); diff --git a/packages/meshbay-hub/src/meshbay_hub/static/platform.js b/packages/meshbay-hub/src/meshbay_hub/static/platform.js index 04e26e6..b1a8ae9 100644 --- a/packages/meshbay-hub/src/meshbay_hub/static/platform.js +++ b/packages/meshbay-hub/src/meshbay_hub/static/platform.js @@ -244,6 +244,8 @@ export const photoSync = { estimate(settings) { return bridge.photoSync.estimate(settings); }, plan() { return bridge.photoSync.plan(); }, sent(token, dir, name) { return bridge.photoSync.sent(token, dir, name); }, + manifest() { return bridge.photoSync.manifest(); }, + manifestSent(token) { return bridge.photoSync.manifestSent(token); }, completed() { return bridge.photoSync.completed(); }, failed(code, text) { return bridge.photoSync.failed(code, text); }, keepAlive(on, text) { return bridge.photoSync.keepAlive(on, text); }, @@ -285,6 +287,8 @@ export const driveSync = { configure(settings) { return driveBridge.configure(settings); }, plan() { return driveBridge.plan(); }, sent(token, dir, name) { return driveBridge.sent(token, dir, name); }, + manifest() { return driveBridge.manifest(); }, + manifestSent(token) { return driveBridge.manifestSent(token); }, completed() { return driveBridge.completed(); }, failed(code, text) { return driveBridge.failed(code, text); }, keepAlive(on, text) { return driveBridge.keepAlive(on, text); }, diff --git a/packages/meshbay-hub/tests/test_drive_sync.py b/packages/meshbay-hub/tests/test_drive_sync.py index d46ac66..7fc35d2 100644 --- a/packages/meshbay-hub/tests/test_drive_sync.py +++ b/packages/meshbay-hub/tests/test_drive_sync.py @@ -75,7 +75,9 @@ fake.transport = { const hubFetch = async () => ({ admin_id: 'u-bob', members: fake.members, invited: [] }); win.meshbay = { driveSync: { status: async () => fake.status, - plan: async () => { fake.log.push(['plan']); return { items: fake.items }; }, + plan: async () => { fake.log.push(['plan']); return { items: fake.items, manifest: false }; }, + manifest: async () => ({ item: null }), + manifestSent: async () => true, sent: async (token, dir, name) => { fake.log.push(['sent', token, dir, name]); return true; }, completed: async () => { fake.log.push(['completed']); fake.status.lastCompleted = Date.now(); return fake.status; }, failed: async (code, text) => { fake.log.push(['failed', code, text]); return true; }, diff --git a/packages/meshbay-hub/tests/test_photo_sync.py b/packages/meshbay-hub/tests/test_photo_sync.py index c422f22..266a111 100644 --- a/packages/meshbay-hub/tests/test_photo_sync.py +++ b/packages/meshbay-hub/tests/test_photo_sync.py @@ -112,7 +112,9 @@ fake.transport = { }; win.meshbay = { photoSync: { status: async () => fake.status, - plan: async () => { fake.log.push(['plan']); return { items: fake.items }; }, + plan: async () => { fake.log.push(['plan']); return { items: fake.items, manifest: !!fake.manifest }; }, + manifest: async () => ({ item: fake.manifest || null }), + manifestSent: async (token) => { fake.log.push(['manifest-sent', token]); fake.manifest = null; return true; }, sent: async (token, dir, name) => { fake.log.push(['sent', token, dir, name]); return true; }, completed: async () => { fake.log.push(['completed']); @@ -310,6 +312,41 @@ def test_another_account_on_the_phone_sends_nothing(tmp_path): assert ["plan"] not in log +# ── the manifest ──────────────────────────────────────────────────────────── + +MANIFEST = """ +fake.manifest = { token: 'mf', name: 'manifest-2026-10-10-090000.jsonl', + dir: 'Media/Photos/Bob/bob-photos/meshbay-manifest', size: 7 }; +""" + + +def test_a_run_ends_with_the_manifest_of_what_it_sent(tmp_path): + log = _run(_start(MANIFEST), tmp_path) + uploads = _of(log, "upload") + assert uploads[-1][:2] == ["Media/Photos/Bob/bob-photos/meshbay-manifest", + "manifest-2026-10-10-090000.jsonl"] + assert ["mkdir", "Media/Photos/Bob/bob-photos/meshbay-manifest"] in log + assert log.index(["manifest-sent", "mf"]) < log.index(["completed"]) + + +def test_a_manifest_left_by_an_interrupted_run_goes_on_its_own(tmp_path): + log = _run(_start(MANIFEST + "fake.items = [];"), tmp_path) + assert [u[1] for u in _of(log, "upload")] == ["manifest-2026-10-10-090000.jsonl"] + assert ["manifest-sent", "mf"] in log and ["completed"] in log + + +def test_a_manifest_the_node_refused_is_kept_for_the_next_run(tmp_path): + log = _run(_start(MANIFEST + """ +const real = fake.transport.uploadFile; +fake.transport.uploadFile = async (file, opts) => { + if (file.name.startsWith('manifest-')) { const e = new Error('full'); e.code = 'disk_full'; throw e; } + return real(file, opts); +};"""), tmp_path) + assert _of(log, "manifest-sent") == [] + assert _of(log, "failed") == [["disk_full"]] + assert ["completed"] not in log + + # ── the account's own group, at every run ─────────────────────────────────── @pytest.mark.parametrize("setup", [ |