diff options
| author | Christophe Besson <cbesson@gmail.com> | 2026-10-10 15:50:39 +0200 |
|---|---|---|
| committer | Christophe Besson <cbesson@gmail.com> | 2026-10-10 15:50:39 +0200 |
| commit | 1c97ffb91bb4928022e995709d7e55b19cfd18eb (patch) | |
| tree | aaa05d17adcd6db4256c01ba79b27dfc7ec96d70 /packages/meshbay-android | |
| parent | 1521cc1ac0cca7ee8786056f84fa399d5cc268f9 (diff) | |
| download | meshbay-1c97ffb91bb4928022e995709d7e55b19cfd18eb.tar.gz | |
feat(android): back up the files of folders the person chooses
A Files section takes folders through the system picker (no storage
permission), refuses DCIM, Pictures and Movies, skips what the photo backup
sends, and runs on the photo backup's own runner into <folder>/<account>-drive.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Diffstat (limited to 'packages/meshbay-android')
10 files changed, 630 insertions, 11 deletions
diff --git a/packages/meshbay-android/app/src/main/assets/bridge/meshbay-bridge.js b/packages/meshbay-android/app/src/main/assets/bridge/meshbay-bridge.js index 80e0e34..32cc4ab 100644 --- a/packages/meshbay-android/app/src/main/assets/bridge/meshbay-bridge.js +++ b/packages/meshbay-android/app/src/main/assets/bridge/meshbay-bridge.js @@ -249,6 +249,20 @@ completed: () => call('calendarsync:completed'), failed: (code, text) => call('calendarsync:failed', code, text), }, + + // Files backup (§9.13): folders the person chose through the system's + // picker, sent like photos, a range at a time from /drivesync/<token>. + driveSync: { + status: () => call('drivesync:status'), + addFolder: () => call('drivesync:add-folder'), + removeFolder: (id) => call('drivesync:remove-folder', id), + configure: (settings) => call('drivesync:configure', settings || null), + plan: () => call('drivesync:plan'), + sent: (token, dir, name) => call('drivesync:sent', token, dir, name), + completed: () => call('drivesync:completed'), + failed: (code, text) => call('drivesync:failed', code, text), + keepAlive: (on, text) => call('drivesync:keep-alive', on === true, text || ''), + }, } : {}), // Messages backup (§9.13), the same way as contacts: the messages added diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/MainActivity.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/MainActivity.kt index 0382534..7c0e79e 100644 --- a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/MainActivity.kt +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/MainActivity.kt @@ -36,6 +36,7 @@ import org.meshbay.client.notify.Notifier import org.meshbay.client.notify.PushChannels import org.meshbay.client.notify.PushState import org.meshbay.client.photos.BackupService +import org.meshbay.client.drive.DriveChannels import org.meshbay.client.phonesync.CalendarSource import org.meshbay.client.phonesync.ContactSource import org.meshbay.client.phonesync.DestinationChannels @@ -66,6 +67,7 @@ class MainActivity : Activity() { private lateinit var channels: Channels private lateinit var photos: PhotoChannels private var docs: List<DocChannels> = emptyList() + private var drive: DriveChannels? = null /** Backups exist in the personal profile only (Profiles.kt). */ private var backups = false private var network: android.net.ConnectivityManager.NetworkCallback? = null @@ -76,7 +78,9 @@ class MainActivity : Activity() { private var shim: ScriptHandler? = null private var casting = false private var playing = false - private var syncing = false + /** Which backups hold the keep-alive now ("photos", "drive"): one stopping leaves the other's. */ + private val syncers = HashSet<String>() + private val syncing get() = syncers.isNotEmpty() private var fullscreen: View? = null private var fullscreenCallback: WebChromeClient.CustomViewCallback? = null private var pendingLink: String? = null @@ -109,7 +113,12 @@ class MainActivity : Activity() { val destinations = DestinationChannels(getSharedPreferences(DestinationChannels.PREFS, Context.MODE_PRIVATE)) photos = PhotoChannels(this, getSharedPreferences(PhotoChannels.PREFS, Context.MODE_PRIVATE), destinations, java.io.File(filesDir, "photosync"), - onKeepAlive = { on, line -> runOnUiThread { backup(on, line) } }) + onKeepAlive = { on, line -> runOnUiThread { backup("photos", on, line) } }) + drive = if (!backups) null else DriveChannels(this, + getSharedPreferences(DriveChannels.PREFS, Context.MODE_PRIVATE), java.io.File(filesDir, "drivesync"), + destinations, pickers, photosBackedUp = { photos.backedUpPaths() }, + unmetered = { photos.unmetered() }, + onKeepAlive = { on, line -> runOnUiThread { backup("drive", on, line) } }) docs = if (!backups) emptyList() else listOf( DocChannels("contactsync", this, getSharedPreferences("contactsync", Context.MODE_PRIVATE), destinations, java.io.File(cacheDir, "contactsync"), ContactSource(this), @@ -130,7 +139,7 @@ class MainActivity : Activity() { Notifier.CHANNEL_CHAT to text.get("push.channel_chat", channels.locale), Notifier.CHANNEL_OTHER to text.get("push.channel_other", channels.locale)) }), photos = photos.takeIf { backups }, destinations = destinations.takeIf { backups }, - docs = docs) + docs = docs, drive = drive) WebViewCompat.addWebMessageListener(web, Bridge.PORT, setOf(UiAssets.ORIGIN), Bridge(channels)) cast.control.warmUp() installShim() @@ -179,6 +188,10 @@ class MainActivity : Activity() { val range = request.requestHeaders.entries.firstOrNull { it.key.equals("Range", true) }?.value return photos.takeIf { backups }?.serve(url.path ?: "", range) ?: refused() } + if (url.host == UiAssets.HOST && url.path?.startsWith(DriveChannels.PATH) == true) { + val range = request.requestHeaders.entries.firstOrNull { it.key.equals("Range", true) }?.value + return drive?.serve(url.path ?: "", range) ?: refused() + } if (url.host == UiAssets.HOST && url.path?.startsWith(DocChannels.PATH) == true) { val path = url.path ?: "" return docs.firstNotNullOfOrNull { it.serve(path) } ?: refused() @@ -318,15 +331,17 @@ class MainActivity : Activity() { } /** - * A photo backup running: its own foreground service, and the page kept - * visible like a cast. Started once; afterwards only its line changes, - * which needs no start — refused from the background on Android 12+. + * A photo or files backup running: its own foreground service, and the + * page kept visible like a cast. Started once; afterwards only its line + * changes, which needs no start (refused from the background on Android + * 12+). Held while any backup holds it. */ - private fun backup(on: Boolean, line: String) { + private fun backup(who: String, on: Boolean, line: String) { val service = Intent(this, BackupService::class.java) - if (on && syncing) { BackupService.update(this, line); return } - if (on == syncing) return - syncing = on + val was = syncing + if (on) syncers += who else syncers -= who + if (on && was) { BackupService.update(this, line); return } + if (syncing == was) return if (on) { try { startForegroundService(service.putExtra(BackupService.EXTRA_TEXT, line)) } catch (e: IllegalStateException) { Log.w(Bridge.TAG, "backup keep-alive refused: $e") } @@ -428,7 +443,7 @@ class MainActivity : Activity() { override fun onDestroy() { if (::cast.isInitialized && cast.relay.active) cast.relay.stop() if (casting || playing) { casting = false; playing = false; keepAlive() } - if (syncing) backup(false, "") + if (syncing) { syncers.clear(); stopService(Intent(this, BackupService::class.java)); keepAlive() } network?.let { try { getSystemService(android.net.ConnectivityManager::class.java).unregisterNetworkCallback(it) } catch (e: Exception) {} } if (::web.isInitialized) { root.removeView(web); web.destroy() } super.onDestroy() diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/bridge/Channels.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/bridge/Channels.kt index 512aa13..9d180c0 100644 --- a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/bridge/Channels.kt +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/bridge/Channels.kt @@ -4,6 +4,7 @@ import org.json.JSONArray import org.meshbay.client.cast.CastChannels import org.meshbay.client.hub.HubClient import org.meshbay.client.notify.PushChannels +import org.meshbay.client.drive.DriveChannels import org.meshbay.client.phonesync.DestinationChannels import org.meshbay.client.phonesync.DocChannels import org.meshbay.client.photos.PhotoChannels @@ -34,6 +35,7 @@ class Channels( private val photos: PhotoChannels? = null, private val destinations: DestinationChannels? = null, private val docs: List<DocChannels> = emptyList(), + private val drive: DriveChannels? = null, ) { @Volatile var locale = "en" private set @@ -64,6 +66,7 @@ class Channels( push != null && push.handles(channel) -> push.call(channel, args) photos != null && photos.handles(channel) -> photos.call(channel, args) destinations != null && destinations.handles(channel) -> destinations.call(channel, args) + drive != null && drive.handles(channel) -> drive.call(channel, args) else -> docs.firstOrNull { it.handles(channel) }?.call(channel, args) ?: throw Refused("Refused: no such channel") } diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveChannels.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveChannels.kt new file mode 100644 index 0000000..4328774 --- /dev/null +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveChannels.kt @@ -0,0 +1,222 @@ +package org.meshbay.client.drive + +import android.app.Activity +import android.content.Intent +import android.content.SharedPreferences +import android.net.Uri +import android.provider.DocumentsContract +import android.webkit.WebResourceResponse +import org.json.JSONArray +import org.json.JSONObject +import org.meshbay.client.bridge.Refused +import org.meshbay.client.phonesync.DestinationChannels +import org.meshbay.client.phonesync.DocPlan +import org.meshbay.client.photos.ByteRange +import org.meshbay.client.photos.PhotoChannels +import org.meshbay.client.shell.Pickers +import java.io.File +import java.security.SecureRandom +import java.util.TimeZone +import java.util.concurrent.ConcurrentHashMap + +/** + * Files backup (docs/MESHBAY_DESIGN.md §9.13): the folders the person chose, + * sent into `<folder>/<account>-drive/<chosen folder>/…` of the destination + * every kind shares, on the photo backup's terms: the page decides when and + * sends, this side lists, keeps the ledger and hands bytes over by token + * (`/drivesync/<token>`, a range at a time). + * + * Media folders cannot be chosen, and files the photo backup sends are left + * out (DrivePlan): nothing is stored twice. + */ +class DriveChannels( + private val activity: Activity, + private val prefs: SharedPreferences, + private val dir: File, + private val destinations: DestinationChannels, + private val pickers: Pickers, + private val photosBackedUp: () -> Set<String>, + private val unmetered: () -> Boolean, + private val onKeepAlive: (Boolean, String) -> Unit, +) { + private val source = DriveSource(activity) + private val random = SecureRandom() + private val tokens = ConcurrentHashMap<String, Issued>() + + private class Issued(val pending: DrivePending, val key: String) + + init { + destinations.onChange { + tokens.clear() + prefs.edit().remove(LAST).remove(FAILURE).remove(FAILURE_AT).remove(NOTIFIED).apply() + } + } + + fun handles(channel: String) = channel.startsWith("drivesync:") + + fun call(channel: String, args: JSONArray): Any? = when (channel) { + "drivesync:status" -> status() + "drivesync:add-folder" -> { addFolder(); status() } + "drivesync:remove-folder" -> { removeFolder(args.optString(0, "")); status() } + "drivesync:configure" -> { configure(args.optJSONObject(0)); status() } + "drivesync:plan" -> plan() + "drivesync:sent" -> { sent(args.optString(0, ""), args.optString(1, ""), args.optString(2, "")); true } + "drivesync:completed" -> { completed(); status() } + "drivesync:failed" -> failed(args.optString(0, ""), args.optString(1, "")) + "drivesync:keep-alive" -> { onKeepAlive(args.optBoolean(0, false), args.optString(1, "").take(200)); true } + else -> throw Refused("Refused: no such channel") + } + + // ── state ──────────────────────────────────────────────────────────────── + + private data class Folder(val uri: String, val name: String) + + private fun folders(): List<Folder> = try { + val a = JSONArray(prefs.getString(FOLDERS, "[]")) + (0 until a.length()).map { a.getJSONObject(it).let { o -> Folder(o.getString("uri"), o.getString("name")) } } + } catch (e: Exception) { emptyList() } + + private fun saveFolders(list: List<Folder>) { + prefs.edit().putString(FOLDERS, JSONArray(list.map { JSONObject().put("uri", it.uri).put("name", it.name) }) + .toString()).apply() + } + + /** Two chosen folders of one name go into two folders on the node: `Notes`, `Notes-2`. */ + private fun topNames(list: List<Folder>): Map<String, String> { + val used = HashMap<String, Int>() + return list.associate { f -> + val n = (used[f.name.lowercase()] ?: 0) + 1 + used[f.name.lowercase()] = n + f.uri to (if (n == 1) f.name else "${f.name}-$n") + } + } + + private fun base(d: org.meshbay.client.phonesync.Destination) = DocPlan.dirFor(d, SUFFIX) + + private fun ledger(d: org.meshbay.client.phonesync.Destination) = + DriveLedger(File(dir, PhotoChannels.hex(PhotoChannels.sha256Of(d.ledgerKey.toByteArray())).take(32) + ".jsonl")) + + fun status(): JSONObject { + val d = destinations.get() + return JSONObject() + .put("on", prefs.getBoolean(ON, false)) + .put("unmetered", unmetered()) + .put("destination", d?.toJson() ?: JSONObject.NULL) + .put("dir", d?.let { base(it) } ?: JSONObject.NULL) + .put("folders", JSONArray(folders().map { JSONObject().put("id", it.uri).put("name", it.name) })) + .put("lastCompleted", if (prefs.contains(LAST)) prefs.getLong(LAST, 0) else JSONObject.NULL) + .put("failure", prefs.getString(FAILURE, null) ?: JSONObject.NULL) + .put("failureAt", if (prefs.contains(FAILURE_AT)) prefs.getLong(FAILURE_AT, 0) else JSONObject.NULL) + .put("sent", if (d != null) ledger(d).size else 0) + .put("now", System.currentTimeMillis()) + } + + /** The system's folder picker; a media folder or a whole volume is refused, with the reason. */ + private fun addFolder() { + val result = pickers.run(Intent(Intent.ACTION_OPEN_DOCUMENT_TREE)) ?: return + val tree = result.data ?: return + DrivePlan.refusal(DocumentsContract.getTreeDocumentId(tree))?.let { throw Refused("Refused: $it") } + activity.contentResolver.takePersistableUriPermission(tree, Intent.FLAG_GRANT_READ_URI_PERMISSION) + val list = folders() + if (list.any { it.uri == tree.toString() }) return + val fallback = DrivePlan.pathOf(DocumentsContract.getTreeDocumentId(tree)).substringAfterLast('/') + saveFolders(list + Folder(tree.toString(), source.nameOf(tree) ?: fallback.ifEmpty { "folder" })) + } + + /** Forgotten here, and its grant given back; what was sent from it stays on the node. */ + private fun removeFolder(uri: String) { + val list = folders() + if (list.none { it.uri == uri }) return + saveFolders(list.filter { it.uri != uri }) + try { + activity.contentResolver.releasePersistableUriPermission(Uri.parse(uri), Intent.FLAG_GRANT_READ_URI_PERMISSION) + } catch (e: Exception) { /* already gone */ } + } + + private fun configure(o: JSONObject?) { + tokens.clear() + if (o == null) { + prefs.edit().remove(ON).remove(LAST).remove(FAILURE).remove(FAILURE_AT).remove(NOTIFIED).apply() + return + } + if (destinations.get() == null) throw Refused("Refused: no destination") + prefs.edit().putBoolean(ON, true).apply() + } + + private fun completed() { + prefs.edit().putLong(LAST, System.currentTimeMillis()).remove(FAILURE).remove(FAILURE_AT).remove(NOTIFIED).apply() + } + + private fun failed(code: String, text: String): Boolean { + val c = code.take(64) + prefs.edit().putString(FAILURE, c).putLong(FAILURE_AT, System.currentTimeMillis()).apply() + if (prefs.getString(NOTIFIED, null) == c || text.isBlank()) return false + prefs.edit().putString(NOTIFIED, c).apply() + PhotoChannels.notice(activity, NOTIFY_ID, text.take(300)) + return true + } + + // ── the files ──────────────────────────────────────────────────────────── + + private fun plan(): JSONObject { + val d = destinations.get()?.takeIf { prefs.getBoolean(ON, false) } ?: throw Refused("Refused: files backup is off") + val list = folders() + val ledger = ledger(d) + // A folder whose grant is gone (removed on the phone, revoked) is + // skipped this run rather than failing the others. + val files = list.flatMap { f -> try { source.files(Uri.parse(f.uri)) } catch (e: Exception) { emptyList() } } + val items = DrivePlan.plan(files, base(d), topNames(list), ledger::get, photosBackedUp(), + TimeZone.getDefault()) { f, sent -> hashOf(f)?.let { it == sent.sha256 } ?: true } + tokens.clear() + val out = JSONArray() + for (p in items) { + val token = PhotoChannels.hex(ByteArray(16).also { random.nextBytes(it) }) + tokens[token] = Issued(p, d.ledgerKey) + out.put(JSONObject().put("token", token).put("name", p.name).put("dir", p.dir) + .put("size", p.file.size).put("edited", p.edited)) + } + return JSONObject().put("items", out) + } + + /** The node took it (or already had it): into the ledger, hashed from the phone now. */ + private fun sent(token: String, dir: String, name: String) { + val issued = tokens[token] ?: throw Refused("Refused: unknown file") + val d = destinations.get()?.takeIf { it.ledgerKey == issued.key } ?: throw Refused("Refused: the backup changed") + val f = issued.pending.file + val sha = hashOf(f) ?: throw Refused("Refused: the file is gone") + ledger(d).record(DriveLedger.Entry(f.docId, f.modified, f.size, sha, dir.take(1024), name.take(256), + System.currentTimeMillis())) + tokens.remove(token) + } + + /** A range of an issued file, for `/drivesync/<token>` on the packaged origin. */ + fun serve(path: String, range: String?): WebResourceResponse? { + val issued = tokens[path.removePrefix(PATH)] ?: return null + val f = issued.pending.file + val r = ByteRange.parse(range, f.size) ?: return WebResourceResponse("text/plain", null, 416, + "Range Not Satisfiable", mapOf("Content-Range" to "bytes */${f.size}"), "".byteInputStream()) + val raw = try { source.open(f) } catch (e: Exception) { null } ?: return null + val headers = mapOf("Cache-Control" to "no-store", "X-Content-Type-Options" to "nosniff", + "Content-Range" to "bytes ${r.first}-${r.last}/${f.size}", + "Content-Length" to (r.last - r.first + 1).toString()) + return WebResourceResponse(f.mime.ifEmpty { "application/octet-stream" }, null, 206, "Partial Content", + headers, ByteRange.slice(raw, r)) + } + + private fun hashOf(f: DriveFile): String? = try { + source.open(f)?.use { PhotoChannels.hex(PhotoChannels.digestOf(it)) } + } catch (e: Exception) { null } + + companion object { + const val PATH = "/drivesync/" + const val SUFFIX = "drive" + const val PREFS = "drivesync" + private const val NOTIFY_ID = 13 + private const val FOLDERS = "folders" + private const val ON = "on" + private const val LAST = "last_completed" + private const val FAILURE = "failure" + private const val FAILURE_AT = "failure_at" + private const val NOTIFIED = "notified" + } +} diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveLedger.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveLedger.kt new file mode 100644 index 0000000..044edc7 --- /dev/null +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveLedger.kt @@ -0,0 +1,69 @@ +package org.meshbay.client.drive + +import org.json.JSONObject +import java.io.File + +/** + * What this phone has sent of its chosen folders, by document id: the memory + * of what was sent, never a mirror (PhotoLedger, the same idea for photos). A + * file deleted on the phone stays on the node; one deleted on the node is not + * sent again. One line of JSON per send, the last line for an id winning, + * compacted on load once dead lines outnumber live ones. + */ +class DriveLedger(private val file: File) { + + data class Entry( + val docId: String, + val modified: Long, + val size: Long, + /** SHA-256 of the bytes sent, hex: a touched file is sent again only if this changed. */ + val sha256: String, + val dir: String, + val name: String, + val sentAt: Long, + ) + + private val entries = HashMap<String, Entry>() + private var lines = 0 + + init { load() } + + val size: Int get() = entries.size + + operator fun get(docId: String): Entry? = entries[docId] + + fun record(entry: Entry) { + entries[entry.docId] = entry + file.parentFile?.mkdirs() + file.appendText(encode(entry) + "\n") + lines += 1 + } + + private fun load() { + if (!file.exists()) return + file.forEachLine { line -> + if (line.isBlank()) return@forEachLine + lines += 1 + decode(line)?.let { entries[it.docId] = it } + } + if (lines > 2 * entries.size + COMPACT_SLACK) { + val tmp = File(file.path + ".tmp") + tmp.writeText(entries.values.joinToString("") { encode(it) + "\n" }) + if (tmp.renameTo(file)) lines = entries.size else tmp.delete() + } + } + + companion object { + private const val COMPACT_SLACK = 64 + + fun encode(e: Entry): String = JSONObject() + .put("id", e.docId).put("m", e.modified).put("s", e.size).put("h", e.sha256) + .put("d", e.dir).put("n", e.name).put("t", e.sentAt).toString() + + fun decode(line: String): Entry? = try { + val o = JSONObject(line) + Entry(o.getString("id"), o.getLong("m"), o.getLong("s"), o.getString("h"), + o.getString("d"), o.getString("n"), o.getLong("t")) + } catch (e: Exception) { null } + } +} diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DrivePlan.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DrivePlan.kt new file mode 100644 index 0000000..660c173 --- /dev/null +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DrivePlan.kt @@ -0,0 +1,112 @@ +package org.meshbay.client.drive + +import org.meshbay.client.photos.PhotoPlan +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Locale +import java.util.TimeZone + +/** One file found in a chosen folder, through the Storage Access Framework. */ +data class DriveFile( + /** The folder it was found under, as granted (a tree URI). */ + val tree: String, + /** Its document id, stable for the life of the file (`primary:Documents/a.pdf`). */ + val docId: String, + /** Its folder inside the chosen one, `/`-separated, empty at the top. */ + val sub: String, + val name: String, + val size: Long, + /** Milliseconds. */ + val modified: Long, + val mime: String, +) + +/** A file to send: a new one, or a new version of one already sent. */ +data class DrivePending(val file: DriveFile, val edited: Boolean, val dir: String, val name: String) + +/** + * The rules of the files backup (docs/MESHBAY_DESIGN.md §9.13), pure, so the + * JVM tests hold them: which folders may be chosen, what a run sends, and the + * names the node will accept. + */ +object DrivePlan { + /** + * The phone's media folders: what lives there is the photo backup's, and + * taking it here too would store every photo twice on the node. + */ + private val MEDIA = setOf("dcim", "pictures", "movies") + + /** `primary:Documents/Scans` → `Documents/Scans`; the part after the volume. */ + fun pathOf(docId: String): String = docId.substringAfter(':', "").trim('/') + + /** + * Why a folder cannot be chosen, or null when it can: the whole of a + * volume (it holds the media folders), or a media folder or anything in one. + */ + fun refusal(treeDocId: String): String? { + val path = pathOf(treeDocId) + if (path.isEmpty()) return "whole_storage" + if (path.split('/').first().lowercase(Locale.ROOT) in MEDIA) return "media_folder" + return null + } + + /** + * A run's files: new ones, and new versions of ones sent (beside them, + * never in their place). `skip` holds the files the photo backup already + * sends, as lowercase `relative/path/name`. Smallest first: a run cut short + * has saved the most files. + */ + fun plan( + files: List<DriveFile>, base: String, folderNames: Map<String, String>, + sent: (String) -> DriveLedger.Entry?, skip: Set<String>, zone: TimeZone, + sameBytes: (DriveFile, DriveLedger.Entry) -> Boolean, + ): List<DrivePending> { + val out = ArrayList<DrivePending>() + for (f in files) { + if (f.name.startsWith('.')) continue + if (skip.contains(pathOf(f.docId).lowercase(Locale.ROOT))) continue + val top = folderNames[f.tree] ?: continue + val dir = dirFor(base, top, f.sub) + val was = sent(f.docId) + if (was == null) { + out += DrivePending(f, false, dir, safeName(f.name)) + } else if (was.modified != f.modified || was.size != f.size) { + if (was.size == f.size && sameBytes(f, was)) continue + out += DrivePending(f, true, dir, editedName(f, zone)) + } + } + return out.sortedBy { it.file.size } + } + + /** `<base>/<chosen folder>/<sub>`, each level a name the node accepts. */ + fun dirFor(base: String, top: String, sub: String): String = + (listOf(base, safeName(top)) + sub.split('/').filter { it.isNotEmpty() }.map(::safeName)) + .joinToString("/") + + /** + * The name as is when the node accepts it (PhotoPlan.UPLOAD_NAME, the + * node's own rule); otherwise its refused characters become `_`, and a + * name that cannot start as the node wants is given a `file-` prefix. + */ + fun safeName(name: String): String { + if (PhotoPlan.UPLOAD_NAME.matches(name)) return name + var s = name.map { c -> if (c.isLetterOrDigit() || c in " _.-()[]'’,&+#@") c else '_' }.joinToString("") + .trimEnd(' ', '.') + if (s.isEmpty() || !s.first().isLetterOrDigit()) s = "file-$s" + s = s.take(MAX_NAME).trimEnd(' ', '.') + return if (PhotoPlan.UPLOAD_NAME.matches(s)) s else "file" + } + + /** A new version lands beside the one sent, its date in its name. */ + fun editedName(f: DriveFile, zone: TimeZone): String { + val base = safeName(f.name) + val dot = base.lastIndexOf('.') + val stem = if (dot > 0) base.substring(0, dot) else base + val ext = if (dot > 0) base.substring(dot) else "" + val stamp = SimpleDateFormat("yyyyMMdd-HHmmss", Locale.ROOT).apply { timeZone = zone }.format(Date(f.modified)) + val suffix = "-modified-$stamp$ext" + return stem.take(MAX_NAME - suffix.length) + suffix + } + + private const val MAX_NAME = 128 +} diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveSource.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveSource.kt new file mode 100644 index 0000000..69077fa --- /dev/null +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveSource.kt @@ -0,0 +1,55 @@ +package org.meshbay.client.drive + +import android.content.Context +import android.net.Uri +import android.provider.DocumentsContract +import android.provider.DocumentsContract.Document +import java.io.InputStream + +/** + * The files of the folders the person chose, read through the Storage Access + * Framework: each folder was granted through the system's own picker, which + * is all the access this needs (no storage permission, which Play keeps for a + * few kinds of application). + */ +class DriveSource(private val context: Context) { + + /** Every file under `tree`, its sub-folders included; hidden ones (`.x`) and their contents skipped. */ + fun files(tree: Uri): List<DriveFile> { + val out = ArrayList<DriveFile>() + val root = DocumentsContract.getTreeDocumentId(tree) + val pending = ArrayDeque(listOf(root to "")) + val cols = arrayOf(Document.COLUMN_DOCUMENT_ID, Document.COLUMN_DISPLAY_NAME, Document.COLUMN_MIME_TYPE, + Document.COLUMN_SIZE, Document.COLUMN_LAST_MODIFIED) + while (pending.isNotEmpty()) { + val (parent, sub) = pending.removeFirst() + val children = DocumentsContract.buildChildDocumentsUriUsingTree(tree, parent) + context.contentResolver.query(children, cols, null, null, null)?.use { c -> + while (c.moveToNext()) { + val id = c.getString(0) ?: continue + val name = c.getString(1) ?: continue + if (name.startsWith('.')) continue + val mime = c.getString(2) ?: "" + if (mime == Document.MIME_TYPE_DIR) { + pending.addLast(id to (if (sub.isEmpty()) name else "$sub/$name")) + } else { + out += DriveFile(tree.toString(), id, sub, name, + if (c.isNull(3)) 0 else c.getLong(3), + if (c.isNull(4)) 0 else c.getLong(4), mime) + } + } + } + } + return out + } + + fun open(f: DriveFile): InputStream? = context.contentResolver.openInputStream( + DocumentsContract.buildDocumentUriUsingTree(Uri.parse(f.tree), f.docId)) + + /** The chosen folder's own name, as the picker showed it. */ + fun nameOf(tree: Uri): String? = try { + val doc = DocumentsContract.buildDocumentUriUsingTree(tree, DocumentsContract.getTreeDocumentId(tree)) + context.contentResolver.query(doc, arrayOf(Document.COLUMN_DISPLAY_NAME), null, null, null) + ?.use { c -> if (c.moveToFirst()) c.getString(0) else null } + } catch (e: Exception) { null } +} diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoChannels.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoChannels.kt index 78f57cb..422103b 100644 --- a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoChannels.kt +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoChannels.kt @@ -244,6 +244,13 @@ class PhotoChannels( source.open(p)?.use { s -> hex(digestOf(s)) } } catch (e: Exception) { null } + /** What the photo backup sends, for the files backup to leave out; empty while it is off. */ + fun backedUpPaths(): Set<String> { + val c = config() ?: return emptySet() + if (permissionState() == "denied") return emptySet() + return try { source.relativePaths(c.albums, c.includeVideos) } catch (e: Exception) { emptySet() } + } + // ── permission and network ─────────────────────────────────────────────── /** For the photos, or with `video` for the videos, which Android 13+ asks about apart. */ diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoSource.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoSource.kt index fe0d183..ce21965 100644 --- a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoSource.kt +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoSource.kt @@ -59,6 +59,32 @@ class PhotoSource(private val context: Context) { return out } + /** + * The files the photo backup sends, as lowercase `relative/path/name` + * from the top of their volume: what the files backup leaves out, so + * nothing is stored twice. + */ + fun relativePaths(albums: List<String>, withVideos: Boolean): Set<String> { + if (albums.isEmpty()) return emptySet() + val out = HashSet<String>() + val where = "${MediaStore.Images.Media.BUCKET_ID} IN (${albums.joinToString(",") { "?" }})" + @Suppress("DEPRECATION") + val cols = if (Build.VERSION.SDK_INT >= 29) + arrayOf(MediaStore.MediaColumns.RELATIVE_PATH, MediaStore.MediaColumns.DISPLAY_NAME) + else arrayOf(MediaStore.MediaColumns.DATA) + for (collection in listOfNotNull(images, videos.takeIf { withVideos })) { + context.contentResolver.query(collection, cols, where, albums.toTypedArray(), null)?.use { c -> + while (c.moveToNext()) { + val path = if (Build.VERSION.SDK_INT >= 29) + (c.getString(0) ?: continue).trim('/') + "/" + (c.getString(1) ?: continue) + else volumeRelative(c.getString(0) ?: continue) + out += path.lowercase(java.util.Locale.ROOT) + } + } + } + return out + } + fun open(p: Photo): InputStream? = context.contentResolver.openInputStream(ContentUris.withAppendedId(if (p.video) videos else images, p.mediaId)) @@ -98,6 +124,12 @@ class PhotoSource(private val context: Context) { } companion object { + /** `/storage/emulated/0/Documents/a.jpg` or `/storage/1234-ABCD/x/a.jpg` → the part after the volume. */ + fun volumeRelative(absolute: String): String { + val m = Regex("^/storage/(?:emulated/\\d+|[^/]+)/(.*)$").matchEntire(absolute) + return (m?.groupValues?.get(1) ?: absolute).trim('/') + } + /** `DCIM/Camera/` (RELATIVE_PATH) or `…/DCIM/Camera/x.jpg` (DATA, before Android 10). */ fun isCamera(path: String): Boolean = path.startsWith("DCIM/Camera") || path.contains("/DCIM/Camera/") diff --git a/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/DriveTest.kt b/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/DriveTest.kt new file mode 100644 index 0000000..e661bd1 --- /dev/null +++ b/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/DriveTest.kt @@ -0,0 +1,90 @@ +package org.meshbay.client + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.meshbay.client.drive.DriveFile +import org.meshbay.client.drive.DriveLedger +import org.meshbay.client.drive.DrivePlan +import org.meshbay.client.photos.PhotoPlan +import org.meshbay.client.photos.PhotoSource +import java.util.TimeZone + +class DriveTest { + @get:Rule val tmp = TemporaryFolder() + + private val utc = TimeZone.getTimeZone("UTC") + private val tree = "content://tree/primary%3ADocuments" + private val names = mapOf(tree to "Documents") + + private fun file(id: String, sub: String = "", size: Long = 10, modified: Long = 1000) = + DriveFile(tree, "primary:Documents/" + (if (sub.isEmpty()) "" else "$sub/") + id, sub, id, size, modified, + "application/pdf") + + private fun plan(files: List<DriveFile>, sent: Map<String, DriveLedger.Entry> = emptyMap(), + skip: Set<String> = emptySet(), same: Boolean = false) = + DrivePlan.plan(files, "Backups/bob-drive", names, { sent[it] }, skip, utc) { _, _ -> same } + + private fun entry(f: DriveFile) = DriveLedger.Entry(f.docId, f.modified, f.size, "h", "d", f.name, 1) + + @Test fun `media folders and a whole volume cannot be chosen`() { + assertEquals("whole_storage", DrivePlan.refusal("primary:")) + assertEquals("whole_storage", DrivePlan.refusal("1234-ABCD:")) + assertEquals("media_folder", DrivePlan.refusal("primary:DCIM")) + assertEquals("media_folder", DrivePlan.refusal("primary:DCIM/Camera")) + assertEquals("media_folder", DrivePlan.refusal("primary:Pictures/Screenshots")) + assertEquals("media_folder", DrivePlan.refusal("1234-ABCD:movies")) + assertNull(DrivePlan.refusal("primary:Documents")) + assertNull(DrivePlan.refusal("primary:Download/Invoices")) + assertNull(DrivePlan.refusal("primary:Documents/Pictures")) + } + + @Test fun `files go under the chosen folder's name, sub-folders kept`() { + val out = plan(listOf(file("a.pdf"), file("b.pdf", sub = "Tax/2026"))) + assertEquals(setOf("Backups/bob-drive/Documents", "Backups/bob-drive/Documents/Tax/2026"), + out.map { it.dir }.toSet()) + } + + @Test fun `smallest first, hidden files and photo-backed files left out`() { + val out = plan(listOf(file("big.pdf", size = 900), file("small.pdf", size = 5), file(".secret"), + file("scan.jpg", sub = "Scans")), + skip = setOf("documents/scans/scan.jpg")) + assertEquals(listOf("small.pdf", "big.pdf"), out.map { it.name }) + } + + @Test fun `a file already sent is not sent again, a changed one goes beside it`() { + val f = file("a.pdf") + assertTrue(plan(listOf(f), sent = mapOf(f.docId to entry(f))).isEmpty()) + val changed = f.copy(size = 20, modified = 86_400_000) + val out = plan(listOf(changed), sent = mapOf(f.docId to entry(f))) + assertEquals("a-modified-19700102-000000.pdf", out.single().name) + assertTrue(out.single().edited) + val touched = f.copy(modified = 5000) + assertTrue(plan(listOf(touched), sent = mapOf(f.docId to entry(f)), same = true).isEmpty()) + } + + @Test fun `names the node would refuse are mended, not dropped`() { + assertEquals("report.pdf", DrivePlan.safeName("report.pdf")) + assertEquals("a_b_c.txt", DrivePlan.safeName("a:b?c.txt")) + assertEquals("file-.env", DrivePlan.safeName(".env")) + assertEquals("file-_draft", DrivePlan.safeName("_draft")) + assertTrue(PhotoPlan.UPLOAD_NAME.matches(DrivePlan.safeName("x".repeat(300) + ".pdf"))) + assertTrue(PhotoPlan.UPLOAD_NAME.matches(DrivePlan.safeName("end. "))) + } + + @Test fun `the ledger remembers across a restart`() { + val path = tmp.newFile("d.jsonl") + DriveLedger(path).apply { record(entry(file("a.pdf"))); record(entry(file("a.pdf")).copy(sha256 = "z")) } + val again = DriveLedger(path) + assertEquals(1, again.size) + assertEquals("z", again["primary:Documents/a.pdf"]!!.sha256) + } + + @Test fun `a path from before Android 10 is read from the top of its volume`() { + assertEquals("Documents/Scans/a.jpg", PhotoSource.volumeRelative("/storage/emulated/0/Documents/Scans/a.jpg")) + assertEquals("x/a.jpg", PhotoSource.volumeRelative("/storage/1234-ABCD/x/a.jpg")) + } +} |