diff options
Diffstat (limited to 'packages/meshbay-android')
23 files changed, 1665 insertions, 76 deletions
diff --git a/packages/meshbay-android/app/src/main/AndroidManifest.xml b/packages/meshbay-android/app/src/main/AndroidManifest.xml index 48a69ea..6b97b45 100644 --- a/packages/meshbay-android/app/src/main/AndroidManifest.xml +++ b/packages/meshbay-android/app/src/main/AndroidManifest.xml @@ -18,11 +18,15 @@ location from the bytes this application reads, so a camera roll sent to a group does not say where its owner lives. --> <uses-permission android:name="android.permission.READ_MEDIA_IMAGES" /> + <!-- Videos of the same albums, when the person ticks them in: asked then. --> + <uses-permission android:name="android.permission.READ_MEDIA_VIDEO" /> <uses-permission android:name="android.permission.READ_MEDIA_VISUAL_USER_SELECTED" /> <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" android:maxSdkVersion="32" /> <uses-permission android:name="android.permission.FOREGROUND_SERVICE_DATA_SYNC" /> <!-- Contacts backup: the address book, asked for when the person turns it on. --> <uses-permission android:name="android.permission.READ_CONTACTS" /> + <!-- Calendar backup: the same, for the calendars. --> + <uses-permission android:name="android.permission.READ_CALENDAR" /> <!-- No backup of any kind: the keys are wrapped by a Keystore key that a restore cannot bring with it, so a backed-up store is one that silently diff --git a/packages/meshbay-android/app/src/main/assets/bridge/meshbay-bridge.js b/packages/meshbay-android/app/src/main/assets/bridge/meshbay-bridge.js index fc7231c..a07399f 100644 --- a/packages/meshbay-android/app/src/main/assets/bridge/meshbay-bridge.js +++ b/packages/meshbay-android/app/src/main/assets/bridge/meshbay-bridge.js @@ -207,12 +207,14 @@ // The page decides when and does the sending. Phone-only, like `push`. photoSync: { status: () => call('photosync:status'), - permit: () => call('photosync:permit'), - albums: () => call('photosync:albums'), + permit: (withVideos) => call('photosync:permit', withVideos === true), + albums: (withVideos) => call('photosync:albums', withVideos === true), configure: (settings) => call('photosync:configure', settings || null), estimate: (settings) => call('photosync:estimate', settings), plan: () => call('photosync:plan'), sent: (token, dir, name) => call('photosync:sent', token, dir, name), + manifest: () => call('photosync:manifest'), + manifestSent: (token) => call('photosync:manifest-sent', token), completed: () => call('photosync:completed'), failed: (code, text) => call('photosync:failed', code, text), keepAlive: (on, text) => call('photosync:keep-alive', on === true, text || ''), @@ -236,6 +238,52 @@ sent: (token, dir, name) => call('contactsync:sent', token, dir, name), completed: () => call('contactsync:completed'), failed: (code, text) => call('contactsync:failed', code, text), + openSettings: () => call('contactsync:open-settings'), + }, + + // Calendar backup (§9.13), the same way: one iCalendar file of every + // calendar the person can write to, when it changed. + calendarSync: { + status: () => call('calendarsync:status'), + permit: () => call('calendarsync:permit'), + configure: (settings) => call('calendarsync:configure', settings || null), + plan: () => call('calendarsync:plan'), + sent: (token, dir, name) => call('calendarsync:sent', token, dir, name), + completed: () => call('calendarsync:completed'), + failed: (code, text) => call('calendarsync:failed', code, text), + openSettings: () => call('calendarsync:open-settings'), + }, + + // Files backup (§9.13): folders the person chose through the system's + // picker, sent like photos, a range at a time from /drivesync/<token>. + driveSync: { + status: () => call('drivesync:status'), + addFolder: () => call('drivesync:add-folder'), + removeFolder: (id) => call('drivesync:remove-folder', id), + configure: (settings) => call('drivesync:configure', settings || null), + plan: () => call('drivesync:plan'), + sent: (token, dir, name) => call('drivesync:sent', token, dir, name), + manifest: () => call('drivesync:manifest'), + manifestSent: (token) => call('drivesync:manifest-sent', token), + completed: () => call('drivesync:completed'), + failed: (code, text) => call('drivesync:failed', code, text), + keepAlive: (on, text) => call('drivesync:keep-alive', on === true, text || ''), + }, + + // WhatsApp's own folder (§9.13), the same way: its encrypted chat + // backups for a restore, and its media if the person ticks them. + whatsappSync: { + status: () => call('wasync:status'), + chooseFolder: () => call('wasync:add-folder'), + forgetFolder: (id) => call('wasync:remove-folder', id), + configure: (settings) => call('wasync:configure', settings || null), + plan: () => call('wasync:plan'), + sent: (token, dir, name) => call('wasync:sent', token, dir, name), + manifest: () => call('wasync:manifest'), + manifestSent: (token) => call('wasync:manifest-sent', token), + completed: () => call('wasync:completed'), + failed: (code, text) => call('wasync:failed', code, text), + keepAlive: (on, text) => call('wasync:keep-alive', on === true, text || ''), }, } : {}), @@ -249,6 +297,7 @@ sent: (token, dir, name) => call('messagesync:sent', token, dir, name), completed: () => call('messagesync:completed'), failed: (code, text) => call('messagesync:failed', code, text), + openSettings: () => call('messagesync:open-settings'), } } : {}), // Where downloads go, chosen once. A display name comes back, never a URI. diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/MainActivity.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/MainActivity.kt index 09cef0e..699e0c3 100644 --- a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/MainActivity.kt +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/MainActivity.kt @@ -36,6 +36,9 @@ import org.meshbay.client.notify.Notifier import org.meshbay.client.notify.PushChannels import org.meshbay.client.notify.PushState import org.meshbay.client.photos.BackupService +import org.meshbay.client.drive.DriveChannels +import org.meshbay.client.drive.DriveKind +import org.meshbay.client.phonesync.CalendarSource import org.meshbay.client.phonesync.ContactSource import org.meshbay.client.phonesync.DestinationChannels import org.meshbay.client.phonesync.DocChannels @@ -65,6 +68,7 @@ class MainActivity : Activity() { private lateinit var channels: Channels private lateinit var photos: PhotoChannels private var docs: List<DocChannels> = emptyList() + private var drives: List<DriveChannels> = emptyList() /** Backups exist in the personal profile only (Profiles.kt). */ private var backups = false private var network: android.net.ConnectivityManager.NetworkCallback? = null @@ -75,7 +79,9 @@ class MainActivity : Activity() { private var shim: ScriptHandler? = null private var casting = false private var playing = false - private var syncing = false + /** Which backups hold the keep-alive now ("photos", "drive"): one stopping leaves the other's. */ + private val syncers = HashSet<String>() + private val syncing get() = syncers.isNotEmpty() private var fullscreen: View? = null private var fullscreenCallback: WebChromeClient.CustomViewCallback? = null private var pendingLink: String? = null @@ -108,11 +114,20 @@ class MainActivity : Activity() { val destinations = DestinationChannels(getSharedPreferences(DestinationChannels.PREFS, Context.MODE_PRIVATE)) photos = PhotoChannels(this, getSharedPreferences(PhotoChannels.PREFS, Context.MODE_PRIVATE), destinations, java.io.File(filesDir, "photosync"), - onKeepAlive = { on, line -> runOnUiThread { backup(on, line) } }) + onKeepAlive = { on, line -> runOnUiThread { backup("photos", on, line) } }) + drives = if (!backups) emptyList() else listOf(DriveKind.FILES, DriveKind.WHATSAPP).map { kind -> + DriveChannels(kind, this, getSharedPreferences(kind.prefix, Context.MODE_PRIVATE), + java.io.File(filesDir, kind.prefix), destinations, pickers, + photosBackedUp = { photos.backedUpPaths() }, unmetered = { photos.unmetered() }, + onKeepAlive = { on, line -> runOnUiThread { backup(kind.suffix, on, line) } }) + } docs = if (!backups) emptyList() else listOf( DocChannels("contactsync", this, getSharedPreferences("contactsync", Context.MODE_PRIVATE), destinations, java.io.File(cacheDir, "contactsync"), ContactSource(this), notifyId = 10, permissionRequest = 4209), + DocChannels("calendarsync", this, getSharedPreferences("calendarsync", Context.MODE_PRIVATE), destinations, + java.io.File(cacheDir, "calendarsync"), CalendarSource(this), + notifyId = 12, permissionRequest = 4211), ) + listOfNotNull(Flavor.messageSource(this)?.let { source -> DocChannels("messagesync", this, getSharedPreferences("messagesync", Context.MODE_PRIVATE), destinations, java.io.File(cacheDir, "messagesync"), source, @@ -126,7 +141,7 @@ class MainActivity : Activity() { Notifier.CHANNEL_CHAT to text.get("push.channel_chat", channels.locale), Notifier.CHANNEL_OTHER to text.get("push.channel_other", channels.locale)) }), photos = photos.takeIf { backups }, destinations = destinations.takeIf { backups }, - docs = docs) + docs = docs, drives = drives) WebViewCompat.addWebMessageListener(web, Bridge.PORT, setOf(UiAssets.ORIGIN), Bridge(channels)) cast.control.warmUp() installShim() @@ -172,7 +187,11 @@ class MainActivity : Activity() { override fun shouldInterceptRequest(view: WebView, request: WebResourceRequest): WebResourceResponse? { val url = request.url if (url.host == UiAssets.HOST && url.path?.startsWith(PhotoChannels.PATH) == true) { - return photos.takeIf { backups }?.serve(url.path ?: "") ?: refused() + return photos.takeIf { backups }?.serve(url.path ?: "", rangeOf(request)) ?: refused() + } + val driveKind = drives.firstOrNull { url.path?.startsWith(it.path) == true } + if (url.host == UiAssets.HOST && driveKind != null) { + return driveKind.serve(url.path ?: "", rangeOf(request)) ?: refused() } if (url.host == UiAssets.HOST && url.path?.startsWith(DocChannels.PATH) == true) { val path = url.path ?: "" @@ -313,15 +332,17 @@ class MainActivity : Activity() { } /** - * A photo backup running: its own foreground service, and the page kept - * visible like a cast. Started once; afterwards only its line changes, - * which needs no start — refused from the background on Android 12+. + * A photo or files backup running: its own foreground service, and the + * page kept visible like a cast. Started once; afterwards only its line + * changes, which needs no start (refused from the background on Android + * 12+). Held while any backup holds it. */ - private fun backup(on: Boolean, line: String) { + private fun backup(who: String, on: Boolean, line: String) { val service = Intent(this, BackupService::class.java) - if (on && syncing) { BackupService.update(this, line); return } - if (on == syncing) return - syncing = on + val was = syncing + if (on) syncers += who else syncers -= who + if (on && was) { BackupService.update(this, line); return } + if (syncing == was) return if (on) { try { startForegroundService(service.putExtra(BackupService.EXTRA_TEXT, line)) } catch (e: IllegalStateException) { Log.w(Bridge.TAG, "backup keep-alive refused: $e") } @@ -360,6 +381,13 @@ class MainActivity : Activity() { super.onRequestPermissionsResult(requestCode, permissions, grantResults) } + /** + * The slice a backup asks for, `?range=a-b` (platform.js): the WebView does + * not hand a fetch's Range header over to the application that answers. + */ + private fun rangeOf(request: WebResourceRequest): String? = + request.url.getQueryParameter("range")?.let { "bytes=$it" } + private fun hasAsset(path: String) = try { assets.open(path).close(); true } catch (e: java.io.IOException) { false } private fun refused() = WebResourceResponse("text/plain", "utf-8", 403, "Forbidden", emptyMap(), "".byteInputStream()) @@ -423,7 +451,7 @@ class MainActivity : Activity() { override fun onDestroy() { if (::cast.isInitialized && cast.relay.active) cast.relay.stop() if (casting || playing) { casting = false; playing = false; keepAlive() } - if (syncing) backup(false, "") + if (syncing) { syncers.clear(); stopService(Intent(this, BackupService::class.java)); keepAlive() } network?.let { try { getSystemService(android.net.ConnectivityManager::class.java).unregisterNetworkCallback(it) } catch (e: Exception) {} } if (::web.isInitialized) { root.removeView(web); web.destroy() } super.onDestroy() diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/bridge/Channels.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/bridge/Channels.kt index 512aa13..e3a81df 100644 --- a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/bridge/Channels.kt +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/bridge/Channels.kt @@ -4,6 +4,7 @@ import org.json.JSONArray import org.meshbay.client.cast.CastChannels import org.meshbay.client.hub.HubClient import org.meshbay.client.notify.PushChannels +import org.meshbay.client.drive.DriveChannels import org.meshbay.client.phonesync.DestinationChannels import org.meshbay.client.phonesync.DocChannels import org.meshbay.client.photos.PhotoChannels @@ -34,6 +35,7 @@ class Channels( private val photos: PhotoChannels? = null, private val destinations: DestinationChannels? = null, private val docs: List<DocChannels> = emptyList(), + private val drives: List<DriveChannels> = emptyList(), ) { @Volatile var locale = "en" private set @@ -64,6 +66,7 @@ class Channels( push != null && push.handles(channel) -> push.call(channel, args) photos != null && photos.handles(channel) -> photos.call(channel, args) destinations != null && destinations.handles(channel) -> destinations.call(channel, args) + drives.any { it.handles(channel) } -> drives.first { it.handles(channel) }.call(channel, args) else -> docs.firstOrNull { it.handles(channel) }?.call(channel, args) ?: throw Refused("Refused: no such channel") } diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveChannels.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveChannels.kt new file mode 100644 index 0000000..c6fb476 --- /dev/null +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveChannels.kt @@ -0,0 +1,297 @@ +package org.meshbay.client.drive + +import android.app.Activity +import android.content.Intent +import android.content.SharedPreferences +import android.net.Uri +import android.provider.DocumentsContract +import android.webkit.WebResourceResponse +import org.json.JSONArray +import org.json.JSONObject +import org.meshbay.client.bridge.Refused +import org.meshbay.client.phonesync.Destination +import org.meshbay.client.phonesync.DestinationChannels +import org.meshbay.client.phonesync.ManifestLog +import org.meshbay.client.phonesync.DocPlan +import org.meshbay.client.photos.PhotoChannels +import org.meshbay.client.shell.Pickers +import java.io.File +import java.security.SecureRandom +import java.util.TimeZone +import java.util.concurrent.ConcurrentHashMap + +/** + * Files backup (docs/MESHBAY_DESIGN.md §9.13): the folders the person chose, + * sent into `<folder>/<account>-drive/<chosen folder>/…` of the destination + * every kind shares, on the photo backup's terms: the page decides when and + * sends, this side lists, keeps the ledger and hands bytes over by token + * (`/drivesync/<token>`, a range at a time). + * + * Media folders cannot be chosen, and files the photo backup sends are left + * out (DrivePlan): nothing is stored twice. + */ +class DriveChannels( + private val kind: DriveKind, + private val activity: Activity, + private val prefs: SharedPreferences, + private val dir: File, + private val destinations: DestinationChannels, + private val pickers: Pickers, + private val photosBackedUp: () -> Set<String>, + private val unmetered: () -> Boolean, + private val onKeepAlive: (Boolean, String) -> Unit, +) { + private val source = DriveSource(activity) + private val random = SecureRandom() + private val tokens = ConcurrentHashMap<String, Issued>() + + private class Issued(val pending: DrivePending, val key: String) + + init { + destinations.onChange { + tokens.clear() + prefs.edit().remove(LAST).remove(FAILURE).remove(FAILURE_AT).remove(NOTIFIED).apply() + } + } + + /** `/drivesync/`, `/wasync/`: where this kind's files are served on the packaged origin. */ + val path: String get() = kind.path + + fun handles(channel: String) = channel.startsWith("${kind.prefix}:") + + fun call(channel: String, args: JSONArray): Any? = when (channel.removePrefix("${kind.prefix}:")) { + "status" -> status() + "add-folder" -> { addFolder(); status() } + "remove-folder" -> { removeFolder(args.optString(0, "")); status() } + "configure" -> { configure(args.optJSONObject(0)); status() } + "plan" -> plan() + "sent" -> { sent(args.optString(0, ""), args.optString(1, ""), args.optString(2, "")); true } + "manifest" -> manifest() + "manifest-sent" -> { manifestSent(args.optString(0, "")); true } + "completed" -> { completed(); status() } + "failed" -> failed(args.optString(0, ""), args.optString(1, "")) + "keep-alive" -> { onKeepAlive(args.optBoolean(0, false), args.optString(1, "").take(200)); true } + else -> throw Refused("Refused: no such channel") + } + + // ── state ──────────────────────────────────────────────────────────────── + + private data class Folder(val uri: String, val name: String) + + private fun folders(): List<Folder> = try { + val a = JSONArray(prefs.getString(FOLDERS, "[]")) + (0 until a.length()).map { a.getJSONObject(it).let { o -> Folder(o.getString("uri"), o.getString("name")) } } + } catch (e: Exception) { emptyList() } + + private fun saveFolders(list: List<Folder>) { + prefs.edit().putString(FOLDERS, JSONArray(list.map { JSONObject().put("uri", it.uri).put("name", it.name) }) + .toString()).apply() + } + + /** Two chosen folders of one name go into two folders on the node: `Notes`, `Notes-2`. */ + private fun topNames(list: List<Folder>): Map<String, String> { + if (!kind.keepTop) return list.associate { it.uri to "" } + val used = HashMap<String, Int>() + return list.associate { f -> + val n = (used[f.name.lowercase()] ?: 0) + 1 + used[f.name.lowercase()] = n + f.uri to (if (n == 1) f.name else "${f.name}-$n") + } + } + + private fun base(d: Destination) = DocPlan.dirFor(d, kind.suffix) + + private fun options(): JSONObject = try { JSONObject(prefs.getString(OPTIONS, "{}")) } catch (e: Exception) { JSONObject() } + + private fun keyOf(d: Destination) = PhotoChannels.hex(PhotoChannels.sha256Of(d.ledgerKey.toByteArray())).take(32) + + private fun ledger(d: Destination) = DriveLedger(File(dir, keyOf(d) + ".jsonl")) + + private fun manifestLog(d: Destination) = ManifestLog(File(dir, keyOf(d) + ".manifest")) + + @Volatile private var manifestToken: Pair<String, File>? = null + + fun status(): JSONObject { + val d = destinations.get() + return JSONObject() + .put("on", prefs.getBoolean(ON, false)) + .put("options", options()) + .put("unmetered", unmetered()) + .put("destination", d?.toJson() ?: JSONObject.NULL) + .put("dir", d?.let { base(it) } ?: JSONObject.NULL) + .put("folders", JSONArray(folders().map { JSONObject().put("id", it.uri).put("name", it.name) })) + .put("lastCompleted", if (prefs.contains(LAST)) prefs.getLong(LAST, 0) else JSONObject.NULL) + .put("failure", prefs.getString(FAILURE, null) ?: JSONObject.NULL) + .put("failureAt", if (prefs.contains(FAILURE_AT)) prefs.getLong(FAILURE_AT, 0) else JSONObject.NULL) + .put("sent", if (d != null) ledger(d).size else 0) + .put("now", System.currentTimeMillis()) + } + + /** The system's folder picker; a media folder or a whole volume is refused, with the reason. */ + private fun addFolder() { + val ask = Intent(Intent.ACTION_OPEN_DOCUMENT_TREE) + kind.initialDocId?.let { + ask.putExtra(DocumentsContract.EXTRA_INITIAL_URI, + DocumentsContract.buildDocumentUri("com.android.externalstorage.documents", it)) + } + val result = pickers.run(ask) ?: return + val tree = result.data ?: return + kind.refusal(DocumentsContract.getTreeDocumentId(tree))?.let { throw Refused("Refused: $it") } + activity.contentResolver.takePersistableUriPermission(tree, Intent.FLAG_GRANT_READ_URI_PERMISSION) + var list = folders() + if (list.any { it.uri == tree.toString() }) return + // One folder only: the one chosen now replaces the last. + if (kind.single) { list.forEach { removeFolder(it.uri) }; list = emptyList() } + val fallback = DrivePlan.pathOf(DocumentsContract.getTreeDocumentId(tree)).substringAfterLast('/') + saveFolders(list + Folder(tree.toString(), source.nameOf(tree) ?: fallback.ifEmpty { "folder" })) + } + + /** Forgotten here, and its grant given back; what was sent from it stays on the node. */ + private fun removeFolder(uri: String) { + val list = folders() + if (list.none { it.uri == uri }) return + saveFolders(list.filter { it.uri != uri }) + try { + activity.contentResolver.releasePersistableUriPermission(Uri.parse(uri), Intent.FLAG_GRANT_READ_URI_PERMISSION) + } catch (e: Exception) { /* already gone */ } + } + + /** `{…options}` turns it on and keeps them (`media` for WhatsApp); null turns it off. */ + private fun configure(o: JSONObject?) { + tokens.clear() + if (o == null) { + prefs.edit().remove(ON).remove(LAST).remove(FAILURE).remove(FAILURE_AT).remove(NOTIFIED).apply() + return + } + if (destinations.get() == null) throw Refused("Refused: no destination") + val options = JSONObject().put("media", o.optBoolean("media", false)) + prefs.edit().putBoolean(ON, true).putString(OPTIONS, options.toString()).apply() + } + + private fun completed() { + prefs.edit().putLong(LAST, System.currentTimeMillis()).remove(FAILURE).remove(FAILURE_AT).remove(NOTIFIED).apply() + } + + private fun failed(code: String, text: String): Boolean { + val c = code.take(64) + prefs.edit().putString(FAILURE, c).putLong(FAILURE_AT, System.currentTimeMillis()).apply() + if (prefs.getString(NOTIFIED, null) == c || text.isBlank()) return false + prefs.edit().putString(NOTIFIED, c).apply() + PhotoChannels.notice(activity, kind.notifyId, text.take(300)) + return true + } + + // ── the files ──────────────────────────────────────────────────────────── + + private fun plan(): JSONObject { + val d = destinations.get()?.takeIf { prefs.getBoolean(ON, false) } ?: throw Refused("Refused: files backup is off") + val list = folders() + val ledger = ledger(d) + // A folder whose grant is gone (removed on the phone, revoked) is + // skipped this run rather than failing the others. + val options = options() + val files = list.flatMap { f -> try { source.files(Uri.parse(f.uri)) } catch (e: Exception) { emptyList() } } + .filter { kind.include(it, options) } + restoreSet = kind.restoreSet?.let { inSet -> files.filter(inSet) } + val items = DrivePlan.plan(files, base(d), topNames(list), ledger::get, photosBackedUp(), + TimeZone.getDefault()) { f, sent -> hashOf(f)?.let { it == sent.sha256 } ?: true } + tokens.clear() + val out = JSONArray() + for (p in items) { + val token = PhotoChannels.hex(ByteArray(16).also { random.nextBytes(it) }) + tokens[token] = Issued(p, d.ledgerKey) + out.put(JSONObject().put("token", token).put("name", p.name).put("dir", p.dir) + .put("size", p.file.size).put("edited", p.edited)) + } + return JSONObject().put("items", out).put("manifest", manifestLog(d).waiting()) + } + + /** The node took it (or already had it): into the ledger, hashed from the phone now. */ + private fun sent(token: String, dir: String, name: String) { + val issued = tokens[token] ?: throw Refused("Refused: unknown file") + val d = destinations.get()?.takeIf { it.ledgerKey == issued.key } ?: throw Refused("Refused: the backup changed") + val f = issued.pending.file + val sha = hashOf(f) ?: throw Refused("Refused: the file is gone") + val now = System.currentTimeMillis() + ledger(d).record(DriveLedger.Entry(f.docId, f.modified, f.size, sha, dir.take(1024), name.take(256), now)) + manifestLog(d).append(JSONObject() + .put("kind", "file") + .put("node", "${dir.take(1024)}/${name.take(256)}") + .put("source", DrivePlan.pathOf(f.docId)).put("name", f.name) + .put("folder", folders().firstOrNull { it.uri == f.tree }?.name ?: "") + .put("modified", f.modified).put("size", f.size).put("sha256", sha).put("mime", f.mime) + .put("edited", issued.pending.edited).put("sentAt", now)) + if (kind.restoreSet?.invoke(f) == true) prefs.edit().putBoolean(SET_CHANGED, true).apply() + tokens.remove(token) + } + + /** The files a restore needs together, as the last plan found them. */ + @Volatile private var restoreSet: List<DriveFile>? = null + + /** + * Once something in the restore set was sent: one manifest line naming, for + * each file of the set as it is now, the copy on the node that a restore + * takes. Without it, a restore could pair this week's full copy with last + * week's increments, which WhatsApp would not open. + */ + private fun noteRestoreSet(d: Destination) { + val set = restoreSet ?: return + if (!prefs.getBoolean(SET_CHANGED, false)) return + val ledger = ledger(d) + val files = JSONArray() + for (f in set) { + val e = ledger[f.docId] ?: return // not all on the node yet: the next run says it + if (e.size != f.size || e.modified != f.modified) return + files.put(JSONObject().put("source", DrivePlan.pathOf(f.docId)).put("node", "${e.dir}/${e.name}") + .put("size", e.size).put("sha256", e.sha256)) + } + manifestLog(d).append(JSONObject().put("kind", "restore-set").put("app", kind.suffix) + .put("at", System.currentTimeMillis()).put("files", files)) + prefs.edit().remove(SET_CHANGED).apply() + } + + /** The manifest of what was sent and not yet described on the node (ManifestLog), or `item: null`. */ + private fun manifest(): JSONObject { + val d = destinations.get()?.takeIf { prefs.getBoolean(ON, false) } ?: throw Refused("Refused: files backup is off") + noteRestoreSet(d) + val file = manifestLog(d).issue() ?: return JSONObject().put("item", JSONObject.NULL) + val token = PhotoChannels.hex(ByteArray(16).also { random.nextBytes(it) }) + manifestToken = token to file + return JSONObject().put("item", JSONObject().put("token", token) + .put("name", ManifestLog.nameFor(System.currentTimeMillis(), TimeZone.getDefault())) + .put("dir", base(d) + "/" + ManifestLog.DIR).put("size", file.length())) + } + + private fun manifestSent(token: String) { + if (manifestToken?.first != token) throw Refused("Refused: unknown manifest") + val d = destinations.get() ?: throw Refused("Refused: the backup changed") + manifestLog(d).confirm() + manifestToken = null + } + + /** A range of an issued file, for `/drivesync/<token>` on the packaged origin. */ + fun serve(path: String, range: String?): WebResourceResponse? { + manifestToken?.takeIf { it.first == path.removePrefix(kind.path) }?.let { (_, file) -> + return PhotoChannels.serveRange(file.inputStream(), file.length(), "application/x-ndjson", range) + } + val issued = tokens[path.removePrefix(kind.path)] ?: return null + val f = issued.pending.file + val raw = try { source.open(f) } catch (e: Exception) { null } ?: return null + return PhotoChannels.serveRange(raw, f.size, f.mime.ifEmpty { "application/octet-stream" }, range) + } + + private fun hashOf(f: DriveFile): String? = try { + source.open(f)?.use { PhotoChannels.hex(PhotoChannels.digestOf(it)) } + } catch (e: Exception) { null } + + companion object { + private const val OPTIONS = "options" + private const val SET_CHANGED = "restore_set_changed" + private const val FOLDERS = "folders" + private const val ON = "on" + private const val LAST = "last_completed" + private const val FAILURE = "failure" + private const val FAILURE_AT = "failure_at" + private const val NOTIFIED = "notified" + } +} diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveKind.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveKind.kt new file mode 100644 index 0000000..69a658a --- /dev/null +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveKind.kt @@ -0,0 +1,79 @@ +package org.meshbay.client.drive + +import org.json.JSONObject +import java.util.Locale + +/** + * A backup of folders chosen through the system's picker: the files backup, + * or WhatsApp's own folder (docs/MESHBAY_DESIGN.md §9.13). One class runs both + * (DriveChannels); this is what differs. + */ +data class DriveKind( + /** Names the channels (`drivesync:status`) and the preferences. */ + val prefix: String, + /** The folder under the destination: `<account>-<suffix>`. */ + val suffix: String, + val notifyId: Int, + /** Why a chosen folder is refused, from its tree document id, or null. */ + val refusal: (String) -> String?, + /** One folder only: choosing again replaces it. */ + val single: Boolean, + /** The chosen folder's own name as the first level on the node. */ + val keepTop: Boolean, + /** Whether a file found there is sent, given the options the person set. */ + val include: (DriveFile, JSONObject) -> Boolean, + /** The files a restore needs together, listed in the manifest as one set; null when none. */ + val restoreSet: ((DriveFile) -> Boolean)?, + /** Where the picker opens. */ + val initialDocId: String?, +) { + val path: String get() = "/$prefix/" + + companion object { + val FILES = DriveKind( + prefix = "drivesync", suffix = "drive", notifyId = 13, + refusal = DrivePlan::refusal, single = false, keepTop = true, + include = { _, _ -> true }, restoreSet = null, initialDocId = null, + ) + + val WHATSAPP = DriveKind( + prefix = "wasync", suffix = "whatsapp", notifyId = 14, + refusal = WhatsApp::refusal, single = true, keepTop = false, + include = { f, options -> WhatsApp.include(f, options.optBoolean("media", false)) }, + restoreSet = WhatsApp::inRestoreSet, + initialDocId = "primary:" + WhatsApp.FOLDER, + ) + } +} + +/** + * WhatsApp's folder, `Android/media/com.whatsapp/WhatsApp` (or the business + * application's): what a restore onto a new phone needs, and the media. + * + * `Databases/` holds the chats, encrypted by WhatsApp with a key only it (or + * the person's end-to-end backup password) opens: a full copy written weekly, + * `msgstore.db.crypt14`, and an increment a night, + * `msgstore-increment-N.db.crypt14`. When a new week starts WhatsApp renames + * the last week's set with its date; those dated copies are what this backup + * already kept as earlier versions, so they are not sent again. `Backups/` + * holds the rest of a restore (contacts, settings, stickers). + */ +object WhatsApp { + const val FOLDER = "Android/media/com.whatsapp/WhatsApp" + private val FOLDERS = setOf(FOLDER, "Android/media/com.whatsapp.w4b/WhatsApp Business") + private val DATED = Regex("^msgstore(-increment-\\d+)?-\\d{4}-\\d{2}-\\d{2}\\..*", RegexOption.IGNORE_CASE) + + fun refusal(treeDocId: String): String? = + if (DrivePlan.pathOf(treeDocId) in FOLDERS) null else "not_whatsapp" + + private fun top(f: DriveFile) = f.sub.substringBefore('/').lowercase(Locale.ROOT) + + fun include(f: DriveFile, media: Boolean): Boolean = when (top(f)) { + "databases" -> !DATED.matches(f.name) + "backups" -> true + "media" -> media + else -> false + } + + fun inRestoreSet(f: DriveFile): Boolean = include(f, false) +} diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveLedger.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveLedger.kt new file mode 100644 index 0000000..044edc7 --- /dev/null +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveLedger.kt @@ -0,0 +1,69 @@ +package org.meshbay.client.drive + +import org.json.JSONObject +import java.io.File + +/** + * What this phone has sent of its chosen folders, by document id: the memory + * of what was sent, never a mirror (PhotoLedger, the same idea for photos). A + * file deleted on the phone stays on the node; one deleted on the node is not + * sent again. One line of JSON per send, the last line for an id winning, + * compacted on load once dead lines outnumber live ones. + */ +class DriveLedger(private val file: File) { + + data class Entry( + val docId: String, + val modified: Long, + val size: Long, + /** SHA-256 of the bytes sent, hex: a touched file is sent again only if this changed. */ + val sha256: String, + val dir: String, + val name: String, + val sentAt: Long, + ) + + private val entries = HashMap<String, Entry>() + private var lines = 0 + + init { load() } + + val size: Int get() = entries.size + + operator fun get(docId: String): Entry? = entries[docId] + + fun record(entry: Entry) { + entries[entry.docId] = entry + file.parentFile?.mkdirs() + file.appendText(encode(entry) + "\n") + lines += 1 + } + + private fun load() { + if (!file.exists()) return + file.forEachLine { line -> + if (line.isBlank()) return@forEachLine + lines += 1 + decode(line)?.let { entries[it.docId] = it } + } + if (lines > 2 * entries.size + COMPACT_SLACK) { + val tmp = File(file.path + ".tmp") + tmp.writeText(entries.values.joinToString("") { encode(it) + "\n" }) + if (tmp.renameTo(file)) lines = entries.size else tmp.delete() + } + } + + companion object { + private const val COMPACT_SLACK = 64 + + fun encode(e: Entry): String = JSONObject() + .put("id", e.docId).put("m", e.modified).put("s", e.size).put("h", e.sha256) + .put("d", e.dir).put("n", e.name).put("t", e.sentAt).toString() + + fun decode(line: String): Entry? = try { + val o = JSONObject(line) + Entry(o.getString("id"), o.getLong("m"), o.getLong("s"), o.getString("h"), + o.getString("d"), o.getString("n"), o.getLong("t")) + } catch (e: Exception) { null } + } +} diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DrivePlan.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DrivePlan.kt new file mode 100644 index 0000000..eb373ba --- /dev/null +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DrivePlan.kt @@ -0,0 +1,112 @@ +package org.meshbay.client.drive + +import org.meshbay.client.photos.PhotoPlan +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Locale +import java.util.TimeZone + +/** One file found in a chosen folder, through the Storage Access Framework. */ +data class DriveFile( + /** The folder it was found under, as granted (a tree URI). */ + val tree: String, + /** Its document id, stable for the life of the file (`primary:Documents/a.pdf`). */ + val docId: String, + /** Its folder inside the chosen one, `/`-separated, empty at the top. */ + val sub: String, + val name: String, + val size: Long, + /** Milliseconds. */ + val modified: Long, + val mime: String, +) + +/** A file to send: a new one, or a new version of one already sent. */ +data class DrivePending(val file: DriveFile, val edited: Boolean, val dir: String, val name: String) + +/** + * The rules of the files backup (docs/MESHBAY_DESIGN.md §9.13), pure, so the + * JVM tests hold them: which folders may be chosen, what a run sends, and the + * names the node will accept. + */ +object DrivePlan { + /** + * The phone's media folders: what lives there is the photo backup's, and + * taking it here too would store every photo twice on the node. + */ + private val MEDIA = setOf("dcim", "pictures", "movies") + + /** `primary:Documents/Scans` → `Documents/Scans`; the part after the volume. */ + fun pathOf(docId: String): String = docId.substringAfter(':', "").trim('/') + + /** + * Why a folder cannot be chosen, or null when it can: the whole of a + * volume (it holds the media folders), or a media folder or anything in one. + */ + fun refusal(treeDocId: String): String? { + val path = pathOf(treeDocId) + if (path.isEmpty()) return "whole_storage" + if (path.split('/').first().lowercase(Locale.ROOT) in MEDIA) return "media_folder" + return null + } + + /** + * A run's files: new ones, and new versions of ones sent (beside them, + * never in their place). `skip` holds the files the photo backup already + * sends, as lowercase `relative/path/name`. Smallest first: a run cut short + * has saved the most files. + */ + fun plan( + files: List<DriveFile>, base: String, folderNames: Map<String, String>, + sent: (String) -> DriveLedger.Entry?, skip: Set<String>, zone: TimeZone, + sameBytes: (DriveFile, DriveLedger.Entry) -> Boolean, + ): List<DrivePending> { + val out = ArrayList<DrivePending>() + for (f in files) { + if (f.name.startsWith('.')) continue + if (skip.contains(pathOf(f.docId).lowercase(Locale.ROOT))) continue + val top = folderNames[f.tree] ?: continue + val dir = dirFor(base, top, f.sub) + val was = sent(f.docId) + if (was == null) { + out += DrivePending(f, false, dir, safeName(f.name)) + } else if (was.modified != f.modified || was.size != f.size) { + if (was.size == f.size && sameBytes(f, was)) continue + out += DrivePending(f, true, dir, editedName(f, zone)) + } + } + return out.sortedBy { it.file.size } + } + + /** `<base>/<chosen folder>/<sub>`, each level a name the node accepts; no chosen-folder level when `top` is empty. */ + fun dirFor(base: String, top: String, sub: String): String = + (listOf(base) + listOf(top).filter { it.isNotEmpty() }.map(::safeName) + + sub.split('/').filter { it.isNotEmpty() }.map(::safeName)).joinToString("/") + + /** + * The name as is when the node accepts it (PhotoPlan.UPLOAD_NAME, the + * node's own rule); otherwise its refused characters become `_`, and a + * name that cannot start as the node wants is given a `file-` prefix. + */ + fun safeName(name: String): String { + if (PhotoPlan.UPLOAD_NAME.matches(name)) return name + var s = name.map { c -> if (c.isLetterOrDigit() || c in " _.-()[]'’,&+#@") c else '_' }.joinToString("") + .trimEnd(' ', '.') + if (s.isEmpty() || !s.first().isLetterOrDigit()) s = "file-$s" + s = s.take(MAX_NAME).trimEnd(' ', '.') + return if (PhotoPlan.UPLOAD_NAME.matches(s)) s else "file" + } + + /** A new version lands beside the one sent, its date in its name. */ + fun editedName(f: DriveFile, zone: TimeZone): String { + val base = safeName(f.name) + val dot = base.lastIndexOf('.') + val stem = if (dot > 0) base.substring(0, dot) else base + val ext = if (dot > 0) base.substring(dot) else "" + val stamp = SimpleDateFormat("yyyyMMdd-HHmmss", Locale.ROOT).apply { timeZone = zone }.format(Date(f.modified)) + val suffix = "-modified-$stamp$ext" + return stem.take(MAX_NAME - suffix.length) + suffix + } + + private const val MAX_NAME = 128 +} diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveSource.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveSource.kt new file mode 100644 index 0000000..69077fa --- /dev/null +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/drive/DriveSource.kt @@ -0,0 +1,55 @@ +package org.meshbay.client.drive + +import android.content.Context +import android.net.Uri +import android.provider.DocumentsContract +import android.provider.DocumentsContract.Document +import java.io.InputStream + +/** + * The files of the folders the person chose, read through the Storage Access + * Framework: each folder was granted through the system's own picker, which + * is all the access this needs (no storage permission, which Play keeps for a + * few kinds of application). + */ +class DriveSource(private val context: Context) { + + /** Every file under `tree`, its sub-folders included; hidden ones (`.x`) and their contents skipped. */ + fun files(tree: Uri): List<DriveFile> { + val out = ArrayList<DriveFile>() + val root = DocumentsContract.getTreeDocumentId(tree) + val pending = ArrayDeque(listOf(root to "")) + val cols = arrayOf(Document.COLUMN_DOCUMENT_ID, Document.COLUMN_DISPLAY_NAME, Document.COLUMN_MIME_TYPE, + Document.COLUMN_SIZE, Document.COLUMN_LAST_MODIFIED) + while (pending.isNotEmpty()) { + val (parent, sub) = pending.removeFirst() + val children = DocumentsContract.buildChildDocumentsUriUsingTree(tree, parent) + context.contentResolver.query(children, cols, null, null, null)?.use { c -> + while (c.moveToNext()) { + val id = c.getString(0) ?: continue + val name = c.getString(1) ?: continue + if (name.startsWith('.')) continue + val mime = c.getString(2) ?: "" + if (mime == Document.MIME_TYPE_DIR) { + pending.addLast(id to (if (sub.isEmpty()) name else "$sub/$name")) + } else { + out += DriveFile(tree.toString(), id, sub, name, + if (c.isNull(3)) 0 else c.getLong(3), + if (c.isNull(4)) 0 else c.getLong(4), mime) + } + } + } + } + return out + } + + fun open(f: DriveFile): InputStream? = context.contentResolver.openInputStream( + DocumentsContract.buildDocumentUriUsingTree(Uri.parse(f.tree), f.docId)) + + /** The chosen folder's own name, as the picker showed it. */ + fun nameOf(tree: Uri): String? = try { + val doc = DocumentsContract.buildDocumentUriUsingTree(tree, DocumentsContract.getTreeDocumentId(tree)) + context.contentResolver.query(doc, arrayOf(Document.COLUMN_DISPLAY_NAME), null, null, null) + ?.use { c -> if (c.moveToFirst()) c.getString(0) else null } + } catch (e: Exception) { null } +} diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/phonesync/CalendarSource.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/phonesync/CalendarSource.kt new file mode 100644 index 0000000..bcef08c --- /dev/null +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/phonesync/CalendarSource.kt @@ -0,0 +1,89 @@ +package org.meshbay.client.phonesync + +import android.Manifest +import android.content.Context +import android.provider.CalendarContract.Attendees +import android.provider.CalendarContract.Calendars +import android.provider.CalendarContract.Events +import android.provider.CalendarContract.Reminders +import java.io.File +import java.security.MessageDigest +import java.util.TimeZone + +/** + * The phone's calendars, as one iCalendar file (Ics.kt) every calendar + * application imports. + * + * Every calendar the person can write to is in it: their own, local ones and + * those of the accounts on the phone. Calendars they only subscribe to (public + * holidays, birthdays from contacts) are left out: they are someone else's and + * come back by subscribing again. Like contacts, every run writes the whole of + * it and sends it only when it differs from the last copy the node took + * (`marker` is its SHA-256, without the DTSTAMP lines that change every run). + */ +class CalendarSource(private val context: Context) : DocSource { + override val permissions = arrayOf(Manifest.permission.READ_CALENDAR) + override val suffix = "calendar" + + override fun export(marker: String?, dir: File, now: Long, zone: TimeZone): Export? { + val resolver = context.contentResolver + val calendars = HashMap<Long, String>() + resolver.query(Calendars.CONTENT_URI, + arrayOf(Calendars._ID, Calendars.CALENDAR_DISPLAY_NAME, Calendars.CALENDAR_ACCESS_LEVEL), + "${Calendars.CALENDAR_ACCESS_LEVEL} >= ?", arrayOf(Calendars.CAL_ACCESS_CONTRIBUTOR.toString()), + null)?.use { c -> while (c.moveToNext()) calendars[c.getLong(0)] = c.getString(1) ?: "" } + if (calendars.isEmpty()) return null + + val reminders = HashMap<Long, MutableList<Int>>() + resolver.query(Reminders.CONTENT_URI, arrayOf(Reminders.EVENT_ID, Reminders.MINUTES), null, null, null) + ?.use { c -> while (c.moveToNext()) reminders.getOrPut(c.getLong(0)) { ArrayList() } += c.getInt(1) } + val attendees = HashMap<Long, MutableList<Attendee>>() + val organizers = HashMap<Long, String>() + resolver.query(Attendees.CONTENT_URI, + arrayOf(Attendees.EVENT_ID, Attendees.ATTENDEE_EMAIL, Attendees.ATTENDEE_NAME, + Attendees.ATTENDEE_RELATIONSHIP), null, null, null)?.use { c -> + while (c.moveToNext()) { + val email = c.getString(1)?.takeIf { it.contains('@') } ?: continue + if (c.getInt(3) == Attendees.RELATIONSHIP_ORGANIZER) organizers[c.getLong(0)] = email + else attendees.getOrPut(c.getLong(0)) { ArrayList() } += Attendee(email, c.getString(2)) + } + } + + val cols = arrayOf( + Events._ID, Events.CALENDAR_ID, Events.UID_2445, Events.TITLE, Events.DESCRIPTION, + Events.EVENT_LOCATION, Events.DTSTART, Events.DTEND, Events.DURATION, Events.ALL_DAY, + Events.EVENT_TIMEZONE, Events.RRULE, Events.RDATE, Events.EXDATE, Events.ORIGINAL_ID, + Events.ORIGINAL_INSTANCE_TIME, Events.ORIGINAL_ALL_DAY, Events.STATUS, + ) + val ids = calendars.keys.joinToString(",") + val events = ArrayList<CalEvent>() + resolver.query(Events.CONTENT_URI, cols, "${Events.DELETED} = 0 AND ${Events.CALENDAR_ID} IN ($ids)", + null, "${Events._ID} ASC")?.use { c -> + fun str(i: Int) = if (c.isNull(i)) null else c.getString(i) + fun long(i: Int) = if (c.isNull(i)) null else c.getLong(i) + while (c.moveToNext()) { + val id = c.getLong(0) + events += CalEvent( + id, calendars[c.getLong(1)] ?: "", str(2), str(3), str(4), str(5), + c.getLong(6), long(7), str(8), c.getInt(9) == 1, str(10), + str(11), str(12), str(13), long(14), long(15), c.getInt(16) == 1, + if (c.isNull(17)) null else c.getInt(17), + reminders[id].orEmpty(), organizers[id], attendees[id].orEmpty(), + ) + } + } + if (events.isEmpty()) return null + + val file = File(dir, "calendar.ics") + val digest = MessageDigest.getInstance("SHA-256") + file.bufferedWriter(Charsets.UTF_8).use { out -> + Ics.write(events, now) { l -> + out.write(l); out.write("\r\n") + if (!l.startsWith("DTSTAMP:")) digest.update((l + "\n").toByteArray(Charsets.UTF_8)) + } + } + val sha = digest.digest().joinToString("") { "%02x".format(it) } + if (sha == marker) { file.delete(); return null } + return Export(file, DocPlan.nameFor("calendar", "ics", now, zone), "text/calendar", sha, events.size) + } +} diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/phonesync/DocChannels.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/phonesync/DocChannels.kt index 2efe558..f4da64f 100644 --- a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/phonesync/DocChannels.kt +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/phonesync/DocChannels.kt @@ -64,6 +64,7 @@ class DocChannels( "sent" -> { sent(args.optString(0, ""), args.optString(1, ""), args.optString(2, "")); true } "completed" -> { completed(); status() } "failed" -> failed(args.optString(0, ""), args.optString(1, "")) + "open-settings" -> { openSettings(); true } else -> throw Refused("Refused: no such channel") } @@ -167,6 +168,19 @@ class DocChannels( permission = null } + /** + * The application's page in the system's settings. Android keeps some + * permissions (text messages) behind "restricted settings" for an + * application installed outside a store; that page is where the person + * lifts it (⋮ > Allow restricted settings), and nowhere in here can. + */ + private fun openSettings() { + val intent = android.content.Intent(android.provider.Settings.ACTION_APPLICATION_DETAILS_SETTINGS, + android.net.Uri.fromParts("package", activity.packageName, null)) + .addFlags(android.content.Intent.FLAG_ACTIVITY_NEW_TASK) + activity.runOnUiThread { activity.startActivity(intent) } + } + /** From Activity.onRequestPermissionsResult; true when the request was ours. */ fun deliverPermission(requestCode: Int): Boolean { if (requestCode != permissionRequest) return false diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/phonesync/Ics.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/phonesync/Ics.kt new file mode 100644 index 0000000..8f151f0 --- /dev/null +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/phonesync/Ics.kt @@ -0,0 +1,176 @@ +package org.meshbay.client.phonesync + +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Locale +import java.util.TimeZone + +/** One event as the calendar provider stores it, as much of it as a backup needs. */ +data class CalEvent( + val id: Long, + val calendar: String, + /** RFC 5545 UID from the provider, when the event came with one. */ + val uid: String?, + val title: String?, + val description: String?, + val location: String?, + /** Milliseconds; for an all-day event, midnight UTC of its first day. */ + val start: Long, + val end: Long?, + /** Android's form, `P3600S` or `P1D`, set instead of `end` on a recurring event. */ + val duration: String?, + val allDay: Boolean, + /** Olson id (`Europe/Paris`), or null. */ + val timeZone: String?, + val rrule: String?, + val rdate: String?, + val exdate: String?, + /** On an exception to a recurring event: the event it changes, and the instance. */ + val originalId: Long?, + val originalTime: Long?, + val originalAllDay: Boolean, + /** 0 tentative, 1 confirmed, 2 cancelled, or null. */ + val status: Int?, + val reminderMinutes: List<Int>, + val organizer: String?, + val attendees: List<Attendee>, +) + +data class Attendee(val email: String, val name: String?) + +/** + * Events written as one iCalendar file (RFC 5545), the format every calendar + * imports. Pure, so the JVM tests hold it; `line` receives each folded line + * without its CRLF, so the caller can write it and leave the DTSTAMP lines out + * of the hash that says whether anything changed. + */ +object Ics { + fun write(events: List<CalEvent>, stamp: Long, line: (String) -> Unit) { + val uids = events.associate { it.id to uidOf(it) } + line("BEGIN:VCALENDAR") + line("VERSION:2.0") + line("PRODID:-//MeshBay//Android backup//EN") + line("CALSCALE:GREGORIAN") + for (e in events) { + val out = ArrayList<String>() + out += "BEGIN:VEVENT" + // An exception carries the UID of the event it changes. + out += "UID:" + text(e.originalId?.let { uids[it] } ?: uids.getValue(e.id)) + out += "DTSTAMP:" + utc(stamp) + if (e.originalId != null && e.originalTime != null) { + out += "RECURRENCE-ID" + time(e.originalTime, e.originalAllDay, e.timeZone) + } + out += "DTSTART" + time(e.start, e.allDay, e.timeZone) + when { + e.end != null && e.end >= e.start -> out += "DTEND" + time(e.end, e.allDay, e.timeZone) + e.duration != null -> duration(e.duration)?.let { out += "DURATION:$it" } + } + e.title?.takeIf { it.isNotEmpty() }?.let { out += "SUMMARY:" + text(it) } + e.description?.takeIf { it.isNotEmpty() }?.let { out += "DESCRIPTION:" + text(it) } + e.location?.takeIf { it.isNotEmpty() }?.let { out += "LOCATION:" + text(it) } + e.rrule?.takeIf { it.isNotBlank() }?.let { out += "RRULE:" + it.trim() } + e.rdate?.let { dates("RDATE", it) }?.let { out += it } + e.exdate?.let { dates("EXDATE", it) }?.let { out += it } + when (e.status) { + 0 -> out += "STATUS:TENTATIVE" + 1 -> out += "STATUS:CONFIRMED" + 2 -> out += "STATUS:CANCELLED" + } + out += "X-MESHBAY-CALENDAR:" + text(e.calendar) + e.organizer?.takeIf { it.isNotBlank() }?.let { out += "ORGANIZER:mailto:" + it.trim() } + for (a in e.attendees) { + val cn = a.name?.takeIf { it.isNotBlank() }?.let { ";CN=" + param(it) } ?: "" + out += "ATTENDEE$cn:mailto:" + a.email.trim() + } + for (m in e.reminderMinutes.distinct()) { + out += "BEGIN:VALARM" + out += "ACTION:DISPLAY" + out += "DESCRIPTION:" + text(e.title ?: "Reminder") + out += "TRIGGER:-PT${maxOf(0, m)}M" + out += "END:VALARM" + } + out += "END:VEVENT" + for (l in out) fold(l, line) + } + line("END:VCALENDAR") + } + + private fun uidOf(e: CalEvent): String = + e.uid?.takeIf { it.isNotBlank() }?.trim() ?: "android-${e.id}@meshbay" + + /** `;VALUE=DATE:20261010`, `;TZID=Europe/Paris:20261010T090000` or `:20261010T070000Z`. */ + fun time(ms: Long, allDay: Boolean, zone: String?): String { + if (allDay) return ";VALUE=DATE:" + fmt("yyyyMMdd", "UTC", ms) + val tz = zone?.takeIf { it.isNotBlank() && it != "UTC" && it != "GMT" && it in zones } + return if (tz == null) ":" + utc(ms) else ";TZID=$tz:" + fmt("yyyyMMdd'T'HHmmss", tz, ms) + } + + private val zones by lazy { TimeZone.getAvailableIDs().toHashSet() } + + fun utc(ms: Long) = fmt("yyyyMMdd'T'HHmmss'Z'", "UTC", ms) + + private fun fmt(pattern: String, zone: String, ms: Long) = + SimpleDateFormat(pattern, Locale.ROOT).apply { timeZone = TimeZone.getTimeZone(zone) }.format(Date(ms)) + + /** Android's `P3600S` is not RFC 5545, which wants `PT3600S`; days and weeks are the same. */ + fun duration(android: String): String? { + val m = Regex("^([+-]?)P(?:(\\d+)W)?(?:(\\d+)D)?(?:T?(?:(\\d+)H)?(?:(\\d+)M)?(?:(\\d+)S)?)?$") + .matchEntire(android.trim()) ?: return null + val (sign, w, d, h, min, s) = m.destructured + if (listOf(w, d, h, min, s).all { it.isEmpty() }) return null + val time = (if (h.isEmpty()) "" else "${h}H") + (if (min.isEmpty()) "" else "${min}M") + + (if (s.isEmpty()) "" else "${s}S") + return sign + "P" + (if (w.isEmpty()) "" else "${w}W") + (if (d.isEmpty()) "" else "${d}D") + + (if (time.isEmpty()) "" else "T$time") + } + + /** + * Android keeps RDATE and EXDATE as `20261010T090000Z,20261017T090000Z`, + * or with a zone first: `Europe/Paris;20261010T090000,…`. + */ + fun dates(name: String, value: String): String? { + val v = value.trim().ifEmpty { return null } + val semi = v.indexOf(';') + if (semi > 0) return "$name;TZID=${v.substring(0, semi)}:${v.substring(semi + 1)}" + return if (v.split(',').all { it.length == 8 }) "$name;VALUE=DATE:$v" else "$name:$v" + } + + /** A TEXT value: backslash, semicolon, comma and line breaks escaped. */ + fun text(s: String): String = buildString { + for (c in s) when (c) { + '\\' -> append("\\\\") + ';' -> append("\\;") + ',' -> append("\\,") + '\n' -> append("\\n") + '\r' -> {} + else -> if (c < ' ' && c != '\t') {} else append(c) + } + } + + /** A parameter value, quoted when it holds what would end it. */ + private fun param(s: String): String { + val clean = s.replace("\"", "'").filter { it >= ' ' } + return if (clean.any { it == ';' || it == ':' || it == ',' }) "\"$clean\"" else clean + } + + /** Lines of at most 75 octets, continued with a space; never inside a character. */ + fun fold(l: String, line: (String) -> Unit) { + var rest = l + var limit = 75 + while (rest.toByteArray(Charsets.UTF_8).size > limit) { + var cut = 0 + var bytes = 0 + while (cut < rest.length) { + val cp = rest.codePointAt(cut) + val n = String(Character.toChars(cp)).toByteArray(Charsets.UTF_8).size + if (bytes + n > limit) break + bytes += n + cut += Character.charCount(cp) + } + line((if (limit == 74) " " else "") + rest.substring(0, cut)) + rest = rest.substring(cut) + limit = 74 + } + line((if (limit == 74) " " else "") + rest) + } +} diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/phonesync/ManifestLog.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/phonesync/ManifestLog.kt new file mode 100644 index 0000000..0d7142c --- /dev/null +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/phonesync/ManifestLog.kt @@ -0,0 +1,56 @@ +package org.meshbay.client.phonesync + +import org.json.JSONObject +import java.io.File +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Locale +import java.util.TimeZone + +/** + * What a backup sent, as the phone knew it, for a restore or a merge later + * (docs/MESHBAY_DESIGN.md §9.12): one JSON line per file the node took, with + * where it came from on the phone, its album or folder, its dates, size and + * SHA-256. The node has the files; only the phone knew these. + * + * Lines wait here until a manifest carrying them is on the node: `issue()` + * gathers them into the file to send, `confirm()` forgets them once the node + * has it. A manifest that did not arrive is sent again, with whatever was + * added since, at the next run: nothing is lost to an interrupted one. + */ +class ManifestLog(private val pending: File) { + private val sending = File(pending.path + ".sending") + + fun append(line: JSONObject) { + pending.parentFile?.mkdirs() + pending.appendText(line.toString() + "\n") + } + + /** True when there are lines no manifest on the node holds yet. */ + fun waiting(): Boolean = (sending.exists() && sending.length() > 0) || (pending.exists() && pending.length() > 0) + + /** The file to send now, holding every waiting line, or null. */ + fun issue(): File? { + if (pending.exists() && pending.length() > 0) { + if (sending.exists()) { sending.appendText(pending.readText()); pending.delete() } + else if (!pending.renameTo(sending)) return null + } + return sending.takeIf { it.exists() && it.length() > 0 } + } + + /** The node has it: those lines are done. */ + fun confirm() { sending.delete() } + + /** Everything forgotten: the destination changed, or the backup was turned off. */ + fun clear() { pending.delete(); sending.delete() } + + companion object { + /** The folder manifests go into, under a kind's own folder. */ + const val DIR = "meshbay-manifest" + + /** `manifest-2026-10-10-143205.jsonl`, in the phone's time: never one name twice. */ + fun nameFor(now: Long, zone: TimeZone): String = + "manifest-" + SimpleDateFormat("yyyy-MM-dd-HHmmss", Locale.ROOT).apply { timeZone = zone }.format(Date(now)) + + ".jsonl" + } +} diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/ByteRange.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/ByteRange.kt new file mode 100644 index 0000000..001bea2 --- /dev/null +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/ByteRange.kt @@ -0,0 +1,40 @@ +package org.meshbay.client.photos + +import java.io.FilterInputStream +import java.io.InputStream + +/** + * One `Range: bytes=a-b` of a file the page fetches a piece at a time, so a + * video of gigabytes never sits whole in the page's memory: the upload reads + * one chunk, sends it, and asks for the next. + */ +object ByteRange { + /** The first and last byte asked for, inside `size`; null for no range or one this does not serve. */ + fun parse(header: String?, size: Long): LongRange? { + val m = Regex("^bytes=(\\d+)-(\\d*)$").matchEntire(header?.trim() ?: return null) ?: return null + val first = m.groupValues[1].toLongOrNull() ?: return null + val last = m.groupValues[2].takeIf { it.isNotEmpty() }?.toLongOrNull() ?: (size - 1) + if (first >= size || last < first) return null + return first..minOf(last, size - 1) + } + + /** `stream` from `range.first`, ending after `range.last`. */ + fun slice(stream: InputStream, range: LongRange): InputStream { + var toSkip = range.first + while (toSkip > 0) { + val n = stream.skip(toSkip) + if (n <= 0) { if (stream.read() < 0) break; toSkip -= 1 } else toSkip -= n + } + return object : FilterInputStream(stream) { + private var left = range.last - range.first + 1 + override fun read(): Int { + if (left <= 0) return -1 + return super.read().also { if (it >= 0) left -= 1 } + } + override fun read(b: ByteArray, off: Int, len: Int): Int { + if (left <= 0) return -1 + return super.read(b, off, minOf(len.toLong(), left).toInt()).also { if (it > 0) left -= it } + } + } + } +} diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoChannels.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoChannels.kt index 3eb0ead..c1de859 100644 --- a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoChannels.kt +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoChannels.kt @@ -22,6 +22,7 @@ import org.meshbay.client.bridge.Refused import org.meshbay.client.notify.Notifier import org.meshbay.client.phonesync.Destination import org.meshbay.client.phonesync.DestinationChannels +import org.meshbay.client.phonesync.ManifestLog import java.io.File import java.io.FilterInputStream import java.io.InputStream @@ -75,12 +76,14 @@ class PhotoChannels( fun call(channel: String, args: JSONArray): Any? = when (channel) { "photosync:status" -> status() - "photosync:permit" -> { permit(); status() } - "photosync:albums" -> { requirePermission(); albums() } + "photosync:permit" -> { permit(args.optBoolean(0, false)); status() } + "photosync:albums" -> { requirePermission(); albums(args.optBoolean(0, false)) } "photosync:configure" -> { configure(args.optJSONObject(0)); status() } "photosync:estimate" -> { requirePermission(); estimate(args.optJSONObject(0) ?: throw Refused("Refused: no settings")) } "photosync:plan" -> { requirePermission(); plan() } "photosync:sent" -> { sent(args.optString(0, ""), args.optString(1, ""), args.optString(2, "")); true } + "photosync:manifest" -> manifest() + "photosync:manifest-sent" -> { manifestSent(args.optString(0, "")); true } "photosync:completed" -> { completed(); status() } "photosync:failed" -> failed(args.optString(0, ""), args.optString(1, "")) "photosync:keep-alive" -> { onKeepAlive(args.optBoolean(0, false), args.optString(1, "").take(200)); true } @@ -97,13 +100,20 @@ class PhotoChannels( return (destinations.get() ?: return null) to c } - private fun ledger(d: Destination) = PhotoLedger(File(dir, hex(sha256Of(d.ledgerKey.toByteArray())).take(32) + ".jsonl")) + private fun keyOf(d: Destination) = hex(sha256Of(d.ledgerKey.toByteArray())).take(32) + + private fun ledger(d: Destination) = PhotoLedger(File(dir, keyOf(d) + ".jsonl")) + + private fun manifestLog(d: Destination) = ManifestLog(File(dir, keyOf(d) + ".manifest")) + + @Volatile private var manifestToken: Pair<String, File>? = null fun status(): JSONObject { val c = config() val d = destinations.get() return JSONObject() .put("permission", permissionState()) + .put("videoPermission", permissionState(video = true)) .put("unmetered", unmetered()) .put("config", c?.toJson() ?: JSONObject.NULL) .put("destination", d?.toJson() ?: JSONObject.NULL) @@ -125,9 +135,11 @@ class PhotoChannels( if (destinations.get() == null) throw Refused("Refused: no destination") val next = SyncConfig.fromJson(o, System.currentTimeMillis(), previous) val edit = prefs.edit().putString(CONFIG, next.toJson().toString()) - // A different scope is a different backup: due at once, and whatever - // the last one was refused for is not this one's problem. - if (previous == null || previous.since != next.since) { + // Anything that asks for more (another scope, an album, the videos) + // is due at once rather than tomorrow: the person just asked for it, + // and waits to see it go. Whatever the last run was refused for is + // not this one's problem. + if (previous == null || previous != next) { edit.remove(LAST).remove(FAILURE).remove(FAILURE_AT).remove(NOTIFIED) } edit.apply() @@ -154,9 +166,10 @@ class PhotoChannels( // ── the phone's photos ─────────────────────────────────────────────────── - private fun albums(): JSONArray = JSONArray().apply { - for (a in source.albums()) put(JSONObject().put("id", a.id).put("name", a.name) - .put("count", a.count).put("bytes", a.bytes).put("camera", a.camera)) + private fun albums(withVideos: Boolean): JSONArray = JSONArray().apply { + for (a in source.albums(withVideos)) put(JSONObject().put("id", a.id).put("name", a.name) + .put("count", a.count).put("bytes", a.bytes).put("camera", a.camera) + .put("videos", a.videos).put("videoBytes", a.videoBytes)) } /** What a backup set up this way would send first: the count and size the confirmation states. */ @@ -164,15 +177,17 @@ class PhotoChannels( val d = destinations.get() ?: throw Refused("Refused: no destination") val c = SyncConfig.fromJson(o, System.currentTimeMillis(), config()) val ledger = ledger(d) - val items = PhotoPlan.plan(source.photos(c.albums), c, PhotoPlan.baseFor(d), ledger::get) { _, _ -> true } + val items = PhotoPlan.plan(source.photos(c.albums, c.includeVideos), c, PhotoPlan.baseFor(d), ledger::get) { _, _ -> true } + val videos = items.filter { it.photo.video } return JSONObject().put("count", items.size).put("bytes", items.sumOf { it.photo.size }) + .put("videos", videos.size).put("videoBytes", videos.sumOf { it.photo.size }) } private fun plan(): JSONObject { val (d, c) = active() ?: throw Refused("Refused: photo backup is off") val ledger = ledger(d) - val items = PhotoPlan.plan(source.photos(c.albums), c, PhotoPlan.baseFor(d), ledger::get) { p, sent -> - hashOf(p.mediaId)?.let { it == sent.sha256 } ?: true + val items = PhotoPlan.plan(source.photos(c.albums, c.includeVideos), c, PhotoPlan.baseFor(d), ledger::get) { p, sent -> + hashOf(p)?.let { it == sent.sha256 } ?: true } // A new plan replaces the last one: tokens are for one run, never kept. tokens.clear() @@ -182,11 +197,12 @@ class PhotoChannels( tokens[token] = Issued(p, d.ledgerKey) out.put(JSONObject().put("token", token).put("name", p.name).put("dir", p.dir) .put("size", p.photo.size).put("edited", p.edited).put("taken", PhotoPlan.whenTaken(p.photo)) + .put("video", p.photo.video) // After a reinstall the ledger is empty, and the page looks in the // folder for what is already there — an edit under its own name too. .put("alsoKnownAs", PhotoPlan.editedName(p.photo, java.util.TimeZone.getDefault()))) } - return JSONObject().put("items", out) + return JSONObject().put("items", out).put("manifest", manifestLog(d).waiting()) } /** The node took it (or already had it): into the ledger, under the name its ack gave. */ @@ -194,19 +210,56 @@ class PhotoChannels( val issued = tokens[token] ?: throw Refused("Refused: unknown photo") val d = active()?.first?.takeIf { it.ledgerKey == issued.key } ?: throw Refused("Refused: the backup changed") val p = issued.pending.photo - val sha = issued.sha256 ?: hashOf(p.mediaId) ?: throw Refused("Refused: the photo is gone") + val sha = issued.sha256 ?: hashOf(p) ?: throw Refused("Refused: the photo is gone") + val now = System.currentTimeMillis() ledger(d).record(PhotoLedger.Entry(p.mediaId, p.modified, p.size, sha, - dir.take(1024), name.take(256), System.currentTimeMillis())) + dir.take(1024), name.take(256), now)) + manifestLog(d).append(JSONObject() + .put("kind", if (p.video) "video" else "photo") + .put("node", "${dir.take(1024)}/${name.take(256)}") + .put("source", p.relPath).put("album", p.album) + .put("taken", PhotoPlan.whenTaken(p)).put("modified", p.modified * 1000) + .put("size", p.size).put("sha256", sha).put("mime", p.mime) + .put("edited", issued.pending.edited).put("sentAt", now)) tokens.remove(token) } /** - * The bytes of an issued photo, for `/photosync/<token>` on the packaged - * origin. Hashed as they go out, so the ledger records exactly what was sent. + * The manifest of what was sent and not yet described on the node, for + * `<base>/meshbay-manifest/` (ManifestLog), or `item: null`. + */ + private fun manifest(): JSONObject { + val (d, _) = active() ?: throw Refused("Refused: photo backup is off") + val file = manifestLog(d).issue() ?: return JSONObject().put("item", JSONObject.NULL) + val token = hex(ByteArray(16).also { random.nextBytes(it) }) + manifestToken = token to file + return JSONObject().put("item", JSONObject().put("token", token) + .put("name", ManifestLog.nameFor(System.currentTimeMillis(), java.util.TimeZone.getDefault())) + .put("dir", PhotoPlan.baseFor(d) + "/" + ManifestLog.DIR).put("size", file.length())) + } + + private fun manifestSent(token: String) { + if (manifestToken?.first != token) throw Refused("Refused: unknown manifest") + val d = destinations.get() ?: throw Refused("Refused: the backup changed") + manifestLog(d).confirm() + manifestToken = null + } + + /** + * The bytes of an issued photo or video, for `/photosync/<token>` on the + * packaged origin. Asked a range at a time (ByteRange), as the upload + * reads them, so a video never sits whole in the page; the ledger's hash + * is then taken when the node has it (`sent`). Asked whole, it is hashed + * as it goes out. */ - fun serve(path: String): WebResourceResponse? { + fun serve(path: String, range: String? = null): WebResourceResponse? { + manifestToken?.takeIf { it.first == path.removePrefix(PATH) }?.let { (_, file) -> + return serveRange(file.inputStream(), file.length(), "application/x-ndjson", range) + } val issued = tokens[path.removePrefix(PATH)] ?: return null - val raw = try { source.open(issued.pending.photo.mediaId) } catch (e: Exception) { null } ?: return null + val raw = try { source.open(issued.pending.photo) } catch (e: Exception) { null } ?: return null + val mime = issued.pending.photo.mime.ifEmpty { "application/octet-stream" } + if (range != null) return serveRange(raw, issued.pending.photo.size, mime, range) val digest = MessageDigest.getInstance("SHA-256") val stream = object : FilterInputStream(raw) { private var done = false @@ -216,20 +269,28 @@ class PhotoChannels( private fun finish() { if (!done) { done = true; issued.sha256 = hex(digest.digest()) } } } val headers = mapOf("Cache-Control" to "no-store", "X-Content-Type-Options" to "nosniff") - return WebResourceResponse(issued.pending.photo.mime.ifEmpty { "application/octet-stream" }, - null, 200, "OK", headers, stream) + return WebResourceResponse(mime, null, 200, "OK", headers, stream) } - private fun hashOf(mediaId: Long): String? = try { - source.open(mediaId)?.use { s -> hex(digestOf(s)) } + private fun hashOf(p: Photo): String? = try { + source.open(p)?.use { s -> hex(digestOf(s)) } } catch (e: Exception) { null } + /** What the photo backup sends, for the files backup to leave out; empty while it is off. */ + fun backedUpPaths(): Set<String> { + val c = config() ?: return emptySet() + if (permissionState() == "denied") return emptySet() + return try { source.relativePaths(c.albums, c.includeVideos) } catch (e: Exception) { emptySet() } + } + // ── permission and network ─────────────────────────────────────────────── - private fun permissionState(): String { + /** For the photos, or with `video` for the videos, which Android 13+ asks about apart. */ + private fun permissionState(video: Boolean = false): String { fun has(p: String) = activity.checkSelfPermission(p) == PackageManager.PERMISSION_GRANTED + val media = if (video) Manifest.permission.READ_MEDIA_VIDEO else Manifest.permission.READ_MEDIA_IMAGES return when { - Build.VERSION.SDK_INT >= 33 && has(Manifest.permission.READ_MEDIA_IMAGES) -> "granted" + Build.VERSION.SDK_INT >= 33 && has(media) -> "granted" Build.VERSION.SDK_INT >= 34 && has(Manifest.permission.READ_MEDIA_VISUAL_USER_SELECTED) -> "partial" Build.VERSION.SDK_INT < 33 && has(Manifest.permission.READ_EXTERNAL_STORAGE) -> "granted" else -> "denied" @@ -241,11 +302,12 @@ class PhotoChannels( } /** Asks, and waits for the answer: the page goes on from what was decided. */ - private fun permit() { + private fun permit(withVideos: Boolean) { + val media = listOf(Manifest.permission.READ_MEDIA_IMAGES) + + (if (withVideos) listOf(Manifest.permission.READ_MEDIA_VIDEO) else emptyList()) val wanted = when { - Build.VERSION.SDK_INT >= 34 -> arrayOf(Manifest.permission.READ_MEDIA_IMAGES, - Manifest.permission.READ_MEDIA_VISUAL_USER_SELECTED) - Build.VERSION.SDK_INT >= 33 -> arrayOf(Manifest.permission.READ_MEDIA_IMAGES) + Build.VERSION.SDK_INT >= 34 -> (media + Manifest.permission.READ_MEDIA_VISUAL_USER_SELECTED).toTypedArray() + Build.VERSION.SDK_INT >= 33 -> media.toTypedArray() else -> arrayOf(Manifest.permission.READ_EXTERNAL_STORAGE) } val latch = CountDownLatch(1) @@ -307,6 +369,21 @@ class PhotoChannels( .setContentIntent(pending).setAutoCancel(true).build()) } + /** `range` of `stream`, `size` bytes long (or a 416 for a range that cannot be served). */ + fun serveRange(stream: InputStream, size: Long, mime: String, range: String?): WebResourceResponse { + val r = ByteRange.parse(range, size) ?: run { + stream.close() + return WebResourceResponse("text/plain", null, 416, "Range Not Satisfiable", + mapOf("Content-Range" to "bytes */$size"), "".byteInputStream()) + } + val headers = mapOf("Cache-Control" to "no-store", "X-Content-Type-Options" to "nosniff", + "Content-Range" to "bytes ${r.first}-${r.last}/$size", + "Content-Length" to (r.last - r.first + 1).toString()) + // 200, not 206: what was asked is in the address, and a 206 to a + // request that carried no Range is not something to rely on. + return WebResourceResponse(mime, null, 200, "OK", headers, ByteRange.slice(stream, r)) + } + fun digestOf(s: InputStream): ByteArray { val d = MessageDigest.getInstance("SHA-256") val buf = ByteArray(64 * 1024) diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoPlan.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoPlan.kt index ee0275e..2cb85e6 100644 --- a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoPlan.kt +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoPlan.kt @@ -21,6 +21,11 @@ data class Photo( val modified: Long, val bucketId: String, val mime: String, + /** From MediaStore's video collection rather than its images. */ + val video: Boolean = false, + /** Where it is on the phone, `DCIM/Camera/PXL_….jpg`, and its album's name, for the manifest. */ + val relPath: String = "", + val album: String = "", ) /** @@ -33,9 +38,12 @@ data class SyncConfig( val includeExisting: Boolean, /** When it was set up or moved, ms: with `includeExisting` off, only photos added since count. */ val since: Long, + /** The videos of the same albums too, beside the photos. Off by default: they are large. */ + val includeVideos: Boolean = false, ) { fun toJson(): JSONObject = JSONObject() .put("albums", JSONArray(albums)).put("includeExisting", includeExisting).put("since", since) + .put("includeVideos", includeVideos) companion object { /** From the page, so checked like any input. */ @@ -49,6 +57,7 @@ data class SyncConfig( (0 until albums.length()).map { albums.optString(it, "").take(64) }.filter { it.isNotEmpty() }.distinct(), includeExisting, if (same) previous!!.since else now, + o.optBoolean("includeVideos", false), ) } @@ -56,7 +65,8 @@ data class SyncConfig( val o = JSONObject(text ?: return null) val albums = o.getJSONArray("albums") SyncConfig((0 until albums.length()).map { albums.getString(it) }, - o.optBoolean("includeExisting", true), o.getLong("since")) + o.optBoolean("includeExisting", true), o.getLong("since"), + o.optBoolean("includeVideos", false)) } catch (e: Exception) { null } } } @@ -92,7 +102,7 @@ object PhotoPlan { val out = ArrayList<Pending>() for (p in photos) { if (p.bucketId !in albums) continue - if (!p.mime.startsWith("image/")) continue + if (!(if (p.video) config.includeVideos && p.mime.startsWith("video/") else p.mime.startsWith("image/"))) continue val sent = ledger(p.mediaId) if (sent == null) { if (!config.includeExisting && p.added * 1000 < config.since) continue @@ -120,7 +130,8 @@ object PhotoPlan { } fun nameFor(p: Photo): String = - p.displayName.takeIf { UPLOAD_NAME.matches(it) } ?: "photo-${p.mediaId}.${extension(p)}" + p.displayName.takeIf { UPLOAD_NAME.matches(it) } + ?: "${if (p.video) "video" else "photo"}-${p.mediaId}.${extension(p)}" /** * An edit lands beside the original under a name that says what it is; left @@ -140,7 +151,10 @@ object PhotoPlan { private fun extension(p: Photo): String = p.displayName.substringAfterLast('.', "").lowercase(Locale.ROOT).takeIf { it.matches(Regex("^[a-z0-9]{1,5}$")) } ?: when (p.mime) { "image/png" -> "png"; "image/heic" -> "heic"; "image/heif" -> "heif" - "image/webp" -> "webp"; "image/gif" -> "gif"; else -> "jpg" } + "image/webp" -> "webp"; "image/gif" -> "gif" + "video/mp4" -> "mp4"; "video/quicktime" -> "mov"; "video/3gpp" -> "3gp" + "video/webm" -> "webm"; "video/x-matroska" -> "mkv" + else -> if (p.video) "mp4" else "jpg" } private const val MAX_NAME = 128 diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoSource.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoSource.kt index f64612f..b3b2834 100644 --- a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoSource.kt +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/photos/PhotoSource.kt @@ -8,54 +8,101 @@ import android.provider.MediaStore import java.io.InputStream /** - * The phone's photos, through MediaStore and nothing else. + * The phone's photos and videos, through MediaStore and nothing else. * * Opened through MediaStore by an application without ACCESS_MEDIA_LOCATION, - * a photo's EXIF location is **redacted by the platform** (Android 10+): a whole - * camera roll going to several people does not say where its owner lives, and - * nobody had to do anything for that. The manifest does not ask for it. + * a file's location is **redacted by the platform** (Android 10+): the EXIF of + * a photo, and the location boxes of an MP4 or MOV video (MediaProvider's + * IsoInterface). A whole camera roll going to several people does not say + * where its owner lives, and nobody had to do anything for that. The manifest + * does not ask for it. + * + * An album is a MediaStore bucket, one directory on the phone; photos and + * videos in it share its id. */ class PhotoSource(private val context: Context) { - data class Album(val id: String, val name: String, val count: Int, val bytes: Long, val camera: Boolean) + data class Album( + val id: String, val name: String, val count: Int, val bytes: Long, val camera: Boolean, + val videos: Int = 0, val videoBytes: Long = 0, + ) - private val collection: Uri = + private val images: Uri = if (Build.VERSION.SDK_INT >= 29) MediaStore.Images.Media.getContentUri(MediaStore.VOLUME_EXTERNAL) else MediaStore.Images.Media.EXTERNAL_CONTENT_URI + private val videos: Uri = + if (Build.VERSION.SDK_INT >= 29) MediaStore.Video.Media.getContentUri(MediaStore.VOLUME_EXTERNAL) + else MediaStore.Video.Media.EXTERNAL_CONTENT_URI - fun albums(): List<Album> { + /** Every album with photos in it, and with videos too when `withVideos`. */ + fun albums(withVideos: Boolean): List<Album> { val by = LinkedHashMap<String, Album>() - query(null, null) { p, name, camera -> + query(images, false, null, null) { p, name, camera -> val a = by[p.bucketId] by[p.bucketId] = if (a == null) Album(p.bucketId, name, 1, p.size, camera) else a.copy(count = a.count + 1, bytes = a.bytes + p.size, camera = a.camera || camera) } - return by.values.sortedWith(compareByDescending<Album> { it.camera }.thenByDescending { it.count }) + if (withVideos) query(videos, true, null, null) { p, name, camera -> + val a = by[p.bucketId] ?: Album(p.bucketId, name, 0, 0, camera) + by[p.bucketId] = a.copy(videos = a.videos + 1, videoBytes = a.videoBytes + p.size, + camera = a.camera || camera) + } + return by.values.sortedWith(compareByDescending<Album> { it.camera }.thenByDescending { it.count + it.videos }) } - fun photos(albums: List<String>): List<Photo> { + fun photos(albums: List<String>, withVideos: Boolean): List<Photo> { if (albums.isEmpty()) return emptyList() val out = ArrayList<Photo>() val where = "${MediaStore.Images.Media.BUCKET_ID} IN (${albums.joinToString(",") { "?" }})" - query(where, albums.toTypedArray()) { p, _, _ -> out += p } + query(images, false, where, albums.toTypedArray()) { p, _, _ -> out += p } + if (withVideos) query(videos, true, where, albums.toTypedArray()) { p, _, _ -> out += p } return out } - fun open(mediaId: Long): InputStream? = - context.contentResolver.openInputStream(ContentUris.withAppendedId(collection, mediaId)) + /** + * The files the photo backup sends, as lowercase `relative/path/name` + * from the top of their volume: what the files backup leaves out, so + * nothing is stored twice. + */ + fun relativePaths(albums: List<String>, withVideos: Boolean): Set<String> { + if (albums.isEmpty()) return emptySet() + val out = HashSet<String>() + val where = "${MediaStore.Images.Media.BUCKET_ID} IN (${albums.joinToString(",") { "?" }})" + @Suppress("DEPRECATION") + val cols = if (Build.VERSION.SDK_INT >= 29) + arrayOf(MediaStore.MediaColumns.RELATIVE_PATH, MediaStore.MediaColumns.DISPLAY_NAME) + else arrayOf(MediaStore.MediaColumns.DATA) + for (collection in listOfNotNull(images, videos.takeIf { withVideos })) { + context.contentResolver.query(collection, cols, where, albums.toTypedArray(), null)?.use { c -> + while (c.moveToNext()) { + val path = if (Build.VERSION.SDK_INT >= 29) + (c.getString(0) ?: continue).trim('/') + "/" + (c.getString(1) ?: continue) + else volumeRelative(c.getString(0) ?: continue) + out += path.lowercase(java.util.Locale.ROOT) + } + } + } + return out + } + + fun open(p: Photo): InputStream? = + context.contentResolver.openInputStream(ContentUris.withAppendedId(if (p.video) videos else images, p.mediaId)) - private fun query(where: String?, args: Array<String>?, each: (Photo, String, Boolean) -> Unit) { + private fun query(collection: Uri, video: Boolean, where: String?, args: Array<String>?, + each: (Photo, String, Boolean) -> Unit) { + // The same columns exist for images and videos (MediaColumns), and on + // Android 8 and 9 under the same names in each collection. val cols = mutableListOf( - MediaStore.Images.Media._ID, MediaStore.Images.Media.DISPLAY_NAME, MediaStore.Images.Media.SIZE, - MediaStore.Images.Media.DATE_TAKEN, MediaStore.Images.Media.DATE_ADDED, - MediaStore.Images.Media.DATE_MODIFIED, MediaStore.Images.Media.BUCKET_ID, - MediaStore.Images.Media.BUCKET_DISPLAY_NAME, MediaStore.Images.Media.MIME_TYPE, + MediaStore.MediaColumns._ID, MediaStore.MediaColumns.DISPLAY_NAME, MediaStore.MediaColumns.SIZE, + MediaStore.Images.Media.DATE_TAKEN, MediaStore.MediaColumns.DATE_ADDED, + MediaStore.MediaColumns.DATE_MODIFIED, MediaStore.Images.Media.BUCKET_ID, + MediaStore.Images.Media.BUCKET_DISPLAY_NAME, MediaStore.MediaColumns.MIME_TYPE, ) @Suppress("DEPRECATION") - val location = if (Build.VERSION.SDK_INT >= 29) MediaStore.Images.Media.RELATIVE_PATH else MediaStore.Images.Media.DATA + val location = if (Build.VERSION.SDK_INT >= 29) MediaStore.MediaColumns.RELATIVE_PATH else MediaStore.MediaColumns.DATA cols += location - // A photo still being written by the camera is not a photo yet. - val pending = if (Build.VERSION.SDK_INT >= 29) "${MediaStore.Images.Media.IS_PENDING} = 0" else null + // A photo or video still being written by the camera is not one yet. + val pending = if (Build.VERSION.SDK_INT >= 29) "${MediaStore.MediaColumns.IS_PENDING} = 0" else null val selection = listOfNotNull(pending, where).joinToString(" AND ").ifEmpty { null } context.contentResolver.query(collection, cols.toTypedArray(), selection, args, null)?.use { c -> val id = c.getColumnIndexOrThrow(cols[0]); val name = c.getColumnIndexOrThrow(cols[1]) @@ -65,18 +112,29 @@ class PhotoSource(private val context: Context) { val mime = c.getColumnIndexOrThrow(cols[8]); val where2 = c.getColumnIndexOrThrow(cols[9]) while (c.moveToNext()) { val bucketId = c.getString(bucket) ?: continue + val path = (c.getString(where2) ?: "").replace('\\', '/') + val display = c.getString(name) ?: "" + val album = c.getString(bucketName) ?: "" + // Where it was on the phone, from the top of its volume: what a + // restore puts it back to (the manifest, §9.12). + val relPath = if (Build.VERSION.SDK_INT >= 29) path.trim('/') + "/" + display else volumeRelative(path) val photo = Photo( - c.getLong(id), c.getString(name) ?: "", c.getLong(size), + c.getLong(id), display, c.getLong(size), if (c.isNull(taken)) 0 else c.getLong(taken), c.getLong(added), c.getLong(modified), - bucketId, c.getString(mime) ?: "", + bucketId, c.getString(mime) ?: "", video, relPath, album, ) - val path = (c.getString(where2) ?: "").replace('\\', '/') - each(photo, c.getString(bucketName) ?: "", isCamera(path)) + each(photo, album, isCamera(path)) } } } companion object { + /** `/storage/emulated/0/Documents/a.jpg` or `/storage/1234-ABCD/x/a.jpg` → the part after the volume. */ + fun volumeRelative(absolute: String): String { + val m = Regex("^/storage/(?:emulated/\\d+|[^/]+)/(.*)$").matchEntire(absolute) + return (m?.groupValues?.get(1) ?: absolute).trim('/') + } + /** `DCIM/Camera/` (RELATIVE_PATH) or `…/DCIM/Camera/x.jpg` (DATA, before Android 10). */ fun isCamera(path: String): Boolean = path.startsWith("DCIM/Camera") || path.contains("/DCIM/Camera/") diff --git a/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/ByteRangeTest.kt b/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/ByteRangeTest.kt new file mode 100644 index 0000000..29d8528 --- /dev/null +++ b/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/ByteRangeTest.kt @@ -0,0 +1,31 @@ +package org.meshbay.client + +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test +import org.meshbay.client.photos.ByteRange + +class ByteRangeTest { + @Test fun `a range is read as the upload asks for it`() { + assertEquals(0L..1023L, ByteRange.parse("bytes=0-1023", 5000)) + assertEquals(4096L..4999L, ByteRange.parse("bytes=4096-", 5000)) + assertEquals("the last chunk may ask past the end", 4096L..4999L, ByteRange.parse("bytes=4096-8191", 5000)) + } + + @Test fun `what cannot be served is refused`() { + assertNull(ByteRange.parse(null, 10)) + assertNull(ByteRange.parse("bytes=10-20", 10)) + assertNull(ByteRange.parse("bytes=5-2", 10)) + assertNull(ByteRange.parse("bytes=0-1,4-5", 10)) + assertNull(ByteRange.parse("bytes=-5", 10)) + assertNull(ByteRange.parse("items=0-1", 10)) + } + + @Test fun `a slice holds exactly those bytes`() { + val data = ByteArray(100_000) { (it % 251).toByte() } + val got = ByteRange.slice(data.inputStream(), 70_000L..99_999L).readBytes() + assertArrayEquals(data.copyOfRange(70_000, 100_000), got) + assertArrayEquals(data.copyOfRange(10, 12), ByteRange.slice(data.inputStream(), 10L..11L).readBytes()) + } +} diff --git a/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/DriveTest.kt b/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/DriveTest.kt new file mode 100644 index 0000000..e661bd1 --- /dev/null +++ b/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/DriveTest.kt @@ -0,0 +1,90 @@ +package org.meshbay.client + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.meshbay.client.drive.DriveFile +import org.meshbay.client.drive.DriveLedger +import org.meshbay.client.drive.DrivePlan +import org.meshbay.client.photos.PhotoPlan +import org.meshbay.client.photos.PhotoSource +import java.util.TimeZone + +class DriveTest { + @get:Rule val tmp = TemporaryFolder() + + private val utc = TimeZone.getTimeZone("UTC") + private val tree = "content://tree/primary%3ADocuments" + private val names = mapOf(tree to "Documents") + + private fun file(id: String, sub: String = "", size: Long = 10, modified: Long = 1000) = + DriveFile(tree, "primary:Documents/" + (if (sub.isEmpty()) "" else "$sub/") + id, sub, id, size, modified, + "application/pdf") + + private fun plan(files: List<DriveFile>, sent: Map<String, DriveLedger.Entry> = emptyMap(), + skip: Set<String> = emptySet(), same: Boolean = false) = + DrivePlan.plan(files, "Backups/bob-drive", names, { sent[it] }, skip, utc) { _, _ -> same } + + private fun entry(f: DriveFile) = DriveLedger.Entry(f.docId, f.modified, f.size, "h", "d", f.name, 1) + + @Test fun `media folders and a whole volume cannot be chosen`() { + assertEquals("whole_storage", DrivePlan.refusal("primary:")) + assertEquals("whole_storage", DrivePlan.refusal("1234-ABCD:")) + assertEquals("media_folder", DrivePlan.refusal("primary:DCIM")) + assertEquals("media_folder", DrivePlan.refusal("primary:DCIM/Camera")) + assertEquals("media_folder", DrivePlan.refusal("primary:Pictures/Screenshots")) + assertEquals("media_folder", DrivePlan.refusal("1234-ABCD:movies")) + assertNull(DrivePlan.refusal("primary:Documents")) + assertNull(DrivePlan.refusal("primary:Download/Invoices")) + assertNull(DrivePlan.refusal("primary:Documents/Pictures")) + } + + @Test fun `files go under the chosen folder's name, sub-folders kept`() { + val out = plan(listOf(file("a.pdf"), file("b.pdf", sub = "Tax/2026"))) + assertEquals(setOf("Backups/bob-drive/Documents", "Backups/bob-drive/Documents/Tax/2026"), + out.map { it.dir }.toSet()) + } + + @Test fun `smallest first, hidden files and photo-backed files left out`() { + val out = plan(listOf(file("big.pdf", size = 900), file("small.pdf", size = 5), file(".secret"), + file("scan.jpg", sub = "Scans")), + skip = setOf("documents/scans/scan.jpg")) + assertEquals(listOf("small.pdf", "big.pdf"), out.map { it.name }) + } + + @Test fun `a file already sent is not sent again, a changed one goes beside it`() { + val f = file("a.pdf") + assertTrue(plan(listOf(f), sent = mapOf(f.docId to entry(f))).isEmpty()) + val changed = f.copy(size = 20, modified = 86_400_000) + val out = plan(listOf(changed), sent = mapOf(f.docId to entry(f))) + assertEquals("a-modified-19700102-000000.pdf", out.single().name) + assertTrue(out.single().edited) + val touched = f.copy(modified = 5000) + assertTrue(plan(listOf(touched), sent = mapOf(f.docId to entry(f)), same = true).isEmpty()) + } + + @Test fun `names the node would refuse are mended, not dropped`() { + assertEquals("report.pdf", DrivePlan.safeName("report.pdf")) + assertEquals("a_b_c.txt", DrivePlan.safeName("a:b?c.txt")) + assertEquals("file-.env", DrivePlan.safeName(".env")) + assertEquals("file-_draft", DrivePlan.safeName("_draft")) + assertTrue(PhotoPlan.UPLOAD_NAME.matches(DrivePlan.safeName("x".repeat(300) + ".pdf"))) + assertTrue(PhotoPlan.UPLOAD_NAME.matches(DrivePlan.safeName("end. "))) + } + + @Test fun `the ledger remembers across a restart`() { + val path = tmp.newFile("d.jsonl") + DriveLedger(path).apply { record(entry(file("a.pdf"))); record(entry(file("a.pdf")).copy(sha256 = "z")) } + val again = DriveLedger(path) + assertEquals(1, again.size) + assertEquals("z", again["primary:Documents/a.pdf"]!!.sha256) + } + + @Test fun `a path from before Android 10 is read from the top of its volume`() { + assertEquals("Documents/Scans/a.jpg", PhotoSource.volumeRelative("/storage/emulated/0/Documents/Scans/a.jpg")) + assertEquals("x/a.jpg", PhotoSource.volumeRelative("/storage/1234-ABCD/x/a.jpg")) + } +} diff --git a/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/IcsTest.kt b/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/IcsTest.kt new file mode 100644 index 0000000..c8f8fa3 --- /dev/null +++ b/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/IcsTest.kt @@ -0,0 +1,117 @@ +package org.meshbay.client + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test +import org.meshbay.client.phonesync.Attendee +import org.meshbay.client.phonesync.CalEvent +import org.meshbay.client.phonesync.Ics + +class IcsTest { + // 2026-10-09 12:00:00 UTC + private val oct9 = 1791547200000L + private val hour = 3600_000L + + private fun event(id: Long, title: String = "Lunch", start: Long = oct9, end: Long? = oct9 + hour, + duration: String? = null, allDay: Boolean = false, zone: String? = "Europe/Paris", + rrule: String? = null, exdate: String? = null, originalId: Long? = null, + originalTime: Long? = null, uid: String? = null, status: Int? = 1, + reminders: List<Int> = emptyList(), attendees: List<Attendee> = emptyList(), + description: String? = null) = + CalEvent(id, "Personal", uid, title, description, null, start, end, duration, allDay, zone, + rrule, null, exdate, originalId, originalTime, false, status, reminders, null, attendees) + + private fun lines(vararg e: CalEvent): List<String> { + val out = ArrayList<String>() + Ics.write(e.toList(), oct9) { out += it } + return out + } + + /** RFC 5545 unfolding: a line starting with a space continues the one before. */ + private fun unfold(l: List<String>): List<String> { + val out = ArrayList<String>() + for (x in l) if (x.startsWith(" ")) out[out.size - 1] = out.last() + x.substring(1) else out += x + return out + } + + private fun events(l: List<String>): List<List<String>> { + val out = ArrayList<List<String>>() + var cur: MutableList<String>? = null + for (x in unfold(l)) when (x) { + "BEGIN:VEVENT" -> cur = ArrayList() + "END:VEVENT" -> { out += cur!!; cur = null } + else -> cur?.add(x) + } + return out + } + + private fun List<String>.prop(name: String) = firstOrNull { it.startsWith("$name:") || it.startsWith("$name;") } + + @Test fun `a calendar wraps the events, with what an importer requires`() { + val l = lines(event(1)) + assertEquals("BEGIN:VCALENDAR", l.first()) + assertEquals("END:VCALENDAR", l.last()) + assertTrue("VERSION:2.0" in l) + val e = events(l).single() + assertEquals("UID:android-1@meshbay", e.prop("UID")) + assertEquals("DTSTAMP:20261009T120000Z", e.prop("DTSTAMP")) + } + + @Test fun `a timed event keeps its zone, so a recurrence follows summer time`() { + val e = events(lines(event(1)))[0] + assertEquals("DTSTART;TZID=Europe/Paris:20261009T140000", e.prop("DTSTART")) + assertEquals("DTEND;TZID=Europe/Paris:20261009T150000", e.prop("DTEND")) + val utc = events(lines(event(2, zone = null)))[0] + assertEquals("DTSTART:20261009T120000Z", utc.prop("DTSTART")) + val unknown = events(lines(event(3, zone = "Not/AZone")))[0] + assertEquals("DTSTART:20261009T120000Z", unknown.prop("DTSTART")) + } + + @Test fun `an all-day event is a date`() { + val day = 1791504000000L // 2026-10-09 00:00 UTC + val e = events(lines(event(1, start = day, end = day + 24 * hour, allDay = true)))[0] + assertEquals("DTSTART;VALUE=DATE:20261009", e.prop("DTSTART")) + assertEquals("DTEND;VALUE=DATE:20261010", e.prop("DTEND")) + } + + @Test fun `a recurring event keeps its rule, and Android's duration is made valid`() { + val e = events(lines(event(1, end = null, duration = "P3600S", rrule = "FREQ=WEEKLY;BYDAY=FR", + exdate = "Europe/Paris;20261016T140000")))[0] + assertEquals("RRULE:FREQ=WEEKLY;BYDAY=FR", e.prop("RRULE")) + assertEquals("DURATION:PT3600S", e.prop("DURATION")) + assertEquals("EXDATE;TZID=Europe/Paris:20261016T140000", e.prop("EXDATE")) + assertNull(e.prop("DTEND")) + assertEquals("P1D", Ics.duration("P1D")) + assertEquals("PT1H30M", Ics.duration("PT1H30M")) + assertNull(Ics.duration("nonsense")) + } + + @Test fun `a changed instance carries the series' UID and says which instance`() { + val master = event(1, uid = "abc@example.org", end = null, duration = "P3600S", rrule = "FREQ=DAILY") + val moved = event(2, start = oct9 + 26 * hour, end = oct9 + 27 * hour, originalId = 1, + originalTime = oct9 + 24 * hour) + val e = events(lines(master, moved)) + assertEquals("UID:abc@example.org", e[0].prop("UID")) + assertEquals("UID:abc@example.org", e[1].prop("UID")) + assertEquals("RECURRENCE-ID;TZID=Europe/Paris:20261010T140000", e[1].prop("RECURRENCE-ID")) + } + + @Test fun `text is escaped and long lines folded without splitting a character`() { + val desc = "a;b,c\\d\nsecond line " + "é".repeat(60) + " 🎉".repeat(20) + val l = lines(event(1, description = desc)) + assertTrue(l.all { it.toByteArray(Charsets.UTF_8).size <= 75 }) + val e = events(l)[0] + assertEquals("DESCRIPTION:" + "a\\;b\\,c\\\\d\\nsecond line " + "é".repeat(60) + " 🎉".repeat(20), + e.prop("DESCRIPTION")) + } + + @Test fun `reminders, attendees and a cancelled status come along`() { + val e = events(lines(event(1, status = 2, reminders = listOf(10, 10, 60), + attendees = listOf(Attendee("ada@example.org", "Ada, L.")))))[0] + assertEquals("STATUS:CANCELLED", e.prop("STATUS")) + assertEquals(listOf("TRIGGER:-PT10M", "TRIGGER:-PT60M"), e.filter { it.startsWith("TRIGGER") }) + assertEquals("ATTENDEE;CN=\"Ada, L.\":mailto:ada@example.org", e.prop("ATTENDEE")) + assertEquals("X-MESHBAY-CALENDAR:Personal", e.prop("X-MESHBAY-CALENDAR")) + } +} diff --git a/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/ManifestLogTest.kt b/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/ManifestLogTest.kt new file mode 100644 index 0000000..bb3a67f --- /dev/null +++ b/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/ManifestLogTest.kt @@ -0,0 +1,52 @@ +package org.meshbay.client + +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.meshbay.client.phonesync.ManifestLog +import java.io.File +import java.util.TimeZone + +class ManifestLogTest { + @get:Rule val tmp = TemporaryFolder() + + private fun line(n: Int) = JSONObject().put("node", "d/$n.jpg").put("source", "DCIM/Camera/$n.jpg") + + private fun nodes(f: File) = f.readLines().filter { it.isNotBlank() }.map { JSONObject(it).getString("node") } + + @Test fun `nothing waiting means nothing to send`() { + val log = ManifestLog(File(tmp.root, "m")) + assertFalse(log.waiting()) + assertNull(log.issue()) + } + + @Test fun `what was sent is described once the node has the manifest, and only then forgotten`() { + val log = ManifestLog(File(tmp.root, "m")) + log.append(line(1)); log.append(line(2)) + assertTrue(log.waiting()) + assertEquals(listOf("d/1.jpg", "d/2.jpg"), nodes(log.issue()!!)) + log.confirm() + assertFalse(log.waiting()) + assertNull(log.issue()) + } + + @Test fun `a manifest that did not arrive goes again, with what came since`() { + val log = ManifestLog(File(tmp.root, "m")) + log.append(line(1)) + log.issue() // sent, never confirmed: the run was cut short + log.append(line(2)) + val again = ManifestLog(File(tmp.root, "m")) // a restart in between + assertTrue(again.waiting()) + assertEquals(listOf("d/1.jpg", "d/2.jpg"), nodes(again.issue()!!)) + } + + @Test fun `manifests are named for when they were written`() { + assertEquals("manifest-2026-10-09-120000.jsonl", + ManifestLog.nameFor(1791547200000L, TimeZone.getTimeZone("UTC"))) + } +} diff --git a/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/PhotoSyncTest.kt b/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/PhotoSyncTest.kt index 7c0b794..a87bffb 100644 --- a/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/PhotoSyncTest.kt +++ b/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/PhotoSyncTest.kt @@ -25,11 +25,12 @@ class PhotoSyncTest { private fun photo(id: Long, name: String = "IMG_$id.jpg", size: Long = 100, taken: Long = oct9, added: Long = oct9 / 1000, modified: Long = oct9 / 1000, bucket: String = "cam", - mime: String = "image/jpeg") = - Photo(id, name, size, taken, added, modified, bucket, mime) + mime: String = "image/jpeg", video: Boolean = false) = + Photo(id, name, size, taken, added, modified, bucket, mime, video) - private fun config(includeExisting: Boolean = true, since: Long = 0, albums: List<String> = listOf("cam")) = - SyncConfig(albums, includeExisting, since) + private fun config(includeExisting: Boolean = true, since: Long = 0, albums: List<String> = listOf("cam"), + videos: Boolean = false) = + SyncConfig(albums, includeExisting, since, videos) private val base = PhotoPlan.baseFor(Destination("bob", "g1", "Mine", "Backups")) @@ -61,6 +62,27 @@ class PhotoSyncTest { assertEquals(listOf(1L), out.map { it.photo.mediaId }) } + @Test fun `videos of the same albums go beside the photos, only when asked for`() { + val clip = photo(5, name = "PXL_20261009_120000.mp4", mime = "video/mp4", video = true) + assertTrue(plan(listOf(clip)).isEmpty()) + val out = plan(listOf(photo(1), clip), config(videos = true)) + assertEquals(setOf(1L, 5L), out.map { it.photo.mediaId }.toSet()) + assertEquals("Backups/bob-photos/2026/2026-10", out.first { it.photo.video }.dir) + assertTrue("a video file is not taken for a photo", plan(listOf(photo(6, mime = "video/mp4")), config(videos = true)).isEmpty()) + assertEquals("video-9.mp4", PhotoPlan.nameFor(photo(9, name = "_clip.mp4", mime = "video/mp4", video = true))) + assertEquals("video-9.mov", PhotoPlan.nameFor(photo(9, name = "", mime = "video/quicktime", video = true))) + } + + @Test fun `ticking videos in keeps the starting point`() { + val o = JSONObject().put("albums", JSONArray(listOf("cam"))).put("includeExisting", false) + val first = SyncConfig.fromJson(o, 100) + val withVideos = SyncConfig.fromJson(o.put("includeVideos", true), 200, first) + assertEquals(100, withVideos.since) + assertTrue(withVideos.includeVideos) + assertEquals(withVideos, SyncConfig.parse(withVideos.toJson().toString())) + assertFalse(SyncConfig.parse("""{"albums":[],"since":1}""")!!.includeVideos) + } + @Test fun `a photo already sent is not sent again`() { val p = photo(1) assertTrue(plan(listOf(p), ledger = mapOf(1L to entry(p))).isEmpty()) diff --git a/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/WhatsAppTest.kt b/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/WhatsAppTest.kt new file mode 100644 index 0000000..f3e76fe --- /dev/null +++ b/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/WhatsAppTest.kt @@ -0,0 +1,57 @@ +package org.meshbay.client + +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test +import org.meshbay.client.drive.DriveFile +import org.meshbay.client.drive.DriveKind +import org.meshbay.client.drive.DrivePlan +import org.meshbay.client.drive.WhatsApp +import java.util.TimeZone + +class WhatsAppTest { + private val tree = "content://tree/wa" + + private fun file(sub: String, name: String, size: Long = 10) = + DriveFile(tree, "primary:${WhatsApp.FOLDER}/$sub/$name", sub, name, size, 1000, "application/octet-stream") + + @Test fun `only WhatsApp's own folder can be chosen`() { + assertNull(WhatsApp.refusal("primary:Android/media/com.whatsapp/WhatsApp")) + assertNull(WhatsApp.refusal("primary:Android/media/com.whatsapp.w4b/WhatsApp Business")) + assertEquals("not_whatsapp", WhatsApp.refusal("primary:Android/media/com.whatsapp")) + assertEquals("not_whatsapp", WhatsApp.refusal("primary:Documents")) + assertEquals("not_whatsapp", WhatsApp.refusal("primary:Android/media/com.whatsapp/WhatsApp/Media")) + } + + @Test fun `the chats as they are now, not the dated copies of earlier weeks`() { + assertTrue(WhatsApp.include(file("Databases", "msgstore.db.crypt14"), false)) + assertTrue(WhatsApp.include(file("Databases", "msgstore-increment-1.db.crypt14"), false)) + assertFalse(WhatsApp.include(file("Databases", "msgstore-2026-10-09.1.db.crypt14"), false)) + assertFalse(WhatsApp.include(file("Databases", "msgstore-increment-3-2026-10-09.1.db.crypt14"), false)) + assertTrue(WhatsApp.include(file("Backups", "wa.db.crypt14"), false)) + assertTrue(WhatsApp.include(file("Backups/Stickers", "s.webp"), false)) + } + + @Test fun `media only when asked for, and nothing else of the folder`() { + val photo = file("Media/WhatsApp Images", "IMG-20261010-WA0001.jpg") + assertFalse(WhatsApp.include(photo, false)) + assertTrue(WhatsApp.include(photo, true)) + assertFalse(WhatsApp.include(file(".Shared", "x"), true)) + assertTrue(DriveKind.WHATSAPP.include(photo, JSONObject().put("media", true))) + } + + @Test fun `what a restore needs together is the chats and the backups, not the media`() { + assertTrue(WhatsApp.inRestoreSet(file("Databases", "msgstore.db.crypt14"))) + assertTrue(WhatsApp.inRestoreSet(file("Backups", "wa.db.crypt14"))) + assertFalse(WhatsApp.inRestoreSet(file("Media/WhatsApp Video", "v.mp4"))) + } + + @Test fun `on the node the folders are WhatsApp's own, with no extra level`() { + val out = DrivePlan.plan(listOf(file("Databases", "msgstore.db.crypt14")), "Backups/bob-whatsapp", + mapOf(tree to ""), { null }, emptySet(), TimeZone.getTimeZone("UTC")) { _, _ -> false } + assertEquals("Backups/bob-whatsapp/Databases", out.single().dir) + } +} |