From 243bcf017b4957237b3ddf556bd0a37296eb8798 Mon Sep 17 00:00:00 2001 From: Christophe Besson Date: Mon, 5 Oct 2026 01:16:12 +0200 Subject: feat(cast): photos on the TV, and a television chosen once for the session A cast button in Videos' toolbar, at the top of Photos, in an album's bar and in the lightbox, in a group and in Search alike. A television chosen there is kept for the session: a film opened plays on it with the player as its remote from the start, and a photo opened in the lightbox is shown on it, scaled to 1920x1080, upright, as JPEG. The lightbox gains a slideshow. The relay serves one photo at /image behind the stream's token, on the desktop and on Android; the shell, not the page, decides that the receiver loads it as a picture. Co-Authored-By: Claude Opus 5.5 --- .../app/src/main/assets/bridge/meshbay-bridge.js | 1 + .../kotlin/org/meshbay/client/cast/CastChannels.kt | 37 ++++++++++++++------ .../kotlin/org/meshbay/client/cast/CastControl.kt | 21 ++++++++---- .../kotlin/org/meshbay/client/cast/CastRelay.kt | 39 +++++++++++++++++++++- .../kotlin/org/meshbay/client/CastRelayTest.kt | 28 ++++++++++++++++ 5 files changed, 108 insertions(+), 18 deletions(-) (limited to 'packages/meshbay-android') diff --git a/packages/meshbay-android/app/src/main/assets/bridge/meshbay-bridge.js b/packages/meshbay-android/app/src/main/assets/bridge/meshbay-bridge.js index 7ace33b..21d27df 100644 --- a/packages/meshbay-android/app/src/main/assets/bridge/meshbay-bridge.js +++ b/packages/meshbay-android/app/src/main/assets/bridge/meshbay-bridge.js @@ -151,6 +151,7 @@ push: (data) => pushSegment(data), stop: () => call('cast:stop'), subtitle: (sub) => call('cast:subtitle', sub), + image: (img) => call('cast:image', { bytes: base64Of(bytesOf(img.bytes)), type: img.type }), finish: () => call('cast:finish'), status: () => call('cast:status'), scan: () => call('cast:scan'), diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/cast/CastChannels.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/cast/CastChannels.kt index 74a086b..6be5d72 100644 --- a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/cast/CastChannels.kt +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/cast/CastChannels.kt @@ -26,8 +26,8 @@ class CastChannels( fun handles(channel: String) = channel.startsWith("cast:") - /** What must reach the relay in the order the page sent it: start, pushes, subtitle, finish, stop. */ - fun ordered(channel: String) = channel in setOf("cast:start", "cast:push", "cast:subtitle", "cast:finish", "cast:stop") + /** What must reach the relay in the order the page sent it: start, pushes, subtitle, photo, finish, stop. */ + fun ordered(channel: String) = channel in setOf("cast:start", "cast:push", "cast:subtitle", "cast:image", "cast:finish", "cast:stop") fun call(channel: String, a: JSONArray): Any? = when (channel) { "cast:start" -> { @@ -38,6 +38,11 @@ class CastChannels( started } "cast:subtitle" -> relay.setSubtitle(a.optJSONObject(0)) + // No foreground service for a photo: the receiver has it once loaded. + "cast:image" -> { + val o = a.optJSONObject(0) ?: JSONObject() + relay.showImage(Base64.getDecoder().decode(o.optString("bytes", "")), o.optString("type", "")) + } "cast:push" -> { // the base64 path, for a WebView without ArrayBuffer messages relay.push(Base64.getDecoder().decode(a.optString(0, ""))); true } @@ -50,11 +55,13 @@ class CastChannels( "cast:devices" -> control.devices() "cast:chromecast:connect" -> { val o = a.optJSONObject(0) ?: JSONObject() - reported { control.connect(o.optString("deviceId", ""), relayUrl(o), subtitleOf(o)) } + val url = relayUrl(o) + reported { control.connect(o.optString("deviceId", ""), url, subtitleFor(url, o), typeOf(url)) } } "cast:chromecast:reload" -> { val o = a.optJSONObject(0) ?: JSONObject() - reported { control.reload(relayUrl(o), subtitleOf(o)) } + val url = relayUrl(o) + reported { control.reload(url, subtitleFor(url, o), typeOf(url)) } } "cast:chromecast:pause" -> control.pause() "cast:chromecast:play" -> control.play() @@ -79,19 +86,27 @@ class CastChannels( /** * The receiver is only ever pointed at this relay. A URL the page names is - * accepted when it is the relay's own, and refused otherwise — a page - * cannot use this application to make a television fetch anything else. + * accepted when it is the relay's own stream or photo, and refused + * otherwise — a page cannot use this application to make a television + * fetch anything else. */ private fun relayUrl(o: JSONObject): String { val asked = o.optString("mediaUrl", "") val ours = relay.url ?: throw Refused("The cast relay is not running") - if (asked != ours) throw Refused("Refused: not this relay's stream") - return ours + if (asked != ours && asked != relay.imageUrl) throw Refused("Refused: not this relay's stream") + return asked } - /** As main.js: no subtitle named means the relay's current one. */ - private fun subtitleOf(o: JSONObject): JSONObject? = - if (o.has("subtitle") && o.get("subtitle") != JSONObject.NULL) o.optJSONObject("subtitle") else relay.subtitleInfo() + /** The relay's photo is loaded as a picture, anything else as the stream; the page does not say which. */ + private fun typeOf(url: String): String = + if (url == relay.imageUrl) relay.imageType ?: "image/jpeg" else "video/mp4" + + /** As main.js: no subtitle named means the relay's current one; a photo has none. */ + private fun subtitleFor(url: String, o: JSONObject): JSONObject? = when { + url == relay.imageUrl -> null + o.has("subtitle") && o.get("subtitle") != JSONObject.NULL -> o.optJSONObject("subtitle") + else -> relay.subtitleInfo() + } /** * The Wi-Fi (or Ethernet) address, never the mobile network's: a TV cannot diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/cast/CastControl.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/cast/CastControl.kt index f574c89..56a1f5e 100644 --- a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/cast/CastControl.kt +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/cast/CastControl.kt @@ -10,6 +10,7 @@ import androidx.mediarouter.media.MediaRouter import com.google.android.gms.cast.CastMediaControlIntent import com.google.android.gms.cast.MediaInfo import com.google.android.gms.cast.MediaLoadRequestData +import com.google.android.gms.cast.MediaMetadata import com.google.android.gms.cast.MediaStatus import com.google.android.gms.cast.MediaTrack import com.google.android.gms.cast.TextTrackStyle @@ -111,7 +112,7 @@ class CastControl(private val context: Context) { } /** Connect, launch the default receiver, load; answers once the receiver has. */ - fun connect(deviceId: String, mediaUrl: String, subtitle: JSONObject?): JSONObject { + fun connect(deviceId: String, mediaUrl: String, subtitle: JSONObject?, contentType: String = "video/mp4"): JSONObject { need() val session = onMain { val cast = CastContext.getSharedInstance(context) @@ -167,7 +168,7 @@ class CastControl(private val context: Context) { // device's name included: read it there, never from this worker. val name = deviceName ?: onMain { session.castDevice?.friendlyName } Log.i(TAG, "session up on '$name', loading the relay stream") - val state = load(session, mediaUrl, subtitle) + val state = load(session, mediaUrl, subtitle, contentType) watch(session) return JSONObject().put("deviceName", name ?: JSONObject.NULL).put("playerState", state) } @@ -200,18 +201,24 @@ class CastControl(private val context: Context) { watched = c } - fun reload(mediaUrl: String, subtitle: JSONObject?): JSONObject { + fun reload(mediaUrl: String, subtitle: JSONObject?, contentType: String = "video/mp4"): JSONObject { need() val session = onMain { CastContext.getSharedInstance(context).sessionManager.currentCastSession } ?: throw Refused("Not connected") - val state = load(session, mediaUrl, subtitle) + val state = load(session, mediaUrl, subtitle, contentType) watch(session) return JSONObject().put("playerState", state) } - private fun load(session: CastSession, mediaUrl: String, subtitle: JSONObject?): String { + private fun load(session: CastSession, mediaUrl: String, subtitle: JSONObject?, contentType: String): String { + val photo = contentType.startsWith("image/") val subUrl = subtitle?.optString("url", "")?.takeIf { it.isNotEmpty() } - val info = MediaInfo.Builder(mediaUrl) + val info = if (photo) { + // A picture, not a stream: nothing to seek, nothing to buffer. + MediaInfo.Builder(mediaUrl).setContentType(contentType) + .setStreamType(MediaInfo.STREAM_TYPE_NONE) + .setMetadata(MediaMetadata(MediaMetadata.MEDIA_TYPE_PHOTO)).build() + } else MediaInfo.Builder(mediaUrl) .setContentType("video/mp4") // LIVE: the relay has no beginning to seek back to — the film's own // timeline lives on this side, and a seek restarts the relay. @@ -258,6 +265,8 @@ class CastControl(private val context: Context) { throw IllegalStateException("The receiver did not load the stream") } if (!ok) throw IllegalStateException("The receiver refused the stream ($reason)") + // A photo has no playing state to wait for: loaded is shown. + if (photo) return onMain { stateName(session.remoteMediaClient?.playerState ?: MediaStatus.PLAYER_STATE_UNKNOWN) } // A load the receiver accepted is not a film on the screen: it answers // IDLE, then fetches the stream, and only then plays — or gives up. // So the answer waits for what the receiver actually did (measured diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/cast/CastRelay.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/cast/CastRelay.kt index 21328db..41bf4bd 100644 --- a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/cast/CastRelay.kt +++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/cast/CastRelay.kt @@ -82,6 +82,7 @@ class CastRelay(private val lanAddress: () -> InetAddress?, private val spoolDir private val spoolSeq = AtomicLong() private class Subtitle(val vtt: ByteArray, val language: String, val label: String) + private class Image(val bytes: ByteArray, val type: String) @Volatile private var server: ServerSocket? = null @Volatile private var port = 0 @@ -92,6 +93,8 @@ class CastRelay(private val lanAddress: () -> InetAddress?, private val spoolDir @Volatile private var headerReady = false @Volatile private var subtitle: Subtitle? = null @Volatile private var subtitleVersion = 0 + @Volatile private var image: Image? = null + @Volatile private var imageVersion = 0 /** How many times a receiver asked for the stream since the relay started. */ @Volatile var streamRequests = 0 private set @@ -113,6 +116,27 @@ class CastRelay(private val lanAddress: () -> InetAddress?, private val spoolDir if (server == null || subtitle == null) null else "http://${hostPart()}:$port/subs.vtt?t=$token&v=$subtitleVersion" + /** Versioned like the subtitle: a receiver handed the same address twice shows what it has. */ + val imageUrl get() = + if (server == null || image == null) null + else "http://${hostPart()}:$port/image?t=$token&v=$imageVersion" + + val imageType get() = image?.type + + /** + * One photo, scaled and re-encoded by the page, served at `/image` until + * the next one. The relay starts for it when nothing else has. + */ + @Synchronized + fun showImage(bytes: ByteArray, type: String): JSONObject { + if (type !in IMAGE_TYPES) throw IllegalArgumentException("Not a picture a receiver shows: $type") + if (server == null) start(null, null) + image = Image(bytes, type) + imageVersion++ + log("image set: ${bytes.size} bytes, $type, v$imageVersion") + return JSONObject().put("url", imageUrl) + } + fun subtitleInfo(): JSONObject? = subtitle?.let { JSONObject().put("url", subtitleUrl).put("language", it.language).put("label", it.label) } @@ -145,6 +169,7 @@ class CastRelay(private val lanAddress: () -> InetAddress?, private val spoolDir spoolDir.mkdirs() spoolDir.listFiles()?.forEach { it.delete() } subtitle = null + image = null setSubtitle(sub) var bound: ServerSocket? = null @@ -251,7 +276,7 @@ class CastRelay(private val lanAddress: () -> InetAddress?, private val spoolDir clients.clear() server?.let { try { it.close() } catch (e: Exception) {} } server = null - port = 0; token = null; initSegment = null; subtitle = null + port = 0; token = null; initSegment = null; subtitle = null; image = null synchronized(ring) { ring.clear(); ringBytes = 0 } accum.reset() } @@ -291,6 +316,7 @@ class CastRelay(private val lanAddress: () -> InetAddress?, private val spoolDir if (token == null || query["t"] != token) return respond(out, socket, 403, emptyMap()) when (path) { "/subs.vtt" -> serveSubtitle(out, socket) + "/image" -> serveImage(out, socket) "/stream.mp4" -> { streamRequests++; serveStream(out, socket) } else -> respond(out, socket, 404, emptyMap()) } @@ -308,6 +334,15 @@ class CastRelay(private val lanAddress: () -> InetAddress?, private val spoolDir ), sub.vtt) } + private fun serveImage(out: OutputStream, socket: Socket) { + val img = image ?: return respond(out, socket, 404, CORS_HEADERS) + respond(out, socket, 200, CORS_HEADERS + mapOf( + "Content-Type" to img.type, + "Content-Length" to img.bytes.size.toString(), + "Cache-Control" to "no-store", + ), img.bytes) + } + private fun serveStream(out: OutputStream, socket: Socket) { // Same headers as the subtitle: a receiver given a side-loaded track // reads the media through the same CORS-checked path. @@ -393,6 +428,8 @@ class CastRelay(private val lanAddress: () -> InetAddress?, private val spoolDir } const val RING_CAP = 64 + // The page re-encodes every photo to JPEG: a guard, not a menu. + val IMAGE_TYPES = setOf("image/jpeg", "image/png", "image/webp") const val RING_MAX_BYTES = 32L * 1024 * 1024 const val SPOOL_MAX_BYTES = 2L * 1024 * 1024 * 1024 private const val HEADER_WAIT_MS = 10_000L diff --git a/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/CastRelayTest.kt b/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/CastRelayTest.kt index 5d3125f..8013e29 100644 --- a/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/CastRelayTest.kt +++ b/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/CastRelayTest.kt @@ -158,6 +158,34 @@ class CastRelayTest { assertEquals(200, get(r.getString("url"), readBytes = 0).status) } + @Test fun `a photo starts the relay and is served behind the token, re-addressed when it changes`() { + val jpeg = byteArrayOf(0xFF.toByte(), 0xD8.toByte(), 1, 2, 3, 0xFF.toByte(), 0xD9.toByte()) + val first = relay.showImage(jpeg, "image/jpeg").getString("url") + assertTrue(relay.active) + val reply = get(first) + assertEquals(200, reply.status) + assertEquals("image/jpeg", reply.headers["content-type"]) + assertEquals("no-store", reply.headers["cache-control"]) + assertArrayEquals(jpeg, reply.body) + assertEquals(403, get(first.replace(Regex("t=[0-9a-f]+"), "t=x")).status) + // A receiver handed the same address twice shows what it already has. + val second = relay.showImage(byteArrayOf(9), "image/jpeg").getString("url") + assertNotEquals(first, second) + assertArrayEquals(byteArrayOf(9), get(second).body) + } + + @Test fun `only a picture is served as one`() { + val refused = try { relay.showImage(byteArrayOf(1), "text/html"); false } catch (e: IllegalArgumentException) { true } + assertTrue(refused) + assertNull(relay.imageUrl) + } + + @Test fun `a film started over a photo takes the photo down`() { + relay.showImage(byteArrayOf(1), "image/jpeg") + relay.start(null, null) + assertNull(relay.imageUrl) + } + @Test fun `the preflight is answered before the token is checked`() { val url = relay.start(null, null).getString("url") val reply = get(url.substringBefore("?"), method = "OPTIONS") -- cgit v1.2.3