feat(cast): photos on the TV, and a television chosen once for the session

A cast button in Videos' toolbar, at the top of Photos, in an album's bar and
in the lightbox, in a group and in Search alike. A television chosen there is
kept for the session: a film opened plays on it with the player as its remote
from the start, and a photo opened in the lightbox is shown on it, scaled to
1920x1080, upright, as JPEG. The lightbox gains a slideshow.

The relay serves one photo at /image behind the stream's token, on the desktop
and on Android; the shell, not the page, decides that the receiver loads it as
a picture.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

5 files changed, 108 insertions, 18 deletions

diff --git a/packages/meshbay-android/app/src/main/assets/bridge/meshbay-bridge.js b/packages/meshbay-android/app/src/main/assets/bridge/meshbay-bridge.js
index 7ace33b..21d27df 100644
--- a/packages/meshbay-android/app/src/main/assets/bridge/meshbay-bridge.js
+++ b/packages/meshbay-android/app/src/main/assets/bridge/meshbay-bridge.js
@@ -151,6 +151,7 @@
       push: (data) => pushSegment(data),
       stop: () => call('cast:stop'),
       subtitle: (sub) => call('cast:subtitle', sub),
+      image: (img) => call('cast:image', { bytes: base64Of(bytesOf(img.bytes)), type: img.type }),
       finish: () => call('cast:finish'),
       status: () => call('cast:status'),
       scan: () => call('cast:scan'),
diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/cast/CastChannels.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/cast/CastChannels.kt
index 74a086b..6be5d72 100644
--- a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/cast/CastChannels.kt
+++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/cast/CastChannels.kt
@@ -26,8 +26,8 @@ class CastChannels(
 
     fun handles(channel: String) = channel.startsWith("cast:")
 
-    /** What must reach the relay in the order the page sent it: start, pushes, subtitle, finish, stop. */
-    fun ordered(channel: String) = channel in setOf("cast:start", "cast:push", "cast:subtitle", "cast:finish", "cast:stop")
+    /** What must reach the relay in the order the page sent it: start, pushes, subtitle, photo, finish, stop. */
+    fun ordered(channel: String) = channel in setOf("cast:start", "cast:push", "cast:subtitle", "cast:image", "cast:finish", "cast:stop")
 
     fun call(channel: String, a: JSONArray): Any? = when (channel) {
         "cast:start" -> {
@@ -38,6 +38,11 @@ class CastChannels(
             started
         }
         "cast:subtitle" -> relay.setSubtitle(a.optJSONObject(0))
+        // No foreground service for a photo: the receiver has it once loaded.
+        "cast:image" -> {
+            val o = a.optJSONObject(0) ?: JSONObject()
+            relay.showImage(Base64.getDecoder().decode(o.optString("bytes", "")), o.optString("type", ""))
+        }
         "cast:push" -> {   // the base64 path, for a WebView without ArrayBuffer messages
             relay.push(Base64.getDecoder().decode(a.optString(0, ""))); true
         }
@@ -50,11 +55,13 @@ class CastChannels(
         "cast:devices" -> control.devices()
         "cast:chromecast:connect" -> {
             val o = a.optJSONObject(0) ?: JSONObject()
-            reported { control.connect(o.optString("deviceId", ""), relayUrl(o), subtitleOf(o)) }
+            val url = relayUrl(o)
+            reported { control.connect(o.optString("deviceId", ""), url, subtitleFor(url, o), typeOf(url)) }
         }
         "cast:chromecast:reload" -> {
             val o = a.optJSONObject(0) ?: JSONObject()
-            reported { control.reload(relayUrl(o), subtitleOf(o)) }
+            val url = relayUrl(o)
+            reported { control.reload(url, subtitleFor(url, o), typeOf(url)) }
         }
         "cast:chromecast:pause" -> control.pause()
         "cast:chromecast:play" -> control.play()
@@ -79,19 +86,27 @@ class CastChannels(
 
     /**
      * The receiver is only ever pointed at this relay. A URL the page names is
-     * accepted when it is the relay's own, and refused otherwise — a page
-     * cannot use this application to make a television fetch anything else.
+     * accepted when it is the relay's own stream or photo, and refused
+     * otherwise — a page cannot use this application to make a television
+     * fetch anything else.
      */
     private fun relayUrl(o: JSONObject): String {
         val asked = o.optString("mediaUrl", "")
         val ours = relay.url ?: throw Refused("The cast relay is not running")
-        if (asked != ours) throw Refused("Refused: not this relay's stream")
-        return ours
+        if (asked != ours && asked != relay.imageUrl) throw Refused("Refused: not this relay's stream")
+        return asked
     }
 
-    /** As main.js: no subtitle named means the relay's current one. */
-    private fun subtitleOf(o: JSONObject): JSONObject? =
-        if (o.has("subtitle") && o.get("subtitle") != JSONObject.NULL) o.optJSONObject("subtitle") else relay.subtitleInfo()
+    /** The relay's photo is loaded as a picture, anything else as the stream; the page does not say which. */
+    private fun typeOf(url: String): String =
+        if (url == relay.imageUrl) relay.imageType ?: "image/jpeg" else "video/mp4"
+
+    /** As main.js: no subtitle named means the relay's current one; a photo has none. */
+    private fun subtitleFor(url: String, o: JSONObject): JSONObject? = when {
+        url == relay.imageUrl -> null
+        o.has("subtitle") && o.get("subtitle") != JSONObject.NULL -> o.optJSONObject("subtitle")
+        else -> relay.subtitleInfo()
+    }
 
     /**
      * The Wi-Fi (or Ethernet) address, never the mobile network's: a TV cannot
diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/cast/CastControl.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/cast/CastControl.kt
index f574c89..56a1f5e 100644
--- a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/cast/CastControl.kt
+++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/cast/CastControl.kt
@@ -10,6 +10,7 @@ import androidx.mediarouter.media.MediaRouter
 import com.google.android.gms.cast.CastMediaControlIntent
 import com.google.android.gms.cast.MediaInfo
 import com.google.android.gms.cast.MediaLoadRequestData
+import com.google.android.gms.cast.MediaMetadata
 import com.google.android.gms.cast.MediaStatus
 import com.google.android.gms.cast.MediaTrack
 import com.google.android.gms.cast.TextTrackStyle
@@ -111,7 +112,7 @@ class CastControl(private val context: Context) {
     }
 
     /** Connect, launch the default receiver, load; answers once the receiver has. */
-    fun connect(deviceId: String, mediaUrl: String, subtitle: JSONObject?): JSONObject {
+    fun connect(deviceId: String, mediaUrl: String, subtitle: JSONObject?, contentType: String = "video/mp4"): JSONObject {
         need()
         val session = onMain {
             val cast = CastContext.getSharedInstance(context)
@@ -167,7 +168,7 @@ class CastControl(private val context: Context) {
         // device's name included: read it there, never from this worker.
         val name = deviceName ?: onMain { session.castDevice?.friendlyName }
         Log.i(TAG, "session up on '$name', loading the relay stream")
-        val state = load(session, mediaUrl, subtitle)
+        val state = load(session, mediaUrl, subtitle, contentType)
         watch(session)
         return JSONObject().put("deviceName", name ?: JSONObject.NULL).put("playerState", state)
     }
@@ -200,18 +201,24 @@ class CastControl(private val context: Context) {
         watched = c
     }
 
-    fun reload(mediaUrl: String, subtitle: JSONObject?): JSONObject {
+    fun reload(mediaUrl: String, subtitle: JSONObject?, contentType: String = "video/mp4"): JSONObject {
         need()
         val session = onMain { CastContext.getSharedInstance(context).sessionManager.currentCastSession }
             ?: throw Refused("Not connected")
-        val state = load(session, mediaUrl, subtitle)
+        val state = load(session, mediaUrl, subtitle, contentType)
         watch(session)
         return JSONObject().put("playerState", state)
     }
 
-    private fun load(session: CastSession, mediaUrl: String, subtitle: JSONObject?): String {
+    private fun load(session: CastSession, mediaUrl: String, subtitle: JSONObject?, contentType: String): String {
+        val photo = contentType.startsWith("image/")
         val subUrl = subtitle?.optString("url", "")?.takeIf { it.isNotEmpty() }
-        val info = MediaInfo.Builder(mediaUrl)
+        val info = if (photo) {
+            // A picture, not a stream: nothing to seek, nothing to buffer.
+            MediaInfo.Builder(mediaUrl).setContentType(contentType)
+                .setStreamType(MediaInfo.STREAM_TYPE_NONE)
+                .setMetadata(MediaMetadata(MediaMetadata.MEDIA_TYPE_PHOTO)).build()
+        } else MediaInfo.Builder(mediaUrl)
             .setContentType("video/mp4")
             // LIVE: the relay has no beginning to seek back to — the film's own
             // timeline lives on this side, and a seek restarts the relay.
@@ -258,6 +265,8 @@ class CastControl(private val context: Context) {
             throw IllegalStateException("The receiver did not load the stream")
         }
         if (!ok) throw IllegalStateException("The receiver refused the stream ($reason)")
+        // A photo has no playing state to wait for: loaded is shown.
+        if (photo) return onMain { stateName(session.remoteMediaClient?.playerState ?: MediaStatus.PLAYER_STATE_UNKNOWN) }
         // A load the receiver accepted is not a film on the screen: it answers
         // IDLE, then fetches the stream, and only then plays — or gives up.
         // So the answer waits for what the receiver actually did (measured
diff --git a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/cast/CastRelay.kt b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/cast/CastRelay.kt
index 21328db..41bf4bd 100644
--- a/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/cast/CastRelay.kt
+++ b/packages/meshbay-android/app/src/main/kotlin/org/meshbay/client/cast/CastRelay.kt
@@ -82,6 +82,7 @@ class CastRelay(private val lanAddress: () -> InetAddress?, private val spoolDir
     private val spoolSeq = AtomicLong()
 
     private class Subtitle(val vtt: ByteArray, val language: String, val label: String)
+    private class Image(val bytes: ByteArray, val type: String)
 
     @Volatile private var server: ServerSocket? = null
     @Volatile private var port = 0
@@ -92,6 +93,8 @@ class CastRelay(private val lanAddress: () -> InetAddress?, private val spoolDir
     @Volatile private var headerReady = false
     @Volatile private var subtitle: Subtitle? = null
     @Volatile private var subtitleVersion = 0
+    @Volatile private var image: Image? = null
+    @Volatile private var imageVersion = 0
     /** How many times a receiver asked for the stream since the relay started. */
     @Volatile var streamRequests = 0
         private set
@@ -113,6 +116,27 @@ class CastRelay(private val lanAddress: () -> InetAddress?, private val spoolDir
         if (server == null || subtitle == null) null
         else "http://${hostPart()}:$port/subs.vtt?t=$token&v=$subtitleVersion"
 
+    /** Versioned like the subtitle: a receiver handed the same address twice shows what it has. */
+    val imageUrl get() =
+        if (server == null || image == null) null
+        else "http://${hostPart()}:$port/image?t=$token&v=$imageVersion"
+
+    val imageType get() = image?.type
+
+    /**
+     * One photo, scaled and re-encoded by the page, served at `/image` until
+     * the next one. The relay starts for it when nothing else has.
+     */
+    @Synchronized
+    fun showImage(bytes: ByteArray, type: String): JSONObject {
+        if (type !in IMAGE_TYPES) throw IllegalArgumentException("Not a picture a receiver shows: $type")
+        if (server == null) start(null, null)
+        image = Image(bytes, type)
+        imageVersion++
+        log("image set: ${bytes.size} bytes, $type, v$imageVersion")
+        return JSONObject().put("url", imageUrl)
+    }
+
     fun subtitleInfo(): JSONObject? = subtitle?.let {
         JSONObject().put("url", subtitleUrl).put("language", it.language).put("label", it.label)
     }
@@ -145,6 +169,7 @@ class CastRelay(private val lanAddress: () -> InetAddress?, private val spoolDir
         spoolDir.mkdirs()
         spoolDir.listFiles()?.forEach { it.delete() }
         subtitle = null
+        image = null
         setSubtitle(sub)
 
         var bound: ServerSocket? = null
@@ -251,7 +276,7 @@ class CastRelay(private val lanAddress: () -> InetAddress?, private val spoolDir
         clients.clear()
         server?.let { try { it.close() } catch (e: Exception) {} }
         server = null
-        port = 0; token = null; initSegment = null; subtitle = null
+        port = 0; token = null; initSegment = null; subtitle = null; image = null
         synchronized(ring) { ring.clear(); ringBytes = 0 }
         accum.reset()
     }
@@ -291,6 +316,7 @@ class CastRelay(private val lanAddress: () -> InetAddress?, private val spoolDir
             if (token == null || query["t"] != token) return respond(out, socket, 403, emptyMap())
             when (path) {
                 "/subs.vtt" -> serveSubtitle(out, socket)
+                "/image" -> serveImage(out, socket)
                 "/stream.mp4" -> { streamRequests++; serveStream(out, socket) }
                 else -> respond(out, socket, 404, emptyMap())
             }
@@ -308,6 +334,15 @@ class CastRelay(private val lanAddress: () -> InetAddress?, private val spoolDir
         ), sub.vtt)
     }
 
+    private fun serveImage(out: OutputStream, socket: Socket) {
+        val img = image ?: return respond(out, socket, 404, CORS_HEADERS)
+        respond(out, socket, 200, CORS_HEADERS + mapOf(
+            "Content-Type" to img.type,
+            "Content-Length" to img.bytes.size.toString(),
+            "Cache-Control" to "no-store",
+        ), img.bytes)
+    }
+
     private fun serveStream(out: OutputStream, socket: Socket) {
         // Same headers as the subtitle: a receiver given a side-loaded track
         // reads the media through the same CORS-checked path.
@@ -393,6 +428,8 @@ class CastRelay(private val lanAddress: () -> InetAddress?, private val spoolDir
         }
 
         const val RING_CAP = 64
+        // The page re-encodes every photo to JPEG: a guard, not a menu.
+        val IMAGE_TYPES = setOf("image/jpeg", "image/png", "image/webp")
         const val RING_MAX_BYTES = 32L * 1024 * 1024
         const val SPOOL_MAX_BYTES = 2L * 1024 * 1024 * 1024
         private const val HEADER_WAIT_MS = 10_000L
diff --git a/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/CastRelayTest.kt b/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/CastRelayTest.kt
index 5d3125f..8013e29 100644
--- a/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/CastRelayTest.kt
+++ b/packages/meshbay-android/app/src/test/kotlin/org/meshbay/client/CastRelayTest.kt
@@ -158,6 +158,34 @@ class CastRelayTest {
         assertEquals(200, get(r.getString("url"), readBytes = 0).status)
     }
 
+    @Test fun `a photo starts the relay and is served behind the token, re-addressed when it changes`() {
+        val jpeg = byteArrayOf(0xFF.toByte(), 0xD8.toByte(), 1, 2, 3, 0xFF.toByte(), 0xD9.toByte())
+        val first = relay.showImage(jpeg, "image/jpeg").getString("url")
+        assertTrue(relay.active)
+        val reply = get(first)
+        assertEquals(200, reply.status)
+        assertEquals("image/jpeg", reply.headers["content-type"])
+        assertEquals("no-store", reply.headers["cache-control"])
+        assertArrayEquals(jpeg, reply.body)
+        assertEquals(403, get(first.replace(Regex("t=[0-9a-f]+"), "t=x")).status)
+        // A receiver handed the same address twice shows what it already has.
+        val second = relay.showImage(byteArrayOf(9), "image/jpeg").getString("url")
+        assertNotEquals(first, second)
+        assertArrayEquals(byteArrayOf(9), get(second).body)
+    }
+
+    @Test fun `only a picture is served as one`() {
+        val refused = try { relay.showImage(byteArrayOf(1), "text/html"); false } catch (e: IllegalArgumentException) { true }
+        assertTrue(refused)
+        assertNull(relay.imageUrl)
+    }
+
+    @Test fun `a film started over a photo takes the photo down`() {
+        relay.showImage(byteArrayOf(1), "image/jpeg")
+        relay.start(null, null)
+        assertNull(relay.imageUrl)
+    }
+
     @Test fun `the preflight is answered before the token is checked`() {
         val url = relay.start(null, null).getString("url")
         val reply = get(url.substringBefore("?"), method = "OPTIONS")