fix: publish storyboard previews in 1.8.0-beta.5

app/build.gradle.kts

@@ -26,8 +26,8 @@ android {
         applicationId = "dev.typetype.android"
         minSdk = 23
         targetSdk = 37
-        versionCode = 10804
-        versionName = "1.8.0-beta.4"
+        versionCode = 10805
+        versionName = "1.8.0-beta.5"
         testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
         resValue("string", "app_name", "TypeType")
     }

app/src/androidTest/java/dev/typetype/android/feature/player/components/PlayerControlsLayoutTest.kt

@@ -8,15 +8,21 @@ import androidx.compose.ui.platform.LocalView
 import android.view.View
 import androidx.activity.ComponentActivity
 import androidx.compose.foundation.layout.Box
+import androidx.compose.foundation.layout.fillMaxWidth
+import androidx.compose.foundation.layout.padding
 import androidx.compose.foundation.layout.size
+import androidx.compose.ui.geometry.Offset
+import androidx.compose.ui.test.assertIsDisplayed
 import androidx.compose.ui.Modifier
 import androidx.compose.ui.platform.testTag
+import androidx.compose.ui.semantics.SemanticsProperties
 import androidx.compose.ui.test.junit4.v2.createAndroidComposeRule
 import androidx.compose.ui.test.onNodeWithTag
 import androidx.compose.ui.unit.dp
 import androidx.compose.ui.unit.Dp
 import androidx.compose.ui.test.assertHeightIsAtLeast
 import androidx.compose.ui.test.onNodeWithContentDescription
+import androidx.compose.ui.test.performTouchInput
 import androidx.compose.ui.test.captureToImage
 import androidx.compose.ui.graphics.asAndroidBitmap
 import androidx.core.view.WindowCompat
@@ -26,6 +32,7 @@ import androidx.core.graphics.Insets
 import android.graphics.Bitmap
 import java.io.File
 import dev.typetype.android.R
+import dev.typetype.android.domain.stream.StreamStoryboard
 import androidx.media3.common.PlaybackParameters
 import androidx.media3.common.Player
 import androidx.media3.common.Timeline
@@ -79,6 +86,49 @@ class PlayerControlsLayoutTest {
         assertEquals(visibleTop, backButtonTopInViewport(), 1f)
     }
 
+    @Test
+    fun timelineScrubShowsStoryboardPreviewAndTimecode() {
+        val player = controlsLayoutPlayer(
+            durationMs = 600_000L,
+            positionMs = 120_000L,
+        )
+        val storyboard = StreamStoryboard(
+            urls = listOf("https://example.test/storyboard.jpg"),
+            frameWidth = 160,
+            frameHeight = 90,
+            totalCount = 100,
+            durationPerFrameMillis = 5_000L,
+            framesPerPageX = 5,
+            framesPerPageY = 5,
+        )
+        composeRule.setContent {
+            PlayerTimeBar(
+                player = player,
+                storyboard = storyboard,
+                modifier = Modifier
+                    .fillMaxWidth()
+                    .padding(top = 200.dp),
+            )
+        }
+
+        composeRule.onNodeWithContentDescription(
+            composeRule.activity.getString(R.string.player_timeline),
+        ).performTouchInput {
+            down(Offset(width * 0.25f, center.y))
+            moveBy(Offset(width * 0.5f, 0f))
+        }
+        composeRule.waitForIdle()
+
+        composeRule.onNodeWithTag(PLAYER_SEEK_STORYBOARD_PREVIEW_TAG)
+            .assertIsDisplayed()
+        val previewTexts = composeRule.onNodeWithTag(PLAYER_SEEK_STORYBOARD_PREVIEW_TAG)
+            .fetchSemanticsNode().children
+            .mapNotNull { child ->
+                child.config.getOrElse(SemanticsProperties.Text) { emptyList() }.singleOrNull()
+            }
+        assertTrue(previewTexts.single().matches(Regex("^\\d{1,2}:\\d{2}$")))
+    }
+
     private fun dispatchStatusBarInsets(top: Int) {
         composeRule.runOnUiThread {
             ViewCompat.dispatchApplyWindowInsets(
@@ -175,13 +225,19 @@ class PlayerControlsLayoutTest {
     }
 }
 
-private fun controlsLayoutPlayer(): Player = Proxy.newProxyInstance(
+private fun controlsLayoutPlayer(
+    durationMs: Long = 0L,
+    positionMs: Long = 0L,
+): Player = Proxy.newProxyInstance(
     Player::class.java.classLoader,
     arrayOf(Player::class.java),
 ) { _, method, _ ->
     when (method.name) {
         "getApplicationLooper" -> Looper.getMainLooper()
         "getAvailableCommands" -> Player.Commands.EMPTY
+        "getDuration" -> durationMs
+        "getCurrentPosition" -> positionMs
+        "getBufferedPosition" -> positionMs
         "getCurrentTimeline" -> Timeline.EMPTY
         "getPlaybackParameters" -> PlaybackParameters.DEFAULT
         "getCurrentTracks" -> Tracks.EMPTY

app/src/main/java/dev/typetype/android/data/stream/StreamRepositoryImpl.kt

@@ -4,6 +4,7 @@ import dev.typetype.android.data.account.AccountScopeProvider
 import dev.typetype.android.core.error.CodedFailure
 import dev.typetype.android.data.network.PlaybackNetworkObserver
 import dev.typetype.android.data.network.dto.AudioStreamItem
+import dev.typetype.android.data.network.dto.PreviewFrameItem
 import dev.typetype.android.data.network.dto.SponsorBlockSegmentItem
 import dev.typetype.android.data.network.dto.StreamResponse
 import dev.typetype.android.data.network.dto.StreamSegmentItem
@@ -19,6 +20,7 @@ import dev.typetype.android.domain.stream.Stream
 import dev.typetype.android.domain.stream.StreamPlaybackContract
 import dev.typetype.android.domain.stream.StreamAudioSource
 import dev.typetype.android.domain.stream.StreamRepository
+import dev.typetype.android.domain.stream.StreamStoryboard
 import dev.typetype.android.domain.stream.StreamRequestScope
 import dev.typetype.android.domain.stream.StreamVideoSource
 import dev.typetype.android.domain.stream.isServerSabrAudioFormat
@@ -176,6 +178,7 @@ internal class StreamRepositoryImpl @Inject constructor(
             subtitles = subtitles.mapIndexedNotNull { index, subtitle ->
                 subtitle.toClientSubtitleSource(index)
             },
+            storyboard = previewFrames.firstNotNullOfOrNull { it.toDomainStoryboard() },
             startPositionMillis = startPosition * 1000L,
             sponsorBlockSegments = sponsorBlockSegments.mapNotNull {
                 it.toDomainSponsorBlockSegment(duration)
@@ -202,6 +205,17 @@ internal class StreamRepositoryImpl @Inject constructor(
             .maxByOrNull { it.height }
             ?.url
 
+    private fun PreviewFrameItem.toDomainStoryboard(): StreamStoryboard? =
+        StreamStoryboard(
+            urls = urls.filter(String::isNotBlank),
+            frameWidth = frameWidth,
+            frameHeight = frameHeight,
+            totalCount = totalCount,
+            durationPerFrameMillis = durationPerFrame.toLong(),
+            framesPerPageX = framesPerPageX,
+            framesPerPageY = framesPerPageY,
+        ).takeIf { it.frameAt(0L) != null }
+
     private fun serverManifestUrl(baseUrl: String, path: String, videoUrl: String): String {
         val normalizedBaseUrl = baseUrl.trimEnd('/')
         val encoded = URLEncoder.encode(videoUrl, StandardCharsets.UTF_8)

app/src/main/java/dev/typetype/android/domain/stream/Stream.kt

@@ -33,6 +33,7 @@ data class Stream(
     val originalAudioTrackId: String? = null,
     val preferredDefaultAudioTrackId: String? = null,
     val subtitles: List<StreamSubtitleSource> = emptyList(),
+    val storyboard: StreamStoryboard? = null,
     val startPositionMillis: Long,
     val sponsorBlockSegments: List<SponsorBlockSegment> = emptyList(),
     val chapters: List<Chapter> = emptyList(),
@@ -43,6 +44,50 @@ data class Stream(
     val category: String? = null,
 )
 
+data class StreamStoryboard(
+    val urls: List<String>,
+    val frameWidth: Int,
+    val frameHeight: Int,
+    val totalCount: Int,
+    val durationPerFrameMillis: Long,
+    val framesPerPageX: Int,
+    val framesPerPageY: Int,
+) {
+    fun frameAt(positionMs: Long): StreamStoryboardFrame? {
+        val totalFrames = totalCount.takeIf { it > 0 } ?: return null
+        val duration = durationPerFrameMillis.takeIf { it > 0L } ?: return null
+        val pageColumns = framesPerPageX.takeIf { it > 0 } ?: return null
+        val pageRows = framesPerPageY.takeIf { it > 0 } ?: return null
+        if (frameWidth <= 0 || frameHeight <= 0 || urls.isEmpty()) return null
+
+        val elapsedFrames = positionMs / duration
+        val frameIndex = when {
+            elapsedFrames <= 0L -> 0
+            elapsedFrames >= totalFrames - 1L -> totalFrames - 1
+            else -> elapsedFrames.toInt()
+        }
+        val framesPerPage = pageColumns * pageRows
+        val pageIndex = (frameIndex / framesPerPage).coerceIn(0, urls.lastIndex)
+        val pageFrameIndex = frameIndex % framesPerPage
+        val url = urls.getOrNull(pageIndex) ?: return null
+        return StreamStoryboardFrame(
+            url = url,
+            x = (pageFrameIndex % pageColumns) * frameWidth,
+            y = (pageFrameIndex / pageColumns) * frameHeight,
+            width = frameWidth,
+            height = frameHeight,
+        )
+    }
+}
+
+data class StreamStoryboardFrame(
+    val url: String,
+    val x: Int,
+    val y: Int,
+    val width: Int,
+    val height: Int,
+)
+
 enum class StreamPlaybackContract {
     ServerSabr,
     ProviderMedia,

app/src/main/java/dev/typetype/android/feature/player/components/PlayerControls.kt

@@ -29,12 +29,14 @@ import androidx.compose.ui.unit.dp
 import androidx.media3.common.Player
 import dev.typetype.android.R
 import dev.typetype.android.domain.stream.SponsorBlockSegment
+import dev.typetype.android.domain.stream.StreamStoryboard
 import dev.typetype.android.feature.player.state.ResizeMode
 
 @Composable
 @OptIn(ExperimentalLayoutApi::class)
 fun PlayerControls(
     player: Player,
+    storyboard: StreamStoryboard? = null,
     title: String,
     onNavigateBack: () -> Unit,
     modifier: Modifier = Modifier,
@@ -108,6 +110,7 @@ fun PlayerControls(
         }
         BottomBar(
             player = player,
+            storyboard = storyboard,
             sponsorBlockSegments = sponsorBlockSegments,
             seekPreviewPositionMs = seekPreviewPositionMs,
             timelineScrubbing = timelineScrubbing,
@@ -170,6 +173,7 @@ private fun BottomScrim(
 @Composable
 private fun BottomBar(
     player: Player,
+    storyboard: StreamStoryboard?,
     sponsorBlockSegments: List<SponsorBlockSegment>,
     seekPreviewPositionMs: Long?,
     timelineScrubbing: Boolean,
@@ -196,6 +200,7 @@ private fun BottomBar(
     ) {
         PlayerTimeBar(
             player = player,
+            storyboard = storyboard,
             segments = sponsorBlockSegments,
             previewPositionMs = seekPreviewPositionMs,
             compact = !isFullscreen && !expanded,

app/src/main/java/dev/typetype/android/feature/player/components/PlayerSurfaceBox.kt

@@ -235,6 +235,7 @@ internal fun PlayerSurfaceBox(
 
         PlayerSurfaceChrome(
             player = player,
+            stream = stream,
             title = stream.title,
             sponsorBlockSegments = sponsorBlockPolicy.visibleSegments,
             seekPreviewPositionMs = gestureState.seekDragTargetMs.longValue

app/src/main/java/dev/typetype/android/feature/player/components/PlayerSurfaceChrome.kt

@@ -10,19 +10,20 @@ import androidx.compose.foundation.layout.fillMaxWidth
 import androidx.compose.foundation.layout.padding
 import androidx.compose.runtime.Composable
 import androidx.compose.material3.Text
-import androidx.compose.ui.graphics.Color
 import androidx.compose.ui.Alignment
 import androidx.compose.ui.Modifier
 import androidx.compose.ui.unit.dp
 import androidx.compose.ui.res.stringResource
 import dev.typetype.android.R
 import androidx.media3.session.MediaController
 import dev.typetype.android.domain.stream.SponsorBlockSegment
+import dev.typetype.android.domain.stream.Stream
 import dev.typetype.android.feature.player.state.ResizeMode
 
 @Composable
 internal fun PlayerSurfaceChrome(
     player: MediaController,
+    stream: Stream,
     title: String,
     sponsorBlockSegments: List<SponsorBlockSegment>,
     seekPreviewPositionMs: Long?,
@@ -56,13 +57,16 @@ internal fun PlayerSurfaceChrome(
             modifier = Modifier.align(Alignment.BottomCenter),
         ) {
             Column(horizontalAlignment = Alignment.CenterHorizontally) {
-                Text(
-                    text = if (fineSeeking) {
-                        stringResource(R.string.player_fine_seek_position, formatPlayerTime(seekDragPositionMs))
+                SeekStoryboardPreview(
+                    frame = stream.storyboard?.frameAt(seekDragPositionMs),
+                    timecode = if (fineSeeking) {
+                        stringResource(
+                            R.string.player_fine_seek_position,
+                            formatPlayerTime(seekDragPositionMs),
+                        )
                     } else {
                         formatPlayerTime(seekDragPositionMs)
                     },
-                    color = Color.White,
                 )
                 PlayerSeekScrubOverlay(
                     player = player,
@@ -91,6 +95,7 @@ internal fun PlayerSurfaceChrome(
         ) {
             PlayerControls(
                 player = player,
+                storyboard = stream.storyboard,
                 title = title,
                 onNavigateBack = onNavigateBack,
                 onOpenOptions = onOpenOptions,