From 9f1dbbc7d06be2f20ff8c4a846664f88139d6404 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Mon, 31 Aug 2026 14:19:48 +0200 Subject: [PATCH 01/45] ci: route workflows to R730 runners --- .github/workflows/ci.yml | 2 +- .github/workflows/coverage.yml | 2 +- .github/workflows/docker.yml | 8 ++++---- .github/workflows/openapi.yml | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 413eb721..5a22891d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ concurrency: jobs: build: - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || fromJSON('["self-hosted","Linux","X64","arko"]') }} + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || fromJSON('["self-hosted","Linux","X64","r730"]') }} steps: - uses: actions/checkout@v7 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 7b38c40d..18d1dfd2 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -13,7 +13,7 @@ concurrency: jobs: coverage: - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || fromJSON('["self-hosted","Linux","X64","arko"]') }} + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || fromJSON('["self-hosted","Linux","X64","r730"]') }} steps: - uses: actions/checkout@v7 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 07c4ac15..a53dc8d6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -21,7 +21,7 @@ concurrency: jobs: prepare-image: - runs-on: [self-hosted, Linux, X64, arko] + runs-on: [self-hosted, Linux, X64, r730] permissions: contents: read outputs: @@ -76,7 +76,7 @@ jobs: build-platform: needs: prepare-image - runs-on: [self-hosted, Linux, X64, arko, docker] + runs-on: [self-hosted, Linux, X64, r730, docker] timeout-minutes: 20 permissions: contents: read @@ -150,7 +150,7 @@ jobs: build-and-push: needs: [prepare-image, build-platform] - runs-on: [self-hosted, Linux, X64, arko, docker] + runs-on: [self-hosted, Linux, X64, r730, docker] timeout-minutes: 10 permissions: contents: read @@ -235,7 +235,7 @@ jobs: notify-orchestrator: needs: [prepare-image, build-and-push] if: github.ref_name == 'dev' || github.ref_name == 'main' || startsWith(github.ref, 'refs/tags/v') - runs-on: [self-hosted, Linux, X64, arko] + runs-on: [self-hosted, Linux, X64, r730] permissions: contents: read env: diff --git a/.github/workflows/openapi.yml b/.github/workflows/openapi.yml index 4abbc6c3..4b50635e 100644 --- a/.github/workflows/openapi.yml +++ b/.github/workflows/openapi.yml @@ -15,7 +15,7 @@ concurrency: jobs: validate: - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || fromJSON('["self-hosted","Linux","X64","arko"]') }} + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || fromJSON('["self-hosted","Linux","X64","r730"]') }} steps: - uses: actions/checkout@v7 - name: Isolate Gradle user home From f74bf02277e12992dbae1c1b2b1cb77b2d864340 Mon Sep 17 00:00:00 2001 From: Tax_Tux <138765817+Priveetee@users.noreply.github.com> Date: Mon, 31 Aug 2026 21:30:53 +0200 Subject: [PATCH 02/45] chore: benchmark dev runners From f604dad788767e35b0b430e5e30981b5160323c7 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 4 Sep 2026 08:24:42 +0200 Subject: [PATCH 03/45] feat: add batch playback progress lookup --- openapi.yaml | 3 ++ openapi/components/progress.yaml | 30 ++++++++++++ openapi/paths/progress.yaml | 33 +++++++++++++ .../kotlin/dev/typetype/server/AppMetrics.kt | 1 + .../typetype/server/routes/ProgressRoutes.kt | 19 +++++++ .../server/services/ProgressService.kt | 18 +++++++ .../dev/typetype/server/ProgressRoutesTest.kt | 49 +++++++++++++++++++ 7 files changed, 153 insertions(+) create mode 100644 openapi/components/progress.yaml create mode 100644 openapi/paths/progress.yaml diff --git a/openapi.yaml b/openapi.yaml index 63aec2da..508706c3 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -56,6 +56,7 @@ paths: /rss/feeds/{id}/regenerate: { $ref: ./openapi/paths/rss.yaml#/RssFeedRegenerate } /rss/feeds/{id}.xml: { $ref: ./openapi/paths/rss.yaml#/RssFeedDocument } /settings: { $ref: ./openapi/paths/access-control.yaml#/Settings } + /progress/batch: { $ref: ./openapi/paths/progress.yaml#/ProgressBatch } /backup/typetype: { $ref: ./openapi/paths/user-backup.yaml#/TypeTypeBackup } /restore/typetype: { $ref: ./openapi/paths/user-backup.yaml#/TypeTypeRestore } /portability/formats: { $ref: ./openapi/paths/portability.yaml#/PortabilityFormats } @@ -173,6 +174,8 @@ components: RssFeedSecretItem: { $ref: ./openapi/components/rss.yaml#/RssFeedSecretItem } AdminRssFeedsPage: { $ref: ./openapi/components/rss.yaml#/AdminRssFeedsPage } SettingsItem: { $ref: ./openapi/components/access-control.yaml#/SettingsItem } + ProgressItem: { $ref: ./openapi/components/progress.yaml#/ProgressItem } + ProgressBatchRequest: { $ref: ./openapi/components/progress.yaml#/ProgressBatchRequest } TypeTypeBackupItem: { $ref: ./openapi/components/user-backup.yaml#/TypeTypeBackupItem } TypeTypeRestoreSummary: { $ref: ./openapi/components/user-backup.yaml#/TypeTypeRestoreSummary } PortabilityAdapterDescriptor: { $ref: ./openapi/components/portability.yaml#/PortabilityAdapterDescriptor } diff --git a/openapi/components/progress.yaml b/openapi/components/progress.yaml new file mode 100644 index 00000000..da01f073 --- /dev/null +++ b/openapi/components/progress.yaml @@ -0,0 +1,30 @@ +ProgressItem: + type: object + required: [videoUrl, position, updatedAt] + properties: + videoUrl: + type: string + format: uri + position: + type: integer + format: int64 + minimum: 0 + description: Playback position in milliseconds. + updatedAt: + type: integer + format: int64 + minimum: 0 + description: Unix timestamp in milliseconds. + +ProgressBatchRequest: + type: object + required: [videoUrls] + properties: + videoUrls: + type: array + minItems: 1 + maxItems: 200 + items: + type: string + format: uri + maxLength: 2048 diff --git a/openapi/paths/progress.yaml b/openapi/paths/progress.yaml new file mode 100644 index 00000000..67c422d4 --- /dev/null +++ b/openapi/paths/progress.yaml @@ -0,0 +1,33 @@ +ProgressBatch: + post: + tags: [user-data] + summary: Read playback progress for multiple videos + security: + - bearerAuth: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: ../components/progress.yaml#/ProgressBatchRequest + responses: + '200': + description: Progress entries in request order. Videos without saved progress have position and updatedAt set to zero. + content: + application/json: + schema: + type: array + items: + $ref: ../components/progress.yaml#/ProgressItem + '400': + description: Invalid request body or video URL list. + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse + '401': + description: Authentication required. + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse diff --git a/src/main/kotlin/dev/typetype/server/AppMetrics.kt b/src/main/kotlin/dev/typetype/server/AppMetrics.kt index 617bfade..b74be9d8 100644 --- a/src/main/kotlin/dev/typetype/server/AppMetrics.kt +++ b/src/main/kotlin/dev/typetype/server/AppMetrics.kt @@ -43,6 +43,7 @@ fun metricPath(path: String): String = when { path.startsWith("/downloader/jobs/") && path.endsWith("/artifact") -> "/downloader/jobs/{id}/artifact" path.startsWith("/downloader/jobs/") && path.endsWith("/cancel") -> "/downloader/jobs/{id}/cancel" path.startsWith("/downloader/jobs/") -> "/downloader/jobs/{id}" + path == "/progress/batch" -> path path.startsWith("/progress/") -> "/progress/{videoUrl}" path.startsWith("/favorites/") -> "/favorites/{videoUrl}" path.startsWith("/watch-later/") -> "/watch-later/{videoUrl}" diff --git a/src/main/kotlin/dev/typetype/server/routes/ProgressRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/ProgressRoutes.kt index 981470a9..c30713f5 100644 --- a/src/main/kotlin/dev/typetype/server/routes/ProgressRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/ProgressRoutes.kt @@ -10,13 +10,29 @@ import io.ktor.server.request.receive import io.ktor.server.response.respond import io.ktor.server.routing.Route import io.ktor.server.routing.get +import io.ktor.server.routing.post import io.ktor.server.routing.put import kotlinx.serialization.Serializable @Serializable internal data class ProgressBody(val position: Long) +@Serializable +internal data class ProgressBatchBody(val videoUrls: List) + fun Route.progressRoutes(progressService: ProgressService, authService: AuthService, settingsService: SettingsService? = null) { + post("/progress/batch") { + call.withJwtAuth(authService) { userId -> + val body = runCatching { call.receive() }.getOrElse { + return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) + } + if (body.videoUrls.isEmpty() || body.videoUrls.size > MAX_PROGRESS_BATCH_SIZE || body.videoUrls.any { it.isBlank() || it.length > MAX_VIDEO_URL_LENGTH }) { + return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid videoUrls")) + } + val videoUrls = body.videoUrls.distinct() + call.respond(progressService.getMany(userId, videoUrls)) + } + } get("/progress/{videoUrl...}") { call.withJwtAuth(authService) { userId -> val videoUrl = call.urlTailParameter("videoUrl") ?: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing videoUrl")) @@ -59,6 +75,9 @@ fun Route.progressRoutes(progressService: ProgressService, authService: AuthServ } } +private const val MAX_PROGRESS_BATCH_SIZE = 200 +private const val MAX_VIDEO_URL_LENGTH = 2_048 + private fun skippedProgress(videoUrl: String, position: Long): ProgressItem = ProgressItem( videoUrl = videoUrl, position = position.coerceAtLeast(0L), diff --git a/src/main/kotlin/dev/typetype/server/services/ProgressService.kt b/src/main/kotlin/dev/typetype/server/services/ProgressService.kt index e73dea44..ae39d248 100644 --- a/src/main/kotlin/dev/typetype/server/services/ProgressService.kt +++ b/src/main/kotlin/dev/typetype/server/services/ProgressService.kt @@ -5,6 +5,7 @@ import dev.typetype.server.db.tables.ProgressTable import dev.typetype.server.models.ProgressItem import org.jetbrains.exposed.v1.core.and import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.inList import org.jetbrains.exposed.v1.jdbc.insert import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.update @@ -38,6 +39,23 @@ class ProgressService { } } + suspend fun getMany(userId: String, videoUrls: List): List = DatabaseFactory.query { + val orderedUrls = videoUrls.distinct() + if (orderedUrls.isEmpty()) return@query emptyList() + val progressByUrl = ProgressTable.selectAll() + .where { (ProgressTable.userId eq userId) and (ProgressTable.videoUrl inList orderedUrls) } + .associate { row -> + row[ProgressTable.videoUrl] to ProgressItem( + videoUrl = row[ProgressTable.videoUrl], + position = row[ProgressTable.position], + updatedAt = row[ProgressTable.updatedAt], + ) + } + orderedUrls.map { videoUrl -> + progressByUrl[videoUrl] ?: ProgressItem(videoUrl = videoUrl, position = 0L) + } + } + suspend fun upsert(userId: String, videoUrl: String, position: Long): ProgressItem { val now = System.currentTimeMillis() val safePosition = position.coerceAtLeast(0L) diff --git a/src/test/kotlin/dev/typetype/server/ProgressRoutesTest.kt b/src/test/kotlin/dev/typetype/server/ProgressRoutesTest.kt index 13f7cbbb..2a64a834 100644 --- a/src/test/kotlin/dev/typetype/server/ProgressRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/ProgressRoutesTest.kt @@ -3,8 +3,10 @@ package dev.typetype.server import dev.typetype.server.routes.progressRoutes import dev.typetype.server.services.AuthService import dev.typetype.server.services.ProgressService +import dev.typetype.server.models.ProgressItem import io.ktor.client.request.get import io.ktor.client.request.headers +import io.ktor.client.request.post import io.ktor.client.request.put import io.ktor.client.request.setBody import io.ktor.client.statement.bodyAsText @@ -17,6 +19,7 @@ import io.ktor.server.plugins.contentnegotiation.ContentNegotiation import io.ktor.server.routing.routing import io.ktor.server.testing.ApplicationTestBuilder import io.ktor.server.testing.testApplication +import kotlinx.serialization.json.Json import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeAll @@ -117,4 +120,50 @@ class ProgressRoutesTest { assertEquals(HttpStatusCode.OK, getResponse.status) assertTrue(getResponse.bodyAsText().contains("\"position\":3300")) } + + @Test + fun `POST progress batch returns exact positions in request order`() = withApp { + service.upsert(TEST_USER_ID, "https://yt.com/watch?v=first", 12_345L) + service.upsert(TEST_USER_ID, "https://yt.com/watch?v=second", 67_890L) + service.upsert("another-user", "https://yt.com/watch?v=private", 99_999L) + + val response = client.post("/progress/batch") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + setBody("""{"videoUrls":["https://yt.com/watch?v=second","https://yt.com/watch?v=missing","https://yt.com/watch?v=first","https://yt.com/watch?v=second","https://yt.com/watch?v=private"]}""") + } + + assertEquals(HttpStatusCode.OK, response.status) + val items = Json.decodeFromString>(response.bodyAsText()) + assertEquals( + listOf( + "https://yt.com/watch?v=second", + "https://yt.com/watch?v=missing", + "https://yt.com/watch?v=first", + "https://yt.com/watch?v=private", + ), + items.map(ProgressItem::videoUrl), + ) + assertEquals(listOf(67_890L, 0L, 12_345L, 0L), items.map(ProgressItem::position)) + } + + @Test + fun `POST progress batch validates authentication and request limits`() = withApp { + val body = """{"videoUrls":["https://yt.com/watch?v=test"]}""" + assertEquals(HttpStatusCode.Unauthorized, client.post("/progress/batch") { + headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + setBody(body) + }.status) + assertEquals(HttpStatusCode.BadRequest, client.post("/progress/batch") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + setBody("""{"videoUrls":[]}""") + }.status) + val tooMany = (0..200).joinToString(",") { "\"https://yt.com/watch?v=$it\"" } + assertEquals(HttpStatusCode.BadRequest, client.post("/progress/batch") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + setBody("""{"videoUrls":[$tooMany]}""") + }.status) + } } From bdd5f045c6a54a7af02b3a0e9dd21e1c2fc42679 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 4 Sep 2026 11:43:46 +0200 Subject: [PATCH 04/45] refactor: add TypeType SABR boundary models --- .../typetype/server/sabr/SabrExceptions.kt | 4 ++ .../dev/typetype/server/sabr/SabrFormat.kt | 30 +++++++++++++ .../dev/typetype/server/sabr/SabrInfo.kt | 30 +++++++++++++ .../dev/typetype/server/sabr/SabrMedia.kt | 45 +++++++++++++++++++ .../dev/typetype/server/sabr/SabrPolicy.kt | 11 +++++ .../dev/typetype/server/sabr/SabrProfile.kt | 18 ++++++++ .../dev/typetype/server/sabr/SabrRange.kt | 40 +++++++++++++++++ .../dev/typetype/server/sabr/SabrRequest.kt | 29 ++++++++++++ .../typetype/server/sabr/SabrTokenProvider.kt | 5 +++ 9 files changed, 212 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/sabr/SabrExceptions.kt create mode 100644 src/main/kotlin/dev/typetype/server/sabr/SabrFormat.kt create mode 100644 src/main/kotlin/dev/typetype/server/sabr/SabrInfo.kt create mode 100644 src/main/kotlin/dev/typetype/server/sabr/SabrMedia.kt create mode 100644 src/main/kotlin/dev/typetype/server/sabr/SabrPolicy.kt create mode 100644 src/main/kotlin/dev/typetype/server/sabr/SabrProfile.kt create mode 100644 src/main/kotlin/dev/typetype/server/sabr/SabrRange.kt create mode 100644 src/main/kotlin/dev/typetype/server/sabr/SabrRequest.kt create mode 100644 src/main/kotlin/dev/typetype/server/sabr/SabrTokenProvider.kt diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrExceptions.kt b/src/main/kotlin/dev/typetype/server/sabr/SabrExceptions.kt new file mode 100644 index 00000000..fd73c3dc --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/sabr/SabrExceptions.kt @@ -0,0 +1,4 @@ +package dev.typetype.server.sabr + +internal typealias SabrProtocolException = org.schabi.newpipe.extractor.services.youtube.sabr.SabrProtocolException +internal typealias SabrRecoverableException = org.schabi.newpipe.extractor.services.youtube.sabr.SabrRecoverableException diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrFormat.kt b/src/main/kotlin/dev/typetype/server/sabr/SabrFormat.kt new file mode 100644 index 00000000..79d46ef0 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/sabr/SabrFormat.kt @@ -0,0 +1,30 @@ +package dev.typetype.server.sabr + +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat as PipeFormat + +/** TypeType's stable representation of a provider media format. */ +internal class YoutubeSabrFormat internal constructor( + internal val delegate: PipeFormat, +) { + val isAudio: Boolean get() = delegate.isAudio + val isVideo: Boolean get() = delegate.isVideo + val itag: Int get() = delegate.itag + val lastModified: Long get() = delegate.lastModified + val xtags: String? get() = delegate.xtags + val mimeType: String? get() = delegate.mimeType + val audioTrackId: String? get() = delegate.audioTrackId + val audioTrackDisplayName: String? get() = delegate.audioTrackDisplayName + val isAudioDefault: Boolean get() = delegate.isAudioDefault + val isOriginalAudio: Boolean get() = delegate.isOriginalAudio + val qualityLabel: String? get() = delegate.qualityLabel + val audioQuality: String? get() = delegate.audioQuality + val isDrc: Boolean get() = delegate.isDrc + val width: Int get() = delegate.width + val height: Int get() = delegate.height + val bitrate: Int get() = delegate.bitrate + val contentLength: Long get() = delegate.contentLength + val approxDurationMs: Long get() = delegate.approxDurationMs + val initializationUrl: String? get() = delegate.initializationUrl + val initRangeStart: Long get() = delegate.initRangeStart + val initRangeEnd: Long get() = delegate.initRangeEnd +} diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrInfo.kt b/src/main/kotlin/dev/typetype/server/sabr/SabrInfo.kt new file mode 100644 index 00000000..68c9785f --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/sabr/SabrInfo.kt @@ -0,0 +1,30 @@ +package dev.typetype.server.sabr + +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo as PipeInfo +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat as PipeFormat +import java.util.IdentityHashMap + +/** Provider-independent SABR metadata used by TypeType's orchestration layer. */ +internal class YoutubeSabrInfo internal constructor( + internal val delegate: PipeInfo, + private val formatCache: IdentityHashMap = IdentityHashMap(), +) { + val profile: YoutubeSabrClientProfile + get() = YoutubeSabrClientProfile.fromDelegate(delegate.profile) + val videoId: String get() = delegate.videoId + val cpn: String get() = delegate.cpn + val clientVersion: String get() = delegate.clientVersion + val visitorData: String? get() = delegate.visitorData + val serverAbrStreamingUrl: String? get() = delegate.serverAbrStreamingUrl + val videoPlaybackUstreamerConfig: String? get() = delegate.videoPlaybackUstreamerConfig + val isPlayerPoTokenAttached: Boolean get() = delegate.isPlayerPoTokenAttached + val formats: List + get() = delegate.formats.map(::format) + + fun findBestAudioFormat(): YoutubeSabrFormat? = delegate.findBestAudioFormat()?.let(::format) + fun findLowestVideoFormat(): YoutubeSabrFormat? = delegate.findLowestVideoFormat()?.let(::format) + fun findFormatByItag(itag: Int): YoutubeSabrFormat? = delegate.findFormatByItag(itag)?.let(::format) + + private fun format(delegateFormat: PipeFormat): YoutubeSabrFormat = + formatCache.getOrPut(delegateFormat) { YoutubeSabrFormat(delegateFormat) } +} diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrMedia.kt b/src/main/kotlin/dev/typetype/server/sabr/SabrMedia.kt new file mode 100644 index 00000000..23b74cee --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/sabr/SabrMedia.kt @@ -0,0 +1,45 @@ +package dev.typetype.server.sabr + +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader as PipeHeader +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment as PipeSegment +import java.io.InputStream + +internal class SabrMediaHeader internal constructor( + private val delegate: PipeHeader, +) { + val headerId: Int get() = delegate.headerId + val videoId: String? get() = delegate.videoId + val itag: Int get() = delegate.itag + val lastModified: Long get() = delegate.lastModified + val xtags: String? get() = delegate.xtags + val startRange: Long get() = delegate.startRange + val compressionAlgorithm: Int get() = delegate.compressionAlgorithm + val isInitSegment: Boolean get() = delegate.isInitSegment + val sequenceNumber: Int get() = delegate.sequenceNumber + val bitrateBps: Long get() = delegate.bitrateBps + val startMs: Long get() = delegate.startMs + val durationMs: Long get() = delegate.durationMs + val contentLength: Long get() = delegate.contentLength + val timeRangeStartTicks: Long get() = delegate.timeRangeStartTicks + val timeRangeDurationTicks: Long get() = delegate.timeRangeDurationTicks + val timeRangeTimescale: Int get() = delegate.timeRangeTimescale + val sequenceLastModified: Long get() = delegate.sequenceLastModified + fun summarize(): String = delegate.summarize() +} + +internal class SabrMediaSegment private constructor( + internal val delegate: PipeSegment, +) { + val header: SabrMediaHeader = SabrMediaHeader(delegate.header) + val data: ByteArray get() = delegate.data + fun openStream(): InputStream = delegate.openStream() + val isDiskBacked: Boolean get() = delegate.isDiskBacked + val isComplete: Boolean get() = delegate.isComplete + val hasFailed: Boolean get() = delegate.hasFailed() + fun delete(): Unit = delegate.delete() + val length: Int get() = delegate.length + + companion object { + internal fun fromDelegate(segment: PipeSegment): SabrMediaSegment = SabrMediaSegment(segment) + } +} diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrPolicy.kt b/src/main/kotlin/dev/typetype/server/sabr/SabrPolicy.kt new file mode 100644 index 00000000..1effadf6 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/sabr/SabrPolicy.kt @@ -0,0 +1,11 @@ +package dev.typetype.server.sabr + +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrNextRequestPolicy as PipePolicy + +internal class SabrNextRequestPolicy internal constructor( + private val delegate: PipePolicy, +) { + val targetAudioReadaheadMs: Int get() = delegate.targetAudioReadaheadMs + val targetVideoReadaheadMs: Int get() = delegate.targetVideoReadaheadMs + val maxTimeSinceLastRequestMs: Int get() = delegate.maxTimeSinceLastRequestMs +} diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrProfile.kt b/src/main/kotlin/dev/typetype/server/sabr/SabrProfile.kt new file mode 100644 index 00000000..0c07023a --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/sabr/SabrProfile.kt @@ -0,0 +1,18 @@ +package dev.typetype.server.sabr + +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrClientProfile as PipeProfile + +internal enum class YoutubeSabrClientProfile(internal val delegate: PipeProfile) { + WEB(PipeProfile.WEB), + MWEB(PipeProfile.MWEB), + WEB_EMBEDDED(PipeProfile.WEB_EMBEDDED), + ANDROID(PipeProfile.ANDROID), + ANDROID_VR(PipeProfile.ANDROID_VR), + IOS(PipeProfile.IOS), + TVHTML5(PipeProfile.TVHTML5); + + companion object { + internal fun fromDelegate(profile: PipeProfile): YoutubeSabrClientProfile = + entries.first { it.delegate == profile } + } +} diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrRange.kt b/src/main/kotlin/dev/typetype/server/sabr/SabrRange.kt new file mode 100644 index 00000000..6a8d95ac --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/sabr/SabrRange.kt @@ -0,0 +1,40 @@ +package dev.typetype.server.sabr + +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrBufferedRange as PipeRange + +internal class SabrBufferedRange internal constructor( + val itag: Int, + val lastModified: Long, + val xtags: String?, + val startTimeMs: Long, + val durationMs: Long, + val startSegmentIndex: Int, + val endSegmentIndex: Int, + val timescale: Int, +) { + internal var delegate = PipeRange( + itag, + lastModified, + xtags, + startTimeMs, + durationMs, + startSegmentIndex, + endSegmentIndex, + timescale, + ) + + fun summarize(): String = delegate.summarize() + + internal companion object { + fun fromDelegate(delegate: PipeRange): SabrBufferedRange = SabrBufferedRange( + delegate.itag, + delegate.lastModified, + delegate.xtags, + delegate.startTimeMs, + delegate.durationMs, + delegate.startSegmentIndex, + delegate.endSegmentIndex, + delegate.timescale, + ).also { it.delegate = delegate } + } +} diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrRequest.kt b/src/main/kotlin/dev/typetype/server/sabr/SabrRequest.kt new file mode 100644 index 00000000..b7685c29 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/sabr/SabrRequest.kt @@ -0,0 +1,29 @@ +package dev.typetype.server.sabr + +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest as PipeRequest + +internal class SabrSegmentRequest private constructor( + val format: YoutubeSabrFormat, + val isInitializationSegment: Boolean, + val sequenceNumber: Int, + private val delegateFactory: () -> PipeRequest, +) { + private val delegateValue: PipeRequest by lazy(LazyThreadSafetyMode.SYNCHRONIZED, delegateFactory) + + internal val delegate: PipeRequest + get() = delegateValue + + companion object { + fun initialization(format: YoutubeSabrFormat): SabrSegmentRequest = + SabrSegmentRequest(format, true, -1) { + PipeRequest.initialization(format.delegate) + } + + fun media(format: YoutubeSabrFormat, sequenceNumber: Int): SabrSegmentRequest = + SabrSegmentRequest(format, false, sequenceNumber) { + PipeRequest.media(format.delegate, sequenceNumber) + }.also { + require(sequenceNumber > 0) { "SABR media sequence number must be positive" } + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrTokenProvider.kt b/src/main/kotlin/dev/typetype/server/sabr/SabrTokenProvider.kt new file mode 100644 index 00000000..2d06cfb3 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/sabr/SabrTokenProvider.kt @@ -0,0 +1,5 @@ +package dev.typetype.server.sabr + +internal fun interface SabrPoTokenProvider { + fun getPoToken(info: YoutubeSabrInfo, streamState: YoutubeSabrStreamState): ByteArray? +} From 02c3621542d35ea013d9b146c2e1c8d3a4a2159f Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 4 Sep 2026 11:43:50 +0200 Subject: [PATCH 05/45] refactor: add TypeType SABR session adapter --- .../dev/typetype/server/sabr/SabrAdapter.kt | 62 +++++++++++++ .../dev/typetype/server/sabr/SabrSession.kt | 89 +++++++++++++++++++ .../typetype/server/sabr/SabrStreamState.kt | 82 +++++++++++++++++ 3 files changed, 233 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/sabr/SabrAdapter.kt create mode 100644 src/main/kotlin/dev/typetype/server/sabr/SabrSession.kt create mode 100644 src/main/kotlin/dev/typetype/server/sabr/SabrStreamState.kt diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrAdapter.kt b/src/main/kotlin/dev/typetype/server/sabr/SabrAdapter.kt new file mode 100644 index 00000000..3ec926d3 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/sabr/SabrAdapter.kt @@ -0,0 +1,62 @@ +package dev.typetype.server.sabr + +import com.grack.nanojson.JsonObject +import org.schabi.newpipe.extractor.localization.ContentCountry +import org.schabi.newpipe.extractor.localization.Localization +import org.schabi.newpipe.extractor.services.youtube.sabr.TypeTypeYoutubeSabrInfoFactory +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrProbe + +/** The only server entry point for PipePipe's SABR extraction API. */ +internal object SabrAdapter { + fun fetchSabrInfo( + videoId: String, + profile: YoutubeSabrClientProfile, + localization: Localization, + contentCountry: ContentCountry, + ): YoutubeSabrInfo = YoutubeSabrInfo( + YoutubeSabrProbe.fetchSabrInfo(videoId, profile.delegate, localization, contentCountry), + ) + + fun fetchSabrInfo( + videoId: String, + profile: YoutubeSabrClientProfile, + localization: Localization, + contentCountry: ContentCountry, + poToken: String, + visitorData: String?, + ): YoutubeSabrInfo = YoutubeSabrInfo( + YoutubeSabrProbe.fetchSabrInfo( + videoId, + profile.delegate, + localization, + contentCountry, + poToken, + visitorData, + ), + ) + + fun fromPlayerResponse( + videoId: String, + profile: YoutubeSabrClientProfile, + cpn: String, + response: JsonObject, + ): YoutubeSabrInfo = YoutubeSabrInfo( + YoutubeSabrProbe.fromPlayerResponse(videoId, profile.delegate, cpn, response), + ) + + fun withPlaybackIdentity( + info: YoutubeSabrInfo, + playbackUrl: String, + clientVersion: String, + cpn: String, + visitorData: String?, + ): YoutubeSabrInfo = YoutubeSabrInfo( + TypeTypeYoutubeSabrInfoFactory.withPlaybackIdentity( + info.delegate, + playbackUrl, + clientVersion, + cpn, + visitorData, + ), + ) +} diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrSession.kt b/src/main/kotlin/dev/typetype/server/sabr/SabrSession.kt new file mode 100644 index 00000000..314820ae --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/sabr/SabrSession.kt @@ -0,0 +1,89 @@ +package dev.typetype.server.sabr + +import org.schabi.newpipe.extractor.localization.Localization +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrPoTokenProvider as PipeProvider +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo as PipeInfo +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession as PipeSession +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState as PipeState + +internal class YoutubeSabrSession( + info: YoutubeSabrInfo, + audioFormat: YoutubeSabrFormat, + videoFormat: YoutubeSabrFormat, + poTokenProvider: SabrPoTokenProvider, +) { + private val delegate = PipeSession( + info.delegate, + audioFormat.delegate, + videoFormat.delegate, + object : PipeProvider { + override fun getPoToken(pipeInfo: PipeInfo, pipeState: PipeState): ByteArray? = + poTokenProvider.getPoToken(YoutubeSabrInfo(pipeInfo), YoutubeSabrStreamState.fromDelegate(pipeState)) + }, + ) + val streamState: YoutubeSabrStreamState = YoutubeSabrStreamState.fromDelegate(delegate.streamState) + + fun fetchSegment(request: SabrSegmentRequest, localization: Localization): SabrMediaSegment? = + SabrMediaSegment.fromDelegate(delegate.fetchSegment(request.delegate, localization)) + + fun addDiagnosticEvent(event: String): Unit = delegate.addDiagnosticEvent(event) + val diagnosticTrace: String get() = delegate.diagnosticTrace + fun pumpOnce(localization: Localization): List = + delegate.pumpOnce(localization).map(SabrMediaSegment::fromDelegate) + fun pumpOnceStreaming(localization: Localization): Int = delegate.pumpOnceStreaming(localization) + fun pumpOnceStreamingForStartup(localization: Localization): Int = + delegate.pumpOnceStreamingForStartup(localization) + fun pumpOnceStreamingUntilCached(localization: Localization, target: SabrSegmentRequest): Int = + delegate.pumpOnceStreamingUntilCached(localization, target.delegate) + fun pumpOnceStreamingForDemand(localization: Localization, target: SabrSegmentRequest): DemandResponseResult = + delegate.pumpOnceStreamingForDemand(localization, target.delegate).let(::DemandResponseResult) + val demandBackoffRemainingMs: Long get() = delegate.demandBackoffRemainingMs + val mediaProgressVersion: Long get() = delegate.mediaProgressVersion + fun setPlayHeadMs(value: Long): Unit = delegate.setPlayHeadMs(value) + val cachedBytes: Long get() = delegate.cachedBytes + val peakCachedBytes: Long get() = delegate.peakCachedBytes + val totalResponseBytes: Long get() = delegate.totalResponseBytes + val maxResponseBytes: Long get() = delegate.maxResponseBytes + val maxUmpPartBytes: Long get() = delegate.maxUmpPartBytes + val maxMediaPartPayloadBytes: Long get() = delegate.maxMediaPartPayloadBytes + val maxSegmentBytes: Long get() = delegate.maxSegmentBytes + val maxSegmentsPerResponse: Int get() = delegate.maxSegmentsPerResponse + val maxStreamProtectionStatus: Int get() = delegate.maxStreamProtectionStatus + val memoryDiagnosticSummary: String get() = delegate.memoryDiagnosticSummary + fun clearCache(): Unit = delegate.clearCache() + fun evictPlayed(): Unit = delegate.evictPlayed() + fun getCachedSegment(request: SabrSegmentRequest): SabrMediaSegment? = + delegate.getCachedSegment(request.delegate)?.let(SabrMediaSegment::fromDelegate) + fun getReadableSegment(request: SabrSegmentRequest): SabrMediaSegment? = + delegate.getReadableSegment(request.delegate)?.let(SabrMediaSegment::fromDelegate) + fun awaitCachedSegment(request: SabrSegmentRequest, timeoutMs: Long): SabrMediaSegment? = + delegate.awaitCachedSegment(request.delegate, timeoutMs)?.let(SabrMediaSegment::fromDelegate) + fun awaitReadableSegment(request: SabrSegmentRequest, timeoutMs: Long): SabrMediaSegment? = + delegate.awaitReadableSegment(request.delegate, timeoutMs)?.let(SabrMediaSegment::fromDelegate) + fun discardCachedSegment(request: SabrSegmentRequest): Unit = delegate.discardCachedSegment(request.delegate) + fun setTraceEnabled(value: Boolean): Unit = delegate.setTraceEnabled(value) + fun isBeyondEnd(request: SabrSegmentRequest): Boolean = delegate.isBeyondEnd(request.delegate) + val isComplete: Boolean get() = delegate.isComplete + val isLive: Boolean get() = delegate.isLive + val liveHeadSequenceNumber: Long get() = delegate.liveHeadSequenceNumber + val isAtLiveEdge: Boolean get() = delegate.isAtLiveEdge + val requestNumber: Int get() = delegate.requestNumber + val sessionPolicyTranscript: List get() = delegate.sessionPolicyTranscript + fun prepareForMediaSegment(request: SabrSegmentRequest): Unit = delegate.prepareForMediaSegment(request.delegate) + fun prepareForInitialization(format: YoutubeSabrFormat): Unit = delegate.prepareForInitialization(format.delegate) + fun bootstrapInitialization(localization: Localization): Unit = delegate.bootstrapInitialization(localization) + fun fetchInitializationData(format: YoutubeSabrFormat, localization: Localization, timeoutMs: Long, poToken: ByteArray): ByteArray = + delegate.fetchInitializationData(format.delegate, localization, timeoutMs, poToken) + fun prepareForRewind(request: SabrSegmentRequest): Unit = delegate.prepareForRewind(request.delegate) + fun prepareForRewind(request: SabrSegmentRequest, value: Long): Unit = delegate.prepareForRewind(request.delegate, value) + fun prepareForForwardJump(request: SabrSegmentRequest): Unit = delegate.prepareForForwardJump(request.delegate) + fun prepareForForwardJump(request: SabrSegmentRequest, value: Long): Unit = delegate.prepareForForwardJump(request.delegate, value) + fun prepareForMissingSegment(request: SabrSegmentRequest): Unit = delegate.prepareForMissingSegment(request.delegate) + + internal class DemandResponseResult internal constructor( + private val delegate: PipeSession.DemandResponseResult, + ) { + val segmentCount: Int get() = delegate.segmentCount + val targetTrackSegmentCount: Int get() = delegate.targetTrackSegmentCount + } +} diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrStreamState.kt b/src/main/kotlin/dev/typetype/server/sabr/SabrStreamState.kt new file mode 100644 index 00000000..5689bd70 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/sabr/SabrStreamState.kt @@ -0,0 +1,82 @@ +package dev.typetype.server.sabr + +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState as PipeState + +internal class YoutubeSabrStreamState private constructor( + internal val delegate: PipeState, +) { + companion object { + const val TRACK_MODE_VIDEO_AND_AUDIO: Int = PipeState.TRACK_MODE_VIDEO_AND_AUDIO + const val TRACK_MODE_AUDIO_ONLY: Int = PipeState.TRACK_MODE_AUDIO_ONLY + const val TRACK_MODE_VIDEO_ONLY: Int = PipeState.TRACK_MODE_VIDEO_ONLY + internal fun fromDelegate(delegate: PipeState): YoutubeSabrStreamState = YoutubeSabrStreamState(delegate) + } + + constructor(audio: YoutubeSabrFormat, video: YoutubeSabrFormat) : this(PipeState(audio.delegate, video.delegate)) + + fun ingest(segment: SabrMediaSegment): Boolean = delegate.ingest(segment.delegate) + fun ingestInitializationData(format: YoutubeSabrFormat, data: ByteArray): Boolean = + delegate.ingestInitializationData(format.delegate, data) + val bufferedRanges: List + get() = delegate.bufferedRanges.map(SabrBufferedRange::fromDelegate) + fun setBufferedRangesOverride(ranges: List?): Unit = + delegate.setBufferedRangesOverride(ranges?.map { it.delegate }) + val playerTimeMs: Long get() = delegate.playerTimeMs + fun getMinBufferedEndMs(): Long = delegate.minBufferedEndMs + fun getBufferedEndMs(format: YoutubeSabrFormat): Long = delegate.getBufferedEndMs(format.delegate) + fun setPlayerTimeMs(value: Long): Unit = delegate.setPlayerTimeMs(value) + fun clearPlayerTimeMsOverride(): Unit = delegate.clearPlayerTimeMsOverride() + val playbackCookie: ByteArray? get() = delegate.playbackCookie + fun setPoToken(value: ByteArray): Unit = delegate.setPoToken(value) + val poToken: ByteArray? get() = delegate.poToken + val isComplete: Boolean get() = delegate.isComplete + val isLive: Boolean get() = delegate.isLive + val isPostLiveDvr: Boolean get() = delegate.isPostLiveDvr + val liveHeadSequenceNumber: Long get() = delegate.liveHeadSequenceNumber + val liveHeadTimeMs: Long get() = delegate.liveHeadTimeMs + fun isAtLiveEdge(audio: YoutubeSabrFormat, video: YoutubeSabrFormat): Boolean = + delegate.isAtLiveEdge(audio.delegate, video.delegate) + fun getMaxSegment(format: YoutubeSabrFormat): Int = delegate.getMaxSegment(format.delegate) + fun getEndSegment(format: YoutubeSabrFormat): Long = delegate.getEndSegment(format.delegate) + fun hasSegmentIndex(format: YoutubeSabrFormat): Boolean = delegate.hasSegmentIndex(format.delegate) + fun isComplete(format: YoutubeSabrFormat): Boolean = delegate.isComplete(format.delegate) + fun assumeBufferedUntil(format: YoutubeSabrFormat, segment: Int): Unit = delegate.assumeBufferedUntil(format.delegate, segment) + fun rewindBufferedTo(format: YoutubeSabrFormat, segment: Int): Unit = delegate.rewindBufferedTo(format.delegate, segment) + fun jumpBufferedTo(format: YoutubeSabrFormat, segment: Int): Unit = delegate.jumpBufferedTo(format.delegate, segment) + fun setFullyBuffered(format: YoutubeSabrFormat, value: Boolean): Unit = delegate.setFullyBuffered(format.delegate, value) + fun setLastOnlyRange(format: YoutubeSabrFormat, value: Boolean): Unit = delegate.setLastOnlyRange(format.delegate, value) + fun setLastOnlyRangesUseObservedTiming(value: Boolean): Unit = delegate.setLastOnlyRangesUseObservedTiming(value) + fun setBufferedRangeSegmentIndexOffset(value: Int): Unit = delegate.setBufferedRangeSegmentIndexOffset(value) + fun setBufferedRangeSegmentIndexOffsets(audio: Int, video: Int): Unit = delegate.setBufferedRangeSegmentIndexOffsets(audio, video) + fun setRequestTrackMode(mode: Int, audio: Boolean, video: Boolean): Unit = delegate.setRequestTrackMode(mode, audio, video) + fun setActiveTrackTypes(video: Boolean, audio: Boolean): Unit = delegate.setActiveTrackTypes(video, audio) + fun setAudioOnlyRequestMode(): Unit = delegate.setAudioOnlyRequestMode() + fun setVideoOnlyRequestMode(): Unit = delegate.setVideoOnlyRequestMode() + fun setVideoAndAudioRequestMode(): Unit = delegate.setVideoAndAudioRequestMode() + fun setClientViewport(width: Int, height: Int): Unit = delegate.setClientViewport(width, height) + fun setBandwidthEstimate(value: Long): Unit = delegate.setBandwidthEstimate(value) + val bandwidthEstimate: Long get() = delegate.bandwidthEstimate + val nextRequestPolicy: SabrNextRequestPolicy? + get() = delegate.nextRequestPolicy?.let(::SabrNextRequestPolicy) + fun setPlaybackRate(value: Float): Unit = delegate.setPlaybackRate(value) + fun setWriteTopLevelPlayerTimeMs(value: Boolean): Unit = delegate.setWriteTopLevelPlayerTimeMs(value) + fun setClientAbrVisibility(value: Int?): Unit = delegate.setClientAbrVisibility(value) + fun setWriteLastManualSelectedResolution(value: Boolean): Unit = delegate.setWriteLastManualSelectedResolution(value) + fun setWriteAllPreferredFormats(value: Boolean): Unit = delegate.setWriteAllPreferredFormats(value) + fun setWriteOfficialWebPreferredFormats(value: Boolean): Unit = delegate.setWriteOfficialWebPreferredFormats(value) + fun setSelectVideoFormatBeforeAudio(value: Boolean): Unit = delegate.setSelectVideoFormatBeforeAudio(value) + fun setWriteBufferedRangeTimeRange(value: Boolean): Unit = delegate.setWriteBufferedRangeTimeRange(value) + fun setStickyResolutionOverride(value: Int?): Unit = delegate.setStickyResolutionOverride(value) + fun setOfficialWebClientAbrTimingOverrides(a: Long?, b: Long?, c: Long?, d: Long?): Unit = + delegate.setOfficialWebClientAbrTimingOverrides(a, b, c, d) + fun setOfficialField68Override(value: Long?): Unit = delegate.setOfficialField68Override(value) + fun setSabrReportRequestCancellationInfoOverride(value: Int?): Unit = + delegate.setSabrReportRequestCancellationInfoOverride(value) + fun setWriteOfficialWebClientAbrFields(value: Boolean): Unit = delegate.setWriteOfficialWebClientAbrFields(value) + fun summarizeBufferedRanges(): String = delegate.summarizeBufferedRanges() + fun getAverageSegmentDurationMs(format: YoutubeSabrFormat): Long = delegate.getAverageSegmentDurationMs(format.delegate) + fun getSegmentStartMs(format: YoutubeSabrFormat, sequence: Int): Long = delegate.getSegmentStartMs(format.delegate, sequence) + fun getSegmentEndMs(format: YoutubeSabrFormat, sequence: Int): Long = delegate.getSegmentEndMs(format.delegate, sequence) + fun getSegmentNumberAtOrAfterTimeMs(format: YoutubeSabrFormat, timeMs: Long): Int = + delegate.getSegmentNumberAtOrAfterTimeMs(format.delegate, timeMs) +} From 41a16fd4495f5c592f7c92cd1ac9d2fef65487d0 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 4 Sep 2026 11:43:58 +0200 Subject: [PATCH 06/45] refactor: route Server SABR code through adapter --- build.gradle.kts | 2 +- .../dev/typetype/server/routes/SabrFormatSelector.kt | 6 +++--- .../dev/typetype/server/routes/SabrManifestHandler.kt | 2 +- .../typetype/server/routes/SabrManifestResponse.kt | 2 +- .../dev/typetype/server/routes/SabrPlaybackHandler.kt | 2 +- .../dev/typetype/server/routes/SabrPlaybackModels.kt | 2 +- .../server/routes/SabrPlaybackStateHandler.kt | 2 +- .../server/routes/SabrPlaybackWindowBuilder.kt | 4 ++-- .../server/routes/SabrPlaybackWindowHandler.kt | 2 +- .../server/routes/SabrPlaybackWindowSegmentTiming.kt | 4 ++-- .../server/routes/SabrPlaybackWindowTiming.kt | 2 +- .../server/routes/SabrProgressivePlaybackWindow.kt | 6 +++--- .../dev/typetype/server/routes/SabrSegmentHandler.kt | 2 +- .../server/routes/SabrSessionDescriptorHandler.kt | 2 +- .../typetype/server/routes/SabrSessionStateHandler.kt | 4 ++-- .../server/routes/SabrStreamContractFilter.kt | 4 ++-- .../server/services/AuthenticatedSabrInfoService.kt | 8 ++++---- .../dev/typetype/server/services/CachedSabrSegment.kt | 2 +- .../typetype/server/services/PipePipeStreamService.kt | 2 +- .../server/services/SabrAdaptiveInitialization.kt | 2 +- .../server/services/SabrCachedSegmentLocator.kt | 8 ++++---- .../server/services/SabrDashManifestBuilder.kt | 4 ++-- .../server/services/SabrDemandAttemptFinisher.kt | 6 +++--- .../server/services/SabrDownloadInitialization.kt | 2 +- .../dev/typetype/server/services/SabrDownloadRange.kt | 4 ++-- .../typetype/server/services/SabrDownloadStreamer.kt | 4 ++-- .../server/services/SabrFallbackStreamMapper.kt | 4 ++-- .../server/services/SabrHlsManifestBuilder.kt | 4 ++-- .../server/services/SabrInFlightDemandTracker.kt | 2 +- .../dev/typetype/server/services/SabrInfoFetcher.kt | 6 +++--- .../typetype/server/services/SabrInfoRepository.kt | 4 ++-- .../typetype/server/services/SabrInfoSharedCache.kt | 2 +- .../server/services/SabrInitializationData.kt | 4 ++-- .../server/services/SabrInitializationPolicy.kt | 2 +- .../services/SabrInitializationSegmentFetcher.kt | 4 ++-- .../server/services/SabrLiveContinuationRequest.kt | 4 ++-- .../dev/typetype/server/services/SabrLivePlayback.kt | 6 +++--- .../server/services/SabrLivePlaybackDiscontinuity.kt | 4 ++-- .../dev/typetype/server/services/SabrLivePumpStep.kt | 6 +++--- .../typetype/server/services/SabrLiveWarmupRequest.kt | 6 +++--- .../typetype/server/services/SabrManifestBuilder.kt | 4 ++-- .../typetype/server/services/SabrManifestTiming.kt | 4 ++-- .../typetype/server/services/SabrMimeAttributes.kt | 2 +- .../dev/typetype/server/services/SabrPendingSeek.kt | 2 +- .../services/SabrPlaybackCachedSegmentLocator.kt | 4 ++-- .../server/services/SabrPlaybackDiagnostics.kt | 4 ++-- .../server/services/SabrPlaybackManifestService.kt | 2 +- .../server/services/SabrPlaybackMediaFetcher.kt | 4 ++-- .../server/services/SabrPlaybackSegmentResult.kt | 2 +- .../server/services/SabrPlaybackSegmentSelection.kt | 2 +- .../server/services/SabrPlaybackSessionService.kt | 4 ++-- .../typetype/server/services/SabrPlaybackStarter.kt | 2 +- .../typetype/server/services/SabrPlaybackWarmer.kt | 2 +- .../server/services/SabrPlayerContextRecovery.kt | 6 +++--- .../typetype/server/services/SabrPlayerInfoProbe.kt | 8 ++++---- .../dev/typetype/server/services/SabrPreparedInfo.kt | 2 +- .../dev/typetype/server/services/SabrPumpLogger.kt | 2 +- .../dev/typetype/server/services/SabrSegmentCache.kt | 6 +++--- .../server/services/SabrSegmentDemandResolution.kt | 4 ++-- .../server/services/SabrSegmentDemandTracker.kt | 2 +- .../typetype/server/services/SabrSessionFactory.kt | 6 +++--- .../dev/typetype/server/services/SabrSessionHolder.kt | 10 +++++----- .../typetype/server/services/SabrSessionIdentity.kt | 6 +++--- .../server/services/SabrSessionMediaFetcher.kt | 4 ++-- .../server/services/SabrSessionPlayerContext.kt | 2 +- .../typetype/server/services/SabrSessionProgress.kt | 4 ++-- .../dev/typetype/server/services/SabrSessionPump.kt | 4 ++-- .../typetype/server/services/SabrSessionPumpLoop.kt | 8 ++++---- .../dev/typetype/server/services/SabrSessionStore.kt | 8 ++++---- .../server/services/SabrSessionTimeRequests.kt | 4 ++-- .../dev/typetype/server/services/SabrTargetRequest.kt | 6 +++--- .../server/services/SabrTargetRequestShape.kt | 8 ++++---- .../dev/typetype/server/services/SabrTokenBundle.kt | 2 +- .../services/SabrUnauthorizedResponseRecovery.kt | 2 +- .../server/services/SabrWindowSegmentFetcher.kt | 4 ++-- .../typetype/server/services/TokenYoutubeSession.kt | 2 +- .../services/TypetypeTokenSabrPoTokenProvider.kt | 8 ++++---- .../services/TypetypeTokenYoutubeSessionClient.kt | 11 +++++------ 78 files changed, 157 insertions(+), 158 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 620ef3ae..9789023e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -44,7 +44,7 @@ dependencies { implementation("io.ktor:ktor-server-call-logging-jvm") implementation("io.ktor:ktor-server-rate-limit-jvm") implementation("ch.qos.logback:logback-classic:1.6.3") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:f156813dd4bbebf3b4dffe541fee6c27ae1dd294") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:ca3280f28f3aa0b980b63a2b2d23c362f7616620") compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") implementation("org.json:json:20260814") implementation("com.squareup.okhttp3:okhttp:5.5.0") diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrFormatSelector.kt b/src/main/kotlin/dev/typetype/server/routes/SabrFormatSelector.kt index 1e8e5cdc..bfb3a215 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrFormatSelector.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrFormatSelector.kt @@ -1,7 +1,7 @@ package dev.typetype.server.routes -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo internal object SabrFormatSelector { fun video(info: YoutubeSabrInfo, itag: Int?): YoutubeSabrFormat? { @@ -23,7 +23,7 @@ internal object SabrFormatSelector { } private fun YoutubeSabrFormat.matchesAudio(itag: Int?, trackId: String?, requireAac: Boolean): Boolean = - itag != null && isAudio && getItag() == itag && (!requireAac || isAac()) && + itag != null && isAudio && this.itag == itag && (!requireAac || isAac()) && (trackId.isNullOrBlank() || audioTrackId == trackId) private fun YoutubeSabrFormat.isAac(): Boolean = diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt b/src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt index f24a01e4..b98707ce 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt @@ -13,7 +13,7 @@ import dev.typetype.server.services.SabrSessionPurpose import dev.typetype.server.services.SabrSessionStore import dev.typetype.server.services.StreamService import dev.typetype.server.services.bothFormatsKnown -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrFormat import io.ktor.http.HttpStatusCode import io.ktor.server.application.ApplicationCall import io.ktor.server.response.respond diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrManifestResponse.kt b/src/main/kotlin/dev/typetype/server/routes/SabrManifestResponse.kt index 329c4a13..faa2ebb0 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrManifestResponse.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrManifestResponse.kt @@ -58,7 +58,7 @@ internal suspend fun ApplicationCall.respondSabrManifest( respondText(manifest, if (hls) HLS_CONTENT_TYPE else DASH_CONTENT_TYPE) } -private fun SabrSessionHolder.startSegment(format: org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat): Int = +private fun SabrSessionHolder.startSegment(format: dev.typetype.server.sabr.YoutubeSabrFormat): Int = key.startTimeMs.takeIf { it > 0L } ?.let { playbackStartSequence(format, it) } ?: 1 diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt index 6326898a..7f7f26dc 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt @@ -19,7 +19,7 @@ import io.ktor.http.HttpStatusCode import io.ktor.server.application.ApplicationCall import io.ktor.server.request.receive import io.ktor.server.response.respond -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrFormat internal class SabrPlaybackHandler( private val sabrSessionStore: SabrSessionStore, diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackModels.kt b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackModels.kt index c4ddf65c..c9ee7140 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackModels.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackModels.kt @@ -1,7 +1,7 @@ package dev.typetype.server.routes import kotlinx.serialization.Serializable -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.SabrSegmentRequest @Serializable internal data class SabrPlaybackRequest( diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackStateHandler.kt b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackStateHandler.kt index a5b8f07b..a51e3cd4 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackStateHandler.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackStateHandler.kt @@ -8,7 +8,7 @@ import dev.typetype.server.services.livePlaybackSnapshot import io.ktor.http.HttpStatusCode import io.ktor.server.application.ApplicationCall import io.ktor.server.response.respond -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.SabrSegmentRequest internal class SabrPlaybackStateHandler(private val sabrSessionStore: SabrSessionStore) { suspend fun get(call: ApplicationCall, sessionId: String) { diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt index 56dd5c8d..183a4a76 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt @@ -12,8 +12,8 @@ import dev.typetype.server.services.livePlaybackSnapshot import dev.typetype.server.services.playbackContinuationSequence import dev.typetype.server.services.playbackSegmentStartMs import dev.typetype.server.services.resolvePlaybackStartMs -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat private const val MAX_SEGMENTS_PER_TRACK = 12 diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowHandler.kt b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowHandler.kt index 52d4d2c9..64241f07 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowHandler.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowHandler.kt @@ -12,7 +12,7 @@ import io.ktor.http.HttpStatusCode import io.ktor.server.application.ApplicationCall import io.ktor.server.request.receive import io.ktor.server.response.respond -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.SabrSegmentRequest internal class SabrPlaybackWindowHandler(private val sabrSessionStore: SabrSessionStore) { private val windowBuilder = SabrPlaybackWindowBuilder(sabrSessionStore) diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowSegmentTiming.kt b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowSegmentTiming.kt index d1174456..cd96e1fd 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowSegmentTiming.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowSegmentTiming.kt @@ -4,8 +4,8 @@ import dev.typetype.server.services.CachedSabrSegment import dev.typetype.server.services.SabrSessionHolder import dev.typetype.server.services.SabrSessionStore import dev.typetype.server.services.playbackSegmentDurationMs -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat internal suspend fun SabrSessionStore.resolvePlaybackDurationMs( holder: SabrSessionHolder, diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTiming.kt b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTiming.kt index c186a231..9d0aee52 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTiming.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTiming.kt @@ -4,7 +4,7 @@ import dev.typetype.server.services.CachedSabrSegment import dev.typetype.server.services.SabrSessionHolder import dev.typetype.server.services.livePlaybackSnapshot import dev.typetype.server.services.playbackSegmentDurationMs -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrFormat internal fun SabrSessionHolder.durationMs(): Long { livePlaybackSnapshot()?.let { live -> diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindow.kt b/src/main/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindow.kt index 35b977e6..5ae51fb1 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindow.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindow.kt @@ -4,9 +4,9 @@ import dev.typetype.server.services.SabrSessionHolder import dev.typetype.server.services.findCachedMediaAt import dev.typetype.server.services.playbackSegmentDurationMs import dev.typetype.server.services.playbackSegmentStartMs -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat internal data class SabrProgressiveWindowSegment( val sequence: Int, diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrSegmentHandler.kt b/src/main/kotlin/dev/typetype/server/routes/SabrSegmentHandler.kt index 8b2de3a3..069c09d6 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrSegmentHandler.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrSegmentHandler.kt @@ -11,7 +11,7 @@ import io.ktor.server.application.ApplicationCall import io.ktor.server.response.respond import kotlinx.coroutines.delay import kotlinx.coroutines.withTimeoutOrNull -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.SabrSegmentRequest internal class SabrSegmentHandler( private val sabrSessionStore: SabrSessionStore, diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrSessionDescriptorHandler.kt b/src/main/kotlin/dev/typetype/server/routes/SabrSessionDescriptorHandler.kt index 92302fa3..acd908d5 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrSessionDescriptorHandler.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrSessionDescriptorHandler.kt @@ -16,7 +16,7 @@ import kotlinx.coroutines.withTimeoutOrNull import kotlinx.serialization.json.buildJsonObject import kotlinx.serialization.json.put import kotlinx.serialization.json.putJsonObject -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrFormat import kotlin.math.max internal class SabrSessionDescriptorHandler( diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrSessionStateHandler.kt b/src/main/kotlin/dev/typetype/server/routes/SabrSessionStateHandler.kt index d68583b8..0dcb823a 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrSessionStateHandler.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrSessionStateHandler.kt @@ -16,8 +16,8 @@ import kotlinx.serialization.json.longOrNull import kotlinx.serialization.json.doubleOrNull import kotlinx.serialization.json.put import kotlinx.serialization.json.putJsonObject -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat internal class SabrSessionStateHandler(private val sabrSessionStore: SabrSessionStore) { suspend fun get(call: ApplicationCall, videoId: String) { diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrStreamContractFilter.kt b/src/main/kotlin/dev/typetype/server/routes/SabrStreamContractFilter.kt index de4379d2..f966d9a4 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrStreamContractFilter.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrStreamContractFilter.kt @@ -4,8 +4,8 @@ import dev.typetype.server.models.AudioStreamItem import dev.typetype.server.models.StreamResponse import dev.typetype.server.models.VideoStreamItem import dev.typetype.server.services.SabrSessionStore -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo internal suspend fun StreamResponse.withPlayableSabrStreams( url: String, diff --git a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt b/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt index 66ecdecc..1768eaec 100644 --- a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt +++ b/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt @@ -8,9 +8,9 @@ import kotlinx.coroutines.runInterruptible import org.schabi.newpipe.extractor.localization.ContentCountry import org.schabi.newpipe.extractor.localization.Localization import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrClientProfile -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrProbe +import dev.typetype.server.sabr.YoutubeSabrClientProfile +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.SabrAdapter import org.slf4j.LoggerFactory internal class AuthenticatedSabrInfoService( @@ -87,7 +87,7 @@ private object PipePipeAuthenticatedSabrProbe : AuthenticatedSabrProbe { private val contentCountry = ContentCountry("US") override fun fetch(videoId: String, token: YoutubeSessionPoToken): YoutubeSabrInfo = - YoutubeSabrProbe.fetchSabrInfo( + SabrAdapter.fetchSabrInfo( videoId, YoutubeSabrClientProfile.WEB, localization, diff --git a/src/main/kotlin/dev/typetype/server/services/CachedSabrSegment.kt b/src/main/kotlin/dev/typetype/server/services/CachedSabrSegment.kt index c92342dc..443c6b6d 100644 --- a/src/main/kotlin/dev/typetype/server/services/CachedSabrSegment.kt +++ b/src/main/kotlin/dev/typetype/server/services/CachedSabrSegment.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrMediaSegment import java.util.Base64 internal class CachedSabrSegment( diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt b/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt index 011f4d46..85209166 100644 --- a/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt +++ b/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt @@ -15,7 +15,7 @@ import org.schabi.newpipe.extractor.NewPipe import org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability import org.schabi.newpipe.extractor.sponsorblock.SponsorBlockApiSettings import org.schabi.newpipe.extractor.sponsorblock.SponsorBlockExtractorHelper -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrInfo import org.schabi.newpipe.extractor.stream.StreamExtractor import org.schabi.newpipe.extractor.stream.StreamInfo diff --git a/src/main/kotlin/dev/typetype/server/services/SabrAdaptiveInitialization.kt b/src/main/kotlin/dev/typetype/server/services/SabrAdaptiveInitialization.kt index de684c89..70b2e088 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrAdaptiveInitialization.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrAdaptiveInitialization.kt @@ -4,7 +4,7 @@ import dev.typetype.server.cache.CacheService import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.runInterruptible import org.schabi.newpipe.extractor.localization.Localization -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrFormat internal object SabrAdaptiveInitialization { private val localization = Localization("en", "US") diff --git a/src/main/kotlin/dev/typetype/server/services/SabrCachedSegmentLocator.kt b/src/main/kotlin/dev/typetype/server/services/SabrCachedSegmentLocator.kt index 7862aaad..a532cdfd 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrCachedSegmentLocator.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrCachedSegmentLocator.kt @@ -1,9 +1,9 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrSession internal fun YoutubeSabrSession.findCachedMediaAt( format: YoutubeSabrFormat, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDashManifestBuilder.kt b/src/main/kotlin/dev/typetype/server/services/SabrDashManifestBuilder.kt index a53a3aa7..6c443d75 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDashManifestBuilder.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrDashManifestBuilder.kt @@ -1,7 +1,7 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrStreamState internal object SabrDashManifestBuilder { fun build( diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt b/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt index 3e733749..e4eccca5 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt @@ -1,8 +1,8 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrSession internal object SabrDemandAttemptFinisher { fun interruptCompletedInFlightDemand(holder: SabrSessionHolder, demand: SabrInFlightDemand): Boolean = diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDownloadInitialization.kt b/src/main/kotlin/dev/typetype/server/services/SabrDownloadInitialization.kt index 62f296e5..f4ed0722 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDownloadInitialization.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrDownloadInitialization.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrFormat internal object SabrDownloadInitialization { suspend fun fetch( diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDownloadRange.kt b/src/main/kotlin/dev/typetype/server/services/SabrDownloadRange.kt index 5117cc1b..f5f80626 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDownloadRange.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrDownloadRange.kt @@ -1,7 +1,7 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrStreamState internal data class SabrDownloadRange( val part: Int = 0, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt b/src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt index 15c3ed98..16fdcc33 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt @@ -6,8 +6,8 @@ import kotlinx.coroutines.runInterruptible import kotlinx.coroutines.withContext import kotlinx.coroutines.withTimeoutOrNull import org.schabi.newpipe.extractor.localization.Localization -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat import org.slf4j.LoggerFactory import java.io.IOException import java.io.OutputStream diff --git a/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamMapper.kt b/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamMapper.kt index 5b8f0151..305edca5 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamMapper.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamMapper.kt @@ -3,8 +3,8 @@ package dev.typetype.server.services import dev.typetype.server.models.AudioStreamItem import dev.typetype.server.models.StreamResponse import dev.typetype.server.models.VideoStreamItem -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo import java.net.URLEncoder import java.nio.charset.StandardCharsets diff --git a/src/main/kotlin/dev/typetype/server/services/SabrHlsManifestBuilder.kt b/src/main/kotlin/dev/typetype/server/services/SabrHlsManifestBuilder.kt index 652e7161..2ef01b4d 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrHlsManifestBuilder.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrHlsManifestBuilder.kt @@ -1,7 +1,7 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.util.Locale internal object SabrHlsManifestBuilder { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrInFlightDemandTracker.kt b/src/main/kotlin/dev/typetype/server/services/SabrInFlightDemandTracker.kt index 97c9b780..e0f6ae20 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrInFlightDemandTracker.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrInFlightDemandTracker.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.SabrSegmentRequest import java.util.concurrent.ConcurrentHashMap internal class SabrInFlightDemand( diff --git a/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt b/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt index e71c0bb5..a1e9b15c 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt @@ -4,9 +4,9 @@ import dev.typetype.server.cache.CacheService import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import kotlinx.coroutines.withTimeoutOrNull -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrClientProfile -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrClientProfile +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo import org.slf4j.LoggerFactory internal class SabrInfoFetcher( diff --git a/src/main/kotlin/dev/typetype/server/services/SabrInfoRepository.kt b/src/main/kotlin/dev/typetype/server/services/SabrInfoRepository.kt index 452aafe5..f0fbd251 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrInfoRepository.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrInfoRepository.kt @@ -1,8 +1,8 @@ package dev.typetype.server.services import dev.typetype.server.cache.CacheService -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo import java.time.Duration internal class SabrInfoRepository( diff --git a/src/main/kotlin/dev/typetype/server/services/SabrInfoSharedCache.kt b/src/main/kotlin/dev/typetype/server/services/SabrInfoSharedCache.kt index 3be48f2a..eabccc84 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrInfoSharedCache.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrInfoSharedCache.kt @@ -1,7 +1,7 @@ package dev.typetype.server.services import dev.typetype.server.cache.CacheService -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrInfo import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream import java.io.ObjectInputStream diff --git a/src/main/kotlin/dev/typetype/server/services/SabrInitializationData.kt b/src/main/kotlin/dev/typetype/server/services/SabrInitializationData.kt index 95b1d8cc..0b9788ca 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrInitializationData.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrInitializationData.kt @@ -3,8 +3,8 @@ package dev.typetype.server.services import dev.typetype.server.cache.CacheService import kotlinx.coroutines.sync.withLock import org.schabi.newpipe.extractor.localization.Localization -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat import java.security.MessageDigest import java.time.Duration import java.util.Base64 diff --git a/src/main/kotlin/dev/typetype/server/services/SabrInitializationPolicy.kt b/src/main/kotlin/dev/typetype/server/services/SabrInitializationPolicy.kt index 2f1a6e58..6f85a8a0 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrInitializationPolicy.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrInitializationPolicy.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrFormat internal object SabrInitializationPolicy { fun warmFormats( diff --git a/src/main/kotlin/dev/typetype/server/services/SabrInitializationSegmentFetcher.kt b/src/main/kotlin/dev/typetype/server/services/SabrInitializationSegmentFetcher.kt index 337677b4..1ccab335 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrInitializationSegmentFetcher.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrInitializationSegmentFetcher.kt @@ -2,8 +2,8 @@ package dev.typetype.server.services import kotlinx.coroutines.sync.withLock import org.schabi.newpipe.extractor.localization.Localization -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest import org.slf4j.LoggerFactory internal suspend fun fetchSabrInitializationSegment( diff --git a/src/main/kotlin/dev/typetype/server/services/SabrLiveContinuationRequest.kt b/src/main/kotlin/dev/typetype/server/services/SabrLiveContinuationRequest.kt index 2b690d87..d89e3736 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrLiveContinuationRequest.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrLiveContinuationRequest.kt @@ -1,7 +1,7 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrBufferedRange -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.SabrBufferedRange +import dev.typetype.server.sabr.YoutubeSabrFormat internal inline fun withLiveContinuationRequestShape( holder: SabrSessionHolder, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt b/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt index f01c902e..0507aa72 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt @@ -1,7 +1,7 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat internal data class SabrLivePlaybackSnapshot( val active: Boolean, @@ -120,7 +120,7 @@ internal fun SabrSessionHolder.isHistoricalLiveRequest(request: SabrSegmentReque internal fun SabrSessionHolder.liveRetryAfterMs(blockedRequests: List = emptyList()): Long = DEFAULT_PLAYBACK_RETRY_MS -private fun org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState.observedEndMs( +private fun dev.typetype.server.sabr.YoutubeSabrStreamState.observedEndMs( format: YoutubeSabrFormat, ): Long { val sequence = runCatching { getMaxSegment(format) }.getOrDefault(0) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrLivePlaybackDiscontinuity.kt b/src/main/kotlin/dev/typetype/server/services/SabrLivePlaybackDiscontinuity.kt index 61c3ad60..91ed693c 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrLivePlaybackDiscontinuity.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrLivePlaybackDiscontinuity.kt @@ -1,7 +1,7 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat internal fun SabrSessionHolder.isLiveDemandOutsideRecoverableWindow(request: SabrSegmentRequest): Boolean { if (request.isInitializationSegment) return false diff --git a/src/main/kotlin/dev/typetype/server/services/SabrLivePumpStep.kt b/src/main/kotlin/dev/typetype/server/services/SabrLivePumpStep.kt index b9abf457..ac33f9f7 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrLivePumpStep.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrLivePumpStep.kt @@ -1,8 +1,8 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat internal suspend fun pumpLiveReadAhead( holder: SabrSessionHolder, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrLiveWarmupRequest.kt b/src/main/kotlin/dev/typetype/server/services/SabrLiveWarmupRequest.kt index 8d0a7c81..189488ab 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrLiveWarmupRequest.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrLiveWarmupRequest.kt @@ -1,8 +1,8 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrBufferedRange -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.SabrBufferedRange +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.YoutubeSabrFormat internal data class SabrLiveWarmupTarget( val sequence: Int, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrManifestBuilder.kt b/src/main/kotlin/dev/typetype/server/services/SabrManifestBuilder.kt index 41043e13..33b45d76 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrManifestBuilder.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrManifestBuilder.kt @@ -1,7 +1,7 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrStreamState internal object SabrManifestBuilder { fun build( diff --git a/src/main/kotlin/dev/typetype/server/services/SabrManifestTiming.kt b/src/main/kotlin/dev/typetype/server/services/SabrManifestTiming.kt index 882a3bbc..433b3aec 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrManifestTiming.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrManifestTiming.kt @@ -1,7 +1,7 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrStreamState internal object SabrManifestTiming { fun videoDurationSec( diff --git a/src/main/kotlin/dev/typetype/server/services/SabrMimeAttributes.kt b/src/main/kotlin/dev/typetype/server/services/SabrMimeAttributes.kt index 50987ed4..4b99aaca 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrMimeAttributes.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrMimeAttributes.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrFormat internal fun splitMime(mime: String): Pair { val parts = mime.split(";", limit = 2) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPendingSeek.kt b/src/main/kotlin/dev/typetype/server/services/SabrPendingSeek.kt index 3cc1c668..43170eca 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPendingSeek.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPendingSeek.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.SabrSegmentRequest internal fun SabrSessionHolder.consumeMatchingSeek(request: SabrSegmentRequest): Boolean { pendingRefetchRequest()?.takeIf { it.matches(request) }?.let { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocator.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocator.kt index 88fa4c0a..1eee1935 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocator.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocator.kt @@ -1,7 +1,7 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat internal suspend fun SabrSessionStore.findCachedPlaybackMediaAt( holder: SabrSessionHolder, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackDiagnostics.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackDiagnostics.kt index 35064731..2c003d40 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackDiagnostics.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackDiagnostics.kt @@ -1,7 +1,7 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest import java.util.concurrent.ConcurrentHashMap internal object SabrPlaybackDiagnostics { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackManifestService.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackManifestService.kt index 0695a916..c713e7a0 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackManifestService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackManifestService.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrFormat internal class SabrPlaybackManifestService { fun build(holder: SabrSessionHolder, mediaBasePath: String): SabrPlaybackManifestResult { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackMediaFetcher.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackMediaFetcher.kt index 4ba20e8e..bb1bd6c9 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackMediaFetcher.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackMediaFetcher.kt @@ -4,8 +4,8 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.delay import kotlinx.coroutines.runInterruptible import kotlinx.coroutines.withTimeoutOrNull -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest internal class SabrPlaybackMediaFetcher(private val sessionStore: SabrSessionStore) { suspend fun fetch( diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentResult.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentResult.kt index 62774b80..3f2cfc7c 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentResult.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentResult.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrMediaSegment internal sealed class SabrPlaybackSegmentResult { data class Ready(val mimeType: String, val bytes: ByteArray) : SabrPlaybackSegmentResult() diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentSelection.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentSelection.kt index 2f957857..5fa0feb9 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentSelection.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentSelection.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrFormat internal fun SabrSessionHolder.playbackStartSequence(format: YoutubeSabrFormat, playerTimeMs: Long): Int { liveSequenceAt(format, playerTimeMs)?.let { return it } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt index 6eb0d40d..b19fc4dd 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt @@ -1,8 +1,8 @@ package dev.typetype.server.services import kotlinx.coroutines.withTimeoutOrNull -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat internal class SabrPlaybackSessionService(private val sessionStore: SabrSessionStore) { private val mediaFetcher = SabrPlaybackMediaFetcher(sessionStore) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackStarter.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackStarter.kt index bc587a5c..e2121ed0 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackStarter.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackStarter.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.SabrSegmentRequest import org.slf4j.LoggerFactory internal object SabrPlaybackStarter { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackWarmer.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackWarmer.kt index 7c803ec0..45c37f49 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackWarmer.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackWarmer.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrMediaSegment import org.slf4j.LoggerFactory internal class SabrPlaybackWarmer { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlayerContextRecovery.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlayerContextRecovery.kt index 76ddc7c6..f28cfbbc 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlayerContextRecovery.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlayerContextRecovery.kt @@ -2,9 +2,9 @@ package dev.typetype.server.services import kotlinx.coroutines.CancellationException import org.schabi.newpipe.extractor.exceptions.AntiBotException -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrProtocolException -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrClientProfile -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.SabrProtocolException +import dev.typetype.server.sabr.YoutubeSabrClientProfile +import dev.typetype.server.sabr.YoutubeSabrInfo internal class SabrPlayerContextRecovery( private val videoId: String, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlayerInfoProbe.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlayerInfoProbe.kt index 44ecba53..c5ba312b 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlayerInfoProbe.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlayerInfoProbe.kt @@ -2,9 +2,9 @@ package dev.typetype.server.services import org.schabi.newpipe.extractor.localization.ContentCountry import org.schabi.newpipe.extractor.localization.Localization -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrClientProfile -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrProbe +import dev.typetype.server.sabr.YoutubeSabrClientProfile +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.SabrAdapter internal fun interface SabrPlayerInfoProbe { fun fetch( @@ -23,6 +23,6 @@ internal object PipePipeSabrPlayerInfoProbe : SabrPlayerInfoProbe { profile: YoutubeSabrClientProfile, token: SabrTokenBundle, ): YoutubeSabrInfo = TypetypeYoutubeSessionPoTokenProvider.withToken(token) { - YoutubeSabrProbe.fetchSabrInfo(videoId, profile, localization, contentCountry) + SabrAdapter.fetchSabrInfo(videoId, profile, localization, contentCountry) } } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfo.kt b/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfo.kt index 23d59c87..d0d661e4 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfo.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfo.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrInfo internal class SabrPreparedInfo( val info: YoutubeSabrInfo, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt b/src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt index d3230963..849a5068 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.SabrSegmentRequest import org.slf4j.LoggerFactory internal object SabrPumpLogger { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSegmentCache.kt b/src/main/kotlin/dev/typetype/server/services/SabrSegmentCache.kt index db315d9d..3e262d83 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSegmentCache.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSegmentCache.kt @@ -1,8 +1,8 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat internal class SabrSegmentCache { fun get(holder: SabrSessionHolder, request: SabrSegmentRequest): CachedSabrSegment? = diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandResolution.kt b/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandResolution.kt index 9acadfeb..8d8ee50b 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandResolution.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandResolution.kt @@ -1,7 +1,7 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest internal fun SabrSessionHolder.resolveSegmentDemand( request: SabrSegmentRequest, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandTracker.kt b/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandTracker.kt index cefd7153..f68e0adc 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandTracker.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandTracker.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.SabrSegmentRequest import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicLong diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionFactory.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionFactory.kt index 5e6ac047..19dbecf3 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionFactory.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionFactory.kt @@ -1,8 +1,8 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession import java.time.Instant internal class SabrSessionFactory( diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt index 626e6b2a..8bb0eb72 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt @@ -1,11 +1,11 @@ package dev.typetype.server.services import kotlinx.coroutines.sync.Mutex -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession import java.time.Instant import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicBoolean diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionIdentity.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionIdentity.kt index f1e86285..a11a99fd 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionIdentity.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionIdentity.kt @@ -2,8 +2,8 @@ package dev.typetype.server.services import okhttp3.HttpUrl.Companion.toHttpUrl import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper -import org.schabi.newpipe.extractor.services.youtube.sabr.TypeTypeYoutubeSabrInfoFactory -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.SabrAdapter +import dev.typetype.server.sabr.YoutubeSabrInfo internal object SabrSessionIdentity { fun fresh(info: YoutubeSabrInfo): YoutubeSabrInfo { @@ -14,7 +14,7 @@ internal object SabrSessionIdentity { .setQueryParameter("cpn", cpn) .build() .toString() - return TypeTypeYoutubeSabrInfoFactory.withPlaybackIdentity( + return SabrAdapter.withPlaybackIdentity( info, url, info.clientVersion, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionMediaFetcher.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionMediaFetcher.kt index 605b5dc6..f0af4faf 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionMediaFetcher.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionMediaFetcher.kt @@ -1,8 +1,8 @@ package dev.typetype.server.services import kotlinx.coroutines.sync.withLock -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest import org.slf4j.LoggerFactory import java.time.Instant diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionPlayerContext.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionPlayerContext.kt index 2d0154d2..db7e1bf3 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionPlayerContext.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionPlayerContext.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrSession internal inline fun SabrSessionHolder.withPlayerContext(crossinline block: YoutubeSabrSession.() -> T): T { val token = playerContextToken ?: return session.block() diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionProgress.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionProgress.kt index cc2cd770..f68876c8 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionProgress.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionProgress.kt @@ -1,7 +1,7 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.YoutubeSabrFormat internal fun SabrSessionHolder.markServed(segment: SabrMediaSegment): Unit { markServed(segment, activeGeneration()) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt index 11280335..052e08d3 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt @@ -3,8 +3,8 @@ package dev.typetype.server.services import kotlinx.coroutines.delay import kotlinx.coroutines.sync.withLock import org.schabi.newpipe.extractor.localization.Localization -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest import java.time.Instant internal class SabrSessionPump( diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt index 3a5fecd7..8f541214 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt @@ -7,10 +7,10 @@ import kotlinx.coroutines.runInterruptible import kotlinx.coroutines.sync.withLock import org.schabi.newpipe.extractor.exceptions.ExtractionException import org.schabi.newpipe.extractor.localization.Localization -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrRecoverableException -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrRecoverableException +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrSession import java.io.IOException internal class SabrSessionPumpLoop( diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt index 57f705e3..56ae348a 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt @@ -9,10 +9,10 @@ import kotlinx.coroutines.cancel import kotlinx.coroutines.delay import kotlinx.coroutines.launch import kotlinx.coroutines.sync.withLock -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo import java.time.Duration import java.time.Instant diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionTimeRequests.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionTimeRequests.kt index 985d3015..3d4c0b1a 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionTimeRequests.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionTimeRequests.kt @@ -1,7 +1,7 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat internal fun SabrSessionHolder.mediaRequestsAt(playerTimeMs: Long): List = mediaRequestsAt(playerTimeMs, activeGeneration()) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrTargetRequest.kt b/src/main/kotlin/dev/typetype/server/services/SabrTargetRequest.kt index 7b49e7a4..8465ed22 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrTargetRequest.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrTargetRequest.kt @@ -1,9 +1,9 @@ package dev.typetype.server.services import org.schabi.newpipe.extractor.localization.Localization -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrSession import org.slf4j.LoggerFactory internal fun YoutubeSabrSession.fetchTargetedSegment( diff --git a/src/main/kotlin/dev/typetype/server/services/SabrTargetRequestShape.kt b/src/main/kotlin/dev/typetype/server/services/SabrTargetRequestShape.kt index 6c7bbb77..1fef82eb 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrTargetRequestShape.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrTargetRequestShape.kt @@ -1,9 +1,9 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrBufferedRange -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrBufferedRange +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrStreamState import org.slf4j.LoggerFactory internal inline fun withTargetedRequestShape( diff --git a/src/main/kotlin/dev/typetype/server/services/SabrTokenBundle.kt b/src/main/kotlin/dev/typetype/server/services/SabrTokenBundle.kt index 001f71a0..f4fa83f9 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrTokenBundle.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrTokenBundle.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import org.json.JSONObject import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrInfo import java.util.Base64 internal class SabrTokenBundle( diff --git a/src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt b/src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt index 8f3d0d12..db9afa70 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrRecoverableException +import dev.typetype.server.sabr.SabrRecoverableException internal class SabrUnauthorizedResponseRecovery( private val refreshPoToken: (SabrSessionHolder) -> SabrTokenBundle?, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrWindowSegmentFetcher.kt b/src/main/kotlin/dev/typetype/server/services/SabrWindowSegmentFetcher.kt index 9969c0f4..a6041877 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrWindowSegmentFetcher.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrWindowSegmentFetcher.kt @@ -1,8 +1,8 @@ package dev.typetype.server.services import org.schabi.newpipe.extractor.localization.Localization -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest import org.slf4j.LoggerFactory internal object SabrWindowSegmentFetcher { diff --git a/src/main/kotlin/dev/typetype/server/services/TokenYoutubeSession.kt b/src/main/kotlin/dev/typetype/server/services/TokenYoutubeSession.kt index 283479f6..713f16a2 100644 --- a/src/main/kotlin/dev/typetype/server/services/TokenYoutubeSession.kt +++ b/src/main/kotlin/dev/typetype/server/services/TokenYoutubeSession.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrInfo internal data class TokenYoutubeSession( val info: YoutubeSabrInfo, diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrPoTokenProvider.kt b/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrPoTokenProvider.kt index 9495bcc1..ec844eb2 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrPoTokenProvider.kt +++ b/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrPoTokenProvider.kt @@ -1,9 +1,9 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrPoTokenProvider -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrRecoverableException -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrPoTokenProvider +import dev.typetype.server.sabr.SabrRecoverableException +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrStreamState internal class TypetypeTokenSabrPoTokenProvider( private val tokenClient: TypetypeTokenSabrTokenClient, diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClient.kt b/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClient.kt index 0629ee93..43074ff1 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClient.kt +++ b/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClient.kt @@ -10,10 +10,9 @@ import okhttp3.Request import okhttp3.Response import org.json.JSONObject import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper -import org.schabi.newpipe.extractor.services.youtube.sabr.TypeTypeYoutubeSabrInfoFactory -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrClientProfile -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrProbe +import dev.typetype.server.sabr.SabrAdapter +import dev.typetype.server.sabr.YoutubeSabrClientProfile +import dev.typetype.server.sabr.YoutubeSabrInfo import java.io.IOException import java.net.URLEncoder import java.nio.charset.StandardCharsets @@ -107,7 +106,7 @@ internal class TypetypeTokenYoutubeSessionClient( ), ), ) - val info = YoutubeSabrProbe.fromPlayerResponse( + val info = SabrAdapter.fromPlayerResponse( videoId, YoutubeSabrClientProfile.MWEB, YoutubeParsingHelper.generateContentPlaybackNonce(), @@ -122,7 +121,7 @@ internal class TypetypeTokenYoutubeSessionClient( ?.queryParameter("cpn") ?.takeIf { it.isNotBlank() } if (playbackUrl != null && clientVersion != null) { - TypeTypeYoutubeSabrInfoFactory.withPlaybackIdentity( + SabrAdapter.withPlaybackIdentity( info, playbackUrl, clientVersion, From df345caaf4771d065f9b75d685ace87e07607467 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 4 Sep 2026 11:44:05 +0200 Subject: [PATCH 07/45] test: migrate SABR fixtures to TypeType boundary --- .../server/SabrPlaybackGranularRoutesTest.kt | 10 +++++----- .../dev/typetype/server/SabrRoutesAccessTest.kt | 8 ++++---- .../server/SabrStreamContractFilterTest.kt | 5 ++--- .../YoutubeAuthenticatedExtractionProbeTest.kt | 2 +- .../server/routes/SabrLiveGapWindowTest.kt | 10 +++++----- .../routes/SabrLivePlaybackContinuityProbeTest.kt | 4 ++-- .../server/routes/SabrLivePlaybackOverlapTest.kt | 10 +++++----- .../routes/SabrLivePlaybackWindowBuilderTest.kt | 14 +++++++------- .../routes/SabrLiveRoundedBoundaryWindowTest.kt | 14 +++++++------- .../routes/SabrPlaybackAudioOnlyWindowTest.kt | 10 +++++----- .../server/routes/SabrPlaybackSeekRouteTest.kt | 8 ++++---- .../server/routes/SabrPlaybackWindowBuilderTest.kt | 14 +++++++------- .../routes/SabrProgressivePlaybackWindowTest.kt | 14 +++++++------- .../server/routes/SabrStreamContractFilterTest.kt | 4 ++-- .../services/AuthenticatedSabrInfoServiceTest.kt | 4 ++-- .../services/AuthenticatedSabrTimeoutTest.kt | 2 +- .../services/SabrAdaptiveInitializationTest.kt | 8 ++++---- .../services/SabrBootstrapStreamServiceTest.kt | 4 ++-- .../services/SabrCachedSegmentLocatorTest.kt | 10 +++++----- .../server/services/SabrDashManifestBuilderTest.kt | 4 ++-- .../server/services/SabrDemandFailurePumpTest.kt | 12 ++++++------ .../services/SabrDemandWatchdogBackoffTest.kt | 8 ++++---- .../services/SabrDemandWatchdogLifecycleTest.kt | 8 ++++---- .../server/services/SabrDownloadRangeTest.kt | 4 ++-- .../server/services/SabrDownloadStreamerTest.kt | 10 +++++----- .../services/SabrFallbackStreamServiceTest.kt | 4 ++-- .../services/SabrInitializationPolicyTest.kt | 2 +- .../services/SabrLiveContinuationRequestTest.kt | 14 +++++++------- .../server/services/SabrLiveFutureRequestTest.kt | 14 +++++++------- .../services/SabrLivePlaybackSessionServiceTest.kt | 12 ++++++------ .../server/services/SabrLivePlaybackTest.kt | 14 +++++++------- .../server/services/SabrLiveProtocolProbeTest.kt | 2 +- .../server/services/SabrLivePumpStepTest.kt | 10 +++++----- .../server/services/SabrLiveSessionWarmupTest.kt | 14 +++++++------- .../services/SabrMissingDemandRecoveryTest.kt | 10 +++++----- .../SabrPlaybackCachedSegmentLocatorTest.kt | 8 ++++---- .../server/services/SabrPlaybackDiagnosticsTest.kt | 4 ++-- .../SabrPlaybackInitializationFailureTest.kt | 8 ++++---- .../services/SabrPlaybackLiveGapServiceTest.kt | 10 +++++----- .../services/SabrPlaybackManifestServiceTest.kt | 8 ++++---- .../services/SabrPlaybackSessionServiceTest.kt | 10 +++++----- .../services/SabrPlayerContextRecoveryTest.kt | 6 +++--- .../server/services/SabrPreparedInfoCacheTest.kt | 8 ++++---- .../server/services/SabrProbeDiagnostics.kt | 6 +++--- .../dev/typetype/server/services/SabrProbeFetch.kt | 4 ++-- .../server/services/SabrProbeFetchResult.kt | 2 +- .../dev/typetype/server/services/SabrProbeTest.kt | 8 ++++---- .../server/services/SabrPumpLauncherTest.kt | 8 ++++---- .../server/services/SabrPumpRuntimeTest.kt | 6 +++--- .../server/services/SabrRandomAccessProbeTest.kt | 6 +++--- .../services/SabrRecoverablePumpFailureTest.kt | 10 +++++----- .../server/services/SabrSeekRepositionPumpTest.kt | 14 +++++++------- .../server/services/SabrSegmentCacheTest.kt | 14 +++++++------- .../services/SabrSegmentDemandResolutionTest.kt | 14 +++++++------- .../services/SabrSegmentDemandTrackerTest.kt | 8 ++++---- .../services/SabrSessionPlayerContextTest.kt | 8 ++++---- .../server/services/SabrSessionPumpLoopTest.kt | 14 +++++++------- .../server/services/SabrSessionPumpTest.kt | 14 +++++++------- .../server/services/SabrSessionRegistryTest.kt | 8 ++++---- .../server/services/SabrSessionStoreTest.kt | 6 +++--- .../server/services/SabrSessionTimeRequestsTest.kt | 14 +++++++------- .../services/SabrTransientDemandFailureTest.kt | 14 +++++++------- .../services/SabrTransitioningLivePlaybackTest.kt | 8 ++++---- .../SabrUnauthorizedResponseRecoveryTest.kt | 8 ++++---- .../services/TypetypeTokenSabrTokenClientTest.kt | 6 +++--- 65 files changed, 279 insertions(+), 280 deletions(-) diff --git a/src/test/kotlin/dev/typetype/server/SabrPlaybackGranularRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SabrPlaybackGranularRoutesTest.kt index 75475493..29e31f97 100644 --- a/src/test/kotlin/dev/typetype/server/SabrPlaybackGranularRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SabrPlaybackGranularRoutesTest.kt @@ -29,11 +29,11 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant class SabrPlaybackGranularRoutesTest { diff --git a/src/test/kotlin/dev/typetype/server/SabrRoutesAccessTest.kt b/src/test/kotlin/dev/typetype/server/SabrRoutesAccessTest.kt index 14c572c7..07c08496 100644 --- a/src/test/kotlin/dev/typetype/server/SabrRoutesAccessTest.kt +++ b/src/test/kotlin/dev/typetype/server/SabrRoutesAccessTest.kt @@ -24,10 +24,10 @@ import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant class SabrRoutesAccessTest { diff --git a/src/test/kotlin/dev/typetype/server/SabrStreamContractFilterTest.kt b/src/test/kotlin/dev/typetype/server/SabrStreamContractFilterTest.kt index 1a502840..f2cf7db4 100644 --- a/src/test/kotlin/dev/typetype/server/SabrStreamContractFilterTest.kt +++ b/src/test/kotlin/dev/typetype/server/SabrStreamContractFilterTest.kt @@ -10,8 +10,8 @@ import io.mockk.mockk import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo class SabrStreamContractFilterTest { @Test @@ -109,7 +109,6 @@ class SabrStreamContractFilterTest { private fun sabrFormat(itag: Int, isAudio: Boolean, mimeType: String): YoutubeSabrFormat { val format = mockk() every { format.itag } returns itag - every { format.getItag() } returns itag every { format.isAudio } returns isAudio every { format.isVideo } returns !isAudio every { format.mimeType } returns mimeType diff --git a/src/test/kotlin/dev/typetype/server/YoutubeAuthenticatedExtractionProbeTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeAuthenticatedExtractionProbeTest.kt index e74fd18f..f68d0bd7 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeAuthenticatedExtractionProbeTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeAuthenticatedExtractionProbeTest.kt @@ -27,7 +27,7 @@ import org.schabi.newpipe.extractor.NewPipe import org.schabi.newpipe.extractor.ServiceList import org.schabi.newpipe.extractor.localization.ContentCountry import org.schabi.newpipe.extractor.localization.Localization -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrSession import java.nio.file.Files import java.nio.file.Path diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrLiveGapWindowTest.kt b/src/test/kotlin/dev/typetype/server/routes/SabrLiveGapWindowTest.kt index 9a7f39de..00e21bb7 100644 --- a/src/test/kotlin/dev/typetype/server/routes/SabrLiveGapWindowTest.kt +++ b/src/test/kotlin/dev/typetype/server/routes/SabrLiveGapWindowTest.kt @@ -13,11 +13,11 @@ import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant class SabrLiveGapWindowTest { diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackContinuityProbeTest.kt b/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackContinuityProbeTest.kt index 3c0fe962..fd4539d2 100644 --- a/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackContinuityProbeTest.kt +++ b/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackContinuityProbeTest.kt @@ -19,8 +19,8 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Tag import org.junit.jupiter.api.Test import org.junit.jupiter.api.condition.EnabledIfSystemProperty -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat @EnabledIfSystemProperty(named = "sabr.probe", matches = "true") @Tag("network") diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackOverlapTest.kt b/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackOverlapTest.kt index 4f5dff49..5d9d1d6b 100644 --- a/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackOverlapTest.kt +++ b/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackOverlapTest.kt @@ -11,11 +11,11 @@ import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant class SabrLivePlaybackOverlapTest { diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackWindowBuilderTest.kt b/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackWindowBuilderTest.kt index bfaa8b02..21613725 100644 --- a/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackWindowBuilderTest.kt +++ b/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackWindowBuilderTest.kt @@ -12,13 +12,13 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrMediaHeader +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant class SabrLivePlaybackWindowBuilderTest { diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrLiveRoundedBoundaryWindowTest.kt b/src/test/kotlin/dev/typetype/server/routes/SabrLiveRoundedBoundaryWindowTest.kt index 3c488ebc..382d1997 100644 --- a/src/test/kotlin/dev/typetype/server/routes/SabrLiveRoundedBoundaryWindowTest.kt +++ b/src/test/kotlin/dev/typetype/server/routes/SabrLiveRoundedBoundaryWindowTest.kt @@ -11,13 +11,13 @@ import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrMediaHeader +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant class SabrLiveRoundedBoundaryWindowTest { diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAudioOnlyWindowTest.kt b/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAudioOnlyWindowTest.kt index 40599f0a..1ea1de7c 100644 --- a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAudioOnlyWindowTest.kt +++ b/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAudioOnlyWindowTest.kt @@ -14,11 +14,11 @@ import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant class SabrPlaybackAudioOnlyWindowTest { diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackSeekRouteTest.kt b/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackSeekRouteTest.kt index 11c5f0ef..9a1dc484 100644 --- a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackSeekRouteTest.kt +++ b/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackSeekRouteTest.kt @@ -25,10 +25,10 @@ import io.mockk.mockk import io.mockk.verify import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant class SabrPlaybackSeekRouteTest { diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt b/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt index 17aeaeac..19ddf9d7 100644 --- a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt +++ b/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt @@ -13,13 +13,13 @@ import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrMediaHeader +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant class SabrPlaybackWindowBuilderTest { diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt b/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt index 6cc81ad7..43989f9d 100644 --- a/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt +++ b/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt @@ -11,13 +11,13 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrMediaHeader +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant class SabrProgressivePlaybackWindowTest { diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrStreamContractFilterTest.kt b/src/test/kotlin/dev/typetype/server/routes/SabrStreamContractFilterTest.kt index f0078556..5672234b 100644 --- a/src/test/kotlin/dev/typetype/server/routes/SabrStreamContractFilterTest.kt +++ b/src/test/kotlin/dev/typetype/server/routes/SabrStreamContractFilterTest.kt @@ -12,8 +12,8 @@ import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo class SabrStreamContractFilterTest { @Test diff --git a/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt index ffdb5440..51448075 100644 --- a/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt @@ -12,8 +12,8 @@ import org.junit.jupiter.api.Assertions.assertSame import org.junit.jupiter.api.Assertions.assertThrows import org.junit.jupiter.api.Test import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo class AuthenticatedSabrInfoServiceTest { @Test diff --git a/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrTimeoutTest.kt b/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrTimeoutTest.kt index 549950a8..32501d89 100644 --- a/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrTimeoutTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrTimeoutTest.kt @@ -12,7 +12,7 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrInfo class AuthenticatedSabrTimeoutTest { @Test diff --git a/src/test/kotlin/dev/typetype/server/services/SabrAdaptiveInitializationTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrAdaptiveInitializationTest.kt index 6c2f8a78..a1e9fde4 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrAdaptiveInitializationTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrAdaptiveInitializationTest.kt @@ -8,10 +8,10 @@ import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Test import org.schabi.newpipe.extractor.localization.Localization -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState class SabrAdaptiveInitializationTest { @Test diff --git a/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamServiceTest.kt index 9be05aad..feea11e9 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamServiceTest.kt @@ -8,8 +8,8 @@ import io.mockk.mockk import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo class SabrBootstrapStreamServiceTest { @Test diff --git a/src/test/kotlin/dev/typetype/server/services/SabrCachedSegmentLocatorTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrCachedSegmentLocatorTest.kt index 4cf003ee..a111f8bd 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrCachedSegmentLocatorTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrCachedSegmentLocatorTest.kt @@ -5,11 +5,11 @@ import io.mockk.mockk import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertSame import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.SabrMediaHeader +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrSession class SabrCachedSegmentLocatorTest { @Test diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDashManifestBuilderTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrDashManifestBuilderTest.kt index 12e7aae0..30f0946e 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrDashManifestBuilderTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrDashManifestBuilderTest.kt @@ -5,8 +5,8 @@ import io.mockk.mockk import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrStreamState class SabrDashManifestBuilderTest { @Test diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDemandFailurePumpTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrDemandFailurePumpTest.kt index 1496c267..880dcb81 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrDemandFailurePumpTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrDemandFailurePumpTest.kt @@ -16,12 +16,12 @@ import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicInteger diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt index facd8627..0541b1e8 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt @@ -11,10 +11,10 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession import java.time.Instant @OptIn(ExperimentalCoroutinesApi::class) diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogLifecycleTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogLifecycleTest.kt index a0be4993..4d183d57 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogLifecycleTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogLifecycleTest.kt @@ -11,10 +11,10 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession import java.time.Instant @OptIn(ExperimentalCoroutinesApi::class) diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDownloadRangeTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrDownloadRangeTest.kt index 3cea29a5..8dc915fd 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrDownloadRangeTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrDownloadRangeTest.kt @@ -6,8 +6,8 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertThrows import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrStreamState class SabrDownloadRangeTest { @Test diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDownloadStreamerTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrDownloadStreamerTest.kt index c6c9225c..0602c2a3 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrDownloadStreamerTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrDownloadStreamerTest.kt @@ -8,11 +8,11 @@ import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Test import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream import java.io.IOException diff --git a/src/test/kotlin/dev/typetype/server/services/SabrFallbackStreamServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrFallbackStreamServiceTest.kt index 2f556726..c5c0c8f0 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrFallbackStreamServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrFallbackStreamServiceTest.kt @@ -10,8 +10,8 @@ import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo class SabrFallbackStreamServiceTest { @Test diff --git a/src/test/kotlin/dev/typetype/server/services/SabrInitializationPolicyTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrInitializationPolicyTest.kt index 1619f081..2ce89bfa 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrInitializationPolicyTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrInitializationPolicyTest.kt @@ -5,7 +5,7 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrFormat class SabrInitializationPolicyTest { @Test diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLiveContinuationRequestTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrLiveContinuationRequestTest.kt index aa9c30c9..4d94d890 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrLiveContinuationRequestTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrLiveContinuationRequestTest.kt @@ -6,13 +6,13 @@ import io.mockk.verify import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrBufferedRange -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrBufferedRange +import dev.typetype.server.sabr.SabrMediaHeader +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant class SabrLiveContinuationRequestTest { diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt index 9151234e..9d9d5711 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt @@ -5,13 +5,13 @@ import io.mockk.mockk import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrMediaHeader +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant class SabrLiveFutureRequestTest { diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackSessionServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackSessionServiceTest.kt index 9b5a88a4..e1b3d56c 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackSessionServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackSessionServiceTest.kt @@ -12,12 +12,12 @@ import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertSame import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrMediaHeader +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant import java.util.concurrent.atomic.AtomicInteger diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackTest.kt index 537cc23e..94fe8c6a 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackTest.kt @@ -8,13 +8,13 @@ import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.SabrMediaHeader +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant class SabrLivePlaybackTest { diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLiveProtocolProbeTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrLiveProtocolProbeTest.kt index df511f17..c4877083 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrLiveProtocolProbeTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrLiveProtocolProbeTest.kt @@ -6,7 +6,7 @@ import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Tag import org.junit.jupiter.api.Test import org.junit.jupiter.api.condition.EnabledIfSystemProperty -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrFormat import java.security.MessageDigest @EnabledIfSystemProperty(named = "sabr.probe", matches = "true") diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLivePumpStepTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrLivePumpStepTest.kt index a7d09976..70915d31 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrLivePumpStepTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrLivePumpStepTest.kt @@ -6,11 +6,11 @@ import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.SabrMediaHeader +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession import java.time.Instant class SabrLivePumpStepTest { diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt index 30e2aa12..53ae9e5b 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt @@ -7,13 +7,13 @@ import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrBufferedRange -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrBufferedRange +import dev.typetype.server.sabr.SabrMediaHeader +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant class SabrLiveSessionWarmupTest { diff --git a/src/test/kotlin/dev/typetype/server/services/SabrMissingDemandRecoveryTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrMissingDemandRecoveryTest.kt index d4a69be9..84a84099 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrMissingDemandRecoveryTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrMissingDemandRecoveryTest.kt @@ -7,11 +7,11 @@ import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant @OptIn(ExperimentalCoroutinesApi::class) diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocatorTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocatorTest.kt index 737d4560..5abc77c0 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocatorTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocatorTest.kt @@ -6,10 +6,10 @@ import io.mockk.mockk import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState class SabrPlaybackCachedSegmentLocatorTest { @Test diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackDiagnosticsTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackDiagnosticsTest.kt index 2bd5dad6..ecf413a9 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackDiagnosticsTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackDiagnosticsTest.kt @@ -4,8 +4,8 @@ import io.mockk.every import io.mockk.mockk import io.mockk.verify import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat class SabrPlaybackDiagnosticsTest { @Test diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInitializationFailureTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInitializationFailureTest.kt index 59b34411..d59079db 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInitializationFailureTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInitializationFailureTest.kt @@ -13,10 +13,10 @@ import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant @OptIn(ExperimentalCoroutinesApi::class) diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackLiveGapServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackLiveGapServiceTest.kt index 36495ca9..1d65b21e 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackLiveGapServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackLiveGapServiceTest.kt @@ -8,11 +8,11 @@ import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant import java.util.Base64 diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackManifestServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackManifestServiceTest.kt index e1e11413..c6dd80f3 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackManifestServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackManifestServiceTest.kt @@ -5,10 +5,10 @@ import io.mockk.mockk import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant class SabrPlaybackManifestServiceTest { diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt index 420f27b0..83e37887 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt @@ -12,11 +12,11 @@ import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertSame import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant import java.util.Base64 import java.util.concurrent.atomic.AtomicInteger diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlayerContextRecoveryTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPlayerContextRecoveryTest.kt index e1f25a8f..8961c0d6 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrPlayerContextRecoveryTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrPlayerContextRecoveryTest.kt @@ -11,9 +11,9 @@ import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Assertions.assertThrows import org.junit.jupiter.api.Test import org.schabi.newpipe.extractor.exceptions.AntiBotException -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrProtocolException -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrClientProfile -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.SabrProtocolException +import dev.typetype.server.sabr.YoutubeSabrClientProfile +import dev.typetype.server.sabr.YoutubeSabrInfo import java.io.IOException class SabrPlayerContextRecoveryTest { diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPreparedInfoCacheTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPreparedInfoCacheTest.kt index bdad3b61..2ce9c2a1 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrPreparedInfoCacheTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrPreparedInfoCacheTest.kt @@ -11,9 +11,9 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertSame import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrClientProfile -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrClientProfile +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo import java.time.Duration class SabrPreparedInfoCacheTest { @@ -157,7 +157,7 @@ class SabrPreparedInfoCacheTest { every { tokenClient.fetch("video", forceRefresh = true, refreshVideo = false) } returns refreshed val probe = SabrPlayerInfoProbe { _, profile, token -> if (token === initial) { - throw org.schabi.newpipe.extractor.services.youtube.sabr.SabrProtocolException( + throw dev.typetype.server.sabr.SabrProtocolException( "Player response has no streamingData for $profile", ) } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrProbeDiagnostics.kt b/src/test/kotlin/dev/typetype/server/services/SabrProbeDiagnostics.kt index 4caf767c..7c28b7f1 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrProbeDiagnostics.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrProbeDiagnostics.kt @@ -1,8 +1,8 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat internal fun printSabrProbeFormat(label: String, format: YoutubeSabrFormat): Unit { println( diff --git a/src/test/kotlin/dev/typetype/server/services/SabrProbeFetch.kt b/src/test/kotlin/dev/typetype/server/services/SabrProbeFetch.kt index be85dc51..86f2cf76 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrProbeFetch.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrProbeFetch.kt @@ -2,8 +2,8 @@ package dev.typetype.server.services import kotlinx.coroutines.CancellationException import kotlinx.coroutines.withTimeoutOrNull -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest internal suspend fun fetchSabrProbeSegment( store: SabrSessionStore, diff --git a/src/test/kotlin/dev/typetype/server/services/SabrProbeFetchResult.kt b/src/test/kotlin/dev/typetype/server/services/SabrProbeFetchResult.kt index 15dccb28..55d799d9 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrProbeFetchResult.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrProbeFetchResult.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrMediaSegment internal data class SabrProbeFetchResult( val segment: SabrMediaSegment?, diff --git a/src/test/kotlin/dev/typetype/server/services/SabrProbeTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrProbeTest.kt index e7dc7e05..9b764213 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrProbeTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrProbeTest.kt @@ -5,9 +5,9 @@ import org.junit.jupiter.api.Test import org.junit.jupiter.api.condition.EnabledIfSystemProperty import org.schabi.newpipe.extractor.localization.ContentCountry import org.schabi.newpipe.extractor.localization.Localization -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrClientProfile -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrProbe -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrClientProfile +import dev.typetype.server.sabr.SabrAdapter +import dev.typetype.server.sabr.YoutubeSabrSession @EnabledIfSystemProperty(named = "sabr.probe", matches = "true") @Tag("network") @@ -39,7 +39,7 @@ class SabrProbeTest { try { val token = tokenClient.fetch(videoId) ?: error("No SABR token") val info = TypetypeYoutubeSessionPoTokenProvider.withToken(token) { - YoutubeSabrProbe.fetchSabrInfo(videoId, profile, loc, country) + SabrAdapter.fetchSabrInfo(videoId, profile, loc, country) } println("serverAbrStreamingUrl present: ${!info.serverAbrStreamingUrl.isNullOrEmpty()}") println("videoPlaybackUstreamerConfig present: ${!info.videoPlaybackUstreamerConfig.isNullOrEmpty()}") diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPumpLauncherTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPumpLauncherTest.kt index a4b8a889..9f021795 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrPumpLauncherTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrPumpLauncherTest.kt @@ -15,10 +15,10 @@ import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession import java.time.Instant @OptIn(ExperimentalCoroutinesApi::class) diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTest.kt index ef44710a..162d2c52 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTest.kt @@ -6,9 +6,9 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrNextRequestPolicy -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrNextRequestPolicy +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState class SabrPumpRuntimeTest { @Test diff --git a/src/test/kotlin/dev/typetype/server/services/SabrRandomAccessProbeTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrRandomAccessProbeTest.kt index 8cfd0190..3b058d58 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrRandomAccessProbeTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrRandomAccessProbeTest.kt @@ -8,9 +8,9 @@ import org.junit.jupiter.api.Tag import org.junit.jupiter.api.Test import org.junit.jupiter.api.condition.EnabledIfSystemProperty import org.schabi.newpipe.extractor.NewPipe -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo import org.schabi.newpipe.extractor.stream.StreamInfo @EnabledIfSystemProperty(named = "sabr.probe", matches = "true") diff --git a/src/test/kotlin/dev/typetype/server/services/SabrRecoverablePumpFailureTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrRecoverablePumpFailureTest.kt index 9f76af83..a477f8dc 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrRecoverablePumpFailureTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrRecoverablePumpFailureTest.kt @@ -7,11 +7,11 @@ import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrRecoverableException -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrRecoverableException +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant @OptIn(ExperimentalCoroutinesApi::class) diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt index 25c24a4c..6ddab8af 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt @@ -8,13 +8,13 @@ import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrMediaHeader +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant class SabrSeekRepositionPumpTest { diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSegmentCacheTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSegmentCacheTest.kt index 4552ef57..d718d8df 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSegmentCacheTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSegmentCacheTest.kt @@ -7,13 +7,13 @@ import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrMediaHeader +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant class SabrSegmentCacheTest { diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandResolutionTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandResolutionTest.kt index a8768057..0eb98055 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandResolutionTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandResolutionTest.kt @@ -9,13 +9,13 @@ import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertSame import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrMediaHeader +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant class SabrSegmentDemandResolutionTest { diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandTrackerTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandTrackerTest.kt index d9a1194d..688eea20 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandTrackerTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandTrackerTest.kt @@ -6,10 +6,10 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession import java.time.Instant class SabrSegmentDemandTrackerTest { diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionPlayerContextTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSessionPlayerContextTest.kt index b95c4f07..20050799 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSessionPlayerContextTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSessionPlayerContextTest.kt @@ -7,10 +7,10 @@ import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Test import org.schabi.newpipe.extractor.localization.ContentCountry import org.schabi.newpipe.extractor.localization.Localization -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant class SabrSessionPlayerContextTest { diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpLoopTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpLoopTest.kt index f8d8a24d..d6e708ec 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpLoopTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpLoopTest.kt @@ -8,13 +8,13 @@ import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNotEquals import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrMediaHeader +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant @OptIn(ExperimentalCoroutinesApi::class) diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpTest.kt index b1b93b14..e1209e0f 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpTest.kt @@ -9,13 +9,13 @@ import org.junit.jupiter.api.Test import io.mockk.every import io.mockk.mockk import io.mockk.verify -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrMediaHeader +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant class SabrSessionPumpTest { diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionRegistryTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSessionRegistryTest.kt index 16c88046..7bc34a9b 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSessionRegistryTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSessionRegistryTest.kt @@ -7,10 +7,10 @@ import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertSame import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant import java.util.concurrent.CountDownLatch diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt index 8ffdeb77..85254a98 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt @@ -4,7 +4,7 @@ import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.Tag import org.junit.jupiter.api.Test import org.junit.jupiter.api.condition.EnabledIfSystemProperty -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.SabrSegmentRequest @EnabledIfSystemProperty(named = "sabr.probe", matches = "true") @Tag("network") @@ -63,8 +63,8 @@ class SabrSessionStoreTest { private fun mediaRequestsForProbe( holder: SabrSessionHolder, - video: org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat, - audio: org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat, + video: dev.typetype.server.sabr.YoutubeSabrFormat, + audio: dev.typetype.server.sabr.YoutubeSabrFormat, playerTimeMs: Long, ): List { val videoSequence = System.getenv("SABR_PROBE_VIDEO_SEQUENCE")?.toIntOrNull() diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionTimeRequestsTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSessionTimeRequestsTest.kt index d62d6ec2..55509e38 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSessionTimeRequestsTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSessionTimeRequestsTest.kt @@ -4,13 +4,13 @@ import io.mockk.every import io.mockk.mockk import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrMediaHeader +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant class SabrSessionTimeRequestsTest { diff --git a/src/test/kotlin/dev/typetype/server/services/SabrTransientDemandFailureTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrTransientDemandFailureTest.kt index b9613340..cce09dc0 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrTransientDemandFailureTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrTransientDemandFailureTest.kt @@ -8,13 +8,13 @@ import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrMediaHeader +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.io.IOException import java.time.Instant diff --git a/src/test/kotlin/dev/typetype/server/services/SabrTransitioningLivePlaybackTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrTransitioningLivePlaybackTest.kt index 0cdbfa61..d7e90ce1 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrTransitioningLivePlaybackTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrTransitioningLivePlaybackTest.kt @@ -10,10 +10,10 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant class SabrTransitioningLivePlaybackTest { diff --git a/src/test/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecoveryTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecoveryTest.kt index e398e8b6..a4aef148 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecoveryTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecoveryTest.kt @@ -5,10 +5,10 @@ import io.mockk.mockk import io.mockk.verify import org.junit.jupiter.api.Assertions.assertThrows import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrRecoverableException -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrRecoverableException +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState class SabrUnauthorizedResponseRecoveryTest { @Test diff --git a/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt b/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt index 6022321e..dda9d1b6 100644 --- a/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt @@ -15,9 +15,9 @@ import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertThrows import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrRecoverableException -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import dev.typetype.server.sabr.SabrRecoverableException +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrStreamState class TypetypeTokenSabrTokenClientTest { From 915991cde7e406e1e7217cc6cdd98b22000f8d3f Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 4 Sep 2026 11:44:09 +0200 Subject: [PATCH 08/45] test: cover Pipe boundary doubles in SABR isolation --- .../SabrPlaybackSessionIsolationTest.kt | 24 ++++++++++++++--- .../services/SabrSessionIdentityTest.kt | 26 +++++++++++++++---- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionIsolationTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionIsolationTest.kt index e36b8f97..1e6223c6 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionIsolationTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionIsolationTest.kt @@ -6,9 +6,12 @@ import org.junit.jupiter.api.Assertions.assertNotEquals import org.junit.jupiter.api.Assertions.assertNotSame import org.junit.jupiter.api.Assertions.assertSame import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrClientProfile -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrClientProfile as PipeProfile +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat as PipeFormat +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo as PipeInfo +import dev.typetype.server.sabr.YoutubeSabrClientProfile +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo class SabrPlaybackSessionIsolationTest { @Test @@ -61,6 +64,16 @@ class SabrPlaybackSessionIsolationTest { every { info.serverAbrStreamingUrl } returns "https://example.com/sabr" every { info.videoPlaybackUstreamerConfig } returns "config" every { info.formats } returns emptyList() + val pipeInfo = mockk(relaxed = true) + every { info.delegate } returns pipeInfo + every { pipeInfo.profile } returns PipeProfile.WEB + every { pipeInfo.videoId } returns "video" + every { pipeInfo.cpn } returns "source-cpn" + every { pipeInfo.clientVersion } returns "1.2.3" + every { pipeInfo.visitorData } returns "visitor" + every { pipeInfo.serverAbrStreamingUrl } returns "https://example.com/sabr" + every { pipeInfo.videoPlaybackUstreamerConfig } returns "config" + every { pipeInfo.formats } returns emptyList() return info } @@ -70,6 +83,11 @@ class SabrPlaybackSessionIsolationTest { every { format.isAudio } returns isAudio every { format.isVideo } returns !isAudio every { format.audioTrackId } returns null + val pipeFormat = mockk(relaxed = true) + every { format.delegate } returns pipeFormat + every { pipeFormat.itag } returns itag + every { pipeFormat.isAudio } returns isAudio + every { pipeFormat.isVideo } returns !isAudio return format } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionIdentityTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSessionIdentityTest.kt index 1545d8a8..1d743c74 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSessionIdentityTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSessionIdentityTest.kt @@ -5,17 +5,20 @@ import io.mockk.mockk import okhttp3.HttpUrl.Companion.toHttpUrl import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNotEquals -import org.junit.jupiter.api.Assertions.assertSame import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrClientProfile -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrClientProfile as PipeProfile +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat as PipeFormat +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo as PipeInfo +import dev.typetype.server.sabr.YoutubeSabrClientProfile +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo class SabrSessionIdentityTest { @Test fun `fresh identities use unique coherent playback nonces`() { val formats = listOf(mockk()) val source = mockk() + every { formats[0].itag } returns 140 every { source.profile } returns YoutubeSabrClientProfile.WEB every { source.videoId } returns "video" every { source.clientVersion } returns "1.2.3" @@ -23,6 +26,19 @@ class SabrSessionIdentityTest { every { source.serverAbrStreamingUrl } returns "https://example.com/sabr?cpn=stale&foo=bar" every { source.videoPlaybackUstreamerConfig } returns "config" every { source.formats } returns formats + val pipeFormat = mockk(relaxed = true) + every { formats[0].delegate } returns pipeFormat + every { pipeFormat.itag } returns 140 + val pipeInfo = mockk(relaxed = true) + every { source.delegate } returns pipeInfo + every { pipeInfo.profile } returns PipeProfile.WEB + every { pipeInfo.videoId } returns "video" + every { pipeInfo.cpn } returns "stale" + every { pipeInfo.clientVersion } returns "1.2.3" + every { pipeInfo.visitorData } returns "visitor" + every { pipeInfo.serverAbrStreamingUrl } returns "https://example.com/sabr?cpn=stale&foo=bar" + every { pipeInfo.videoPlaybackUstreamerConfig } returns "config" + every { pipeInfo.formats } returns listOf(pipeFormat) val first = SabrSessionIdentity.fresh(source) val second = SabrSessionIdentity.fresh(source) @@ -31,6 +47,6 @@ class SabrSessionIdentityTest { assertEquals(first.cpn, first.serverAbrStreamingUrl!!.toHttpUrl().queryParameter("cpn")) assertEquals(second.cpn, second.serverAbrStreamingUrl!!.toHttpUrl().queryParameter("cpn")) assertEquals("bar", first.serverAbrStreamingUrl!!.toHttpUrl().queryParameter("foo")) - assertSame(formats[0], first.formats[0]) + assertEquals(formats[0].itag, first.formats[0].itag) } } From 3a60872c457478640a7f7c1af7f8c61ed6fd5a91 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 4 Sep 2026 11:56:58 +0200 Subject: [PATCH 09/45] test: enforce SABR adapter boundary --- build.gradle.kts | 21 +++++++ .../server/sabr/SabrBoundaryContractTest.kt | 60 +++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/sabr/SabrBoundaryContractTest.kt diff --git a/build.gradle.kts b/build.gradle.kts index 9789023e..c77bb2e5 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -145,8 +145,29 @@ tasks.jacocoTestCoverageVerification { } } +val verifySabrBoundary = tasks.register("verifySabrBoundary") { + doLast { + val adapterRoot = file("src/main/kotlin/dev/typetype/server/sabr") + .canonicalFile + .toPath() + val violations = fileTree("src/main/kotlin") + .matching { include("**/*.kt") } + .files + .filterNot { it.canonicalFile.toPath().startsWith(adapterRoot) } + .flatMap { source -> + source.readLines().withIndex() + .filter { it.value.contains("org.schabi.newpipe.extractor.services.youtube.sabr") } + .map { "${source.path}:${it.index + 1}" } + } + check(violations.isEmpty()) { + "PipePipe SABR imports must stay in the TypeType adapter: ${violations.joinToString()}" + } + } +} + tasks.check { dependsOn(tasks.jacocoTestCoverageVerification) + dependsOn(verifySabrBoundary) } kotlin { diff --git a/src/test/kotlin/dev/typetype/server/sabr/SabrBoundaryContractTest.kt b/src/test/kotlin/dev/typetype/server/sabr/SabrBoundaryContractTest.kt new file mode 100644 index 00000000..a6b4da07 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/sabr/SabrBoundaryContractTest.kt @@ -0,0 +1,60 @@ +package dev.typetype.server.sabr + +import io.mockk.every +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrBufferedRange as PipeRange +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat as PipeFormat + +class SabrBoundaryContractTest { + @Test + fun `format exposes the current extractor contract without loss`() { + val pipe = mockk(relaxed = true) + every { pipe.isAudio } returns false + every { pipe.isVideo } returns true + every { pipe.itag } returns 137 + every { pipe.width } returns 1920 + every { pipe.height } returns 1080 + every { pipe.bitrate } returns 4_000_000 + every { pipe.mimeType } returns "video/mp4" + every { pipe.initializationUrl } returns "https://example.test/init" + + val format = YoutubeSabrFormat(pipe) + + assertFalse(format.isAudio) + assertTrue(format.isVideo) + assertEquals(137, format.itag) + assertEquals(1920, format.width) + assertEquals(1080, format.height) + assertEquals(4_000_000, format.bitrate) + assertEquals("video/mp4", format.mimeType) + assertEquals("https://example.test/init", format.initializationUrl) + } + + @Test + fun `buffered range preserves timing and segment identity`() { + val pipe = mockk(relaxed = true) + every { pipe.itag } returns 137 + every { pipe.lastModified } returns 42L + every { pipe.xtags } returns "xtags" + every { pipe.startTimeMs } returns 1_000L + every { pipe.durationMs } returns 4_000L + every { pipe.startSegmentIndex } returns 3 + every { pipe.endSegmentIndex } returns 7 + every { pipe.timescale } returns 1_000 + + val range = SabrBufferedRange.fromDelegate(pipe) + + assertEquals(137, range.itag) + assertEquals(42L, range.lastModified) + assertEquals("xtags", range.xtags) + assertEquals(1_000L, range.startTimeMs) + assertEquals(4_000L, range.durationMs) + assertEquals(3, range.startSegmentIndex) + assertEquals(7, range.endSegmentIndex) + assertEquals(1_000, range.timescale) + } +} From 9f570c74bf92f1bd3f1d58f92a483638d06492f0 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 6 Sep 2026 12:34:00 +0200 Subject: [PATCH 10/45] fix: return a typed error for oversized portability uploads --- openapi/paths/portability.yaml | 8 +++- .../kotlin/dev/typetype/server/Plugins.kt | 7 ++++ .../routes/PortabilityMultipartFailure.kt | 10 +++++ .../server/routes/PortabilityRouteSupport.kt | 5 +++ .../server/PortabilityMultipartFailureTest.kt | 38 +++++++++++++++++++ 5 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 src/main/kotlin/dev/typetype/server/routes/PortabilityMultipartFailure.kt create mode 100644 src/test/kotlin/dev/typetype/server/PortabilityMultipartFailureTest.kt diff --git a/openapi/paths/portability.yaml b/openapi/paths/portability.yaml index 060de475..db938723 100644 --- a/openapi/paths/portability.yaml +++ b/openapi/paths/portability.yaml @@ -16,6 +16,11 @@ PortabilityImports: post: tags: [portability] summary: Upload and analyze an account backup + description: | + Accepts one backup of at most 512 MiB. For large YouTube Takeout exports, + clients can package only the YouTube CSV, HTML and JSON metadata, preserving + entry paths and excluding uploaded media and unrelated Google products. + The server still validates the archive and parses all imported records. parameters: - name: format in: query @@ -39,7 +44,8 @@ PortabilityImports: schema: { $ref: ../components/portability.yaml#/PortabilityJobSnapshot } '400': { description: Invalid or unrecognized backup } '401': { description: Missing or invalid token } - '413': { description: Upload exceeds the configured limit } + '413': + description: Upload exceeds 512 MiB (portability_upload_too_large) PortabilityExports: post: tags: [portability] diff --git a/src/main/kotlin/dev/typetype/server/Plugins.kt b/src/main/kotlin/dev/typetype/server/Plugins.kt index 63c900d5..ef71d1cf 100644 --- a/src/main/kotlin/dev/typetype/server/Plugins.kt +++ b/src/main/kotlin/dev/typetype/server/Plugins.kt @@ -1,6 +1,8 @@ package dev.typetype.server import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.routes.isMultipartSizeLimit +import dev.typetype.server.routes.respondPortabilityError import dev.typetype.server.services.AuthService import io.ktor.http.HttpHeaders import io.ktor.http.HttpMethod @@ -118,6 +120,11 @@ internal fun Application.configureStatusPages() { exception { call, cause -> if (cause is io.ktor.utils.io.ClosedWriteChannelException) return@exception if (cause is kotlinx.coroutines.CancellationException) throw cause + // Ktor's multipart producer can fail outside the route's receive block. + if (call.request.path() == "/portability/imports" && cause.isMultipartSizeLimit()) { + call.respondPortabilityError(dev.typetype.server.portability.PortabilityUploadTooLargeException()) + return@exception + } log.error("Unhandled exception requestId=${call.requestId()} path=${call.request.path()}", cause) call.respond(HttpStatusCode.InternalServerError, ErrorResponse("Internal server error", "internal_error")) } diff --git a/src/main/kotlin/dev/typetype/server/routes/PortabilityMultipartFailure.kt b/src/main/kotlin/dev/typetype/server/routes/PortabilityMultipartFailure.kt new file mode 100644 index 00000000..a9751289 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/PortabilityMultipartFailure.kt @@ -0,0 +1,10 @@ +package dev.typetype.server.routes + +import java.io.IOException + +internal fun Throwable.isMultipartSizeLimit(): Boolean = this is IOException && ( + message?.let { + (it.startsWith("Multipart content length exceeds limit ") && "formFieldLimit" in it) || + (it.startsWith("Limit of ") && " bytes exceeded while searching for " in it) + } == true +) diff --git a/src/main/kotlin/dev/typetype/server/routes/PortabilityRouteSupport.kt b/src/main/kotlin/dev/typetype/server/routes/PortabilityRouteSupport.kt index 28ac1d61..11c9bd47 100644 --- a/src/main/kotlin/dev/typetype/server/routes/PortabilityRouteSupport.kt +++ b/src/main/kotlin/dev/typetype/server/routes/PortabilityRouteSupport.kt @@ -17,6 +17,11 @@ internal fun parsePortabilityFormat(value: String?): PortabilityFormat? { } internal suspend fun ApplicationCall.respondPortabilityError(error: Exception) { + if (error is kotlinx.coroutines.CancellationException) throw error + if (error.isMultipartSizeLimit()) { + respondPortabilityError(PortabilityUploadTooLargeException()) + return + } val status = when (error) { is PortabilityJobNotFoundException -> HttpStatusCode.NotFound is PortabilityUploadTooLargeException -> HttpStatusCode.PayloadTooLarge diff --git a/src/test/kotlin/dev/typetype/server/PortabilityMultipartFailureTest.kt b/src/test/kotlin/dev/typetype/server/PortabilityMultipartFailureTest.kt new file mode 100644 index 00000000..62357677 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/PortabilityMultipartFailureTest.kt @@ -0,0 +1,38 @@ +package dev.typetype.server + +import dev.typetype.server.cache.CacheJson +import io.ktor.client.request.post +import io.ktor.client.statement.bodyAsText +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.post +import io.ktor.server.routing.routing +import io.ktor.server.testing.testApplication +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.io.IOException + +class PortabilityMultipartFailureTest { + @Test + fun `multipart scanner limit is a typed 413 not a 500`() = testApplication { + application { + install(ContentNegotiation) { json(CacheJson) } + configureStatusPages() + routing { + post("/portability/imports") { + throw IOException("Limit of 536870912 bytes exceeded while searching for boundary") + } + post("/unrelated") { + throw IOException("disk failure") + } + } + } + val response = client.post("/portability/imports") + assertEquals(HttpStatusCode.PayloadTooLarge, response.status) + assertTrue(response.bodyAsText().contains("portability_upload_too_large")) + assertEquals(HttpStatusCode.InternalServerError, client.post("/unrelated").status) + } +} From d797660167605682af52ab308355b932e5d1804a Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 6 Sep 2026 12:34:00 +0200 Subject: [PATCH 11/45] test: add opt-in Takeout archive upload reproduction --- .../PortabilityArchiveReproductionTest.kt | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/PortabilityArchiveReproductionTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PortabilityArchiveReproductionTest.kt b/src/test/kotlin/dev/typetype/server/PortabilityArchiveReproductionTest.kt new file mode 100644 index 00000000..56ff39a1 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/PortabilityArchiveReproductionTest.kt @@ -0,0 +1,90 @@ +package dev.typetype.server + +import dev.typetype.server.cache.CacheJson +import dev.typetype.server.portability.* +import dev.typetype.server.routes.portabilityRoutes +import dev.typetype.server.services.AuthService +import io.ktor.client.request.forms.* +import io.ktor.client.request.* +import io.ktor.client.statement.bodyAsText +import io.ktor.http.* +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.testApplication +import kotlinx.coroutines.* +import kotlinx.io.asSource +import kotlinx.io.buffered +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path + +@EnabledIfEnvironmentVariable(named = "PORTABILITY_REPRO_ARCHIVE", matches = ".+") +class PortabilityArchiveReproductionTest { + @TempDir + lateinit var directory: Path + + @Test + fun `real archive reaches preview without account writes`() = testApplication { + val archive = Path.of(System.getenv("PORTABILITY_REPRO_ARCHIVE")) + val data = object : PortabilityDataPort { + override suspend fun import( + userId: String, + source: PortabilityRecordSource, + request: PortabilityImportRequest, + onCategoryComplete: (PortabilityCategory, Long) -> Unit, + ): Map = error("Account writes are forbidden in this probe") + + override suspend fun export( + userId: String, + categories: Set, + sink: PortabilityRecordSink, + onCategoryComplete: (PortabilityCategory, Long) -> Unit, + ) = error("Account reads are forbidden in this probe") + } + val engine = PortabilityEngine( + PortabilityRegistry(listOf(YoutubeTakeoutPortabilityAdapter())), data, + PortabilityJobStore(directory.resolve("jobs")), + CoroutineScope(SupervisorJob() + Dispatchers.Default), + ) + application { + install(ContentNegotiation) { json(CacheJson) } + configureStatusPages() + routing { portabilityRoutes(engine, AuthService.fixed("owner")) } + } + try { + val response = client.post("/portability/imports?format=youtube-takeout") { + header(HttpHeaders.Authorization, "Bearer test-jwt") + setBody(MultiPartFormDataContent(formData { + append("file", InputProvider(Files.size(archive)) { + Files.newInputStream(archive).asSource().buffered() + }, Headers.build { + append(HttpHeaders.ContentDisposition, "filename=takeout.zip") + append(HttpHeaders.ContentType, "application/zip") + }) + })) + } + val body = response.bodyAsText() + if (Files.size(archive) > PortabilityLimits.MAX_UPLOAD_BYTES) { + assertEquals(HttpStatusCode.PayloadTooLarge, response.status, body) + return@testApplication + } + assertEquals(HttpStatusCode.Accepted, response.status, body) + val id = CacheJson.decodeFromString(body).id + withTimeout(120_000) { + while (engine.snapshot("owner", id).state in setOf( + PortabilityJobState.QUEUED, PortabilityJobState.ANALYZING, + )) delay(100) + } + val result = engine.snapshot("owner", id) + assertEquals(PortabilityJobState.READY, result.state, result.errorMessage) + println("Archive preview counts: ${result.preview?.counts}") + } finally { + engine.close() + } + } +} From 484ecad66d3fca6f8f080d83ff5ae58341f08260 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Mon, 7 Sep 2026 21:53:03 +0200 Subject: [PATCH 12/45] feat: add RSS video thumbnails --- openapi/paths/rss.yaml | 2 +- .../server/services/RssDocumentRenderer.kt | 16 +++++ .../server/RssDocumentRendererTest.kt | 60 +++++++++++++++++++ .../server/RssFeedReaderRoutesTest.kt | 2 +- 4 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 src/test/kotlin/dev/typetype/server/RssDocumentRendererTest.kt diff --git a/openapi/paths/rss.yaml b/openapi/paths/rss.yaml index bd8e610c..de483019 100644 --- a/openapi/paths/rss.yaml +++ b/openapi/paths/rss.yaml @@ -108,7 +108,7 @@ RssFeedDocument: - { name: token, in: query, required: true, schema: { type: string } } responses: '200': - description: RSS document + description: RSS 2.0 document. Video items include a media:thumbnail element when a HTTP(S) thumbnail URL is available. headers: ETag: { schema: { type: string } } Last-Modified: { schema: { type: string } } diff --git a/src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt b/src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt index c84e9665..169281fa 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt +++ b/src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt @@ -3,6 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.models.RssFeedItem import dev.typetype.server.models.VideoItem import java.io.ByteArrayOutputStream +import java.net.URI import java.net.URLEncoder import java.nio.charset.StandardCharsets import java.time.Instant @@ -23,6 +24,7 @@ internal object RssDocumentRenderer { writer.writeStartDocument(StandardCharsets.UTF_8.name(), "1.0") writer.writeStartElement("rss") writer.writeAttribute("version", "2.0") + writer.writeNamespace("media", MEDIA_NAMESPACE) writer.writeStartElement("channel") writer.element("title", feed.name) writer.element("link", publicBaseUrl) @@ -52,6 +54,11 @@ internal object RssDocumentRenderer { writeEndElement() element("author", video.uploaderName) video.shortDescription?.takeIf(String::isNotBlank)?.let { element("description", it) } + video.thumbnailUrl.httpUrlOrNull()?.let { thumbnailUrl -> + writeStartElement("media", "thumbnail", MEDIA_NAMESPACE) + writeAttribute("url", thumbnailUrl) + writeEndElement() + } RssVideoMetadata.publishedAtMillis(video).takeIf { it > 0 } ?.let { element("pubDate", RFC_1123.format(Instant.ofEpochMilli(it))) } writeEndElement() @@ -63,5 +70,14 @@ internal object RssDocumentRenderer { writeEndElement() } + private fun String.httpUrlOrNull(): String? = runCatching { + URI(this).takeIf { uri -> + uri.isAbsolute && uri.host != null && + (uri.scheme.equals("http", ignoreCase = true) || + uri.scheme.equals("https", ignoreCase = true)) + }?.toString() + }.getOrNull() + + private const val MEDIA_NAMESPACE = "http://search.yahoo.com/mrss/" private val RFC_1123 = DateTimeFormatter.RFC_1123_DATE_TIME.withZone(ZoneOffset.UTC) } diff --git a/src/test/kotlin/dev/typetype/server/RssDocumentRendererTest.kt b/src/test/kotlin/dev/typetype/server/RssDocumentRendererTest.kt new file mode 100644 index 00000000..ea4148a5 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/RssDocumentRendererTest.kt @@ -0,0 +1,60 @@ +package dev.typetype.server + +import dev.typetype.server.models.RssFeedItem +import dev.typetype.server.services.RssDocumentRenderer +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class RssDocumentRendererTest { + @Test + fun `renders a media thumbnail for HTTP(S) video thumbnails`() { + val video = testVideoItem().copy(thumbnailUrl = "https://img.example/thumb.jpg") + + val xml = RssDocumentRenderer.render(feed(), listOf(video), "https://video.example", 1_000L) + .toString(Charsets.UTF_8) + + assertTrue(xml.contains("xmlns:media=\"http://search.yahoo.com/mrss/\"")) + assertTrue(xml.contains("")) + assertTrue(first.bodyAsText().contains(", + val activeProfileId: String, + val defaultProfileId: String, +) diff --git a/src/main/kotlin/dev/typetype/server/models/ProfileNameRequest.kt b/src/main/kotlin/dev/typetype/server/models/ProfileNameRequest.kt new file mode 100644 index 00000000..9a369431 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/ProfileNameRequest.kt @@ -0,0 +1,6 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class ProfileNameRequest(val name: String) diff --git a/src/main/kotlin/dev/typetype/server/models/ProfileSwitchResponse.kt b/src/main/kotlin/dev/typetype/server/models/ProfileSwitchResponse.kt new file mode 100644 index 00000000..27756aeb --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/ProfileSwitchResponse.kt @@ -0,0 +1,9 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class ProfileSwitchResponse( + val accessToken: String, + val profile: AccountProfileItem, +) diff --git a/src/test/kotlin/dev/typetype/server/TestDatabase.kt b/src/test/kotlin/dev/typetype/server/TestDatabase.kt index be1dc356..92da90a8 100644 --- a/src/test/kotlin/dev/typetype/server/TestDatabase.kt +++ b/src/test/kotlin/dev/typetype/server/TestDatabase.kt @@ -32,7 +32,15 @@ import dev.typetype.server.db.tables.YoutubeSessionPairingsTable import dev.typetype.server.db.tables.YoutubeSessionsTable import dev.typetype.server.db.tables.UsersTable import dev.typetype.server.db.tables.UserAvatarsTable +import dev.typetype.server.db.tables.UserChannelInterestTable +import dev.typetype.server.db.tables.UserTopicInterestTable +import dev.typetype.server.db.tables.RecommendationEventsTable +import dev.typetype.server.db.tables.RecommendationFeedHistoryTable +import dev.typetype.server.db.tables.RecommendationFeedbackTable +import dev.typetype.server.db.tables.RecommendationOnboardingPreferencesTable +import dev.typetype.server.db.tables.RecommendationOnboardingStateTable import dev.typetype.server.db.tables.WatchLaterTable +import dev.typetype.server.db.tables.ProfileAccountsTable import dev.typetype.server.services.AdminSettingsService import org.jetbrains.exposed.v1.jdbc.deleteAll import org.jetbrains.exposed.v1.jdbc.transactions.transaction @@ -126,6 +134,14 @@ object TestDatabase { YoutubeSessionsTable.deleteAll() BugReportsTable.deleteAll() NotificationStatesTable.deleteAll() + UserChannelInterestTable.deleteAll() + UserTopicInterestTable.deleteAll() + RecommendationEventsTable.deleteAll() + RecommendationFeedHistoryTable.deleteAll() + RecommendationFeedbackTable.deleteAll() + RecommendationOnboardingPreferencesTable.deleteAll() + RecommendationOnboardingStateTable.deleteAll() + ProfileAccountsTable.deleteAll() AdminSettingsService.clearCache() } } From bf6a6eeee9115057fde97f4fa7c8641be46f5fc5 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 8 Sep 2026 11:37:56 +0200 Subject: [PATCH 14/45] feat: add isolated profile lifecycle --- .../server/services/ProfileAccountService.kt | 166 ++++++++++++++++++ .../services/ProfileDataDeletionService.kt | 102 +++++++++++ 2 files changed, 268 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/services/ProfileAccountService.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/ProfileDataDeletionService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/ProfileAccountService.kt b/src/main/kotlin/dev/typetype/server/services/ProfileAccountService.kt new file mode 100644 index 00000000..6cbb5356 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/ProfileAccountService.kt @@ -0,0 +1,166 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.ProfileAccountsTable +import dev.typetype.server.db.tables.UsersTable +import dev.typetype.server.models.AccountProfileItem +import dev.typetype.server.models.AccountProfilesResponse +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insertIgnore +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.update +import java.util.UUID + +class ProfileAccountService { + suspend fun ownerUserId(profileId: String): String? = DatabaseFactory.query { + ownerIdInTransaction(profileId) + } + + suspend fun list(activeProfileId: String): AccountProfilesResponse? = DatabaseFactory.query { + val ownerId = ownerIdInTransaction(activeProfileId) ?: return@query null + val rows = ProfileAccountsTable.selectAll().where { ProfileAccountsTable.ownerUserId eq ownerId } + .map { row -> profileRow(row, activeProfileId) } + .sortedWith(compareByDescending { it.isDefault }.thenBy { it.name.lowercase() }) + val defaultId = rows.firstOrNull { it.isDefault }?.id ?: activeProfileId + AccountProfilesResponse(rows, activeProfileId, defaultId) + } + + suspend fun resolveSignInProfile(ownerUserId: String): String = DatabaseFactory.query { + val ownerId = ownerIdInTransaction(ownerUserId) ?: return@query ownerUserId + val rows = ProfileAccountsTable.selectAll().where { ProfileAccountsTable.ownerUserId eq ownerId }.toList() + val selected = rows.filter { it[ProfileAccountsTable.lastUsedAt] > 0L } + .maxByOrNull { it[ProfileAccountsTable.lastUsedAt] } + ?: rows.firstOrNull { it[ProfileAccountsTable.isDefault] } + ?: rows.firstOrNull() + val profileId = selected?.get(ProfileAccountsTable.profileId) ?: ownerId + markUsedInTransaction(profileId, System.currentTimeMillis()) + profileId + } + + suspend fun create(activeProfileId: String, name: String): ProfileMutationResult = DatabaseFactory.query { + val normalized = normalizeName(name) ?: return@query ProfileMutationResult.InvalidName + val ownerId = ownerIdInTransaction(activeProfileId) ?: return@query ProfileMutationResult.NotFound + val profileId = UUID.randomUUID().toString() + val now = System.currentTimeMillis() + UsersTable.insertIgnore { + it[id] = profileId + it[email] = "profile-$profileId@profiles.invalid" + it[passwordHash] = "profile:$profileId" + it[UsersTable.name] = normalized + it[role] = "user" + it[verified] = true + it[createdAt] = now + it[updatedAt] = now + } + ProfileAccountsTable.insertIgnore { + it[ProfileAccountsTable.profileId] = profileId + it[ownerUserId] = ownerId + it[displayName] = normalized + it[isDefault] = false + it[lastUsedAt] = 0L + it[createdAt] = now + it[updatedAt] = now + } + ProfileMutationResult.Success(profileRowById(profileId, activeProfileId)) + } + + suspend fun rename(activeProfileId: String, profileId: String, name: String): ProfileMutationResult = DatabaseFactory.query { + val normalized = normalizeName(name) ?: return@query ProfileMutationResult.InvalidName + val ownerId = ownerIdInTransaction(activeProfileId) ?: return@query ProfileMutationResult.NotFound + val target = targetInTransaction(ownerId, profileId) ?: return@query ProfileMutationResult.NotFound + ProfileAccountsTable.update({ ProfileAccountsTable.profileId eq profileId }) { + it[displayName] = normalized + it[updatedAt] = System.currentTimeMillis() + } + ProfileMutationResult.Success(profileRowById(target, activeProfileId)) + } + + suspend fun setDefault(activeProfileId: String, profileId: String): ProfileMutationResult = DatabaseFactory.query { + val ownerId = ownerIdInTransaction(activeProfileId) ?: return@query ProfileMutationResult.NotFound + val target = targetInTransaction(ownerId, profileId) ?: return@query ProfileMutationResult.NotFound + ProfileAccountsTable.update({ ProfileAccountsTable.ownerUserId eq ownerId }) { it[isDefault] = false } + ProfileAccountsTable.update({ ProfileAccountsTable.profileId eq target }) { + it[isDefault] = true + it[updatedAt] = System.currentTimeMillis() + } + ProfileMutationResult.Success(profileRowById(target, activeProfileId)) + } + + suspend fun switch(activeProfileId: String, profileId: String): AccountProfileItem? = DatabaseFactory.query { + val ownerId = ownerIdInTransaction(activeProfileId) ?: return@query null + val target = targetInTransaction(ownerId, profileId) ?: return@query null + markUsedInTransaction(target, System.currentTimeMillis()) + profileRowById(target, target) + } + + suspend fun delete(activeProfileId: String, profileId: String): ProfileMutationResult = DatabaseFactory.query { + val ownerId = ownerIdInTransaction(activeProfileId) ?: return@query ProfileMutationResult.NotFound + val target = targetInTransaction(ownerId, profileId) ?: return@query ProfileMutationResult.NotFound + if (target == ownerId) return@query ProfileMutationResult.CannotDeleteOwner + if (target == activeProfileId) return@query ProfileMutationResult.CannotDeleteActive + ProfileDataDeletionService.deleteUser(target) + ProfileAccountsTable.deleteWhere { ProfileAccountsTable.profileId eq target } + ProfileMutationResult.Deleted + } + + private fun ownerIdInTransaction(profileId: String): String? { + if (profileId.startsWith("guest:")) return null + ProfileAccountsTable.selectAll().where { ProfileAccountsTable.profileId eq profileId } + .singleOrNull()?.let { return it[ProfileAccountsTable.ownerUserId] } + val userExists = UsersTable.selectAll().where { UsersTable.id eq profileId }.any() + if (!userExists) return null + val now = System.currentTimeMillis() + ProfileAccountsTable.insertIgnore { + it[ProfileAccountsTable.profileId] = profileId + it[ownerUserId] = profileId + it[displayName] = "Profile" + it[isDefault] = true + it[lastUsedAt] = 0L + it[createdAt] = now + it[updatedAt] = now + } + return profileId + } + + private fun targetInTransaction(ownerId: String, profileId: String): String? = + ProfileAccountsTable.selectAll().where { + (ProfileAccountsTable.ownerUserId eq ownerId) and (ProfileAccountsTable.profileId eq profileId) + }.singleOrNull()?.get(ProfileAccountsTable.profileId) + + private fun profileRowById(profileId: String, activeProfileId: String): AccountProfileItem = + ProfileAccountsTable.selectAll().where { ProfileAccountsTable.profileId eq profileId } + .single().let { profileRow(it, activeProfileId) } + + private fun profileRow(row: org.jetbrains.exposed.v1.core.ResultRow, activeProfileId: String): AccountProfileItem { + val profileId = row[ProfileAccountsTable.profileId] + val user = UsersTable.selectAll().where { UsersTable.id eq profileId }.single() + return AccountProfileItem( + id = profileId, + name = row[ProfileAccountsTable.displayName], + isActive = profileId == activeProfileId, + isDefault = row[ProfileAccountsTable.isDefault], + lastUsedAt = row[ProfileAccountsTable.lastUsedAt], + publicUsername = user[UsersTable.publicUsername], + avatarUrl = user[UsersTable.avatarUrl], + avatarType = user[UsersTable.avatarType], + avatarCode = user[UsersTable.avatarCode], + ) + } + + private fun markUsedInTransaction(profileId: String, now: Long) { + ProfileAccountsTable.update({ ProfileAccountsTable.profileId eq profileId }) { it[lastUsedAt] = now } + } + + private fun normalizeName(value: String): String? = value.trim().takeIf { it.length in 1..40 } +} + +sealed interface ProfileMutationResult { + data class Success(val profile: AccountProfileItem) : ProfileMutationResult + data object Deleted : ProfileMutationResult + data object InvalidName : ProfileMutationResult + data object CannotDeleteOwner : ProfileMutationResult + data object CannotDeleteActive : ProfileMutationResult + data object NotFound : ProfileMutationResult +} diff --git a/src/main/kotlin/dev/typetype/server/services/ProfileDataDeletionService.kt b/src/main/kotlin/dev/typetype/server/services/ProfileDataDeletionService.kt new file mode 100644 index 00000000..6a418afc --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/ProfileDataDeletionService.kt @@ -0,0 +1,102 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.tables.AllowedChannelsTable +import dev.typetype.server.db.tables.AllowedPlaylistsTable +import dev.typetype.server.db.tables.BlockedChannelsTable +import dev.typetype.server.db.tables.BlockedKeywordsTable +import dev.typetype.server.db.tables.BlockedVideosTable +import dev.typetype.server.db.tables.BugReportsTable +import dev.typetype.server.db.tables.FavoritesTable +import dev.typetype.server.db.tables.HistoryTable +import dev.typetype.server.db.tables.NotificationStatesTable +import dev.typetype.server.db.tables.PasswordResetTable +import dev.typetype.server.db.tables.PlaylistVideosTable +import dev.typetype.server.db.tables.PlaylistsTable +import dev.typetype.server.db.tables.ProgressTable +import dev.typetype.server.db.tables.RecommendationEventsTable +import dev.typetype.server.db.tables.RecommendationFeedbackTable +import dev.typetype.server.db.tables.RecommendationFeedHistoryTable +import dev.typetype.server.db.tables.RecommendationOnboardingPreferencesTable +import dev.typetype.server.db.tables.RecommendationOnboardingStateTable +import dev.typetype.server.db.tables.RssFeedChannelsTable +import dev.typetype.server.db.tables.RssFeedServicesTable +import dev.typetype.server.db.tables.RssFeedsTable +import dev.typetype.server.db.tables.RssUserPoliciesTable +import dev.typetype.server.db.tables.SavedPlaylistsTable +import dev.typetype.server.db.tables.SearchHistoryTable +import dev.typetype.server.db.tables.SessionsTable +import dev.typetype.server.db.tables.SettingsTable +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable +import dev.typetype.server.db.tables.SubscriptionGroupsTable +import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.db.tables.UserAvatarsTable +import dev.typetype.server.db.tables.UserChannelInterestTable +import dev.typetype.server.db.tables.UserTopicInterestTable +import dev.typetype.server.db.tables.WatchLaterTable +import dev.typetype.server.db.tables.YoutubeSessionPairingsTable +import dev.typetype.server.db.tables.YoutubeSessionsTable +import dev.typetype.server.db.tables.YoutubeTakeoutImportJobsTable +import dev.typetype.server.db.tables.YoutubeTakeoutPlaylistKeysTable +import dev.typetype.server.db.tables.UsersTable +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.selectAll + +internal object ProfileDataDeletionService { + fun deleteUser(userId: String) { + val feedIds = RssFeedsTable.selectAll().where { RssFeedsTable.userId eq userId } + .map { it[RssFeedsTable.id] } + feedIds.forEach { feedId -> + RssFeedChannelsTable.deleteWhere { RssFeedChannelsTable.feedId eq feedId } + RssFeedServicesTable.deleteWhere { RssFeedServicesTable.feedId eq feedId } + } + RssFeedsTable.deleteWhere { RssFeedsTable.userId eq userId } + + val groupIds = SubscriptionGroupsTable.selectAll().where { SubscriptionGroupsTable.userId eq userId } + .map { it[SubscriptionGroupsTable.id] } + SubscriptionGroupMembershipsTable.deleteWhere { SubscriptionGroupMembershipsTable.userId eq userId } + groupIds.forEach { groupId -> + SubscriptionGroupMembershipsTable.deleteWhere { SubscriptionGroupMembershipsTable.groupId eq groupId } + } + + PlaylistVideosTable.deleteWhere { PlaylistVideosTable.userId eq userId } + PlaylistsTable.deleteWhere { PlaylistsTable.userId eq userId } + deleteUserRows(userId) + UsersTable.deleteWhere { UsersTable.id eq userId } + } + + private fun deleteUserRows(userId: String) { + listOf( + { HistoryTable.deleteWhere { HistoryTable.userId eq userId } }, + { FavoritesTable.deleteWhere { FavoritesTable.userId eq userId } }, + { ProgressTable.deleteWhere { ProgressTable.userId eq userId } }, + { WatchLaterTable.deleteWhere { WatchLaterTable.userId eq userId } }, + { SubscriptionsTable.deleteWhere { SubscriptionsTable.userId eq userId } }, + { SubscriptionGroupsTable.deleteWhere { SubscriptionGroupsTable.userId eq userId } }, + { SavedPlaylistsTable.deleteWhere { SavedPlaylistsTable.userId eq userId } }, + { SearchHistoryTable.deleteWhere { SearchHistoryTable.userId eq userId } }, + { SettingsTable.deleteWhere { SettingsTable.userId eq userId } }, + { AllowedChannelsTable.deleteWhere { AllowedChannelsTable.userId eq userId } }, + { AllowedPlaylistsTable.deleteWhere { AllowedPlaylistsTable.userId eq userId } }, + { BlockedChannelsTable.deleteWhere { BlockedChannelsTable.userId eq userId } }, + { BlockedKeywordsTable.deleteWhere { BlockedKeywordsTable.userId eq userId } }, + { BlockedVideosTable.deleteWhere { BlockedVideosTable.userId eq userId } }, + { NotificationStatesTable.deleteWhere { NotificationStatesTable.userId eq userId } }, + { PasswordResetTable.deleteWhere { PasswordResetTable.userId eq userId } }, + { SessionsTable.deleteWhere { SessionsTable.userId eq userId } }, + { UserAvatarsTable.deleteWhere { UserAvatarsTable.userId eq userId } }, + { UserChannelInterestTable.deleteWhere { UserChannelInterestTable.userId eq userId } }, + { UserTopicInterestTable.deleteWhere { UserTopicInterestTable.userId eq userId } }, + { YoutubeSessionsTable.deleteWhere { YoutubeSessionsTable.userId eq userId } }, + { YoutubeSessionPairingsTable.deleteWhere { YoutubeSessionPairingsTable.userId eq userId } }, + { YoutubeTakeoutImportJobsTable.deleteWhere { YoutubeTakeoutImportJobsTable.userId eq userId } }, + { YoutubeTakeoutPlaylistKeysTable.deleteWhere { YoutubeTakeoutPlaylistKeysTable.userId eq userId } }, + { RecommendationEventsTable.deleteWhere { RecommendationEventsTable.userId eq userId } }, + { RecommendationFeedHistoryTable.deleteWhere { RecommendationFeedHistoryTable.userId eq userId } }, + { RecommendationFeedbackTable.deleteWhere { RecommendationFeedbackTable.userId eq userId } }, + { RecommendationOnboardingPreferencesTable.deleteWhere { RecommendationOnboardingPreferencesTable.userId eq userId } }, + { RecommendationOnboardingStateTable.deleteWhere { RecommendationOnboardingStateTable.userId eq userId } }, + { RssUserPoliciesTable.deleteWhere { RssUserPoliciesTable.userId eq userId } }, + ).forEach { it() } + } +} From 9a8f06b8ec78c70c7370fdad0d15a04f01dadf8d Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 8 Sep 2026 11:38:09 +0200 Subject: [PATCH 15/45] feat: expose account profile routes --- openapi.yaml | 20 ++++ openapi/components/profiles.yaml | 33 ++++++ openapi/paths/profiles.yaml | 102 +++++++++++++++++ .../kotlin/dev/typetype/server/Application.kt | 6 +- .../dev/typetype/server/ApplicationRoutes.kt | 4 + .../dev/typetype/server/ServiceRegistry.kt | 4 +- .../server/routes/AccountProfilesRoutes.kt | 103 ++++++++++++++++++ 7 files changed, 270 insertions(+), 2 deletions(-) create mode 100644 openapi/components/profiles.yaml create mode 100644 openapi/paths/profiles.yaml create mode 100644 src/main/kotlin/dev/typetype/server/routes/AccountProfilesRoutes.kt diff --git a/openapi.yaml b/openapi.yaml index 508706c3..cabda1d9 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -50,6 +50,13 @@ paths: /subscriptions/groups/{groupId}: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroup } /subscriptions/groups/{groupId}/channels: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroupChannels } /subscriptions/feed: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionFeed } + /profiles: { $ref: ./openapi/paths/profiles.yaml#/AccountProfiles } + /profiles/{profileId}: { $ref: ./openapi/paths/profiles.yaml#/AccountProfile } + /profiles/{profileId}/default: { $ref: ./openapi/paths/profiles.yaml#/AccountProfileDefault } + /profiles/{profileId}/switch: { $ref: ./openapi/paths/profiles.yaml#/AccountProfileSwitch } + /notifications: { $ref: ./openapi/paths/notifications.yaml#/Notifications } + /notifications/unread-count: { $ref: ./openapi/paths/notifications.yaml#/NotificationsUnreadCount } + /notifications/read-all: { $ref: ./openapi/paths/notifications.yaml#/NotificationsReadAll } /rss/feeds: { $ref: ./openapi/paths/rss.yaml#/RssFeeds } /rss/feeds/{id}: { $ref: ./openapi/paths/rss.yaml#/RssFeed } /rss/feeds/{id}/enabled: { $ref: ./openapi/paths/rss.yaml#/RssFeedEnabled } @@ -114,6 +121,11 @@ paths: /internal/youtube-session/browser/complete: $ref: ./openapi/paths/youtube-session.yaml#/InternalBrowserComplete components: + securitySchemes: + bearerAuth: + type: http + scheme: bearer + bearerFormat: JWT schemas: ErrorResponse: $ref: ./openapi/components/common.yaml#/ErrorResponse @@ -169,6 +181,14 @@ components: SubscriptionGroupMembershipBatchRequest: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionGroupMembershipBatchRequest } SubscriptionFeedResponse: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionFeedResponse } SubscriptionFeedPreparingResponse: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionFeedPreparingResponse } + AccountProfileItem: { $ref: ./openapi/components/profiles.yaml#/AccountProfileItem } + AccountProfilesResponse: { $ref: ./openapi/components/profiles.yaml#/AccountProfilesResponse } + ProfileNameRequest: { $ref: ./openapi/components/profiles.yaml#/ProfileNameRequest } + ProfileSwitchResponse: { $ref: ./openapi/components/profiles.yaml#/ProfileSwitchResponse } + NotificationItem: { $ref: ./openapi/components/notifications.yaml#/NotificationItem } + NotificationsResponse: { $ref: ./openapi/components/notifications.yaml#/NotificationsResponse } + UnreadCountResponse: { $ref: ./openapi/components/notifications.yaml#/UnreadCountResponse } + MarkNotificationsReadResponse: { $ref: ./openapi/components/notifications.yaml#/MarkNotificationsReadResponse } RssFeedRequest: { $ref: ./openapi/components/rss.yaml#/RssFeedRequest } RssFeedItem: { $ref: ./openapi/components/rss.yaml#/RssFeedItem } RssFeedSecretItem: { $ref: ./openapi/components/rss.yaml#/RssFeedSecretItem } diff --git a/openapi/components/profiles.yaml b/openapi/components/profiles.yaml new file mode 100644 index 00000000..695c8b0c --- /dev/null +++ b/openapi/components/profiles.yaml @@ -0,0 +1,33 @@ +AccountProfileItem: + type: object + required: [id, name, isActive, isDefault, lastUsedAt] + properties: + id: { type: string, format: uuid } + name: { type: string, minLength: 1, maxLength: 40 } + isActive: { type: boolean } + isDefault: { type: boolean } + lastUsedAt: { type: integer, format: int64 } + publicUsername: { type: string, nullable: true } + avatarUrl: { type: string, nullable: true } + avatarType: { type: string, nullable: true } + avatarCode: { type: string, nullable: true } +AccountProfilesResponse: + type: object + required: [profiles, activeProfileId, defaultProfileId] + properties: + profiles: + type: array + items: { $ref: '#/AccountProfileItem' } + activeProfileId: { type: string, format: uuid } + defaultProfileId: { type: string, format: uuid } +ProfileNameRequest: + type: object + required: [name] + properties: + name: { type: string, minLength: 1, maxLength: 40 } +ProfileSwitchResponse: + type: object + required: [accessToken, profile] + properties: + accessToken: { type: string } + profile: { $ref: '#/AccountProfileItem' } diff --git a/openapi/paths/profiles.yaml b/openapi/paths/profiles.yaml new file mode 100644 index 00000000..c562e3b8 --- /dev/null +++ b/openapi/paths/profiles.yaml @@ -0,0 +1,102 @@ +AccountProfiles: + get: + tags: [user-data] + summary: List the profiles available to the authenticated account + security: [{ bearerAuth: [] }] + responses: + '200': + description: Account profiles and the active/default profile IDs. + content: + application/json: + schema: { $ref: ../components/profiles.yaml#/AccountProfilesResponse } + '401': { $ref: ../components/common.yaml#/JsonError } + '403': { $ref: ../components/common.yaml#/JsonError } + post: + tags: [user-data] + summary: Create an isolated account profile + security: [{ bearerAuth: [] }] + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/profiles.yaml#/ProfileNameRequest } + responses: + '201': + description: The newly created profile. + content: + application/json: + schema: { $ref: ../components/profiles.yaml#/AccountProfileItem } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + +AccountProfile: + parameters: + - name: profileId + in: path + required: true + schema: { type: string, format: uuid } + put: + tags: [user-data] + summary: Rename an account profile + security: [{ bearerAuth: [] }] + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/profiles.yaml#/ProfileNameRequest } + responses: + '200': + description: Renamed profile. + content: + application/json: + schema: { $ref: ../components/profiles.yaml#/AccountProfileItem } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + delete: + tags: [user-data] + summary: Delete an account profile and its private data + security: [{ bearerAuth: [] }] + responses: + '204': { description: Profile deleted. } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + '409': { $ref: ../components/common.yaml#/JsonError } + +AccountProfileDefault: + parameters: + - name: profileId + in: path + required: true + schema: { type: string, format: uuid } + post: + tags: [user-data] + summary: Set the default profile used after sign-in + security: [{ bearerAuth: [] }] + responses: + '200': + description: The selected default profile. + content: + application/json: + schema: { $ref: ../components/profiles.yaml#/AccountProfileItem } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + +AccountProfileSwitch: + parameters: + - name: profileId + in: path + required: true + schema: { type: string, format: uuid } + post: + tags: [user-data] + summary: Switch the authenticated session to another profile + security: [{ bearerAuth: [] }] + responses: + '200': + description: A new profile-scoped access token and profile projection. + content: + application/json: + schema: { $ref: ../components/profiles.yaml#/ProfileSwitchResponse } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } diff --git a/src/main/kotlin/dev/typetype/server/Application.kt b/src/main/kotlin/dev/typetype/server/Application.kt index 05786bc2..c5312b65 100644 --- a/src/main/kotlin/dev/typetype/server/Application.kt +++ b/src/main/kotlin/dev/typetype/server/Application.kt @@ -11,6 +11,7 @@ import dev.typetype.server.services.DownloaderGatewayService import dev.typetype.server.services.GitHubIssueService import dev.typetype.server.services.PasswordResetService import dev.typetype.server.services.ProfileService +import dev.typetype.server.services.ProfileAccountService import dev.typetype.server.services.PipePipeBackupImporterService import dev.typetype.server.services.OpenMojiProxyService import dev.typetype.server.services.InstanceService @@ -44,7 +45,8 @@ fun Application.module() { DatabaseFactory.init(dbUrl, dbUser, dbPassword) val jwtSecret = System.getenv("JWT_SECRET") ?: UUID.randomUUID().toString() val authSessionConfig = AuthSessionConfig.fromEnvironment() - val authService = AuthService(jwtSecret, sessionConfig = authSessionConfig) + val profileAccountService = ProfileAccountService() + val authService = AuthService(jwtSecret, sessionConfig = authSessionConfig, profileAccountService = profileAccountService) val oidcAuthService = OidcAuthService(OidcConfigLoader.fromEnvironment(), jwtSecret, authService) val userAdminService = UserAdminService() val passwordResetService = PasswordResetService() @@ -68,6 +70,7 @@ fun Application.module() { jwtSecret, adminSettingsService, youtubeProxySelector, + profileAccountService, ) val youtubeRemoteBrowserConfig = YoutubeRemoteBrowserConfig.fromEnvironment(subtitleServiceUrl) val youtubeRemoteLoginReadinessService = YoutubeRemoteLoginReadinessService( @@ -107,6 +110,7 @@ fun Application.module() { oidcAuthService = oidcAuthService, passwordResetService = passwordResetService, profileService = profileService, + profileAccountService = profileAccountService, userAdminService = userAdminService, avatarService = avatarService, openMojiProxyService = openMojiProxyService, diff --git a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt index 3a930d92..24bdb477 100644 --- a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt @@ -7,6 +7,7 @@ import dev.typetype.server.routes.adminRssRoutes import dev.typetype.server.routes.adminIdentityRoutes import dev.typetype.server.routes.adminSessionRoutes import dev.typetype.server.routes.authRoutes +import dev.typetype.server.routes.accountProfilesRoutes import dev.typetype.server.routes.avatarRoutes import dev.typetype.server.routes.bulletCommentRoutes import dev.typetype.server.routes.channelRoutes @@ -40,6 +41,7 @@ import dev.typetype.server.services.OpenMojiProxyService import dev.typetype.server.services.PasswordResetService import dev.typetype.server.services.PipePipeBackupImporterService import dev.typetype.server.services.ProfileService +import dev.typetype.server.services.ProfileAccountService import dev.typetype.server.services.UserAdminService import dev.typetype.server.services.YoutubeRemoteBrowserService import dev.typetype.server.portability.PortabilityEngine @@ -59,6 +61,7 @@ internal fun Application.installApplicationRoutes( oidcAuthService: OidcAuthService, passwordResetService: PasswordResetService, profileService: ProfileService, + profileAccountService: ProfileAccountService, userAdminService: UserAdminService, avatarService: AvatarService, openMojiProxyService: OpenMojiProxyService, @@ -110,6 +113,7 @@ internal fun Application.installApplicationRoutes( svc.homeRecommendationWarmupService, authSessionConfig, ) + accountProfilesRoutes(profileAccountService, authService, authSessionConfig) adminRoutes(authService, userAdminService, passwordResetService, adminSettingsService) adminRssRoutes(svc.rssFeedManagementService, authService) adminIdentityRoutes(svc.accountIdentityService, authService) diff --git a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt index de8e6a0b..92b9f6a1 100644 --- a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt @@ -16,6 +16,7 @@ import dev.typetype.server.services.FavoritesService import dev.typetype.server.services.HistoryService import dev.typetype.server.services.HomeRecommendationService import dev.typetype.server.services.NotificationsService +import dev.typetype.server.services.ProfileAccountService import dev.typetype.server.services.PlaylistService import dev.typetype.server.services.ProgressService import dev.typetype.server.services.RssFeedManagementService @@ -45,9 +46,10 @@ internal class ServiceRegistry( jwtSecret: String, adminSettingsService: AdminSettingsService, youtubeProxySelector: ProxySelector? = null, + profileAccountService: ProfileAccountService? = null, ) { val publicHlsManifestTokenService = PublicHlsManifestTokenService(jwtSecret) - val accountIdentityService = AccountIdentityService() + val accountIdentityService = AccountIdentityService(profileAccountService) val customAvatarService = CustomAvatarService() val deArrowService = DeArrowService(cache) private val extraction = ExtractionServiceRegistry( diff --git a/src/main/kotlin/dev/typetype/server/routes/AccountProfilesRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/AccountProfilesRoutes.kt new file mode 100644 index 00000000..d921a688 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/AccountProfilesRoutes.kt @@ -0,0 +1,103 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.models.ProfileNameRequest +import dev.typetype.server.models.ProfileSwitchResponse +import dev.typetype.server.services.AuthCookieHelpers +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.AuthSessionConfig +import dev.typetype.server.services.ProfileAccountService +import dev.typetype.server.services.ProfileMutationResult +import io.ktor.http.HttpStatusCode +import io.ktor.server.request.receive +import io.ktor.server.response.respond +import io.ktor.server.routing.Route +import io.ktor.server.routing.delete +import io.ktor.server.routing.get +import io.ktor.server.routing.post +import io.ktor.server.routing.put + +fun Route.accountProfilesRoutes( + profileService: ProfileAccountService, + authService: AuthService, + sessionConfig: AuthSessionConfig, +) { + get("/profiles") { + call.withJwtAuth(authService) { userId -> + val profiles = profileService.list(userId) + if (profiles == null) call.respond(HttpStatusCode.Forbidden, ErrorResponse("Profiles are unavailable")) + else call.respond(profiles) + } + } + + post("/profiles") { + call.withJwtAuth(authService) { userId -> + val body = call.receiveNameOrNull() ?: return@withJwtAuth + when (val result = profileService.create(userId, body.name)) { + is ProfileMutationResult.Success -> call.respond(HttpStatusCode.Created, result.profile) + ProfileMutationResult.InvalidName -> call.respond(HttpStatusCode.BadRequest, ErrorResponse("PROFILE_NAME_INVALID")) + ProfileMutationResult.NotFound -> call.respond(HttpStatusCode.NotFound, ErrorResponse("Profile not found")) + else -> call.respond(HttpStatusCode.BadRequest, ErrorResponse("Profile operation failed")) + } + } + } + + put("/profiles/{profileId}") { + call.withJwtAuth(authService) { userId -> + val profileId = call.parameters["profileId"] ?: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing profileId")) + val body = call.receiveNameOrNull() ?: return@withJwtAuth + call.respondMutation(profileService.rename(userId, profileId, body.name)) + } + } + + post("/profiles/{profileId}/default") { + call.withJwtAuth(authService) { userId -> + val profileId = call.parameters["profileId"] ?: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing profileId")) + call.respondMutation(profileService.setDefault(userId, profileId)) + } + } + + post("/profiles/{profileId}/switch") { + call.withJwtAuth(authService) { userId -> + val profileId = call.parameters["profileId"] ?: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing profileId")) + val profile = profileService.switch(userId, profileId) + if (profile == null) { + call.respond(HttpStatusCode.NotFound, ErrorResponse("Profile not found")) + return@withJwtAuth + } + val token = authService.issueSession(profile.id) + if (token == null) { + call.respond(HttpStatusCode.InternalServerError, ErrorResponse("Failed to create session")) + return@withJwtAuth + } + AuthCookieHelpers.setRefreshCookie(call.response, token.refreshToken, sessionConfig) + call.respond(ProfileSwitchResponse(token.accessToken, profile)) + } + } + + delete("/profiles/{profileId}") { + call.withJwtAuth(authService) { userId -> + val profileId = call.parameters["profileId"] ?: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing profileId")) + when (profileService.delete(userId, profileId)) { + ProfileMutationResult.Deleted -> call.respond(HttpStatusCode.NoContent) + ProfileMutationResult.CannotDeleteOwner -> call.respond(HttpStatusCode.Conflict, ErrorResponse("The account profile cannot be deleted")) + ProfileMutationResult.CannotDeleteActive -> call.respond(HttpStatusCode.Conflict, ErrorResponse("Switch profiles before deleting the active profile")) + ProfileMutationResult.NotFound -> call.respond(HttpStatusCode.NotFound, ErrorResponse("Profile not found")) + else -> call.respond(HttpStatusCode.BadRequest, ErrorResponse("Profile deletion failed")) + } + } + } +} + +private suspend fun io.ktor.server.application.ApplicationCall.receiveNameOrNull(): ProfileNameRequest? = + runCatching { receive() }.getOrElse { + respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) + null + } + +private suspend fun io.ktor.server.application.ApplicationCall.respondMutation(result: ProfileMutationResult): Unit = when (result) { + is ProfileMutationResult.Success -> respond(result.profile) + ProfileMutationResult.InvalidName -> respond(HttpStatusCode.BadRequest, ErrorResponse("PROFILE_NAME_INVALID")) + ProfileMutationResult.NotFound -> respond(HttpStatusCode.NotFound, ErrorResponse("Profile not found")) + else -> respond(HttpStatusCode.BadRequest, ErrorResponse("Profile operation failed")) +} From e70df6317998d5049711b406909c08a6d4d27f15 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 8 Sep 2026 11:38:20 +0200 Subject: [PATCH 16/45] feat: bind authentication to active profile --- .../server/services/AccountIdentityService.kt | 14 +- .../typetype/server/services/AuthService.kt | 17 +- .../server/services/OidcUserService.kt | 2 +- .../server/AccountProfilesRoutesTest.kt | 145 ++++++++++++++++++ 4 files changed, 167 insertions(+), 11 deletions(-) create mode 100644 src/test/kotlin/dev/typetype/server/AccountProfilesRoutesTest.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AccountIdentityService.kt b/src/main/kotlin/dev/typetype/server/services/AccountIdentityService.kt index 98ce5bc8..4c6b6e2a 100644 --- a/src/main/kotlin/dev/typetype/server/services/AccountIdentityService.kt +++ b/src/main/kotlin/dev/typetype/server/services/AccountIdentityService.kt @@ -11,14 +11,17 @@ import org.jetbrains.exposed.v1.core.neq import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.update -class AccountIdentityService { - suspend fun get(userId: String): AccountIdentityItem? = DatabaseFactory.query { - UsersTable.selectAll().where { UsersTable.id eq userId }.singleOrNull()?.let { +class AccountIdentityService(private val profileAccountService: ProfileAccountService? = null) { + suspend fun get(userId: String): AccountIdentityItem? { + val identityUserId = profileAccountService?.ownerUserId(userId) ?: userId + return DatabaseFactory.query { + UsersTable.selectAll().where { UsersTable.id eq identityUserId }.singleOrNull()?.let { AccountIdentityItem( email = it[UsersTable.email], name = it[UsersTable.name], managedByOidc = it[UsersTable.oidcIssuer] != null, ) + } } } @@ -29,8 +32,9 @@ class AccountIdentityService { currentPassword: String, ): AccountIdentityUpdateResult { val normalized = validate(email, name) ?: return AccountIdentityUpdateResult.InvalidInput + val identityUserId = profileAccountService?.ownerUserId(userId) ?: userId val credentials = DatabaseFactory.query { - UsersTable.selectAll().where { UsersTable.id eq userId }.singleOrNull()?.let { + UsersTable.selectAll().where { UsersTable.id eq identityUserId }.singleOrNull()?.let { Credentials(it[UsersTable.passwordHash], it[UsersTable.oidcIssuer] != null) } } ?: return AccountIdentityUpdateResult.UserNotFound @@ -38,7 +42,7 @@ class AccountIdentityService { if (!Password.check(currentPassword, credentials.passwordHash).withArgon2()) { return AccountIdentityUpdateResult.InvalidPassword } - return update(userId, normalized) + return update(identityUserId, normalized) } suspend fun updateAdmin(userId: String, email: String, name: String): AccountIdentityUpdateResult { diff --git a/src/main/kotlin/dev/typetype/server/services/AuthService.kt b/src/main/kotlin/dev/typetype/server/services/AuthService.kt index 54a81a8e..10ccb51b 100644 --- a/src/main/kotlin/dev/typetype/server/services/AuthService.kt +++ b/src/main/kotlin/dev/typetype/server/services/AuthService.kt @@ -19,6 +19,7 @@ open class AuthService( private val jwtSecret: String, private val hasUsersProbe: (() -> Boolean)? = null, sessionConfig: AuthSessionConfig = AuthSessionConfig(), + private val profileAccountService: ProfileAccountService? = null, ) { private val accessCodec = AuthAccessTokenCodec(jwtSecret) private val sessionStore = AuthSessionStore() @@ -48,9 +49,8 @@ open class AuthService( it[UsersTable.updatedAt] = now } } - return DatabaseFactory.blocking { - tokenIssuer.issue(userId) ?: throw IllegalStateException("Failed to create session") - } + val sessionUserId = profileAccountService?.resolveSignInProfile(userId) ?: userId + return DatabaseFactory.blocking { tokenIssuer.issue(sessionUserId) ?: throw IllegalStateException("Failed to create session") } } suspend fun login(identifier: String, password: String): AuthSessionTokens? { @@ -71,7 +71,8 @@ open class AuthService( val verified = withContext(passwordDispatcher) { Password.check(password, hashed).withArgon2() } if (!verified) return null - return DatabaseFactory.blocking { tokenIssuer.issue(user[UsersTable.id]) } + val sessionUserId = profileAccountService?.resolveSignInProfile(user[UsersTable.id]) ?: user[UsersTable.id] + return DatabaseFactory.blocking { tokenIssuer.issue(sessionUserId) } } suspend fun refreshSession(refreshToken: String): AuthSessionTokens? = DatabaseFactory.blocking { @@ -82,6 +83,11 @@ open class AuthService( tokenIssuer.issue(userId) } + suspend fun issueSessionForOwner(ownerUserId: String): AuthSessionTokens? { + val sessionUserId = profileAccountService?.resolveSignInProfile(ownerUserId) ?: ownerUserId + return issueSession(sessionUserId) + } + suspend fun logout(refreshToken: String?): Unit = DatabaseFactory.blocking { sessionRevoker.revokeByRefreshToken(refreshToken) } @@ -102,8 +108,9 @@ open class AuthService( suspend fun getUserRole(userId: String): String? { if (userId.startsWith("guest:")) return "user" + val roleUserId = profileAccountService?.ownerUserId(userId) ?: userId return DatabaseFactory.query { - UsersTable.selectAll().where { UsersTable.id eq userId }.singleOrNull() + UsersTable.selectAll().where { UsersTable.id eq roleUserId }.singleOrNull() }?.get(UsersTable.role) } diff --git a/src/main/kotlin/dev/typetype/server/services/OidcUserService.kt b/src/main/kotlin/dev/typetype/server/services/OidcUserService.kt index d6c50d06..67db26cc 100644 --- a/src/main/kotlin/dev/typetype/server/services/OidcUserService.kt +++ b/src/main/kotlin/dev/typetype/server/services/OidcUserService.kt @@ -13,7 +13,7 @@ import java.util.UUID class OidcUserService(private val authService: AuthService) { suspend fun login(identity: OidcIdentity): AuthSessionTokens { val userId = transaction { resolveUserId(identity) } - return authService.issueSession(userId) ?: throw IllegalStateException("Failed to create session") + return authService.issueSessionForOwner(userId) ?: throw IllegalStateException("Failed to create session") } private fun resolveUserId(identity: OidcIdentity): String { diff --git a/src/test/kotlin/dev/typetype/server/AccountProfilesRoutesTest.kt b/src/test/kotlin/dev/typetype/server/AccountProfilesRoutesTest.kt new file mode 100644 index 00000000..01b8c3fb --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/AccountProfilesRoutesTest.kt @@ -0,0 +1,145 @@ +package dev.typetype.server + +import dev.typetype.server.db.tables.UsersTable +import dev.typetype.server.models.AccountProfileItem +import dev.typetype.server.routes.accountProfilesRoutes +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.AuthSessionConfig +import dev.typetype.server.services.ProfileAccountService +import dev.typetype.server.services.ProfileMutationResult +import io.ktor.client.request.delete +import io.ktor.client.request.get +import io.ktor.client.request.headers +import io.ktor.client.request.post +import io.ktor.client.request.put +import io.ktor.client.request.setBody +import io.ktor.client.statement.bodyAsText +import io.ktor.http.ContentType +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.testApplication +import kotlinx.serialization.json.Json +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.transactions.transaction +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class AccountProfilesRoutesTest { + private val profileService = ProfileAccountService() + private val auth = AuthService.fixed(TEST_USER_ID) + + companion object { + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() { + TestDatabase.truncateAll() + transaction { + UsersTable.insert { + it[id] = TEST_USER_ID + it[email] = "profiles@test.local" + it[passwordHash] = "hash" + it[name] = "Profile" + it[role] = "user" + it[createdAt] = 0L + it[updatedAt] = 0L + } + } + } + + @Test + fun `profiles can be created renamed and made default`() = withApp { + val initial = client.get("/profiles") { bearer() } + assertEquals(HttpStatusCode.OK, initial.status) + assertTrue(initial.bodyAsText().contains("\"isDefault\":true")) + + val created = client.post("/profiles") { + bearer() + contentTypeJson() + setBody("{\"name\":\"Tech\"}") + } + assertEquals(HttpStatusCode.Created, created.status) + val profile = Json.decodeFromString(created.bodyAsText()) + + val renamed = client.put("/profiles/${profile.id}") { + bearer() + contentTypeJson() + setBody("{\"name\":\"Bricolage\"}") + } + assertEquals(HttpStatusCode.OK, renamed.status) + + val defaulted = client.post("/profiles/${profile.id}/default") { bearer() } + assertEquals(HttpStatusCode.OK, defaulted.status) + val listed = client.get("/profiles") { bearer() }.bodyAsText() + assertTrue(listed.contains("Bricolage")) + assertTrue(listed.contains("\"defaultProfileId\":\"${profile.id}\"")) + } + + @Test + fun `profile routes isolate owners and protect active deletion`() = withApp { + val created = client.post("/profiles") { + bearer() + contentTypeJson() + setBody("{\"name\":\"Private\"}") + } + val profile = Json.decodeFromString(created.bodyAsText()) + + transaction { + UsersTable.insert { + it[id] = "other-user-id" + it[email] = "other@test.local" + it[passwordHash] = "hash" + it[name] = "Other" + it[role] = "user" + it[createdAt] = 0L + it[updatedAt] = 0L + } + } + val foreignProfiles = profileService.list("other-user-id") + assertTrue(foreignProfiles?.profiles?.none { it.id == profile.id } == true) + + val activeDelete = client.delete("/profiles/${profile.id}") { bearer() } + assertEquals(HttpStatusCode.NoContent, activeDelete.status) + } + + @Test + fun `switch returns a new session and active profile cannot be deleted`() = withApp { + val created = client.post("/profiles") { + bearer() + contentTypeJson() + setBody("{\"name\":\"Travel\"}") + } + val profile = Json.decodeFromString(created.bodyAsText()) + val switched = client.post("/profiles/${profile.id}/switch") { bearer() } + assertEquals(HttpStatusCode.OK, switched.status) + assertTrue(switched.bodyAsText().contains("accessToken")) + + assertEquals(ProfileMutationResult.CannotDeleteActive, profileService.delete(profile.id, profile.id)) + } + + private fun withApp(block: suspend io.ktor.server.testing.ApplicationTestBuilder.() -> Unit) = testApplication { + application { + install(ContentNegotiation) { json() } + routing { accountProfilesRoutes(profileService, auth, AuthSessionConfig()) } + } + block() + } + + private fun io.ktor.client.request.HttpRequestBuilder.bearer() { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + } + + private fun io.ktor.client.request.HttpRequestBuilder.contentTypeJson() { + headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + } +} From d599514b4751b4937911e9164b47c5db6f00d59e Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 8 Sep 2026 11:38:34 +0200 Subject: [PATCH 17/45] feat: add profile scoped service notifications --- openapi/components/notifications.yaml | 37 ++++++ openapi/paths/notifications.yaml | 47 ++++++++ .../models/MarkNotificationsReadResponse.kt | 1 + .../server/models/NotificationItem.kt | 2 + .../server/models/NotificationsResponse.kt | 1 + .../server/models/UnreadCountResponse.kt | 1 + .../server/services/NotificationsService.kt | 110 +++++++++++------- .../services/SubscriptionFeedService.kt | 22 +++- ...icationItemPublishedAtSerializationTest.kt | 2 + .../server/NotificationsRoutesTest.kt | 21 +++- 10 files changed, 191 insertions(+), 53 deletions(-) create mode 100644 openapi/components/notifications.yaml create mode 100644 openapi/paths/notifications.yaml diff --git a/openapi/components/notifications.yaml b/openapi/components/notifications.yaml new file mode 100644 index 00000000..51103b15 --- /dev/null +++ b/openapi/components/notifications.yaml @@ -0,0 +1,37 @@ +NotificationItem: + type: object + required: [type, title, createdAt, publishedAt, channelUrl, channelName, channelAvatarUrl, serviceId, serviceName, video] + properties: + type: { type: string } + title: { type: string } + createdAt: { type: integer, format: int64 } + publishedAt: { type: integer, format: int64 } + channelUrl: { type: string } + channelName: { type: string } + channelAvatarUrl: { type: string } + serviceId: { type: integer, format: int32, minimum: 0 } + serviceName: { type: string } + video: { $ref: ./media.yaml#/VideoItem } +NotificationsResponse: + type: object + required: [items, unreadCount, nextpage, available] + properties: + items: + type: array + items: { $ref: '#/NotificationItem' } + unreadCount: { type: integer, minimum: 0 } + nextpage: { type: string, nullable: true } + available: { type: boolean } +UnreadCountResponse: + type: object + required: [unreadCount, available] + properties: + unreadCount: { type: integer, minimum: 0 } + available: { type: boolean } +MarkNotificationsReadResponse: + type: object + required: [readAt, unreadCount, available] + properties: + readAt: { type: integer, format: int64 } + unreadCount: { type: integer, minimum: 0 } + available: { type: boolean } diff --git a/openapi/paths/notifications.yaml b/openapi/paths/notifications.yaml new file mode 100644 index 00000000..a5492916 --- /dev/null +++ b/openapi/paths/notifications.yaml @@ -0,0 +1,47 @@ +Notifications: + get: + tags: [user-data] + summary: List new videos from the current profile's subscriptions + security: [{ bearerAuth: [] }] + parameters: + - name: page + in: query + required: false + schema: { type: integer, minimum: 0, default: 0 } + - name: limit + in: query + required: false + schema: { type: integer, minimum: 1, maximum: 100, default: 20 } + responses: + '200': + description: Profile-scoped notifications. available is false when the feed could not be refreshed. + content: + application/json: + schema: { $ref: ../components/notifications.yaml#/NotificationsResponse } + '401': { $ref: ../components/common.yaml#/JsonError } + +NotificationsUnreadCount: + get: + tags: [user-data] + summary: Read the current profile's unread notification count + security: [{ bearerAuth: [] }] + responses: + '200': + description: Unread count and feed availability. + content: + application/json: + schema: { $ref: ../components/notifications.yaml#/UnreadCountResponse } + '401': { $ref: ../components/common.yaml#/JsonError } + +NotificationsReadAll: + post: + tags: [user-data] + summary: Mark all current-profile notifications as read + security: [{ bearerAuth: [] }] + responses: + '200': + description: Read marker and resulting unread count. + content: + application/json: + schema: { $ref: ../components/notifications.yaml#/MarkNotificationsReadResponse } + '401': { $ref: ../components/common.yaml#/JsonError } diff --git a/src/main/kotlin/dev/typetype/server/models/MarkNotificationsReadResponse.kt b/src/main/kotlin/dev/typetype/server/models/MarkNotificationsReadResponse.kt index 93e0f41d..5b9cb89e 100644 --- a/src/main/kotlin/dev/typetype/server/models/MarkNotificationsReadResponse.kt +++ b/src/main/kotlin/dev/typetype/server/models/MarkNotificationsReadResponse.kt @@ -6,4 +6,5 @@ import kotlinx.serialization.Serializable data class MarkNotificationsReadResponse( val readAt: Long, val unreadCount: Int, + val available: Boolean = true, ) diff --git a/src/main/kotlin/dev/typetype/server/models/NotificationItem.kt b/src/main/kotlin/dev/typetype/server/models/NotificationItem.kt index f188f0af..65a871eb 100644 --- a/src/main/kotlin/dev/typetype/server/models/NotificationItem.kt +++ b/src/main/kotlin/dev/typetype/server/models/NotificationItem.kt @@ -11,5 +11,7 @@ data class NotificationItem( val channelUrl: String, val channelName: String, val channelAvatarUrl: String, + val serviceId: Int, + val serviceName: String, val video: VideoItem, ) diff --git a/src/main/kotlin/dev/typetype/server/models/NotificationsResponse.kt b/src/main/kotlin/dev/typetype/server/models/NotificationsResponse.kt index 229cef5c..9cadd471 100644 --- a/src/main/kotlin/dev/typetype/server/models/NotificationsResponse.kt +++ b/src/main/kotlin/dev/typetype/server/models/NotificationsResponse.kt @@ -7,4 +7,5 @@ data class NotificationsResponse( val items: List, val unreadCount: Int, val nextpage: String?, + val available: Boolean = true, ) diff --git a/src/main/kotlin/dev/typetype/server/models/UnreadCountResponse.kt b/src/main/kotlin/dev/typetype/server/models/UnreadCountResponse.kt index f75b59df..26b77b89 100644 --- a/src/main/kotlin/dev/typetype/server/models/UnreadCountResponse.kt +++ b/src/main/kotlin/dev/typetype/server/models/UnreadCountResponse.kt @@ -5,4 +5,5 @@ import kotlinx.serialization.Serializable @Serializable data class UnreadCountResponse( val unreadCount: Int, + val available: Boolean = true, ) diff --git a/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt b/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt index fb50c816..66f5016d 100644 --- a/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt @@ -19,25 +19,35 @@ class NotificationsService( private val unreadCache = ConcurrentHashMap() suspend fun getNotifications(userId: String, page: Int, limit: Int): NotificationsResponse { - val items = buildItems(userId) - val unreadCount = unreadCount(items, userId, refresh = true) + val feed = loadFeed(userId) + val items = buildItems(feed.videos) + val unreadCount = if (feed.available) unreadCount(items, userId) else cachedUnread(userId) val from = page * limit - if (from >= items.size) return NotificationsResponse(items = emptyList(), unreadCount = unreadCount, nextpage = null) + if (from >= items.size) { + return NotificationsResponse(emptyList(), unreadCount, null, feed.available) + } val to = minOf(from + limit, items.size) val nextpage = if (to < items.size) (page + 1).toString() else null - return NotificationsResponse(items = items.subList(from, to), unreadCount = unreadCount, nextpage = nextpage) + return NotificationsResponse(items.subList(from, to), unreadCount, nextpage, feed.available) } suspend fun getUnreadCount(userId: String): UnreadCountResponse { + val cached = unreadCache[userId] val now = System.currentTimeMillis() - unreadCache[userId]?.takeIf { it.expiresAt > now }?.let { return UnreadCountResponse(unreadCount = it.value) } - val items = buildItems(userId) - return UnreadCountResponse(unreadCount = unreadCount(items, userId, refresh = true)) + if (cached != null && cached.expiresAt > now) return UnreadCountResponse(cached.value, true) + val feed = loadFeed(userId) + if (!feed.available) return UnreadCountResponse(cachedUnread(userId), false) + val value = unreadCount(buildItems(feed.videos), userId) + return UnreadCountResponse(value, true) } suspend fun markAllRead(userId: String): MarkNotificationsReadResponse { + val feed = loadFeed(userId) + if (!feed.available) { + return MarkNotificationsReadResponse(System.currentTimeMillis(), cachedUnread(userId), false) + } + val latestUploaded = buildItems(feed.videos).firstOrNull()?.createdAt ?: 0L val now = System.currentTimeMillis() - val latestUploaded = buildItems(userId).firstOrNull()?.createdAt ?: 0L DatabaseFactory.query { val updated = NotificationStatesTable.update({ NotificationStatesTable.userId eq userId }) { it[subscriptionLastSeenUploaded] = latestUploaded @@ -46,62 +56,72 @@ class NotificationsService( if (updated == 0) { NotificationStatesTable.insert { it[NotificationStatesTable.userId] = userId - it[subscriptionLastSeenUploaded] = latestUploaded - it[updatedAt] = now + it[NotificationStatesTable.subscriptionLastSeenUploaded] = latestUploaded + it[NotificationStatesTable.updatedAt] = now } } } - unreadCache[userId] = CachedUnread(value = 0, expiresAt = now + UNREAD_CACHE_TTL_MS) - return MarkNotificationsReadResponse(readAt = now, unreadCount = 0) + unreadCache[userId] = CachedUnread(0, now + UNREAD_CACHE_TTL_MS) + return MarkNotificationsReadResponse(now, 0, true) } - private suspend fun buildItems(userId: String): List = subscriptionFeedService.getAll(userId) - .asSequence() + private suspend fun loadFeed(userId: String): SubscriptionFeedAvailability = + runCatching { subscriptionFeedService.getAllWithAvailability(userId) } + .getOrElse { SubscriptionFeedAvailability(emptyList(), false) } + + private fun buildItems(videos: List): List = videos.asSequence() .filter { it.uploaded > 0L } - .groupBy { notificationKey(it) } - .values - .mapNotNull { group -> group.maxByOrNull { it.uploaded } } + .distinctBy(::notificationKey) .sortedByDescending { it.uploaded } .map { it.toNotificationItem() } .toList() - private suspend fun getLastSeenUploaded(userId: String): Long = DatabaseFactory.query { - NotificationStatesTable.selectAll().where { NotificationStatesTable.userId eq userId } - .singleOrNull()?.get(NotificationStatesTable.subscriptionLastSeenUploaded) ?: 0L - } - - private suspend fun unreadCount(items: List, userId: String, refresh: Boolean): Int { - if (!refresh) { - val now = System.currentTimeMillis() - unreadCache[userId]?.takeIf { it.expiresAt > now }?.let { return it.value } + private suspend fun unreadCount(items: List, userId: String): Int { + val lastSeenUploaded = DatabaseFactory.query { + NotificationStatesTable.selectAll().where { NotificationStatesTable.userId eq userId } + .singleOrNull()?.get(NotificationStatesTable.subscriptionLastSeenUploaded) ?: 0L } - val lastSeenUploaded = getLastSeenUploaded(userId) val value = items.count { it.createdAt > lastSeenUploaded } - val now = System.currentTimeMillis() - unreadCache[userId] = CachedUnread(value = value, expiresAt = now + UNREAD_CACHE_TTL_MS) + unreadCache[userId] = CachedUnread(value, System.currentTimeMillis() + UNREAD_CACHE_TTL_MS) return value } - private fun notificationKey(video: VideoItem): String = - video.uploaderUrl.ifBlank { video.uploaderName.ifBlank { video.url } } + private fun cachedUnread(userId: String): Int = unreadCache[userId]?.value ?: 0 - private fun VideoItem.toNotificationItem(): NotificationItem = NotificationItem( - type = "subscription_new_video", - title = "$uploaderName uploaded a new video", - createdAt = uploaded, - publishedAt = uploaded, - channelUrl = uploaderUrl, - channelName = uploaderName, - channelAvatarUrl = uploaderAvatarUrl, - video = this, - ) + private fun notificationKey(video: VideoItem): String { + val serviceId = RssVideoMetadata.serviceId(video) + val videoIdentity = video.url.ifBlank { video.id } + return "$serviceId:${video.uploaderUrl}:$videoIdentity" + } + + private fun VideoItem.toNotificationItem(): NotificationItem { + val serviceId = RssVideoMetadata.serviceId(this) + return NotificationItem( + type = "subscription_new_video", + title = "$uploaderName uploaded a new video", + createdAt = uploaded, + publishedAt = uploaded, + channelUrl = uploaderUrl, + channelName = uploaderName, + channelAvatarUrl = uploaderAvatarUrl, + serviceId = serviceId, + serviceName = serviceName(serviceId), + video = this, + ) + } - private data class CachedUnread( - val value: Int, - val expiresAt: Long, - ) + private data class CachedUnread(val value: Int, val expiresAt: Long) private companion object { const val UNREAD_CACHE_TTL_MS = 30_000L + + fun serviceName(serviceId: Int): String = when (serviceId) { + YOUTUBE_SERVICE_ID -> "YouTube" + BILIBILI_SERVICE_ID -> "BiliBili" + NICONICO_SERVICE_ID -> "NicoNico" + SOUNDCLOUD_SERVICE_ID -> "SoundCloud" + MEDIA_CCC_SERVICE_ID -> "MediaCCC" + else -> "Video service" + } } } diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt index 7581eb66..21501bef 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt @@ -99,13 +99,26 @@ class SubscriptionFeedService( } suspend fun getAll(userId: String): List { + return getAllWithAvailability(userId).videos + } + + suspend fun getAllWithAvailability(userId: String): SubscriptionFeedAvailability { val snapshot = store.current(userId) if (snapshot == null || snapshot.stale || clock() - snapshot.generatedAt >= FRESHNESS_MS) { scheduleRefresh(userId, currentRequestId()) } - if (snapshot != null) return snapshot.videos + if (snapshot != null) { + return SubscriptionFeedAvailability( + videos = snapshot.videos, + available = !snapshot.stale && clock() - snapshot.generatedAt < FRESHNESS_MS, + ) + } withTimeoutOrNull(INTERNAL_COLD_WAIT_MS) { awaitRefresh(userId) } - return store.current(userId)?.videos.orEmpty() + val refreshed = store.current(userId) + return SubscriptionFeedAvailability( + videos = refreshed?.videos.orEmpty(), + available = refreshed != null && !refreshed.stale, + ) } suspend fun getCachedFeed(userId: String, page: Int, limit: Int): SubscriptionFeedResponse? { @@ -201,3 +214,8 @@ class SubscriptionFeedService( private val logger = LoggerFactory.getLogger(SubscriptionFeedService::class.java) } } + +data class SubscriptionFeedAvailability( + val videos: List, + val available: Boolean, +) diff --git a/src/test/kotlin/dev/typetype/server/NotificationItemPublishedAtSerializationTest.kt b/src/test/kotlin/dev/typetype/server/NotificationItemPublishedAtSerializationTest.kt index cbdaaefe..7c50cf17 100644 --- a/src/test/kotlin/dev/typetype/server/NotificationItemPublishedAtSerializationTest.kt +++ b/src/test/kotlin/dev/typetype/server/NotificationItemPublishedAtSerializationTest.kt @@ -19,6 +19,8 @@ class NotificationItemPublishedAtSerializationTest { channelUrl = "https://yt.com/c/a", channelName = "A", channelAvatarUrl = "", + serviceId = 0, + serviceName = "YouTube", video = VideoItem( id = "id", title = "video", diff --git a/src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt b/src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt index 92d21d32..8c522f38 100644 --- a/src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt @@ -64,17 +64,26 @@ class NotificationsRoutesTest { } @Test - fun `GET notifications returns latest per channel with unread count`() = withApp { + fun `GET notifications returns every new video with service identity`() = withApp { subscriptionsService.add(TEST_USER_ID, subscription("https://yt.com/c/a", "A")) subscriptionsService.add(TEST_USER_ID, subscription("https://yt.com/c/b", "B")) - coEvery { channelService.getChannel("https://yt.com/c/a", null) } returns channel(video(1000L, "A"), video(3000L, "A")) - coEvery { channelService.getChannel("https://yt.com/c/b", null) } returns channel(video(2000L, "B")) + coEvery { channelService.getChannel("https://yt.com/c/a", null) } returns channel( + video(1000L, "A", "https://www.youtube.com/watch?v=yt-old"), + video(3000L, "A", "https://www.youtube.com/watch?v=yt-new"), + ) + coEvery { channelService.getChannel("https://yt.com/c/b", null) } returns channel( + video(2000L, "A", "https://www.bilibili.com/video/av2000"), + video(4000L, "A", "https://www.nicovideo.jp/watch/sm4000"), + ) val body = client.get("/notifications?page=0&limit=10") { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") }.bodyAsText() - assertTrue(body.contains("\"unreadCount\":2")) - assertTrue(body.indexOf("3000") < body.indexOf("2000")) - assertTrue(!body.contains("1000")) + assertTrue(body.contains("\"unreadCount\":4")) + assertTrue(body.indexOf("4000") < body.indexOf("3000")) + assertTrue(body.contains("yt-new")) + assertTrue(body.contains("yt-old")) + assertTrue(body.contains("\"serviceName\":\"BiliBili\"")) + assertTrue(body.contains("\"serviceName\":\"NicoNico\"")) } @Test From 9fd2efd918e09338de2133242832d74659cc18cb Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 8 Sep 2026 11:38:38 +0200 Subject: [PATCH 18/45] chore: bump server version to 1.8.1 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 2de0665a..3e32a72b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ org.gradle.jvmargs=-Xmx2g -XX:+UseG1GC kotlin.code.style=official -appVersion=1.7.1 +appVersion=1.8.1 systemProp.sun.net.client.defaultReadTimeout=180000 systemProp.sun.net.client.defaultConnectTimeout=60000 From 9162c3c51a915fdd17f3e8bfe845f78edbb78434 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 8 Sep 2026 13:01:41 +0200 Subject: [PATCH 19/45] feat: add UnifiedPush channel notifications --- .env.example | 4 + openapi.yaml | 10 ++ openapi/components/instance.yaml | 14 ++ openapi/components/notifications.yaml | 4 +- openapi/components/push-notifications.yaml | 58 +++++++ openapi/paths/notifications.yaml | 19 ++ openapi/paths/push-notifications.yaml | 85 +++++++++ .../kotlin/dev/typetype/server/Application.kt | 7 + .../dev/typetype/server/ApplicationRoutes.kt | 1 + .../dev/typetype/server/ServiceRegistry.kt | 12 ++ .../dev/typetype/server/db/DatabaseFactory.kt | 14 ++ .../ChannelNotificationPreferencesTable.kt | 16 ++ .../db/tables/NotificationReadItemsTable.kt | 15 ++ .../server/db/tables/PushDevicesTable.kt | 22 +++ .../tables/PushNotificationBaselinesTable.kt | 10 ++ .../tables/PushNotificationDeliveriesTable.kt | 20 +++ .../db/tables/PushNotificationEventsTable.kt | 24 +++ .../tables/PushNotificationSeenVideosTable.kt | 17 ++ .../server/models/InstanceResponse.kt | 1 + .../server/models/NotificationItem.kt | 2 + .../server/models/PushNotificationModels.kt | 57 ++++++ .../server/routes/NotificationsRoutes.kt | 11 ++ .../server/routes/PushNotificationRoutes.kt | 110 ++++++++++++ .../server/routes/SubscriptionsRoutes.kt | 14 +- .../typetype/server/routes/UserDataRoutes.kt | 2 + .../ChannelNotificationPreferenceService.kt | 82 +++++++++ .../server/services/InstanceService.kt | 3 + .../server/services/NotificationsService.kt | 103 ++++++++--- .../services/ProfileDataDeletionService.kt | 12 ++ .../server/services/PushDeviceRegistry.kt | 146 ++++++++++++++++ .../services/PushNotificationDeliveryStore.kt | 106 +++++++++++ .../services/PushNotificationScheduler.kt | 40 +++++ .../services/PushNotificationService.kt | 164 ++++++++++++++++++ .../services/PushNotificationSupport.kt | 52 ++++++ .../services/SubscriptionFeedAvailability.kt | 14 ++ .../services/SubscriptionFeedService.kt | 8 +- .../services/UnifiedPushEndpointValidator.kt | 67 +++++++ .../server/services/UnifiedPushSender.kt | 68 ++++++++ .../typetype/server/PushDeviceRegistryTest.kt | 45 +++++ .../PushNotificationDeliveryStoreTest.kt | 31 ++++ .../dev/typetype/server/TestDatabase.kt | 14 ++ .../UnifiedPushEndpointValidatorTest.kt | 32 ++++ 42 files changed, 1502 insertions(+), 34 deletions(-) create mode 100644 openapi/components/push-notifications.yaml create mode 100644 openapi/paths/push-notifications.yaml create mode 100644 src/main/kotlin/dev/typetype/server/db/tables/ChannelNotificationPreferencesTable.kt create mode 100644 src/main/kotlin/dev/typetype/server/db/tables/NotificationReadItemsTable.kt create mode 100644 src/main/kotlin/dev/typetype/server/db/tables/PushDevicesTable.kt create mode 100644 src/main/kotlin/dev/typetype/server/db/tables/PushNotificationBaselinesTable.kt create mode 100644 src/main/kotlin/dev/typetype/server/db/tables/PushNotificationDeliveriesTable.kt create mode 100644 src/main/kotlin/dev/typetype/server/db/tables/PushNotificationEventsTable.kt create mode 100644 src/main/kotlin/dev/typetype/server/db/tables/PushNotificationSeenVideosTable.kt create mode 100644 src/main/kotlin/dev/typetype/server/models/PushNotificationModels.kt create mode 100644 src/main/kotlin/dev/typetype/server/routes/PushNotificationRoutes.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/ChannelNotificationPreferenceService.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/PushDeviceRegistry.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/PushNotificationDeliveryStore.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/PushNotificationScheduler.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/PushNotificationService.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/PushNotificationSupport.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/SubscriptionFeedAvailability.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/UnifiedPushEndpointValidator.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/UnifiedPushSender.kt create mode 100644 src/test/kotlin/dev/typetype/server/PushDeviceRegistryTest.kt create mode 100644 src/test/kotlin/dev/typetype/server/PushNotificationDeliveryStoreTest.kt create mode 100644 src/test/kotlin/dev/typetype/server/UnifiedPushEndpointValidatorTest.kt diff --git a/.env.example b/.env.example index 9f061443..0c395a83 100644 --- a/.env.example +++ b/.env.example @@ -12,4 +12,8 @@ YOUTUBE_REMOTE_LOGIN_INTERNAL_TOKEN=replace-with-shared-internal-token YOUTUBE_REMOTE_LOGIN_TTL_MS=480000 YOUTUBE_REMOTE_LOGIN_MAX_SESSIONS=2 +TYPE_TYPE_INSTANCE_ID=typetype +TYPE_TYPE_PUSH_NOTIFICATIONS_ENABLED=true +TYPE_TYPE_PUSH_NOTIFICATIONS_INTERVAL_SECONDS=300 + ALLOWED_ORIGINS=http://localhost:5173 diff --git a/openapi.yaml b/openapi.yaml index cabda1d9..2a953e3e 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -57,6 +57,10 @@ paths: /notifications: { $ref: ./openapi/paths/notifications.yaml#/Notifications } /notifications/unread-count: { $ref: ./openapi/paths/notifications.yaml#/NotificationsUnreadCount } /notifications/read-all: { $ref: ./openapi/paths/notifications.yaml#/NotificationsReadAll } + /notifications/{notificationId}/read: { $ref: ./openapi/paths/notifications.yaml#/NotificationsRead } + /notifications/channel-preferences: { $ref: ./openapi/paths/push-notifications.yaml#/ChannelNotificationPreferences } + /notifications/push/devices: { $ref: ./openapi/paths/push-notifications.yaml#/PushDevices } + /notifications/push/devices/{deviceId}: { $ref: ./openapi/paths/push-notifications.yaml#/PushDevice } /rss/feeds: { $ref: ./openapi/paths/rss.yaml#/RssFeeds } /rss/feeds/{id}: { $ref: ./openapi/paths/rss.yaml#/RssFeed } /rss/feeds/{id}/enabled: { $ref: ./openapi/paths/rss.yaml#/RssFeedEnabled } @@ -189,6 +193,12 @@ components: NotificationsResponse: { $ref: ./openapi/components/notifications.yaml#/NotificationsResponse } UnreadCountResponse: { $ref: ./openapi/components/notifications.yaml#/UnreadCountResponse } MarkNotificationsReadResponse: { $ref: ./openapi/components/notifications.yaml#/MarkNotificationsReadResponse } + PushNotificationCapability: { $ref: ./openapi/components/instance.yaml#/PushNotificationCapability } + PushDeviceRegistrationRequest: { $ref: ./openapi/components/push-notifications.yaml#/PushDeviceRegistrationRequest } + PushDeviceRegistrationResponse: { $ref: ./openapi/components/push-notifications.yaml#/PushDeviceRegistrationResponse } + ChannelNotificationPreferenceRequest: { $ref: ./openapi/components/push-notifications.yaml#/ChannelNotificationPreferenceRequest } + ChannelNotificationPreference: { $ref: ./openapi/components/push-notifications.yaml#/ChannelNotificationPreference } + UnifiedPushNotificationPayload: { $ref: ./openapi/components/push-notifications.yaml#/UnifiedPushNotificationPayload } RssFeedRequest: { $ref: ./openapi/components/rss.yaml#/RssFeedRequest } RssFeedItem: { $ref: ./openapi/components/rss.yaml#/RssFeedItem } RssFeedSecretItem: { $ref: ./openapi/components/rss.yaml#/RssFeedSecretItem } diff --git a/openapi/components/instance.yaml b/openapi/components/instance.yaml index c40a33cc..f2bb25aa 100644 --- a/openapi/components/instance.yaml +++ b/openapi/components/instance.yaml @@ -22,6 +22,7 @@ InstanceResponse: - youtubeRemoteLoginReady - parentalControlsEnabled - rss + - pushNotifications properties: name: { type: string, example: TypeType } tagline: { type: string, nullable: true } @@ -56,6 +57,8 @@ InstanceResponse: description: True when the instance-wide allow-list policy is enabled. rss: $ref: '#/RssInstanceCapability' + pushNotifications: + $ref: '#/PushNotificationCapability' RssInstanceCapability: type: object required: [enabled, maxFeedsPerUser, maxItems, minimumPollMinutes, rateLimitPerMinute] @@ -65,3 +68,14 @@ RssInstanceCapability: maxItems: { type: integer } minimumPollMinutes: { type: integer } rateLimitPerMinute: { type: integer } + +PushNotificationCapability: + type: object + required: [enabled, provider, eventTypes, maxDevicesPerAccount] + properties: + enabled: { type: boolean } + provider: { type: string, enum: [unifiedpush] } + eventTypes: + type: array + items: { type: string } + maxDevicesPerAccount: { type: integer, minimum: 0 } diff --git a/openapi/components/notifications.yaml b/openapi/components/notifications.yaml index 51103b15..30841545 100644 --- a/openapi/components/notifications.yaml +++ b/openapi/components/notifications.yaml @@ -1,7 +1,8 @@ NotificationItem: type: object - required: [type, title, createdAt, publishedAt, channelUrl, channelName, channelAvatarUrl, serviceId, serviceName, video] + required: [id, type, title, createdAt, publishedAt, channelUrl, channelName, channelAvatarUrl, serviceId, serviceName, read, video] properties: + id: { type: string } type: { type: string } title: { type: string } createdAt: { type: integer, format: int64 } @@ -11,6 +12,7 @@ NotificationItem: channelAvatarUrl: { type: string } serviceId: { type: integer, format: int32, minimum: 0 } serviceName: { type: string } + read: { type: boolean } video: { $ref: ./media.yaml#/VideoItem } NotificationsResponse: type: object diff --git a/openapi/components/push-notifications.yaml b/openapi/components/push-notifications.yaml new file mode 100644 index 00000000..fda7e6f1 --- /dev/null +++ b/openapi/components/push-notifications.yaml @@ -0,0 +1,58 @@ +PushDeviceRegistrationRequest: + type: object + required: [deviceId, endpoint] + properties: + deviceId: { type: string, minLength: 1, maxLength: 128 } + platform: { type: string, enum: [android], default: android } + endpoint: { type: string, format: uri, minLength: 1, maxLength: 2048 } + expiresAt: { type: integer, format: int64, nullable: true } +PushDeviceRegistrationResponse: + type: object + required: [id, deviceId, platform, expiresAt, updatedAt] + properties: + id: { type: string } + deviceId: { type: string } + platform: { type: string, enum: [android] } + expiresAt: { type: integer, format: int64, nullable: true } + updatedAt: { type: integer, format: int64 } +ChannelNotificationPreferenceRequest: + type: object + required: [channelUrl, enabled] + properties: + channelUrl: { type: string, minLength: 1 } + enabled: { type: boolean } +ChannelNotificationPreference: + type: object + required: [channelUrl, enabled, updatedAt] + properties: + channelUrl: { type: string } + enabled: { type: boolean } + updatedAt: { type: integer, format: int64 } +UnifiedPushNotificationPayload: + type: object + required: + - version + - eventType + - eventId + - videoId + - videoUrl + - channelId + - channelName + - channelAvatarUrl + - instanceId + - accountId + - publishedAt + - title + properties: + version: { type: integer, minimum: 1 } + eventType: { type: string, enum: [subscription_new_video] } + eventId: { type: string } + videoId: { type: string } + videoUrl: { type: string } + channelId: { type: string } + channelName: { type: string } + channelAvatarUrl: { type: string } + instanceId: { type: string } + accountId: { type: string } + publishedAt: { type: integer, format: int64 } + title: { type: string } diff --git a/openapi/paths/notifications.yaml b/openapi/paths/notifications.yaml index a5492916..5a60699f 100644 --- a/openapi/paths/notifications.yaml +++ b/openapi/paths/notifications.yaml @@ -45,3 +45,22 @@ NotificationsReadAll: application/json: schema: { $ref: ../components/notifications.yaml#/MarkNotificationsReadResponse } '401': { $ref: ../components/common.yaml#/JsonError } + +NotificationsRead: + post: + tags: [user-data] + summary: Mark one current-profile notification as read + security: [{ bearerAuth: [] }] + parameters: + - name: notificationId + in: path + required: true + schema: { type: string } + responses: + '200': + description: Read marker and resulting unread count. + content: + application/json: + schema: { $ref: ../components/notifications.yaml#/MarkNotificationsReadResponse } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } diff --git a/openapi/paths/push-notifications.yaml b/openapi/paths/push-notifications.yaml new file mode 100644 index 00000000..30bd33a4 --- /dev/null +++ b/openapi/paths/push-notifications.yaml @@ -0,0 +1,85 @@ +ChannelNotificationPreferences: + get: + tags: [user-data] + summary: List notification preferences for followed channels + security: [{ bearerAuth: [] }] + responses: + '200': + description: Channel notification preferences for the current profile. + content: + application/json: + schema: + type: array + items: { $ref: ../components/push-notifications.yaml#/ChannelNotificationPreference } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + put: + tags: [user-data] + summary: Enable or disable notifications for a followed channel + security: [{ bearerAuth: [] }] + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/push-notifications.yaml#/ChannelNotificationPreferenceRequest } + responses: + '200': + description: Updated channel notification preference. + content: + application/json: + schema: { $ref: ../components/push-notifications.yaml#/ChannelNotificationPreference } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + '409': { $ref: ../components/common.yaml#/JsonError } + +PushDevices: + get: + tags: [user-data] + summary: List registered UnifiedPush devices + security: [{ bearerAuth: [] }] + responses: + '200': + description: Devices registered by the current profile. + content: + application/json: + schema: + type: array + items: { $ref: ../components/push-notifications.yaml#/PushDeviceRegistrationResponse } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + post: + tags: [user-data] + summary: Register or replace a UnifiedPush device endpoint + security: [{ bearerAuth: [] }] + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/push-notifications.yaml#/PushDeviceRegistrationRequest } + responses: + '201': + description: Registered device. + content: + application/json: + schema: { $ref: ../components/push-notifications.yaml#/PushDeviceRegistrationResponse } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + '409': { $ref: ../components/common.yaml#/JsonError } + '422': { $ref: ../components/common.yaml#/JsonError } + +PushDevice: + delete: + tags: [user-data] + summary: Unregister a UnifiedPush device + security: [{ bearerAuth: [] }] + parameters: + - name: deviceId + in: path + required: true + schema: { type: string } + responses: + '204': { description: Device unregistered. } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } diff --git a/src/main/kotlin/dev/typetype/server/Application.kt b/src/main/kotlin/dev/typetype/server/Application.kt index c5312b65..a84be3ce 100644 --- a/src/main/kotlin/dev/typetype/server/Application.kt +++ b/src/main/kotlin/dev/typetype/server/Application.kt @@ -25,6 +25,7 @@ import dev.typetype.server.services.UserAdminService import dev.typetype.server.services.YoutubeRemoteBrowserConfig import dev.typetype.server.services.YoutubeRemoteBrowserService import dev.typetype.server.services.YoutubeRemoteLoginReadinessService +import dev.typetype.server.services.PushNotificationScheduler import dev.typetype.server.portability.PortabilityEngineFactory import io.ktor.server.application.Application import io.ktor.server.application.ApplicationStopped @@ -71,7 +72,12 @@ fun Application.module() { adminSettingsService, youtubeProxySelector, profileAccountService, + instanceId = System.getenv("TYPE_TYPE_INSTANCE_ID")?.trim().takeUnless { it.isNullOrBlank() } ?: "typetype", + pushNotificationsEnabled = System.getenv("TYPE_TYPE_PUSH_NOTIFICATIONS_ENABLED")?.toBooleanStrictOrNull() ?: true, ) + val pushNotificationScheduler = PushNotificationScheduler(svc.pushNotificationService) + pushNotificationScheduler.start() + monitor.subscribe(ApplicationStopped) { pushNotificationScheduler.close() } val youtubeRemoteBrowserConfig = YoutubeRemoteBrowserConfig.fromEnvironment(subtitleServiceUrl) val youtubeRemoteLoginReadinessService = YoutubeRemoteLoginReadinessService( youtubeRemoteBrowserConfig, @@ -82,6 +88,7 @@ fun Application.module() { adminSettingsService, youtubeRemoteLoginStatusProvider = youtubeRemoteLoginReadinessService::status, oidcConfigProvider = oidcAuthService::publicConfig, + pushNotificationCapabilityProvider = svc.pushNotificationService::capability, ) val youtubeRemoteBrowserService = YoutubeRemoteBrowserService( youtubeRemoteBrowserConfig, diff --git a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt index 24bdb477..2e52330d 100644 --- a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt @@ -19,6 +19,7 @@ import dev.typetype.server.routes.oidcAuthRoutes import dev.typetype.server.routes.podcastRoutes import dev.typetype.server.routes.publicMetadataRoutes import dev.typetype.server.routes.publicPlaylistRoutes +import dev.typetype.server.routes.pushNotificationRoutes import dev.typetype.server.routes.rssPublicRoutes import dev.typetype.server.routes.sabrRoutes import dev.typetype.server.routes.searchRoutes diff --git a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt index 92b9f6a1..fbf9f139 100644 --- a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt @@ -16,6 +16,8 @@ import dev.typetype.server.services.FavoritesService import dev.typetype.server.services.HistoryService import dev.typetype.server.services.HomeRecommendationService import dev.typetype.server.services.NotificationsService +import dev.typetype.server.services.ChannelNotificationPreferenceService +import dev.typetype.server.services.PushNotificationService import dev.typetype.server.services.ProfileAccountService import dev.typetype.server.services.PlaylistService import dev.typetype.server.services.ProgressService @@ -47,6 +49,8 @@ internal class ServiceRegistry( adminSettingsService: AdminSettingsService, youtubeProxySelector: ProxySelector? = null, profileAccountService: ProfileAccountService? = null, + private val instanceId: String = "typetype", + private val pushNotificationsEnabled: Boolean = true, ) { val publicHlsManifestTokenService = PublicHlsManifestTokenService(jwtSecret) val accountIdentityService = AccountIdentityService(profileAccountService) @@ -101,6 +105,14 @@ internal class ServiceRegistry( ) } val notificationsService = NotificationsService(subscriptionFeedService) + val channelNotificationPreferenceService = ChannelNotificationPreferenceService(subscriptionsService) + val pushNotificationService = PushNotificationService( + subscriptionsService = subscriptionsService, + subscriptionFeedService = subscriptionFeedService, + preferenceService = channelNotificationPreferenceService, + instanceId = instanceId, + enabled = pushNotificationsEnabled, + ) val playlistService = PlaylistService() val videoMetadataRepairService = UserVideoMetadataRepairService(VideoMetadataResolver(streamService)) val savedPlaylistService = SavedPlaylistService() diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt index e65a7e91..1368eb0f 100644 --- a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt +++ b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt @@ -25,6 +25,13 @@ import dev.typetype.server.db.tables.AdminSettingsTable import dev.typetype.server.db.tables.AllowedChannelsTable import dev.typetype.server.db.tables.PasswordResetTable import dev.typetype.server.db.tables.NotificationStatesTable +import dev.typetype.server.db.tables.NotificationReadItemsTable +import dev.typetype.server.db.tables.ChannelNotificationPreferencesTable +import dev.typetype.server.db.tables.PushDevicesTable +import dev.typetype.server.db.tables.PushNotificationBaselinesTable +import dev.typetype.server.db.tables.PushNotificationSeenVideosTable +import dev.typetype.server.db.tables.PushNotificationEventsTable +import dev.typetype.server.db.tables.PushNotificationDeliveriesTable import dev.typetype.server.db.tables.ProfileAccountsTable import dev.typetype.server.db.tables.RecommendationEventsTable import dev.typetype.server.db.tables.RecommendationFeedHistoryTable @@ -93,6 +100,13 @@ object DatabaseFactory { YoutubeSessionPairingsTable, BugReportsTable, NotificationStatesTable, + NotificationReadItemsTable, + ChannelNotificationPreferencesTable, + PushDevicesTable, + PushNotificationBaselinesTable, + PushNotificationSeenVideosTable, + PushNotificationEventsTable, + PushNotificationDeliveriesTable, UserChannelInterestTable, UserTopicInterestTable, RecommendationEventsTable, diff --git a/src/main/kotlin/dev/typetype/server/db/tables/ChannelNotificationPreferencesTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/ChannelNotificationPreferencesTable.kt new file mode 100644 index 00000000..7b5b7ef9 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/ChannelNotificationPreferencesTable.kt @@ -0,0 +1,16 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object ChannelNotificationPreferencesTable : Table("channel_notification_preferences") { + val userId = text("user_id") + val channelUrl = text("channel_url") + val enabled = bool("enabled") + val updatedAt = long("updated_at") + + init { + index(false, userId) + } + + override val primaryKey = PrimaryKey(userId, channelUrl) +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/NotificationReadItemsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/NotificationReadItemsTable.kt new file mode 100644 index 00000000..dcc10129 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/NotificationReadItemsTable.kt @@ -0,0 +1,15 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object NotificationReadItemsTable : Table("notification_read_items") { + val userId = text("user_id") + val notificationId = text("notification_id") + val readAt = long("read_at") + + init { + index(false, userId, readAt) + } + + override val primaryKey = PrimaryKey(userId, notificationId) +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PushDevicesTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/PushDevicesTable.kt new file mode 100644 index 00000000..b72410e3 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/PushDevicesTable.kt @@ -0,0 +1,22 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object PushDevicesTable : Table("push_devices") { + val id = text("id") + val userId = text("user_id") + val deviceId = text("device_id") + val platform = text("platform") + val endpoint = text("endpoint") + val endpointHash = text("endpoint_hash").uniqueIndex() + val expiresAt = long("expires_at").nullable() + val createdAt = long("created_at") + val updatedAt = long("updated_at") + + init { + index(false, userId) + uniqueIndex(userId, deviceId) + } + + override val primaryKey = PrimaryKey(id) +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationBaselinesTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationBaselinesTable.kt new file mode 100644 index 00000000..e3f696a7 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationBaselinesTable.kt @@ -0,0 +1,10 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object PushNotificationBaselinesTable : Table("push_notification_baselines") { + val userId = text("user_id") + val initializedAt = long("initialized_at") + + override val primaryKey = PrimaryKey(userId) +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationDeliveriesTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationDeliveriesTable.kt new file mode 100644 index 00000000..b94f3ac9 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationDeliveriesTable.kt @@ -0,0 +1,20 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object PushNotificationDeliveriesTable : Table("push_notification_deliveries") { + val eventId = text("event_id") + val userId = text("user_id") + val deviceId = text("device_id") + val status = text("status") + val attempts = integer("attempts") + val lastError = text("last_error").nullable() + val updatedAt = long("updated_at") + + init { + index(false, userId, status) + index(false, deviceId) + } + + override val primaryKey = PrimaryKey(eventId, userId, deviceId) +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationEventsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationEventsTable.kt new file mode 100644 index 00000000..7d14c8fc --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationEventsTable.kt @@ -0,0 +1,24 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object PushNotificationEventsTable : Table("push_notification_events") { + val eventId = text("event_id") + val eventType = text("event_type") + val instanceId = text("instance_id") + val serviceId = integer("service_id") + val channelId = text("channel_id") + val videoId = text("video_id") + val videoUrl = text("video_url") + val title = text("title") + val channelName = text("channel_name") + val channelAvatarUrl = text("channel_avatar_url") + val publishedAt = long("published_at") + val createdAt = long("created_at") + + init { + index(false, createdAt) + } + + override val primaryKey = PrimaryKey(eventId) +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationSeenVideosTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationSeenVideosTable.kt new file mode 100644 index 00000000..b4df2b01 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationSeenVideosTable.kt @@ -0,0 +1,17 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object PushNotificationSeenVideosTable : Table("push_notification_seen_videos") { + val userId = text("user_id") + val serviceId = integer("service_id") + val channelId = text("channel_id") + val videoId = text("video_id") + val firstSeenAt = long("first_seen_at") + + init { + index(false, userId) + } + + override val primaryKey = PrimaryKey(userId, serviceId, channelId, videoId) +} diff --git a/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt b/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt index 133514e6..dd9cf4b1 100644 --- a/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt +++ b/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt @@ -26,6 +26,7 @@ data class InstanceResponse( val youtubeRemoteLoginUnavailableReason: String? = null, val parentalControlsEnabled: Boolean = false, val rss: RssInstanceCapability = RssInstanceCapability(), + val pushNotifications: PushNotificationCapability = PushNotificationCapability(), ) @Serializable diff --git a/src/main/kotlin/dev/typetype/server/models/NotificationItem.kt b/src/main/kotlin/dev/typetype/server/models/NotificationItem.kt index 65a871eb..eb7bcde7 100644 --- a/src/main/kotlin/dev/typetype/server/models/NotificationItem.kt +++ b/src/main/kotlin/dev/typetype/server/models/NotificationItem.kt @@ -4,6 +4,7 @@ import kotlinx.serialization.Serializable @Serializable data class NotificationItem( + val id: String = "", val type: String, val title: String, val createdAt: Long, @@ -13,5 +14,6 @@ data class NotificationItem( val channelAvatarUrl: String, val serviceId: Int, val serviceName: String, + val read: Boolean = false, val video: VideoItem, ) diff --git a/src/main/kotlin/dev/typetype/server/models/PushNotificationModels.kt b/src/main/kotlin/dev/typetype/server/models/PushNotificationModels.kt new file mode 100644 index 00000000..3a3ee31b --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/PushNotificationModels.kt @@ -0,0 +1,57 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class PushNotificationCapability( + val enabled: Boolean = false, + val provider: String = "unifiedpush", + val eventTypes: List = emptyList(), + val maxDevicesPerAccount: Int = 0, +) + +@Serializable +data class PushDeviceRegistrationRequest( + val deviceId: String, + val platform: String = "android", + val endpoint: String, + val expiresAt: Long? = null, +) + +@Serializable +data class PushDeviceRegistrationResponse( + val id: String, + val deviceId: String, + val platform: String, + val expiresAt: Long? = null, + val updatedAt: Long, +) + +@Serializable +data class ChannelNotificationPreferenceRequest( + val channelUrl: String, + val enabled: Boolean, +) + +@Serializable +data class ChannelNotificationPreference( + val channelUrl: String, + val enabled: Boolean, + val updatedAt: Long, +) + +@Serializable +data class UnifiedPushNotificationPayload( + val version: Int = 1, + val eventType: String, + val eventId: String, + val videoId: String, + val videoUrl: String, + val channelId: String, + val channelName: String, + val channelAvatarUrl: String, + val instanceId: String, + val accountId: String, + val publishedAt: Long, + val title: String, +) diff --git a/src/main/kotlin/dev/typetype/server/routes/NotificationsRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/NotificationsRoutes.kt index 9c408ac1..50fe1477 100644 --- a/src/main/kotlin/dev/typetype/server/routes/NotificationsRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/NotificationsRoutes.kt @@ -29,4 +29,15 @@ fun Route.notificationsRoutes(notificationsService: NotificationsService, authSe call.respond(notificationsService.markAllRead(userId)) } } + + post("/notifications/{notificationId}/read") { + call.withJwtAuth(authService) { userId -> + val notificationId = call.parameters["notificationId"] + if (notificationId.isNullOrBlank()) { + call.respond(io.ktor.http.HttpStatusCode.BadRequest, dev.typetype.server.models.ErrorResponse("Missing notificationId")) + return@withJwtAuth + } + call.respond(notificationsService.markRead(userId, notificationId)) + } + } } diff --git a/src/main/kotlin/dev/typetype/server/routes/PushNotificationRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/PushNotificationRoutes.kt new file mode 100644 index 00000000..abd8ac9d --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/PushNotificationRoutes.kt @@ -0,0 +1,110 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ChannelNotificationPreferenceRequest +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.models.PushDeviceRegistrationRequest +import dev.typetype.server.services.DeviceRegistrationResult +import dev.typetype.server.services.PreferenceUpdateResult +import dev.typetype.server.services.PushNotificationService +import dev.typetype.server.services.AuthService +import io.ktor.http.HttpStatusCode +import io.ktor.server.request.receive +import io.ktor.server.response.respond +import io.ktor.server.routing.Route +import io.ktor.server.routing.delete +import io.ktor.server.routing.get +import io.ktor.server.routing.post +import io.ktor.server.routing.put + +internal fun Route.pushNotificationRoutes( + service: PushNotificationService, + authService: AuthService, +) { + get("/notifications/channel-preferences") { + call.withJwtAuth(authService) { userId -> + if (!call.requirePushEnabled(service)) return@withJwtAuth + call.respond(service.listPreferences(userId)) + } + } + put("/notifications/channel-preferences") { + call.withJwtAuth(authService) { userId -> + if (!call.requirePushEnabled(service)) return@withJwtAuth + val request = runCatching { call.receive() }.getOrNull() + ?: return@withJwtAuth call.respond( + HttpStatusCode.BadRequest, + ErrorResponse("Invalid channel notification preference", "push_request_invalid"), + ) + when (val result = service.setPreference(userId, request.channelUrl, request.enabled)) { + is PreferenceUpdateResult.Updated -> call.respond(result.preference) + PreferenceUpdateResult.NotSubscribed -> call.respond( + HttpStatusCode.Conflict, + ErrorResponse("Channel is not subscribed", "channel_not_subscribed"), + ) + } + } + } + get("/notifications/push/devices") { + call.withJwtAuth(authService) { userId -> + if (!call.requirePushEnabled(service)) return@withJwtAuth + call.respond(service.listDevices(userId)) + } + } + post("/notifications/push/devices") { + call.withJwtAuth(authService) { userId -> + if (!call.requirePushEnabled(service)) return@withJwtAuth + val request = runCatching { call.receive() }.getOrNull() + ?: return@withJwtAuth call.respond( + HttpStatusCode.BadRequest, + ErrorResponse("Invalid push device registration", "push_request_invalid"), + ) + when (val result = service.registerDevice(userId, request)) { + is DeviceRegistrationResult.Success -> call.respond(HttpStatusCode.Created, result.response) + is DeviceRegistrationResult.Invalid -> call.respond( + HttpStatusCode.UnprocessableEntity, + ErrorResponse("Invalid push device registration: ${result.reason}", result.errorCode()), + ) + DeviceRegistrationResult.UnsupportedPlatform -> call.respond( + HttpStatusCode.UnprocessableEntity, + ErrorResponse("Push platform is not supported", "push_platform_unsupported"), + ) + DeviceRegistrationResult.EndpointConflict -> call.respond( + HttpStatusCode.Conflict, + ErrorResponse("Push endpoint is already registered", "push_endpoint_conflict"), + ) + DeviceRegistrationResult.LimitReached -> call.respond( + HttpStatusCode.Conflict, + ErrorResponse("Push device limit reached", "push_device_limit_reached"), + ) + } + } + } + delete("/notifications/push/devices/{deviceId}") { + call.withJwtAuth(authService) { userId -> + if (!call.requirePushEnabled(service)) return@withJwtAuth + val deviceId = call.parameters["deviceId"]?.takeIf { it.isNotBlank() } + ?: return@withJwtAuth call.respond( + HttpStatusCode.BadRequest, + ErrorResponse("Missing deviceId", "push_request_invalid"), + ) + if (service.unregisterDevice(userId, deviceId)) { + call.respond(HttpStatusCode.NoContent) + } else { + call.respond(HttpStatusCode.NotFound, ErrorResponse("Push device not found", "push_device_not_found")) + } + } + } +} + +private fun DeviceRegistrationResult.Invalid.errorCode(): String = when (reason) { + "device_id" -> "push_device_id_invalid" + "expires_at" -> "push_expiry_invalid" + else -> "push_endpoint_invalid" +} + +private suspend fun io.ktor.server.application.ApplicationCall.requirePushEnabled( + service: PushNotificationService, +): Boolean { + if (service.capability.enabled) return true + respond(HttpStatusCode.NotFound, ErrorResponse("Push notifications are unavailable", "push_notifications_unavailable")) + return false +} diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt index 33aac871..1903cc99 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt @@ -9,6 +9,7 @@ import dev.typetype.server.services.NoopHomeRecommendationWarmup import dev.typetype.server.services.SubscriptionsService import dev.typetype.server.services.SubscriptionGroupsService import dev.typetype.server.services.SubscriptionSelection +import dev.typetype.server.services.PushNotificationService import io.ktor.http.HttpStatusCode import io.ktor.server.application.ApplicationCall import io.ktor.server.request.receive @@ -21,11 +22,12 @@ import io.ktor.server.routing.post import java.net.URLDecoder import java.nio.charset.StandardCharsets -fun Route.subscriptionsRoutes( +internal fun Route.subscriptionsRoutes( subscriptionsService: SubscriptionsService, authService: AuthService, warmupService: HomeRecommendationWarmup = NoopHomeRecommendationWarmup, groupsService: SubscriptionGroupsService = SubscriptionGroupsService(), + pushNotificationService: PushNotificationService? = null, ) { get("/subscriptions/group-memberships") { call.withJwtAuth(authService) { userId -> @@ -65,14 +67,14 @@ fun Route.subscriptionsRoutes( call.withJwtAuth(authService) { userId -> val channelUrl = call.request.queryParameters["url"]?.takeIf { it.isNotBlank() } ?: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing channelUrl")) - call.respondDeleteResult(subscriptionsService, warmupService, userId, channelUrl) + call.respondDeleteResult(subscriptionsService, warmupService, pushNotificationService, userId, channelUrl) } } delete("/subscriptions/{channelUrl...}") { call.withJwtAuth(authService) { userId -> val channelUrl = call.extractDeleteChannelUrl() ?: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing channelUrl")) - call.respondDeleteResult(subscriptionsService, warmupService, userId, channelUrl) + call.respondDeleteResult(subscriptionsService, warmupService, pushNotificationService, userId, channelUrl) } } } @@ -80,11 +82,15 @@ fun Route.subscriptionsRoutes( private suspend fun ApplicationCall.respondDeleteResult( subscriptionsService: SubscriptionsService, warmupService: HomeRecommendationWarmup, + pushNotificationService: PushNotificationService?, userId: String, channelUrl: String, ) { val deleted = subscriptionsService.delete(userId, channelUrl) - if (deleted) warmupService.invalidateAndWarm(userId) + if (deleted) { + warmupService.invalidateAndWarm(userId) + pushNotificationService?.onSubscriptionRemoved(userId, channelUrl) + } if (deleted) respond(HttpStatusCode.NoContent) else respond(HttpStatusCode.NotFound, ErrorResponse("Not found")) } diff --git a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt index 9da6de79..e5c98696 100644 --- a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt @@ -25,6 +25,7 @@ internal fun Route.userDataRoutes( authService, svc.homeRecommendationWarmupService, svc.subscriptionGroupsService, + svc.pushNotificationService, ) subscriptionFeedRoutes( svc.subscriptionFeedService, @@ -44,6 +45,7 @@ internal fun Route.userDataRoutes( allowedChannelsRoutes(svc.allowedChannelsService, authService) blockedRoutes(svc.blockedService, authService) notificationsRoutes(svc.notificationsService, authService) + pushNotificationRoutes(svc.pushNotificationService, authService) youtubeSessionRoutes(svc.youtubeSessionService, authService) youtubeTakeoutImportRoutes(svc.youtubeTakeoutImportService, authService) profileRoutes(profileService, avatarService, svc.customAvatarService, authService) diff --git a/src/main/kotlin/dev/typetype/server/services/ChannelNotificationPreferenceService.kt b/src/main/kotlin/dev/typetype/server/services/ChannelNotificationPreferenceService.kt new file mode 100644 index 00000000..10968e4e --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/ChannelNotificationPreferenceService.kt @@ -0,0 +1,82 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.ChannelNotificationPreferencesTable +import dev.typetype.server.models.ChannelNotificationPreference +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insertIgnore +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.update + +internal class ChannelNotificationPreferenceService( + private val subscriptionsService: SubscriptionsService, +) { + suspend fun list(userId: String): List { + val subscriptions = subscriptionsService.getAll(userId) + val urls = subscriptions.map { ChannelUrlCanonicalizer.canonicalize(it.channelUrl) } + if (urls.isEmpty()) return emptyList() + val stored = DatabaseFactory.query { + ChannelNotificationPreferencesTable.selectAll().where { + ChannelNotificationPreferencesTable.userId eq userId + }.associateBy { it[ChannelNotificationPreferencesTable.channelUrl] } + } + return urls.map { channelUrl -> + val row = stored[channelUrl] + ChannelNotificationPreference( + channelUrl, + row?.get(ChannelNotificationPreferencesTable.enabled) ?: true, + row?.get(ChannelNotificationPreferencesTable.updatedAt) ?: 0L, + ) + } + } + + suspend fun set(userId: String, rawChannelUrl: String, enabled: Boolean): PreferenceUpdateResult { + val channelUrl = ChannelUrlCanonicalizer.canonicalize(rawChannelUrl) + if (!isSubscribed(userId, channelUrl)) return PreferenceUpdateResult.NotSubscribed + val now = System.currentTimeMillis() + DatabaseFactory.query { + val inserted = ChannelNotificationPreferencesTable.insertIgnore { + it[ChannelNotificationPreferencesTable.userId] = userId + it[ChannelNotificationPreferencesTable.channelUrl] = channelUrl + it[ChannelNotificationPreferencesTable.enabled] = enabled + it[ChannelNotificationPreferencesTable.updatedAt] = now + }.insertedCount + if (inserted == 0) { + ChannelNotificationPreferencesTable.update({ + (ChannelNotificationPreferencesTable.userId eq userId) and + (ChannelNotificationPreferencesTable.channelUrl eq channelUrl) + }) { + it[ChannelNotificationPreferencesTable.enabled] = enabled + it[ChannelNotificationPreferencesTable.updatedAt] = now + } + } + } + return PreferenceUpdateResult.Updated(ChannelNotificationPreference(channelUrl, enabled, now)) + } + + suspend fun remove(userId: String, rawChannelUrl: String) { + val channelUrl = ChannelUrlCanonicalizer.canonicalize(rawChannelUrl) + DatabaseFactory.query { + ChannelNotificationPreferencesTable.deleteWhere { + (ChannelNotificationPreferencesTable.userId eq userId) and + (ChannelNotificationPreferencesTable.channelUrl eq channelUrl) + } + } + } + + suspend fun enabledChannelUrls(userId: String): Set = list(userId) + .filter(ChannelNotificationPreference::enabled) + .mapTo(mutableSetOf(), ChannelNotificationPreference::channelUrl) + + private suspend fun isSubscribed(userId: String, channelUrl: String): Boolean = + subscriptionsService.getAll(userId).any { + ChannelUrlCanonicalizer.canonicalize(it.channelUrl) == channelUrl + } +} + +internal sealed interface PreferenceUpdateResult { + data class Updated(val preference: ChannelNotificationPreference) : PreferenceUpdateResult + data object NotSubscribed : PreferenceUpdateResult +} diff --git a/src/main/kotlin/dev/typetype/server/services/InstanceService.kt b/src/main/kotlin/dev/typetype/server/services/InstanceService.kt index ab42ccb1..96456634 100644 --- a/src/main/kotlin/dev/typetype/server/services/InstanceService.kt +++ b/src/main/kotlin/dev/typetype/server/services/InstanceService.kt @@ -8,6 +8,7 @@ import dev.typetype.server.models.InstanceResponse import dev.typetype.server.models.OidcPublicConfig import dev.typetype.server.models.YoutubeRemoteLoginStatus import dev.typetype.server.models.RssInstanceCapability +import dev.typetype.server.models.PushNotificationCapability class InstanceService( private val authService: AuthService, @@ -21,6 +22,7 @@ class InstanceService( else -> YoutubeRemoteLoginStatus.NotConfigured } }, + private val pushNotificationCapabilityProvider: () -> PushNotificationCapability = { PushNotificationCapability() }, ) { suspend fun getInstance(): InstanceResponse { @@ -56,6 +58,7 @@ class InstanceService( minimumPollMinutes = settings.rssMinimumPollMinutes, rateLimitPerMinute = settings.rssRateLimitPerMinute, ), + pushNotifications = pushNotificationCapabilityProvider(), ) } diff --git a/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt b/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt index 66f5016d..b630c02e 100644 --- a/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt @@ -1,6 +1,7 @@ package dev.typetype.server.services import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.NotificationReadItemsTable import dev.typetype.server.db.tables.NotificationStatesTable import dev.typetype.server.models.MarkNotificationsReadResponse import dev.typetype.server.models.NotificationItem @@ -9,9 +10,11 @@ import dev.typetype.server.models.UnreadCountResponse import dev.typetype.server.models.VideoItem import java.util.concurrent.ConcurrentHashMap import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.insertIgnore import org.jetbrains.exposed.v1.jdbc.insert import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.update +import java.security.MessageDigest class NotificationsService( private val subscriptionFeedService: SubscriptionFeedService, @@ -20,15 +23,18 @@ class NotificationsService( suspend fun getNotifications(userId: String, page: Int, limit: Int): NotificationsResponse { val feed = loadFeed(userId) - val items = buildItems(feed.videos) - val unreadCount = if (feed.available) unreadCount(items, userId) else cachedUnread(userId) + if (!feed.available) { + return NotificationsResponse(emptyList(), cachedUnread(userId), null, false) + } + val items = withReadState(buildItems(feed.videos), userId) + val unreadCount = unreadCount(items, userId) val from = page * limit if (from >= items.size) { return NotificationsResponse(emptyList(), unreadCount, null, feed.available) } val to = minOf(from + limit, items.size) val nextpage = if (to < items.size) (page + 1).toString() else null - return NotificationsResponse(items.subList(from, to), unreadCount, nextpage, feed.available) + return NotificationsResponse(items.subList(from, to), unreadCount, nextpage, true) } suspend fun getUnreadCount(userId: String): UnreadCountResponse { @@ -37,7 +43,7 @@ class NotificationsService( if (cached != null && cached.expiresAt > now) return UnreadCountResponse(cached.value, true) val feed = loadFeed(userId) if (!feed.available) return UnreadCountResponse(cachedUnread(userId), false) - val value = unreadCount(buildItems(feed.videos), userId) + val value = unreadCount(withReadState(buildItems(feed.videos), userId), userId) return UnreadCountResponse(value, true) } @@ -46,9 +52,17 @@ class NotificationsService( if (!feed.available) { return MarkNotificationsReadResponse(System.currentTimeMillis(), cachedUnread(userId), false) } - val latestUploaded = buildItems(feed.videos).firstOrNull()?.createdAt ?: 0L + val items = buildItems(feed.videos) val now = System.currentTimeMillis() DatabaseFactory.query { + items.forEach { item -> + NotificationReadItemsTable.insertIgnore { + it[NotificationReadItemsTable.userId] = userId + it[notificationId] = item.id + it[readAt] = now + } + } + val latestUploaded = items.maxOfOrNull { it.createdAt } ?: 0L val updated = NotificationStatesTable.update({ NotificationStatesTable.userId eq userId }) { it[subscriptionLastSeenUploaded] = latestUploaded it[updatedAt] = now @@ -65,23 +79,54 @@ class NotificationsService( return MarkNotificationsReadResponse(now, 0, true) } + suspend fun markRead(userId: String, notificationId: String): MarkNotificationsReadResponse { + val feed = loadFeed(userId) + if (!feed.available) { + return MarkNotificationsReadResponse(System.currentTimeMillis(), cachedUnread(userId), false) + } + val items = buildItems(feed.videos) + val now = System.currentTimeMillis() + if (items.any { it.id == notificationId }) { + DatabaseFactory.query { + NotificationReadItemsTable.insertIgnore { + it[NotificationReadItemsTable.userId] = userId + it[NotificationReadItemsTable.notificationId] = notificationId + it[readAt] = now + } + } + } + val unread = unreadCount(withReadState(items, userId), userId) + return MarkNotificationsReadResponse(now, unread, true) + } + private suspend fun loadFeed(userId: String): SubscriptionFeedAvailability = runCatching { subscriptionFeedService.getAllWithAvailability(userId) } .getOrElse { SubscriptionFeedAvailability(emptyList(), false) } + private suspend fun withReadState(items: List, userId: String): List { + val readIds = DatabaseFactory.query { + NotificationReadItemsTable.selectAll().where { NotificationReadItemsTable.userId eq userId } + .mapTo(HashSet()) { it[NotificationReadItemsTable.notificationId] } + } + val legacyWatermark = DatabaseFactory.query { + NotificationStatesTable.selectAll().where { NotificationStatesTable.userId eq userId } + .singleOrNull()?.get(NotificationStatesTable.subscriptionLastSeenUploaded) ?: 0L + } + return items.map { item -> + item.copy(read = item.id in readIds || (readIds.isEmpty() && item.createdAt <= legacyWatermark)) + } + } + private fun buildItems(videos: List): List = videos.asSequence() - .filter { it.uploaded > 0L } - .distinctBy(::notificationKey) - .sortedByDescending { it.uploaded } + .map { video -> video to RssVideoMetadata.publishedAtMillis(video) } + .filter { (_, createdAt) -> createdAt > 0L } + .distinctBy { (video, _) -> notificationKey(video) } + .sortedByDescending { it.second } .map { it.toNotificationItem() } .toList() private suspend fun unreadCount(items: List, userId: String): Int { - val lastSeenUploaded = DatabaseFactory.query { - NotificationStatesTable.selectAll().where { NotificationStatesTable.userId eq userId } - .singleOrNull()?.get(NotificationStatesTable.subscriptionLastSeenUploaded) ?: 0L - } - val value = items.count { it.createdAt > lastSeenUploaded } + val value = items.count { !it.read } unreadCache[userId] = CachedUnread(value, System.currentTimeMillis() + UNREAD_CACHE_TTL_MS) return value } @@ -90,26 +135,36 @@ class NotificationsService( private fun notificationKey(video: VideoItem): String { val serviceId = RssVideoMetadata.serviceId(video) - val videoIdentity = video.url.ifBlank { video.id } - return "$serviceId:${video.uploaderUrl}:$videoIdentity" + val channelIdentity = video.uploaderUrl.trim().ifBlank { video.uploaderAvatarUrl.trim() } + .ifBlank { video.uploaderName.trim() } + val videoIdentity = video.url.trim().ifBlank { video.id.trim() } + .ifBlank { "${video.title.trim()}:${RssVideoMetadata.publishedAtMillis(video)}" } + return "$serviceId|$channelIdentity|$videoIdentity" } - private fun VideoItem.toNotificationItem(): NotificationItem { - val serviceId = RssVideoMetadata.serviceId(this) + private fun Pair.toNotificationItem(): NotificationItem { + val video = first + val createdAt = second + val serviceId = RssVideoMetadata.serviceId(video) return NotificationItem( + id = notificationId(video), type = "subscription_new_video", - title = "$uploaderName uploaded a new video", - createdAt = uploaded, - publishedAt = uploaded, - channelUrl = uploaderUrl, - channelName = uploaderName, - channelAvatarUrl = uploaderAvatarUrl, + title = "${video.uploaderName} uploaded a new video", + createdAt = createdAt, + publishedAt = createdAt, + channelUrl = video.uploaderUrl, + channelName = video.uploaderName, + channelAvatarUrl = video.uploaderAvatarUrl, serviceId = serviceId, serviceName = serviceName(serviceId), - video = this, + video = video, ) } + private fun notificationId(video: VideoItem): String = + MessageDigest.getInstance("SHA-256").digest(notificationKey(video).toByteArray()) + .joinToString("") { byte -> "%02x".format(byte) } + private data class CachedUnread(val value: Int, val expiresAt: Long) private companion object { diff --git a/src/main/kotlin/dev/typetype/server/services/ProfileDataDeletionService.kt b/src/main/kotlin/dev/typetype/server/services/ProfileDataDeletionService.kt index 6a418afc..9e286c6c 100644 --- a/src/main/kotlin/dev/typetype/server/services/ProfileDataDeletionService.kt +++ b/src/main/kotlin/dev/typetype/server/services/ProfileDataDeletionService.kt @@ -9,6 +9,12 @@ import dev.typetype.server.db.tables.BugReportsTable import dev.typetype.server.db.tables.FavoritesTable import dev.typetype.server.db.tables.HistoryTable import dev.typetype.server.db.tables.NotificationStatesTable +import dev.typetype.server.db.tables.NotificationReadItemsTable +import dev.typetype.server.db.tables.ChannelNotificationPreferencesTable +import dev.typetype.server.db.tables.PushDevicesTable +import dev.typetype.server.db.tables.PushNotificationBaselinesTable +import dev.typetype.server.db.tables.PushNotificationSeenVideosTable +import dev.typetype.server.db.tables.PushNotificationDeliveriesTable import dev.typetype.server.db.tables.PasswordResetTable import dev.typetype.server.db.tables.PlaylistVideosTable import dev.typetype.server.db.tables.PlaylistsTable @@ -82,6 +88,12 @@ internal object ProfileDataDeletionService { { BlockedKeywordsTable.deleteWhere { BlockedKeywordsTable.userId eq userId } }, { BlockedVideosTable.deleteWhere { BlockedVideosTable.userId eq userId } }, { NotificationStatesTable.deleteWhere { NotificationStatesTable.userId eq userId } }, + { NotificationReadItemsTable.deleteWhere { NotificationReadItemsTable.userId eq userId } }, + { PushNotificationDeliveriesTable.deleteWhere { PushNotificationDeliveriesTable.userId eq userId } }, + { PushNotificationSeenVideosTable.deleteWhere { PushNotificationSeenVideosTable.userId eq userId } }, + { PushNotificationBaselinesTable.deleteWhere { PushNotificationBaselinesTable.userId eq userId } }, + { PushDevicesTable.deleteWhere { PushDevicesTable.userId eq userId } }, + { ChannelNotificationPreferencesTable.deleteWhere { ChannelNotificationPreferencesTable.userId eq userId } }, { PasswordResetTable.deleteWhere { PasswordResetTable.userId eq userId } }, { SessionsTable.deleteWhere { SessionsTable.userId eq userId } }, { UserAvatarsTable.deleteWhere { UserAvatarsTable.userId eq userId } }, diff --git a/src/main/kotlin/dev/typetype/server/services/PushDeviceRegistry.kt b/src/main/kotlin/dev/typetype/server/services/PushDeviceRegistry.kt new file mode 100644 index 00000000..5ad17817 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/PushDeviceRegistry.kt @@ -0,0 +1,146 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.PushDevicesTable +import dev.typetype.server.models.PushDeviceRegistrationRequest +import dev.typetype.server.models.PushDeviceRegistrationResponse +import java.security.MessageDigest +import java.util.UUID +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.inList +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.update + +internal class PushDeviceRegistry( + private val endpointValidator: UnifiedPushEndpointValidator = UnifiedPushEndpointValidator(), + private val maxDevicesPerAccount: Int = DEFAULT_MAX_DEVICES, +) { + suspend fun register(userId: String, request: PushDeviceRegistrationRequest): DeviceRegistrationResult { + val deviceId = request.deviceId.trim() + val platform = request.platform.trim().lowercase() + val endpoint = request.endpoint.trim() + if (!DEVICE_ID.matches(deviceId)) return DeviceRegistrationResult.Invalid("device_id") + if (platform != "android") return DeviceRegistrationResult.UnsupportedPlatform + val expiresAt = request.expiresAt + val now = System.currentTimeMillis() + if (expiresAt != null && (expiresAt <= now || expiresAt > now + MAX_EXPIRY_MS)) { + return DeviceRegistrationResult.Invalid("expires_at") + } + val uri = when (val result = endpointValidator.validate(endpoint)) { + is EndpointValidationResult.Valid -> result.uri + is EndpointValidationResult.Invalid -> return DeviceRegistrationResult.Invalid(result.reason) + } + val hash = sha256(uri.toString()) + return DatabaseFactory.query { + removeExpired(userId, now) + val existingDevice = PushDevicesTable.selectAll().where { + (PushDevicesTable.userId eq userId) and (PushDevicesTable.deviceId eq deviceId) + }.singleOrNull() + val existingEndpoint = PushDevicesTable.selectAll().where { PushDevicesTable.endpointHash eq hash }.singleOrNull() + if (existingEndpoint != null && existingEndpoint[PushDevicesTable.id] != existingDevice?.get(PushDevicesTable.id)) { + return@query DeviceRegistrationResult.EndpointConflict + } + if (existingDevice == null) { + val activeCount = PushDevicesTable.selectAll().where { PushDevicesTable.userId eq userId }.count() + if (activeCount >= maxDevicesPerAccount) return@query DeviceRegistrationResult.LimitReached + val id = UUID.randomUUID().toString() + PushDevicesTable.insert { + it[PushDevicesTable.id] = id + it[PushDevicesTable.userId] = userId + it[PushDevicesTable.deviceId] = deviceId + it[PushDevicesTable.platform] = platform + it[PushDevicesTable.endpoint] = uri.toString() + it[PushDevicesTable.endpointHash] = hash + it[PushDevicesTable.expiresAt] = expiresAt + it[PushDevicesTable.createdAt] = now + it[PushDevicesTable.updatedAt] = now + } + DeviceRegistrationResult.Success(PushDeviceRegistrationResponse(id, deviceId, platform, expiresAt, now)) + } else { + val id = existingDevice[PushDevicesTable.id] + PushDevicesTable.update({ PushDevicesTable.id eq id }) { + it[PushDevicesTable.platform] = platform + it[PushDevicesTable.endpoint] = uri.toString() + it[PushDevicesTable.endpointHash] = hash + it[PushDevicesTable.expiresAt] = expiresAt + it[PushDevicesTable.updatedAt] = now + } + DeviceRegistrationResult.Success(PushDeviceRegistrationResponse(id, deviceId, platform, expiresAt, now)) + } + } + } + + suspend fun unregister(userId: String, deviceId: String): Boolean = DatabaseFactory.query { + PushDevicesTable.deleteWhere { + (PushDevicesTable.userId eq userId) and (PushDevicesTable.deviceId eq deviceId) + } > 0 + } + + suspend fun list(userId: String): List = DatabaseFactory.query { + val now = System.currentTimeMillis() + removeExpired(userId, now) + PushDevicesTable.selectAll().where { PushDevicesTable.userId eq userId } + .map { row -> + PushDeviceRegistrationResponse( + id = row[PushDevicesTable.id], + deviceId = row[PushDevicesTable.deviceId], + platform = row[PushDevicesTable.platform], + expiresAt = row[PushDevicesTable.expiresAt], + updatedAt = row[PushDevicesTable.updatedAt], + ) + } + } + + suspend fun removeById(id: String): Boolean = DatabaseFactory.query { + PushDevicesTable.deleteWhere { PushDevicesTable.id eq id } > 0 + } + + suspend fun activeDevices(userId: String, now: Long = System.currentTimeMillis()): List = DatabaseFactory.query { + removeExpired(userId, now) + PushDevicesTable.selectAll().where { PushDevicesTable.userId eq userId }.map { row -> + PushDevice( + id = row[PushDevicesTable.id], + userId = row[PushDevicesTable.userId], + endpoint = row[PushDevicesTable.endpoint], + ) + } + } + + suspend fun registeredUserIds(): List = DatabaseFactory.query { + PushDevicesTable.selectAll().map { it[PushDevicesTable.userId] }.distinct() + } + + private fun sha256(value: String): String = MessageDigest.getInstance("SHA-256") + .digest(value.toByteArray()) + .joinToString("") { byte -> "%02x".format(byte) } + + private fun removeExpired(userId: String, now: Long) { + val expiredIds = PushDevicesTable.selectAll().where { + PushDevicesTable.userId eq userId + }.mapNotNull { row -> + row[PushDevicesTable.expiresAt]?.takeIf { it <= now }?.let { row[PushDevicesTable.id] } + } + if (expiredIds.isNotEmpty()) { + PushDevicesTable.deleteWhere { PushDevicesTable.id inList expiredIds } + } + } + + companion object { + const val DEFAULT_MAX_DEVICES = 8 + private const val MAX_EXPIRY_MS = 366L * 24 * 60 * 60 * 1000 + private val DEVICE_ID = Regex("[A-Za-z0-9._:-]{1,128}") + } +} + +internal data class PushDevice(val id: String, val userId: String, val endpoint: String) + +internal sealed interface DeviceRegistrationResult { + data class Success(val response: PushDeviceRegistrationResponse) : DeviceRegistrationResult + data class Invalid(val reason: String) : DeviceRegistrationResult + data object UnsupportedPlatform : DeviceRegistrationResult + data object EndpointConflict : DeviceRegistrationResult + data object LimitReached : DeviceRegistrationResult +} diff --git a/src/main/kotlin/dev/typetype/server/services/PushNotificationDeliveryStore.kt b/src/main/kotlin/dev/typetype/server/services/PushNotificationDeliveryStore.kt new file mode 100644 index 00000000..5d5315e3 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/PushNotificationDeliveryStore.kt @@ -0,0 +1,106 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.PushNotificationDeliveriesTable +import dev.typetype.server.db.tables.PushNotificationEventsTable +import dev.typetype.server.db.tables.PushNotificationSeenVideosTable +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.inList +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.insertIgnore +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.update + +internal class PushNotificationDeliveryStore( + private val clock: () -> Long, +) { + fun markSeen(userId: String, candidate: PushCandidate, now: Long): Boolean = + PushNotificationSeenVideosTable.insertIgnore { + it[PushNotificationSeenVideosTable.userId] = userId + it[PushNotificationSeenVideosTable.serviceId] = candidate.serviceId + it[PushNotificationSeenVideosTable.channelId] = candidate.channelId + it[PushNotificationSeenVideosTable.videoId] = candidate.videoId + it[PushNotificationSeenVideosTable.firstSeenAt] = now + }.insertedCount == 1 + + fun claim(eventId: String, userId: String, deviceId: String, createIfMissing: Boolean, now: Long): Boolean { + val existing = PushNotificationDeliveriesTable.selectAll().where { + (PushNotificationDeliveriesTable.eventId eq eventId) and + (PushNotificationDeliveriesTable.userId eq userId) and + (PushNotificationDeliveriesTable.deviceId eq deviceId) + }.singleOrNull() + if (existing == null) { + if (!createIfMissing) return false + return PushNotificationDeliveriesTable.insertIgnore { + it[PushNotificationDeliveriesTable.eventId] = eventId + it[PushNotificationDeliveriesTable.userId] = userId + it[PushNotificationDeliveriesTable.deviceId] = deviceId + it[PushNotificationDeliveriesTable.status] = PushNotificationDeliveryStatus.PENDING + it[PushNotificationDeliveriesTable.attempts] = 1 + it[PushNotificationDeliveriesTable.updatedAt] = now + }.insertedCount == 1 + } + val attempts = existing[PushNotificationDeliveriesTable.attempts] + val status = existing[PushNotificationDeliveriesTable.status] + if (status !in PushNotificationDeliveryStatus.RETRYABLE || attempts >= MAX_ATTEMPTS || + existing[PushNotificationDeliveriesTable.updatedAt] > now - RETRY_DELAY_MS + ) return false + return PushNotificationDeliveriesTable.update({ + (PushNotificationDeliveriesTable.eventId eq eventId) and + (PushNotificationDeliveriesTable.userId eq userId) and + (PushNotificationDeliveriesTable.deviceId eq deviceId) and + (PushNotificationDeliveriesTable.status inList PushNotificationDeliveryStatus.RETRYABLE) and + (PushNotificationDeliveriesTable.attempts eq attempts) and + (PushNotificationDeliveriesTable.updatedAt eq existing[PushNotificationDeliveriesTable.updatedAt]) + }) { + it[PushNotificationDeliveriesTable.status] = PushNotificationDeliveryStatus.PENDING + it[PushNotificationDeliveriesTable.attempts] = attempts + 1 + it[PushNotificationDeliveriesTable.lastError] = null + it[PushNotificationDeliveriesTable.updatedAt] = now + } == 1 + } + + suspend fun update(work: DeliveryWork, status: String, error: String?) { + DatabaseFactory.query { + PushNotificationDeliveriesTable.update({ + (PushNotificationDeliveriesTable.eventId eq work.eventId) and + (PushNotificationDeliveriesTable.userId eq work.device.userId) and + (PushNotificationDeliveriesTable.deviceId eq work.device.id) + }) { + it[PushNotificationDeliveriesTable.status] = status + it[PushNotificationDeliveriesTable.lastError] = error + it[PushNotificationDeliveriesTable.updatedAt] = clock() + } + } + } + + suspend fun clearPending(userId: String, channelId: String) { + val eventIds = DatabaseFactory.query { + PushNotificationEventsTable.selectAll().where { PushNotificationEventsTable.channelId eq channelId } + .mapTo(mutableSetOf()) { it[PushNotificationEventsTable.eventId] } + } + if (eventIds.isEmpty()) return + DatabaseFactory.query { + PushNotificationDeliveriesTable.deleteWhere { + (PushNotificationDeliveriesTable.userId eq userId) and + (PushNotificationDeliveriesTable.eventId inList eventIds) and + (PushNotificationDeliveriesTable.status inList PushNotificationDeliveryStatus.RETRYABLE) + } + } + } + + companion object { + private const val MAX_ATTEMPTS = 3 + private const val RETRY_DELAY_MS = 30_000L + } +} + +internal object PushNotificationDeliveryStatus { + const val PENDING = "pending" + const val DELIVERED = "delivered" + const val INVALID = "invalid" + const val FAILED = "failed" + val RETRYABLE = listOf(PENDING, FAILED) +} diff --git a/src/main/kotlin/dev/typetype/server/services/PushNotificationScheduler.kt b/src/main/kotlin/dev/typetype/server/services/PushNotificationScheduler.kt new file mode 100644 index 00000000..e82a5285 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/PushNotificationScheduler.kt @@ -0,0 +1,40 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.delay +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch + +internal class PushNotificationScheduler( + private val service: PushNotificationService, + private val intervalMs: Long = configuredIntervalMs(), + private val initialDelayMs: Long = 10_000L, +) { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + private var job: Job? = null + + fun start() { + if (job?.isActive == true) return + job = scope.launch { + delay(initialDelayMs) + while (isActive) { + service.pollRegisteredAccounts() + delay(intervalMs) + } + } + } + + fun close() { + scope.cancel() + } + + private companion object { + fun configuredIntervalMs(): Long = + (System.getenv("TYPE_TYPE_PUSH_NOTIFICATIONS_INTERVAL_SECONDS")?.toLongOrNull() ?: 300L) + .coerceIn(30L, 86_400L) * 1_000L + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/PushNotificationService.kt b/src/main/kotlin/dev/typetype/server/services/PushNotificationService.kt new file mode 100644 index 00000000..dd0ad996 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/PushNotificationService.kt @@ -0,0 +1,164 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.PushNotificationBaselinesTable +import dev.typetype.server.db.tables.PushNotificationEventsTable +import dev.typetype.server.models.ChannelNotificationPreference +import dev.typetype.server.models.PushDeviceRegistrationRequest +import dev.typetype.server.models.PushNotificationCapability +import dev.typetype.server.models.VideoItem +import kotlinx.coroutines.CancellationException +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.insertIgnore +import org.jetbrains.exposed.v1.jdbc.selectAll + +internal class PushNotificationService( + private val subscriptionsService: SubscriptionsService, + private val subscriptionFeedService: SubscriptionFeedService, + private val preferenceService: ChannelNotificationPreferenceService, + private val instanceId: String, + private val enabled: Boolean = true, + private val deviceRegistry: PushDeviceRegistry = PushDeviceRegistry(), + private val sender: PushEndpointSender = UnifiedPushSender(), + private val clock: () -> Long = System::currentTimeMillis, +) { + private val deliveryStore = PushNotificationDeliveryStore(clock) + + val capability: PushNotificationCapability = PushNotificationCapability( + enabled = enabled, + provider = "unifiedpush", + eventTypes = listOf(EVENT_TYPE_NEW_VIDEO), + maxDevicesPerAccount = PushDeviceRegistry.DEFAULT_MAX_DEVICES, + ) + + suspend fun registerDevice(userId: String, request: PushDeviceRegistrationRequest) = + deviceRegistry.register(userId, request) + + suspend fun listDevices(userId: String) = deviceRegistry.list(userId) + + suspend fun unregisterDevice(userId: String, deviceId: String): Boolean = + deviceRegistry.unregister(userId, deviceId) + + suspend fun listPreferences(userId: String): List = + preferenceService.list(userId) + + suspend fun setPreference(userId: String, channelUrl: String, value: Boolean): PreferenceUpdateResult { + val result = preferenceService.set(userId, channelUrl, value) + if (result is PreferenceUpdateResult.Updated && !value) { + deliveryStore.clearPending(userId, result.preference.channelUrl) + } + return result + } + + suspend fun onSubscriptionRemoved(userId: String, channelUrl: String) { + val canonical = ChannelUrlCanonicalizer.canonicalize(channelUrl) + preferenceService.remove(userId, canonical) + deliveryStore.clearPending(userId, canonical) + } + + suspend fun pollRegisteredAccounts() { + if (!enabled) return + try { + deviceRegistry.registeredUserIds().forEach { userId -> + try { + pollUser(userId) + } catch (error: CancellationException) { + throw error + } catch (error: Throwable) { + logger.warn("push_notifications event=poll_failed user={} error={}", userKey(userId), error.message) + } + } + } catch (error: CancellationException) { + throw error + } catch (error: Throwable) { + logger.warn("push_notifications event=registry_failed error={}", error.message) + } + } + + internal suspend fun pollUser(userId: String) { + val subscriptions = subscriptionsService.getAll(userId) + if (subscriptions.isEmpty()) return + val devices = deviceRegistry.activeDevices(userId, clock()) + if (devices.isEmpty()) return + val feed = subscriptionFeedService.getAllWithSources(userId) + if (!feed.available) return + val followedSince = subscriptions.associate { + ChannelUrlCanonicalizer.canonicalize(it.channelUrl) to it.subscribedAt + } + val enabledChannels = preferenceService.enabledChannelUrls(userId) + val candidates = feed.videos.flatMap { video -> + PushNotificationSupport.candidates(video, feed.sourceChannelUrls, followedSince) + } + .distinctBy { it.serviceId to (it.channelId to it.videoId) } + val work = DatabaseFactory.query { + val baseline = PushNotificationBaselinesTable.selectAll() + .where { PushNotificationBaselinesTable.userId eq userId }.singleOrNull() + if (baseline == null) { + PushNotificationBaselinesTable.insert { + it[PushNotificationBaselinesTable.userId] = userId + it[PushNotificationBaselinesTable.initializedAt] = clock() + } + candidates.forEach { deliveryStore.markSeen(userId, it, clock()) } + return@query emptyList() + } + candidates.mapNotNull candidateLoop@{ candidate -> + val isNew = deliveryStore.markSeen(userId, candidate, clock()) + if (candidate.channelId !in enabledChannels) return@candidateLoop null + val eventId = PushNotificationSupport.eventId(instanceId, candidate) + PushNotificationEventsTable.insertIgnore { + it[PushNotificationEventsTable.eventId] = eventId + it[PushNotificationEventsTable.eventType] = EVENT_TYPE_NEW_VIDEO + it[PushNotificationEventsTable.instanceId] = this@PushNotificationService.instanceId + it[PushNotificationEventsTable.serviceId] = candidate.serviceId + it[PushNotificationEventsTable.channelId] = candidate.channelId + it[PushNotificationEventsTable.videoId] = candidate.videoId + it[PushNotificationEventsTable.videoUrl] = candidate.video.url + it[PushNotificationEventsTable.title] = candidate.video.title + it[PushNotificationEventsTable.channelName] = candidate.video.uploaderName + it[PushNotificationEventsTable.channelAvatarUrl] = candidate.video.uploaderAvatarUrl + it[PushNotificationEventsTable.publishedAt] = candidate.publishedAt + it[PushNotificationEventsTable.createdAt] = clock() + } + devices.mapNotNull { device -> + if (deliveryStore.claim(eventId, userId, device.id, isNew, clock())) { + DeliveryWork(eventId, device, PushNotificationSupport.payload(instanceId, eventId, candidate, userId)) + } else null + } + }.flatten() + } + work.forEach { delivery -> deliver(delivery) } + } + + private suspend fun deliver(work: DeliveryWork) { + when (val result = sender.send(work.device.endpoint, work.eventId, work.payload)) { + PushSendResult.Delivered -> deliveryStore.update(work, PushNotificationDeliveryStatus.DELIVERED, null) + PushSendResult.InvalidEndpoint -> { + deviceRegistry.removeById(work.device.id) + deliveryStore.update(work, PushNotificationDeliveryStatus.INVALID, "endpoint_invalid") + } + is PushSendResult.Retry -> deliveryStore.update( + work, + PushNotificationDeliveryStatus.FAILED, + result.statusCode?.toString() ?: "network", + ) + } + } + + private fun userKey(userId: String): String = userId.take(8) + + private companion object { + const val EVENT_TYPE_NEW_VIDEO = "subscription_new_video" + val logger = org.slf4j.LoggerFactory.getLogger(PushNotificationService::class.java) + } +} + +internal data class PushCandidate( + val serviceId: Int, + val channelId: String, + val videoId: String, + val publishedAt: Long, + val video: VideoItem, +) + +internal data class DeliveryWork(val eventId: String, val device: PushDevice, val payload: String) diff --git a/src/main/kotlin/dev/typetype/server/services/PushNotificationSupport.kt b/src/main/kotlin/dev/typetype/server/services/PushNotificationSupport.kt new file mode 100644 index 00000000..060c1cc3 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/PushNotificationSupport.kt @@ -0,0 +1,52 @@ +package dev.typetype.server.services + +import dev.typetype.server.cache.CacheJson +import dev.typetype.server.models.UnifiedPushNotificationPayload +import dev.typetype.server.models.VideoItem +import java.security.MessageDigest + +internal object PushNotificationSupport { + fun candidates( + video: VideoItem, + sources: Map>, + followedSince: Map, + ): List { + if (video.isShortFormContent || video.isLiveContent || video.isPostLive || video.isLive) return emptyList() + val channelUrls = sources[video.subscriptionFeedKey()].orEmpty().ifEmpty { listOf(video.uploaderUrl) } + val publishedAt = RssVideoMetadata.publishedAtMillis(video) + if (publishedAt <= 0L) return emptyList() + val videoId = video.id.trim().ifBlank { video.url.trim() } + if (videoId.isBlank()) return emptyList() + return channelUrls.map { ChannelUrlCanonicalizer.canonicalize(it) } + .filter { channel -> + val subscribedAt = followedSince[channel] ?: return@filter false + subscribedAt <= 0L || publishedAt >= subscribedAt + } + .distinct() + .map { channel -> PushCandidate(RssVideoMetadata.serviceId(video), channel, videoId, publishedAt, video) } + } + + fun payload(instanceId: String, eventId: String, candidate: PushCandidate, userId: String): String = CacheJson.encodeToString( + UnifiedPushNotificationPayload.serializer(), + UnifiedPushNotificationPayload( + eventType = "subscription_new_video", + eventId = eventId, + videoId = candidate.videoId, + videoUrl = candidate.video.url, + channelId = candidate.channelId, + channelName = candidate.video.uploaderName, + channelAvatarUrl = candidate.video.uploaderAvatarUrl, + instanceId = instanceId, + accountId = userId, + publishedAt = candidate.publishedAt, + title = candidate.video.title, + ), + ) + + fun eventId(instanceId: String, candidate: PushCandidate): String = sha256( + "$instanceId|subscription_new_video|${candidate.serviceId}|${candidate.channelId}|${candidate.videoId}", + ) + + private fun sha256(value: String): String = MessageDigest.getInstance("SHA-256") + .digest(value.toByteArray()).joinToString("") { byte -> "%02x".format(byte) } +} diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedAvailability.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedAvailability.kt new file mode 100644 index 00000000..e29e0beb --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedAvailability.kt @@ -0,0 +1,14 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.VideoItem + +data class SubscriptionFeedAvailability( + val videos: List, + val available: Boolean, +) + +internal data class SubscriptionFeedWithSources( + val videos: List, + val available: Boolean, + val sourceChannelUrls: Map>, +) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt index 21501bef..8dffcf16 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt @@ -121,6 +121,9 @@ class SubscriptionFeedService( ) } + internal suspend fun getAllWithSources(userId: String): SubscriptionFeedWithSources = + getAllWithAvailability(userId).let { SubscriptionFeedWithSources(it.videos, it.available, store.current(userId)?.sourceChannelUrls.orEmpty()) } + suspend fun getCachedFeed(userId: String, page: Int, limit: Int): SubscriptionFeedResponse? { val snapshot = store.current(userId) ?: return null return snapshot.page(page * limit, limit, isRefreshing(userId)) @@ -214,8 +217,3 @@ class SubscriptionFeedService( private val logger = LoggerFactory.getLogger(SubscriptionFeedService::class.java) } } - -data class SubscriptionFeedAvailability( - val videos: List, - val available: Boolean, -) diff --git a/src/main/kotlin/dev/typetype/server/services/UnifiedPushEndpointValidator.kt b/src/main/kotlin/dev/typetype/server/services/UnifiedPushEndpointValidator.kt new file mode 100644 index 00000000..4b59038e --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/UnifiedPushEndpointValidator.kt @@ -0,0 +1,67 @@ +package dev.typetype.server.services + +import java.net.InetAddress +import java.net.URI +import java.net.UnknownHostException + +internal class UnifiedPushEndpointValidator( + private val resolver: (String) -> Array = InetAddress::getAllByName, +) { + fun validate(raw: String): EndpointValidationResult { + if (raw.length !in 1..2048) return EndpointValidationResult.Invalid("endpoint_length") + val uri = runCatching { URI(raw) }.getOrNull() + ?: return EndpointValidationResult.Invalid("endpoint_uri") + if (uri.scheme?.lowercase() != "https" || uri.userInfo != null || uri.fragment != null) { + return EndpointValidationResult.Invalid("endpoint_scheme") + } + val host = uri.host?.trim()?.trim('[', ']')?.lowercase()?.takeIf { it.isNotBlank() } + ?: return EndpointValidationResult.Invalid("endpoint_host") + if (uri.port == 0 || uri.port < -1 || uri.port > 65535) { + return EndpointValidationResult.Invalid("endpoint_port") + } + val addresses = try { + resolver(host).toList() + } catch (_: UnknownHostException) { + return EndpointValidationResult.Invalid("endpoint_unresolvable") + } catch (_: Exception) { + return EndpointValidationResult.Invalid("endpoint_unresolvable") + } + if (addresses.isEmpty() || addresses.any(::isBlockedAddress)) { + return EndpointValidationResult.Invalid("endpoint_private_address") + } + return EndpointValidationResult.Valid(uri, addresses) + } + + private fun isBlockedAddress(address: InetAddress): Boolean { + if (address.isAnyLocalAddress || address.isLoopbackAddress || address.isLinkLocalAddress || + address.isSiteLocalAddress || address.isMulticastAddress + ) return true + val bytes = address.address + if (bytes.size == 16 && bytes.take(10).all { it == 0.toByte() } && bytes[10] == 0xff.toByte() && bytes[11] == 0xff.toByte()) { + return isBlockedIpv4(bytes.copyOfRange(12, 16)) + } + if (bytes.size == 16) { + val first = bytes[0].toInt() and 0xff + return first in 0xfc..0xfd + } + return bytes.size == 4 && isBlockedIpv4(bytes) + } + + private fun isBlockedIpv4(bytes: ByteArray): Boolean { + val first = bytes[0].toInt() and 0xff + val second = bytes[1].toInt() and 0xff + return first == 0 || first == 10 || first == 127 || + (first == 100 && second in 64..127) || + (first == 169 && second == 254) || + (first == 172 && second in 16..31) || + (first == 192 && second == 0) || + (first == 192 && second == 168) || + (first == 198 && second in 18..19) || + first >= 224 + } +} + +internal sealed interface EndpointValidationResult { + data class Valid(val uri: URI, val addresses: List) : EndpointValidationResult + data class Invalid(val reason: String) : EndpointValidationResult +} diff --git a/src/main/kotlin/dev/typetype/server/services/UnifiedPushSender.kt b/src/main/kotlin/dev/typetype/server/services/UnifiedPushSender.kt new file mode 100644 index 00000000..ac0a1d74 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/UnifiedPushSender.kt @@ -0,0 +1,68 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody +import java.net.InetAddress +import java.net.UnknownHostException + +internal class UnifiedPushSender( + private val endpointValidator: UnifiedPushEndpointValidator = UnifiedPushEndpointValidator(), + client: OkHttpClient = OkHttpClient(), +) : PushEndpointSender { + private val client = client.newBuilder() + .followRedirects(false) + .followSslRedirects(false) + .build() + + override suspend fun send(endpoint: String, eventId: String, payload: String): PushSendResult = withContext(Dispatchers.IO) { + val validated = endpointValidator.validate(endpoint) + if (validated !is EndpointValidationResult.Valid) return@withContext PushSendResult.InvalidEndpoint + val request = Request.Builder() + .url(validated.uri.toString()) + .header("Content-Type", JSON_MEDIA_TYPE.toString()) + .header("X-TypeType-Event-Id", eventId) + .post(payload.toRequestBody(JSON_MEDIA_TYPE)) + .build() + val requestClient = client.newBuilder() + .dns(StaticPushDns(validated.uri.host.orEmpty(), validated.addresses)) + .build() + runCatching { requestClient.newCall(request).execute().use { response -> response.code } } + .fold( + onSuccess = { code -> + when { + code in 200..299 -> PushSendResult.Delivered + code == 404 || code == 410 -> PushSendResult.InvalidEndpoint + else -> PushSendResult.Retry(code) + } + }, + onFailure = { PushSendResult.Retry(null) }, + ) + } + + private class StaticPushDns( + private val validatedHost: String, + private val addresses: List, + ) : okhttp3.Dns { + override fun lookup(hostname: String): List = + if (hostname.equals(validatedHost, ignoreCase = true)) addresses + else throw UnknownHostException("Unexpected push endpoint host") + } + + private companion object { + val JSON_MEDIA_TYPE = "application/json".toMediaType() + } +} + +internal interface PushEndpointSender { + suspend fun send(endpoint: String, eventId: String, payload: String): PushSendResult +} + +internal sealed interface PushSendResult { + data object Delivered : PushSendResult + data object InvalidEndpoint : PushSendResult + data class Retry(val statusCode: Int?) : PushSendResult +} diff --git a/src/test/kotlin/dev/typetype/server/PushDeviceRegistryTest.kt b/src/test/kotlin/dev/typetype/server/PushDeviceRegistryTest.kt new file mode 100644 index 00000000..0e51f6a9 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/PushDeviceRegistryTest.kt @@ -0,0 +1,45 @@ +package dev.typetype.server + +import dev.typetype.server.models.PushDeviceRegistrationRequest +import dev.typetype.server.services.DeviceRegistrationResult +import dev.typetype.server.services.EndpointValidationResult +import dev.typetype.server.services.PushDeviceRegistry +import dev.typetype.server.services.UnifiedPushEndpointValidator +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import java.net.InetAddress + +class PushDeviceRegistryTest { + private val validator = UnifiedPushEndpointValidator { + arrayOf(InetAddress.getByAddress(byteArrayOf(93.toByte(), 184.toByte(), 216.toByte(), 34))) + } + private val registry = PushDeviceRegistry(validator) + + companion object { + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() = TestDatabase.truncateAll() + + @Test + fun `registration replaces one device and prevents endpoint sharing`() = runTest { + val first = registry.register("user-a", request("device-a", "https://push.example/a")) + assertTrue(first is DeviceRegistrationResult.Success) + val replacement = registry.register("user-a", request("device-a", "https://push.example/b")) + assertTrue(replacement is DeviceRegistrationResult.Success) + assertEquals(1, registry.list("user-a").size) + assertEquals( + DeviceRegistrationResult.EndpointConflict, + registry.register("user-b", request("device-b", "https://push.example/b")), + ) + } + + private fun request(deviceId: String, endpoint: String) = PushDeviceRegistrationRequest(deviceId, endpoint = endpoint) +} diff --git a/src/test/kotlin/dev/typetype/server/PushNotificationDeliveryStoreTest.kt b/src/test/kotlin/dev/typetype/server/PushNotificationDeliveryStoreTest.kt new file mode 100644 index 00000000..a5038b89 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/PushNotificationDeliveryStoreTest.kt @@ -0,0 +1,31 @@ +package dev.typetype.server + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.services.PushNotificationDeliveryStore +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class PushNotificationDeliveryStoreTest { + private val store = PushNotificationDeliveryStore { 1_000L } + + companion object { + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() = TestDatabase.truncateAll() + + @Test + fun `does not create historical delivery and retries an existing pending delivery`() = runTest { + assertFalse(DatabaseFactory.query { store.claim("event", "user", "device", false, 1_000L) }) + assertTrue(DatabaseFactory.query { store.claim("event", "user", "device", true, 1_000L) }) + assertFalse(DatabaseFactory.query { store.claim("event", "user", "device", false, 1_001L) }) + assertTrue(DatabaseFactory.query { store.claim("event", "user", "device", false, 31_001L) }) + } +} diff --git a/src/test/kotlin/dev/typetype/server/TestDatabase.kt b/src/test/kotlin/dev/typetype/server/TestDatabase.kt index 92da90a8..de24cde7 100644 --- a/src/test/kotlin/dev/typetype/server/TestDatabase.kt +++ b/src/test/kotlin/dev/typetype/server/TestDatabase.kt @@ -20,6 +20,13 @@ import dev.typetype.server.db.tables.RssFeedsTable import dev.typetype.server.db.tables.RssUserPoliciesTable import dev.typetype.server.db.tables.SavedPlaylistsTable import dev.typetype.server.db.tables.NotificationStatesTable +import dev.typetype.server.db.tables.NotificationReadItemsTable +import dev.typetype.server.db.tables.ChannelNotificationPreferencesTable +import dev.typetype.server.db.tables.PushDevicesTable +import dev.typetype.server.db.tables.PushNotificationBaselinesTable +import dev.typetype.server.db.tables.PushNotificationSeenVideosTable +import dev.typetype.server.db.tables.PushNotificationEventsTable +import dev.typetype.server.db.tables.PushNotificationDeliveriesTable import dev.typetype.server.db.tables.SearchHistoryTable import dev.typetype.server.db.tables.SettingsTable import dev.typetype.server.db.tables.SessionsTable @@ -134,6 +141,13 @@ object TestDatabase { YoutubeSessionsTable.deleteAll() BugReportsTable.deleteAll() NotificationStatesTable.deleteAll() + NotificationReadItemsTable.deleteAll() + PushNotificationDeliveriesTable.deleteAll() + PushNotificationEventsTable.deleteAll() + PushNotificationSeenVideosTable.deleteAll() + PushNotificationBaselinesTable.deleteAll() + PushDevicesTable.deleteAll() + ChannelNotificationPreferencesTable.deleteAll() UserChannelInterestTable.deleteAll() UserTopicInterestTable.deleteAll() RecommendationEventsTable.deleteAll() diff --git a/src/test/kotlin/dev/typetype/server/UnifiedPushEndpointValidatorTest.kt b/src/test/kotlin/dev/typetype/server/UnifiedPushEndpointValidatorTest.kt new file mode 100644 index 00000000..a09b8279 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/UnifiedPushEndpointValidatorTest.kt @@ -0,0 +1,32 @@ +package dev.typetype.server + +import dev.typetype.server.services.EndpointValidationResult +import dev.typetype.server.services.UnifiedPushEndpointValidator +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.net.InetAddress + +class UnifiedPushEndpointValidatorTest { + private val publicAddress = InetAddress.getByAddress(byteArrayOf(93.toByte(), 184.toByte(), 216.toByte(), 34)) + private val resolver = UnifiedPushEndpointValidator { arrayOf(publicAddress) } + + @Test + fun `accepts https endpoint and rejects non https forms`() { + assertTrue(resolver.validate("https://push.example.invalid/endpoint?id=1") is EndpointValidationResult.Valid) + assertEquals("endpoint_scheme", reason(resolver.validate("http://push.example.invalid/endpoint"))) + assertEquals("endpoint_scheme", reason(resolver.validate("https://user:pass@push.example.invalid/endpoint"))) + assertEquals("endpoint_scheme", reason(resolver.validate("https://push.example.invalid/endpoint#fragment"))) + } + + @Test + fun `rejects an address that resolves into a private network`() { + val privateValidator = UnifiedPushEndpointValidator { + arrayOf(InetAddress.getByAddress(byteArrayOf(10, 0, 0, 8))) + } + assertEquals("endpoint_private_address", reason(privateValidator.validate("https://push.example.invalid/endpoint"))) + } + + private fun reason(result: EndpointValidationResult): String = + (result as EndpointValidationResult.Invalid).reason +} From f85957e4ccb5965d4ed58d4c9783451526376219 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 8 Sep 2026 13:18:49 +0200 Subject: [PATCH 20/45] fix: harden profiles and notification state --- openapi/components/push-notifications.yaml | 4 ++ .../db/DatabaseProfileAccountsMigration.kt | 20 +++++++ .../server/models/PushNotificationModels.kt | 2 + .../typetype/server/services/AuthService.kt | 4 ++ .../server/services/NotificationsService.kt | 13 ++--- .../server/services/ProfileAccountService.kt | 25 +++++++++ .../services/ProfileDataDeletionService.kt | 1 + .../services/PushNotificationSupport.kt | 11 ++++ .../server/AccountProfilesRoutesTest.kt | 17 ++++++ .../typetype/server/AuthServiceCoreTest.kt | 14 +++++ .../server/NotificationsRoutesTest.kt | 53 +++++++++++++++++++ .../server/PushNotificationSupportTest.kt | 33 ++++++++++++ 12 files changed, 188 insertions(+), 9 deletions(-) create mode 100644 src/test/kotlin/dev/typetype/server/PushNotificationSupportTest.kt diff --git a/openapi/components/push-notifications.yaml b/openapi/components/push-notifications.yaml index fda7e6f1..29501691 100644 --- a/openapi/components/push-notifications.yaml +++ b/openapi/components/push-notifications.yaml @@ -33,6 +33,8 @@ UnifiedPushNotificationPayload: required: - version - eventType + - serviceId + - serviceName - eventId - videoId - videoUrl @@ -46,6 +48,8 @@ UnifiedPushNotificationPayload: properties: version: { type: integer, minimum: 1 } eventType: { type: string, enum: [subscription_new_video] } + serviceId: { type: integer, minimum: 0 } + serviceName: { type: string } eventId: { type: string } videoId: { type: string } videoUrl: { type: string } diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseProfileAccountsMigration.kt b/src/main/kotlin/dev/typetype/server/db/DatabaseProfileAccountsMigration.kt index c7c69496..8ec4421b 100644 --- a/src/main/kotlin/dev/typetype/server/db/DatabaseProfileAccountsMigration.kt +++ b/src/main/kotlin/dev/typetype/server/db/DatabaseProfileAccountsMigration.kt @@ -13,6 +13,26 @@ object DatabaseProfileAccountsMigration { """.trimIndent(), ) exec("CREATE INDEX IF NOT EXISTS profile_accounts_owner_idx ON profile_accounts (owner_user_id)") + exec( + """ + WITH ranked AS ( + SELECT profile_id, + row_number() OVER ( + PARTITION BY owner_user_id + ORDER BY is_default DESC, last_used_at DESC, created_at ASC, profile_id ASC + ) AS profile_rank + FROM profile_accounts + ) + UPDATE profile_accounts AS profiles + SET is_default = (ranked.profile_rank = 1) + FROM ranked + WHERE profiles.profile_id = ranked.profile_id + """.trimIndent(), + ) + exec( + "CREATE UNIQUE INDEX IF NOT EXISTS profile_accounts_one_default_idx " + + "ON profile_accounts (owner_user_id) WHERE is_default", + ) } private fun exec(sql: String) { diff --git a/src/main/kotlin/dev/typetype/server/models/PushNotificationModels.kt b/src/main/kotlin/dev/typetype/server/models/PushNotificationModels.kt index 3a3ee31b..5a2126ab 100644 --- a/src/main/kotlin/dev/typetype/server/models/PushNotificationModels.kt +++ b/src/main/kotlin/dev/typetype/server/models/PushNotificationModels.kt @@ -44,6 +44,8 @@ data class ChannelNotificationPreference( data class UnifiedPushNotificationPayload( val version: Int = 1, val eventType: String, + val serviceId: Int, + val serviceName: String, val eventId: String, val videoId: String, val videoUrl: String, diff --git a/src/main/kotlin/dev/typetype/server/services/AuthService.kt b/src/main/kotlin/dev/typetype/server/services/AuthService.kt index 10ccb51b..9c56e7d0 100644 --- a/src/main/kotlin/dev/typetype/server/services/AuthService.kt +++ b/src/main/kotlin/dev/typetype/server/services/AuthService.kt @@ -67,6 +67,10 @@ open class AuthService( query.singleOrNull() } ?: return null + if (profileAccountService != null && !profileAccountService.isOwnerProfile(user[UsersTable.id])) { + return null + } + val hashed = user[UsersTable.passwordHash] val verified = withContext(passwordDispatcher) { Password.check(password, hashed).withArgon2() } if (!verified) return null diff --git a/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt b/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt index b630c02e..28fba0cf 100644 --- a/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt @@ -38,9 +38,6 @@ class NotificationsService( } suspend fun getUnreadCount(userId: String): UnreadCountResponse { - val cached = unreadCache[userId] - val now = System.currentTimeMillis() - if (cached != null && cached.expiresAt > now) return UnreadCountResponse(cached.value, true) val feed = loadFeed(userId) if (!feed.available) return UnreadCountResponse(cachedUnread(userId), false) val value = unreadCount(withReadState(buildItems(feed.videos), userId), userId) @@ -75,7 +72,7 @@ class NotificationsService( } } } - unreadCache[userId] = CachedUnread(0, now + UNREAD_CACHE_TTL_MS) + unreadCache[userId] = CachedUnread(0) return MarkNotificationsReadResponse(now, 0, true) } @@ -113,7 +110,7 @@ class NotificationsService( .singleOrNull()?.get(NotificationStatesTable.subscriptionLastSeenUploaded) ?: 0L } return items.map { item -> - item.copy(read = item.id in readIds || (readIds.isEmpty() && item.createdAt <= legacyWatermark)) + item.copy(read = item.id in readIds || item.createdAt <= legacyWatermark) } } @@ -127,7 +124,7 @@ class NotificationsService( private suspend fun unreadCount(items: List, userId: String): Int { val value = items.count { !it.read } - unreadCache[userId] = CachedUnread(value, System.currentTimeMillis() + UNREAD_CACHE_TTL_MS) + unreadCache[userId] = CachedUnread(value) return value } @@ -165,11 +162,9 @@ class NotificationsService( MessageDigest.getInstance("SHA-256").digest(notificationKey(video).toByteArray()) .joinToString("") { byte -> "%02x".format(byte) } - private data class CachedUnread(val value: Int, val expiresAt: Long) + private data class CachedUnread(val value: Int) private companion object { - const val UNREAD_CACHE_TTL_MS = 30_000L - fun serviceName(serviceId: Int): String = when (serviceId) { YOUTUBE_SERVICE_ID -> "YouTube" BILIBILI_SERVICE_ID -> "BiliBili" diff --git a/src/main/kotlin/dev/typetype/server/services/ProfileAccountService.kt b/src/main/kotlin/dev/typetype/server/services/ProfileAccountService.kt index 6cbb5356..bc74cd43 100644 --- a/src/main/kotlin/dev/typetype/server/services/ProfileAccountService.kt +++ b/src/main/kotlin/dev/typetype/server/services/ProfileAccountService.kt @@ -18,8 +18,15 @@ class ProfileAccountService { ownerIdInTransaction(profileId) } + suspend fun isOwnerProfile(profileId: String): Boolean = DatabaseFactory.query { + if (profileId.startsWith("guest:")) return@query false + ProfileAccountsTable.selectAll().where { ProfileAccountsTable.profileId eq profileId } + .singleOrNull()?.let { it[ProfileAccountsTable.ownerUserId] == profileId } ?: true + } + suspend fun list(activeProfileId: String): AccountProfilesResponse? = DatabaseFactory.query { val ownerId = ownerIdInTransaction(activeProfileId) ?: return@query null + ensureDefaultInTransaction(ownerId) val rows = ProfileAccountsTable.selectAll().where { ProfileAccountsTable.ownerUserId eq ownerId } .map { row -> profileRow(row, activeProfileId) } .sortedWith(compareByDescending { it.isDefault }.thenBy { it.name.lowercase() }) @@ -29,6 +36,7 @@ class ProfileAccountService { suspend fun resolveSignInProfile(ownerUserId: String): String = DatabaseFactory.query { val ownerId = ownerIdInTransaction(ownerUserId) ?: return@query ownerUserId + ensureDefaultInTransaction(ownerId) val rows = ProfileAccountsTable.selectAll().where { ProfileAccountsTable.ownerUserId eq ownerId }.toList() val selected = rows.filter { it[ProfileAccountsTable.lastUsedAt] > 0L } .maxByOrNull { it[ProfileAccountsTable.lastUsedAt] } @@ -102,6 +110,7 @@ class ProfileAccountService { if (target == activeProfileId) return@query ProfileMutationResult.CannotDeleteActive ProfileDataDeletionService.deleteUser(target) ProfileAccountsTable.deleteWhere { ProfileAccountsTable.profileId eq target } + ensureDefaultInTransaction(ownerId) ProfileMutationResult.Deleted } @@ -153,6 +162,22 @@ class ProfileAccountService { ProfileAccountsTable.update({ ProfileAccountsTable.profileId eq profileId }) { it[lastUsedAt] = now } } + private fun ensureDefaultInTransaction(ownerId: String) { + val rows = ProfileAccountsTable.selectAll().where { ProfileAccountsTable.ownerUserId eq ownerId }.toList() + if (rows.isEmpty()) return + val selected = rows.filter { it[ProfileAccountsTable.isDefault] } + .maxByOrNull { it[ProfileAccountsTable.lastUsedAt] } + ?: rows.minByOrNull { it[ProfileAccountsTable.createdAt] } + ?: return + val selectedId = selected[ProfileAccountsTable.profileId] + ProfileAccountsTable.update({ ProfileAccountsTable.ownerUserId eq ownerId }) { + it[isDefault] = false + } + ProfileAccountsTable.update({ ProfileAccountsTable.profileId eq selectedId }) { + it[isDefault] = true + } + } + private fun normalizeName(value: String): String? = value.trim().takeIf { it.length in 1..40 } } diff --git a/src/main/kotlin/dev/typetype/server/services/ProfileDataDeletionService.kt b/src/main/kotlin/dev/typetype/server/services/ProfileDataDeletionService.kt index 9e286c6c..f4f261e2 100644 --- a/src/main/kotlin/dev/typetype/server/services/ProfileDataDeletionService.kt +++ b/src/main/kotlin/dev/typetype/server/services/ProfileDataDeletionService.kt @@ -87,6 +87,7 @@ internal object ProfileDataDeletionService { { BlockedChannelsTable.deleteWhere { BlockedChannelsTable.userId eq userId } }, { BlockedKeywordsTable.deleteWhere { BlockedKeywordsTable.userId eq userId } }, { BlockedVideosTable.deleteWhere { BlockedVideosTable.userId eq userId } }, + { BugReportsTable.deleteWhere { BugReportsTable.userId eq userId } }, { NotificationStatesTable.deleteWhere { NotificationStatesTable.userId eq userId } }, { NotificationReadItemsTable.deleteWhere { NotificationReadItemsTable.userId eq userId } }, { PushNotificationDeliveriesTable.deleteWhere { PushNotificationDeliveriesTable.userId eq userId } }, diff --git a/src/main/kotlin/dev/typetype/server/services/PushNotificationSupport.kt b/src/main/kotlin/dev/typetype/server/services/PushNotificationSupport.kt index 060c1cc3..fc1d548e 100644 --- a/src/main/kotlin/dev/typetype/server/services/PushNotificationSupport.kt +++ b/src/main/kotlin/dev/typetype/server/services/PushNotificationSupport.kt @@ -30,6 +30,8 @@ internal object PushNotificationSupport { UnifiedPushNotificationPayload.serializer(), UnifiedPushNotificationPayload( eventType = "subscription_new_video", + serviceId = candidate.serviceId, + serviceName = serviceName(candidate.serviceId), eventId = eventId, videoId = candidate.videoId, videoUrl = candidate.video.url, @@ -49,4 +51,13 @@ internal object PushNotificationSupport { private fun sha256(value: String): String = MessageDigest.getInstance("SHA-256") .digest(value.toByteArray()).joinToString("") { byte -> "%02x".format(byte) } + + private fun serviceName(serviceId: Int): String = when (serviceId) { + YOUTUBE_SERVICE_ID -> "YouTube" + BILIBILI_SERVICE_ID -> "BiliBili" + NICONICO_SERVICE_ID -> "NicoNico" + SOUNDCLOUD_SERVICE_ID -> "SoundCloud" + MEDIA_CCC_SERVICE_ID -> "MediaCCC" + else -> "Video service" + } } diff --git a/src/test/kotlin/dev/typetype/server/AccountProfilesRoutesTest.kt b/src/test/kotlin/dev/typetype/server/AccountProfilesRoutesTest.kt index 01b8c3fb..53641f7f 100644 --- a/src/test/kotlin/dev/typetype/server/AccountProfilesRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/AccountProfilesRoutesTest.kt @@ -127,6 +127,23 @@ class AccountProfilesRoutesTest { assertEquals(ProfileMutationResult.CannotDeleteActive, profileService.delete(profile.id, profile.id)) } + @Test + fun `deleting the default profile promotes the owner profile`() = withApp { + val created = client.post("/profiles") { + bearer() + contentTypeJson() + setBody("{\"name\":\"Temporary\"}") + } + val profile = Json.decodeFromString(created.bodyAsText()) + assertEquals(HttpStatusCode.OK, client.post("/profiles/${profile.id}/default") { bearer() }.status) + assertEquals(HttpStatusCode.OK, client.post("/profiles/${profile.id}/switch") { bearer() }.status) + assertEquals(HttpStatusCode.OK, client.post("/profiles/$TEST_USER_ID/switch") { bearer() }.status) + + assertEquals(ProfileMutationResult.Deleted, profileService.delete(TEST_USER_ID, profile.id)) + val listed = profileService.list(TEST_USER_ID) + assertEquals(TEST_USER_ID, listed?.defaultProfileId) + } + private fun withApp(block: suspend io.ktor.server.testing.ApplicationTestBuilder.() -> Unit) = testApplication { application { install(ContentNegotiation) { json() } diff --git a/src/test/kotlin/dev/typetype/server/AuthServiceCoreTest.kt b/src/test/kotlin/dev/typetype/server/AuthServiceCoreTest.kt index 56741b55..69d64acb 100644 --- a/src/test/kotlin/dev/typetype/server/AuthServiceCoreTest.kt +++ b/src/test/kotlin/dev/typetype/server/AuthServiceCoreTest.kt @@ -4,6 +4,8 @@ import dev.typetype.server.db.tables.UsersTable import dev.typetype.server.db.tables.SessionsTable import dev.typetype.server.services.AuthService import dev.typetype.server.services.AuthSessionConfig +import dev.typetype.server.services.ProfileAccountService +import dev.typetype.server.services.ProfileMutationResult import kotlinx.coroutines.test.runTest import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.selectAll @@ -111,6 +113,18 @@ class AuthServiceCoreTest { assertEquals(userId, byUsername?.let { service.verify(it.accessToken) }) } + @Test + fun `secondary profile credentials cannot be used as a local login`() = runTest { + val profiles = ProfileAccountService() + val service = AuthService("test-secret", profileAccountService = profiles) + val ownerSession = service.register("profile-owner@test.local", "secret-1", "Owner") + val ownerId = service.verify(ownerSession.accessToken) ?: error("missing owner id") + val child = profiles.create(ownerId, "Child") + val childId = (child as ProfileMutationResult.Success).profile.id + + assertNull(service.login("profile-$childId@profiles.invalid", "profile:$childId")) + } + @Test fun `guest token verifies and has user role`() = runTest { val service = AuthService("test-secret") diff --git a/src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt b/src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt index 8c522f38..40c0cb0e 100644 --- a/src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt @@ -4,9 +4,14 @@ import dev.typetype.server.SubscriptionFeedTestFixtures.channel import dev.typetype.server.SubscriptionFeedTestFixtures.subscription import dev.typetype.server.SubscriptionFeedTestFixtures.video import dev.typetype.server.routes.notificationsRoutes +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.NotificationReadItemsTable +import dev.typetype.server.db.tables.NotificationStatesTable +import dev.typetype.server.models.NotificationsResponse import dev.typetype.server.services.AuthService import dev.typetype.server.services.ChannelService import dev.typetype.server.services.NotificationsService +import dev.typetype.server.services.SubscriptionFeedAvailability import dev.typetype.server.services.SubscriptionFeedService import dev.typetype.server.services.SubscriptionsService import io.ktor.client.request.get @@ -23,6 +28,9 @@ import io.ktor.server.testing.ApplicationTestBuilder import io.ktor.server.testing.testApplication import io.mockk.coEvery import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import kotlinx.serialization.json.Json +import org.jetbrains.exposed.v1.jdbc.insert import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeAll @@ -123,4 +131,49 @@ class NotificationsRoutesTest { }.bodyAsText() assertTrue(after.contains("\"unreadCount\":0")) } + + @Test + fun `read watermark remains effective when one notification has an explicit read row`() = withApp { + subscriptionsService.add(TEST_USER_ID, subscription("https://yt.com/c/a", "A")) + coEvery { channelService.getChannel("https://yt.com/c/a", null) } returns channel( + video(1000L, "A", "https://www.youtube.com/watch?v=old"), + video(2000L, "A", "https://www.youtube.com/watch?v=new"), + ) + val initial = Json.decodeFromString(client.get("/notifications?page=0&limit=10") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + }.bodyAsText()) + val latest = initial.items.maxBy { it.createdAt } + DatabaseFactory.query { + NotificationStatesTable.insert { + it[userId] = TEST_USER_ID + it[subscriptionLastSeenUploaded] = initial.items.maxOf { item -> item.createdAt } + it[updatedAt] = System.currentTimeMillis() + } + NotificationReadItemsTable.insert { + it[userId] = TEST_USER_ID + it[notificationId] = latest.id + it[readAt] = System.currentTimeMillis() + } + } + val result = Json.decodeFromString(client.get("/notifications?page=0&limit=10") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + }.bodyAsText()) + assertTrue(result.items.all { it.read }) + assertEquals(0, result.unreadCount) + } + + @Test + fun `unread count reports unavailable instead of trusting a fresh cache`() = runTest { + val feed = mockk() + val availableVideo = video(1000L, "A") + coEvery { feed.getAllWithAvailability(TEST_USER_ID) } returnsMany listOf( + SubscriptionFeedAvailability(listOf(availableVideo), true), + SubscriptionFeedAvailability(emptyList(), false), + ) + val service = NotificationsService(feed) + assertEquals(1, service.getUnreadCount(TEST_USER_ID).unreadCount) + val unavailable = service.getUnreadCount(TEST_USER_ID) + assertEquals(1, unavailable.unreadCount) + assertTrue(!unavailable.available) + } } diff --git a/src/test/kotlin/dev/typetype/server/PushNotificationSupportTest.kt b/src/test/kotlin/dev/typetype/server/PushNotificationSupportTest.kt new file mode 100644 index 00000000..3b6a18fe --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/PushNotificationSupportTest.kt @@ -0,0 +1,33 @@ +package dev.typetype.server + +import dev.typetype.server.models.UnifiedPushNotificationPayload +import dev.typetype.server.services.PushCandidate +import dev.typetype.server.services.PushNotificationSupport +import kotlinx.serialization.json.Json +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class PushNotificationSupportTest { + @Test + fun `payload identifies the provider independently of channel names`() { + val candidate = PushCandidate( + serviceId = 5, + channelId = "https://www.bilibili.com/space/42", + videoId = "BV1", + publishedAt = 1_000L, + video = SubscriptionFeedTestFixtures.video( + uploaded = 1L, + channel = "Shared name", + url = "https://www.bilibili.com/video/BV1", + ), + ) + + val payload = Json.decodeFromString( + PushNotificationSupport.payload("instance", "event", candidate, "profile"), + ) + assertEquals(5, payload.serviceId) + assertEquals("BiliBili", payload.serviceName) + assertEquals("event", payload.eventId) + assertEquals("profile", payload.accountId) + } +} From 407f87a44c9fad95cd23a9bef3716d62e657146d Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 8 Sep 2026 13:38:13 +0200 Subject: [PATCH 21/45] fix: retain notifications during feed refresh --- openapi/paths/notifications.yaml | 2 +- .../server/services/NotificationsService.kt | 8 +-- .../services/SubscriptionFeedService.kt | 3 +- .../server/NotificationsRoutesTest.kt | 15 +++++ .../SubscriptionFeedAvailabilityTest.kt | 56 +++++++++++++++++++ 5 files changed, 78 insertions(+), 6 deletions(-) create mode 100644 src/test/kotlin/dev/typetype/server/SubscriptionFeedAvailabilityTest.kt diff --git a/openapi/paths/notifications.yaml b/openapi/paths/notifications.yaml index 5a60699f..eda74832 100644 --- a/openapi/paths/notifications.yaml +++ b/openapi/paths/notifications.yaml @@ -14,7 +14,7 @@ Notifications: schema: { type: integer, minimum: 1, maximum: 100, default: 20 } responses: '200': - description: Profile-scoped notifications. available is false when the feed could not be refreshed. + description: Profile-scoped notifications. Cached items remain visible during refresh. If refresh fails, available is false and previously collected items are retained. content: application/json: schema: { $ref: ../components/notifications.yaml#/NotificationsResponse } diff --git a/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt b/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt index 28fba0cf..bc713ee4 100644 --- a/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt @@ -23,7 +23,7 @@ class NotificationsService( suspend fun getNotifications(userId: String, page: Int, limit: Int): NotificationsResponse { val feed = loadFeed(userId) - if (!feed.available) { + if (!feed.available && feed.videos.isEmpty()) { return NotificationsResponse(emptyList(), cachedUnread(userId), null, false) } val items = withReadState(buildItems(feed.videos), userId) @@ -34,14 +34,14 @@ class NotificationsService( } val to = minOf(from + limit, items.size) val nextpage = if (to < items.size) (page + 1).toString() else null - return NotificationsResponse(items.subList(from, to), unreadCount, nextpage, true) + return NotificationsResponse(items.subList(from, to), unreadCount, nextpage, feed.available) } suspend fun getUnreadCount(userId: String): UnreadCountResponse { val feed = loadFeed(userId) - if (!feed.available) return UnreadCountResponse(cachedUnread(userId), false) + if (!feed.available && feed.videos.isEmpty()) return UnreadCountResponse(cachedUnread(userId), false) val value = unreadCount(withReadState(buildItems(feed.videos), userId), userId) - return UnreadCountResponse(value, true) + return UnreadCountResponse(value, feed.available) } suspend fun markAllRead(userId: String): MarkNotificationsReadResponse { diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt index 8dffcf16..21e2627f 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt @@ -110,7 +110,7 @@ class SubscriptionFeedService( if (snapshot != null) { return SubscriptionFeedAvailability( videos = snapshot.videos, - available = !snapshot.stale && clock() - snapshot.generatedAt < FRESHNESS_MS, + available = !snapshot.stale, ) } withTimeoutOrNull(INTERNAL_COLD_WAIT_MS) { awaitRefresh(userId) } @@ -176,6 +176,7 @@ class SubscriptionFeedService( if (store.invalidationToken(userId) != invalidation) return true val valid = result.successfulSources > 0 || subscriptions.isEmpty() if (!valid) { + if (previous != null) store.markStale(userId) logger.warn( "subscription_feed event=refresh_kept_previous user={} durationMs={} failedSources={}", userKey(userId), clock() - startedAt, result.failedSources, diff --git a/src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt b/src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt index 40c0cb0e..add61cbf 100644 --- a/src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt @@ -162,6 +162,21 @@ class NotificationsRoutesTest { assertEquals(0, result.unreadCount) } + @Test + fun `cached notifications remain visible when refresh fails`() = runTest { + val feed = mockk() + coEvery { feed.getAllWithAvailability(TEST_USER_ID) } returns + SubscriptionFeedAvailability(listOf(video(1000L, "A")), false) + val service = NotificationsService(feed) + val response = service.getNotifications(TEST_USER_ID, 0, 20) + assertEquals(1, response.items.size) + assertEquals(1, response.unreadCount) + assertTrue(!response.available) + val count = service.getUnreadCount(TEST_USER_ID) + assertEquals(1, count.unreadCount) + assertTrue(!count.available) + } + @Test fun `unread count reports unavailable instead of trusting a fresh cache`() = runTest { val feed = mockk() diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedAvailabilityTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedAvailabilityTest.kt new file mode 100644 index 00000000..56726d94 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedAvailabilityTest.kt @@ -0,0 +1,56 @@ +package dev.typetype.server + +import dev.typetype.server.SubscriptionFeedTestFixtures.channel +import dev.typetype.server.SubscriptionFeedTestFixtures.subscription +import dev.typetype.server.SubscriptionFeedTestFixtures.video +import dev.typetype.server.services.ChannelService +import dev.typetype.server.services.SubscriptionFeedService +import dev.typetype.server.services.SubscriptionsService +import io.mockk.coEvery +import io.mockk.mockk +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test + +class SubscriptionFeedAvailabilityTest { + companion object { @BeforeAll @JvmStatic fun initDb() = TestDatabase.setup() } + + @Test + fun `aged snapshot stays available during refresh and survives refresh failure`() = runTest { + TestDatabase.truncateAll() + val subscriptions = SubscriptionsService() + subscriptions.add(TEST_USER_ID, subscription("https://yt.com/c/a", "A")) + val channels = mockk() + val gate = CompletableDeferred() + var now = 1000L + var failRefresh = false + coEvery { channels.getChannel(any(), null) } coAnswers { + if (failRefresh) { + gate.await() + error("source unavailable") + } + channel(video(1000L, "A")) + } + val feed = SubscriptionFeedService(subscriptions, channels, FakeCacheService(), clock = { now }) + feed.getAllWithAvailability(TEST_USER_ID) + feed.awaitRefresh(TEST_USER_ID) + failRefresh = true + now += 61_000 + try { + val refreshing = feed.getAllWithAvailability(TEST_USER_ID) + assertTrue(refreshing.available) + assertEquals(1, refreshing.videos.size) + } finally { + gate.complete(Unit) + feed.awaitRefresh(TEST_USER_ID) + } + val failed = feed.getAllWithAvailability(TEST_USER_ID) + feed.awaitRefresh(TEST_USER_ID) + assertFalse(failed.available) + assertEquals(1, failed.videos.size) + } +} From 2a8078a6c75e3de56b11241c80a4d536b9017e93 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 8 Sep 2026 14:40:11 +0200 Subject: [PATCH 22/45] perf: expire warmup tracking and deduplicate concurrent scheduling --- .../HomeRecommendationWarmupService.kt | 15 ++---- .../server/services/HomeWarmupTracker.kt | 29 +++++++++++ .../server/services/HomeWarmupTrackerTest.kt | 48 +++++++++++++++++++ 3 files changed, 82 insertions(+), 10 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/services/HomeWarmupTracker.kt create mode 100644 src/test/kotlin/dev/typetype/server/services/HomeWarmupTrackerTest.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmupService.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmupService.kt index 59d9365a..6d079362 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmupService.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmupService.kt @@ -7,15 +7,13 @@ import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.delay import kotlinx.coroutines.isActive import kotlinx.coroutines.launch -import java.util.concurrent.ConcurrentHashMap class HomeRecommendationWarmupService( private val recommendationService: HomeRecommendationService, private val cache: CacheService, ) : HomeRecommendationWarmup { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) - private val activeUsers = ConcurrentHashMap() - private val warmupStartedAt = ConcurrentHashMap() + private val tracker = HomeWarmupTracker(WARMUP_THROTTLE_MS, ACTIVE_TTL_MS) private val poolCache = HomeRecommendationPoolCache(cache) init { @@ -23,12 +21,12 @@ class HomeRecommendationWarmupService( } override fun markActive(userId: String) { - activeUsers[userId] = System.currentTimeMillis() + tracker.markActive(userId, System.currentTimeMillis()) schedule(userId) } override fun invalidateAndWarm(userId: String) { - activeUsers[userId] = System.currentTimeMillis() + tracker.markActive(userId, System.currentTimeMillis()) scope.launch { invalidate(userId) SubscriptionFeedCacheInvalidation.awaitRefresh(userId) @@ -38,9 +36,7 @@ class HomeRecommendationWarmupService( private fun schedule(userId: String, force: Boolean = false) { val now = System.currentTimeMillis() - val previous = warmupStartedAt[userId] - if (!force && previous != null && now - previous < WARMUP_THROTTLE_MS) return - warmupStartedAt[userId] = now + if (!tracker.trySchedule(userId, now, force)) return scope.launch { warm(userId) } } @@ -64,8 +60,7 @@ class HomeRecommendationWarmupService( while (scope.isActive) { delay(REFRESH_INTERVAL_MS) val now = System.currentTimeMillis() - activeUsers.entries.removeIf { now - it.value > ACTIVE_TTL_MS } - activeUsers.keys.forEach { schedule(it) } + tracker.activeUsers(now).forEach { schedule(it) } } } diff --git a/src/main/kotlin/dev/typetype/server/services/HomeWarmupTracker.kt b/src/main/kotlin/dev/typetype/server/services/HomeWarmupTracker.kt new file mode 100644 index 00000000..a78aea45 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/HomeWarmupTracker.kt @@ -0,0 +1,29 @@ +package dev.typetype.server.services + +internal class HomeWarmupTracker( + private val throttleMs: Long, + private val activeTtlMs: Long, +) { + private val active = mutableMapOf() + private val started = mutableMapOf() + + @Synchronized + fun markActive(userId: String, now: Long) { + active[userId] = now + } + + @Synchronized + fun trySchedule(userId: String, now: Long, force: Boolean): Boolean { + val previous = started[userId] + if (!force && previous != null && now - previous < throttleMs) return false + started[userId] = now + return true + } + + @Synchronized + fun activeUsers(now: Long): List { + active.entries.removeIf { now - it.value > activeTtlMs } + started.keys.retainAll(active.keys) + return active.keys.toList() + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/HomeWarmupTrackerTest.kt b/src/test/kotlin/dev/typetype/server/services/HomeWarmupTrackerTest.kt new file mode 100644 index 00000000..996e349e --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/HomeWarmupTrackerTest.kt @@ -0,0 +1,48 @@ +package dev.typetype.server.services + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.util.concurrent.Callable +import java.util.concurrent.Executors + +class HomeWarmupTrackerTest { + @Test + fun `ordinary concurrent requests start only one warmup`() { + val tracker = HomeWarmupTracker(100L, 1_000L) + tracker.markActive("user", 0L) + Executors.newFixedThreadPool(8).use { executor -> + val results = executor.invokeAll(List(100) { + Callable { tracker.trySchedule("user", 0L, false) } + }) + assertEquals(1, results.count { it.get() }) + } + assertFalse(tracker.trySchedule("user", 99L, false)) + assertTrue(tracker.trySchedule("user", 100L, false)) + } + + @Test + fun `forced invalidation retains its immediate warmup behavior`() { + val tracker = HomeWarmupTracker(100L, 1_000L) + assertTrue(tracker.trySchedule("user", 0L, false)) + assertTrue(tracker.trySchedule("user", 1L, true)) + assertFalse(tracker.trySchedule("user", 2L, false)) + } + + @Test + fun `expiry removes both activity and throttle entries`() { + val tracker = HomeWarmupTracker(10_000L, 1_000L) + repeat(10_000) { index -> + tracker.markActive("user-$index", 0L) + tracker.trySchedule("user-$index", 0L, false) + } + tracker.markActive("retained", 1_000L) + assertEquals(listOf("retained"), tracker.activeUsers(1_001L)) + repeat(10_000) { index -> + assertTrue(tracker.trySchedule("user-$index", 1_001L, false)) + } + assertEquals(listOf("retained"), tracker.activeUsers(2_000L)) + assertTrue(tracker.activeUsers(2_001L).isEmpty()) + } +} From f73f1b13052843d9136b2f14e441c687b1f8fb7d Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 9 Sep 2026 08:06:46 +0200 Subject: [PATCH 23/45] perf: skip cache expiry scans before the next deadline --- .../server/services/BoundedExpiringCache.kt | 14 +++++- .../services/BoundedExpiringCacheTest.kt | 48 +++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/services/BoundedExpiringCache.kt b/src/main/kotlin/dev/typetype/server/services/BoundedExpiringCache.kt index cba0a255..7a35ef16 100644 --- a/src/main/kotlin/dev/typetype/server/services/BoundedExpiringCache.kt +++ b/src/main/kotlin/dev/typetype/server/services/BoundedExpiringCache.kt @@ -13,6 +13,8 @@ internal class BoundedExpiringCache( private val ttlMs = ttl.toMillis() private val entries = LinkedHashMap>(maxEntries.coerceAtMost(64), 0.75f, true) private var weight = 0L + // Removals may leave an earlier bound, causing one extra scan but never delaying expiry. + private var nextExpiryMs = Long.MAX_VALUE init { require(maxEntries > 0) { "maxEntries must be positive" } @@ -33,7 +35,9 @@ internal class BoundedExpiringCache( removeEntry(key) val entryWeight = weigher(value).coerceAtLeast(0L) if (entryWeight > maxWeight) return - entries[key] = Entry(value, expiresAt(now), entryWeight) + val expiry = expiresAt(now) + entries[key] = Entry(value, expiry, entryWeight) + nextExpiryMs = minOf(nextExpiryMs, expiry) weight += entryWeight trim() } @@ -61,6 +65,7 @@ internal class BoundedExpiringCache( fun clear() { entries.clear() weight = 0L + nextExpiryMs = Long.MAX_VALUE } @Synchronized @@ -73,10 +78,15 @@ internal class BoundedExpiringCache( if (Long.MAX_VALUE - now < ttlMs) Long.MAX_VALUE else now + ttlMs private fun evictExpired(now: Long) { + if (now < nextExpiryMs) return + nextExpiryMs = Long.MAX_VALUE val iterator = entries.iterator() while (iterator.hasNext()) { val entry = iterator.next().value - if (entry.expiresAtMs > now) continue + if (entry.expiresAtMs > now) { + nextExpiryMs = minOf(nextExpiryMs, entry.expiresAtMs) + continue + } weight -= entry.weight iterator.remove() } diff --git a/src/test/kotlin/dev/typetype/server/services/BoundedExpiringCacheTest.kt b/src/test/kotlin/dev/typetype/server/services/BoundedExpiringCacheTest.kt index bf593670..f7c658fe 100644 --- a/src/test/kotlin/dev/typetype/server/services/BoundedExpiringCacheTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/BoundedExpiringCacheTest.kt @@ -6,6 +6,54 @@ import org.junit.jupiter.api.Test import java.time.Duration class BoundedExpiringCacheTest { + @Test + fun `reads expire entries independently of access order`() { + var now = 0L + val cache = BoundedExpiringCache(10, ttl = Duration.ofMillis(10), clock = { now }) + cache.put("first", "1") + now = 5L + cache.put("second", "2") + assertEquals("1", cache.get("first")) + now = 10L + + assertEquals("2", cache.get("second")) + assertNull(cache.get("first")) + assertEquals(1, cache.size()) + assertEquals(1L, cache.weight()) + now = 15L + assertNull(cache.get("second")) + assertEquals(0L, cache.weight()) + } + + @Test + fun `replacement keeps its new expiry when the original expiry passes`() { + var now = 0L + val cache = BoundedExpiringCache(10, ttl = Duration.ofMillis(10), clock = { now }) + cache.put("key", "old") + now = 5L + cache.put("key", "new") + now = 10L + cache.evictExpired() + assertEquals("new", cache.get("key")) + now = 15L + assertNull(cache.get("key")) + } + + @Test + fun `new entries honor their expiry after clear and a backward clock change`() { + var now = 100L + val cache = BoundedExpiringCache(10, ttl = Duration.ofMillis(10), clock = { now }) + cache.put("old", "1") + cache.clear() + cache.put("later", "2") + now = 50L + cache.put("earlier", "3") + now = 60L + assertNull(cache.get("earlier")) + assertEquals("2", cache.get("later")) + assertEquals(1, cache.size()) + } + @Test fun `least recently used entry is removed at capacity`() { val cache = BoundedExpiringCache( From 3b8816ad7ca8b1bfedde218c6dc029adc44e5246 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 9 Sep 2026 08:53:14 +0200 Subject: [PATCH 24/45] perf: stream BiliBili ranges with bounded retry memory --- .../server/services/BilibiliRangeProxy.kt | 56 ++++---- .../server/services/OkHttpProxyService.kt | 15 ++- .../services/RetryingProxyInputStream.kt | 122 ++++++++++++++++++ 3 files changed, 157 insertions(+), 36 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/services/RetryingProxyInputStream.kt diff --git a/src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt b/src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt index 3268ea8e..0f04f09a 100644 --- a/src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt +++ b/src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt @@ -4,7 +4,6 @@ import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.ProxyResponse import okhttp3.Request import okhttp3.Response -import java.io.ByteArrayInputStream import java.io.IOException private const val BILIBILI_RANGE_ATTEMPTS = 3 @@ -12,40 +11,33 @@ private const val BILIBILI_RANGE_ATTEMPTS = 3 internal fun readBilibiliRangeWithRetry( execute: (Request) -> Response, request: Request, + checkActive: () -> Unit = {}, ): ExtractionResult { var lastMessage = "Proxy fetch failed" for (attempt in 1..BILIBILI_RANGE_ATTEMPTS) { - runCatching { execute(request) } - .onSuccess { response -> - response.use { - val result = it.readBilibiliRangeBytes() - when (result) { - is ExtractionResult.Success -> return result - is ExtractionResult.BadRequest -> return result - is ExtractionResult.Failure -> lastMessage = result.message - } - } - } - .onFailure { lastMessage = it.message ?: "Proxy fetch failed" } - if (attempt == BILIBILI_RANGE_ATTEMPTS) break + checkActive() + val response = try { + execute(request) + } catch (error: IOException) { + lastMessage = error.message ?: "Proxy fetch failed" + continue + } + if (!response.isSuccessful) { + lastMessage = "Upstream returned ${response.code}" + response.close() + continue + } + val stream = RetryingProxyInputStream(execute, request, response, BILIBILI_RANGE_ATTEMPTS - attempt, checkActive) + return ExtractionResult.Success(ProxyResponse( + status = response.code, + contentType = response.header("Content-Type") ?: "application/octet-stream", + contentLength = response.body.contentLength().takeIf { it >= 0 }, + contentRange = response.header("Content-Range"), + acceptRanges = response.header("Accept-Ranges"), + cacheControl = response.header("Cache-Control"), + stream = stream, + close = stream::close, + )) } return ExtractionResult.Failure(lastMessage) } - -private fun Response.readBilibiliRangeBytes(): ExtractionResult { - if (!isSuccessful && code != 206) return ExtractionResult.Failure("Upstream returned $code") - val bytes = try { - body.bytes() - } catch (e: IOException) { - return ExtractionResult.Failure(e.message ?: "Proxy fetch failed") - } - return ExtractionResult.Success(ProxyResponse( - status = code, - contentType = header("Content-Type") ?: "application/octet-stream", - contentLength = bytes.size.toLong(), - contentRange = header("Content-Range"), - acceptRanges = header("Accept-Ranges"), - stream = ByteArrayInputStream(bytes), - close = {}, - )) -} diff --git a/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt b/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt index ebde6bc6..24a120c7 100644 --- a/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt +++ b/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt @@ -3,6 +3,8 @@ package dev.typetype.server.services import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.ProxyResponse import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.ensureActive import kotlinx.coroutines.withContext import okhttp3.OkHttpClient import okhttp3.Request @@ -24,8 +26,9 @@ internal fun rewriteHlsManifest(manifest: String): String = class OkHttpProxyService(client: OkHttpClient) : ProxyService { private val executor = ProxyHttpExecutor(client) - override suspend fun pipe(url: String, rangeHeader: String?, domandBid: String?): ExtractionResult = - withContext(Dispatchers.IO) { + override suspend fun pipe(url: String, rangeHeader: String?, domandBid: String?): ExtractionResult { + val requestContext = currentCoroutineContext() + return withContext(Dispatchers.IO) { val hashIdx = url.indexOf('#') val fetchUrl = if (hashIdx >= 0) url.substring(0, hashIdx) else url val fragment = if (hashIdx >= 0) url.substring(hashIdx + 1) else "" @@ -46,7 +49,7 @@ class OkHttpProxyService(client: OkHttpClient) : ProxyService { if (rangeHeader != null) builder.header("Range", rangeHeader) val request = builder.build() if (bilibili && rangeHeader != null) { - return@withContext readBilibiliRangeWithRetry(executor::execute, request) + return@withContext readBilibiliRangeWithRetry(executor::execute, request, requestContext::ensureActive) } executor.execute(request) }.fold( @@ -98,9 +101,13 @@ class OkHttpProxyService(client: OkHttpClient) : ProxyService { } } }, - onFailure = { ExtractionResult.Failure(it.message ?: "Proxy fetch failed") } + onFailure = { + requestContext.ensureActive() + ExtractionResult.Failure(it.message ?: "Proxy fetch failed") + } ) } + } private fun isBilibili(url: String): Boolean { val host = runCatching { java.net.URI(url).host ?: "" }.getOrElse { "" } diff --git a/src/main/kotlin/dev/typetype/server/services/RetryingProxyInputStream.kt b/src/main/kotlin/dev/typetype/server/services/RetryingProxyInputStream.kt new file mode 100644 index 00000000..5dbe0158 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/RetryingProxyInputStream.kt @@ -0,0 +1,122 @@ +package dev.typetype.server.services + +import okhttp3.Request +import okhttp3.Response +import java.io.EOFException +import java.io.IOException +import java.io.InputStream +import java.security.MessageDigest +import java.util.Objects + +internal class RetryingProxyInputStream( + private val execute: (Request) -> Response, + private val request: Request, + initialResponse: Response, + private var attemptsRemaining: Int, + private val checkActive: () -> Unit = {}, +) : InputStream() { + private var response = initialResponse + private var source = response.body.byteStream() + private val status = response.code + private val length = response.body.contentLength() + private val headers = listOf("Content-Range", "Content-Type", "Content-Encoding", "ETag") + .associateWith(response::header) + private var digest = if (response.header("ETag")?.let { + it.length >= 2 && it.startsWith('"') && it.endsWith('"') + } == true) { + null + } else { + MessageDigest.getInstance("SHA-256") + } + private var delivered = 0L + private var closed = false + private val singleByte = ByteArray(1) + + override fun read(): Int = if (read(singleByte, 0, 1) == -1) -1 else singleByte[0].toInt() and 0xff + + override fun read(bytes: ByteArray, offset: Int, count: Int): Int { + Objects.checkFromIndexSize(offset, count, bytes.size) + checkOpen() + if (count == 0) return 0 + if (length >= 0 && delivered == length) return -1 + while (true) { + try { + val size = if (length < 0) count else minOf(count.toLong(), length - delivered).toInt() + val read = source.read(bytes, offset, size) + if (read < 0 && length >= 0 && delivered < length) throw EOFException("Incomplete proxy body") + if (read > 0) { + digest?.update(bytes, offset, read) + delivered += read + } + return read + } catch (error: IOException) { + recover(error) + } + } + } + + private fun recover(initialError: IOException) { + response.close() + val expectedPrefix = digest?.digest() + var lastError = initialError + while (attemptsRemaining > 0) { + checkOpen() + if (Thread.currentThread().isInterrupted) throw lastError + attemptsRemaining-- + var candidate: Response? = null + try { + val opened = execute(request) + candidate = opened + if (!opened.isSuccessful) throw IOException("Upstream returned ${opened.code}") + if (opened.code != status || opened.body.contentLength() != length || + headers.any { (name, value) -> opened.header(name) != value } + ) throw ChangedProxyBodyException() + val nextSource = candidate.body.byteStream() + val nextDigest = expectedPrefix?.let { MessageDigest.getInstance("SHA-256") } + verifyPrefix(nextSource, nextDigest, expectedPrefix) + response = candidate + source = nextSource + digest = nextDigest + candidate = null + return + } catch (error: IOException) { + if (error is ChangedProxyBodyException) throw error + lastError = error + } finally { + candidate?.close() + } + } + throw lastError + } + + private fun verifyPrefix(input: InputStream, nextDigest: MessageDigest?, expected: ByteArray?) { + // Replaying the original range also works when the CDN supplies no strong validator. + val buffer = ByteArray(64 * 1024) + val verificationDigest = expected?.let { MessageDigest.getInstance("SHA-256") } + var remaining = delivered + while (remaining > 0) { + checkOpen() + val count = input.read(buffer, 0, minOf(remaining, buffer.size.toLong()).toInt()) + if (count < 0) throw EOFException("Incomplete proxy retry prefix") + nextDigest?.update(buffer, 0, count) + verificationDigest?.update(buffer, 0, count) + remaining -= count + } + if (expected != null && !MessageDigest.isEqual(expected, verificationDigest?.digest())) { + throw ChangedProxyBodyException() + } + } + + private fun checkOpen() { + if (closed) throw IOException("Proxy stream is closed") + checkActive() + } + + override fun close() { + if (closed) return + closed = true + response.close() + } +} + +private class ChangedProxyBodyException : IOException("Upstream proxy body changed during retry") From 78a796b8364058f29200496a2a95004813ccbfc2 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 9 Sep 2026 08:53:15 +0200 Subject: [PATCH 25/45] test: verify bounded range streaming and safe retries --- .../typetype/server/BilibiliRangeProxyTest.kt | 2 +- .../server/RetryingProxyInputStreamTest.kt | 264 ++++++++++++++++++ 2 files changed, 265 insertions(+), 1 deletion(-) create mode 100644 src/test/kotlin/dev/typetype/server/RetryingProxyInputStreamTest.kt diff --git a/src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt b/src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt index 796c59cc..8e5e463b 100644 --- a/src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt +++ b/src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt @@ -53,6 +53,6 @@ class BilibiliRangeProxyTest { assertEquals(2, calls) assertEquals(206, result.data.status) assertEquals("bytes 0-3/4", result.data.contentRange) - assertArrayEquals(bytes, result.data.stream.readBytes()) + result.data.stream.use { assertArrayEquals(bytes, it.readBytes()) } } } diff --git a/src/test/kotlin/dev/typetype/server/RetryingProxyInputStreamTest.kt b/src/test/kotlin/dev/typetype/server/RetryingProxyInputStreamTest.kt new file mode 100644 index 00000000..5f6a17e1 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/RetryingProxyInputStreamTest.kt @@ -0,0 +1,264 @@ +package dev.typetype.server + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.services.readBilibiliRangeWithRetry +import kotlinx.coroutines.CancellationException +import okhttp3.MediaType +import okhttp3.Protocol +import okhttp3.Request +import okhttp3.Response +import okhttp3.ResponseBody +import okio.Buffer +import okio.Source +import okio.Timeout +import okio.buffer +import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable +import java.io.IOException +import java.io.OutputStream +import java.lang.management.ManagementFactory + +class RetryingProxyInputStreamTest { + private val request = Request.Builder().url("https://video.bilivideo.com/test.m4s") + .header("Range", "bytes=0-").build() + + @Test + fun `large range is read on demand without a body-sized allocation`() { + val body = GeneratedBody(3L * 1024 * 1024 * 1024) + val proxy = open { response(body) } + assertEquals(0L, body.readBytes) + assertEquals(body.length, proxy.contentLength) + val bytes = ByteArray(64 * 1024) + assertTrue(proxy.stream.read(bytes) > 0) + assertTrue(body.readBytes <= bytes.size) + proxy.close() + assertTrue(body.closed) + } + + @Test + fun `truncated response replays verified prefix without duplicate bytes`() { + val bodies = listOf(GeneratedBody(100_000, failAt = 20_000), GeneratedBody(100_000)) + var calls = 0 + val proxy = open { next -> + assertEquals(request.header("Range"), next.header("Range")) + response(bodies[calls++]) + } + proxy.stream.use { assertArrayEquals(ByteArray(100_000) { 42 }, it.readBytes()) } + assertEquals(2, calls) + assertTrue(bodies.all { it.closed }) + } + + @Test + fun `strong etag permits replay without hashing all delivered media`() { + val bodies = listOf(GeneratedBody(100, failAt = 20), GeneratedBody(100)) + var calls = 0 + val proxy = open { response(bodies[calls++]).newBuilder().header("ETag", "\"stable\"").build() } + proxy.stream.use { assertArrayEquals(ByteArray(100) { 42 }, it.readBytes()) } + assertEquals(2, calls) + assertTrue(bodies.all { it.closed }) + } + + @Test + fun `weak etag still requires identical replay bytes`() { + val bodies = listOf(GeneratedBody(100, failAt = 20), GeneratedBody(100, value = 43)) + var calls = 0 + val proxy = open { response(bodies[calls++]).newBuilder().header("ETag", "W/\"stable\"").build() } + proxy.stream.use { + assertEquals(20, it.read(ByteArray(100))) + assertThrows(IOException::class.java) { it.read(ByteArray(100)) } + } + assertEquals(2, calls) + assertTrue(bodies.all { it.closed }) + } + + @Test + fun `changed replay prefix is rejected before any new bytes are sent`() { + val bodies = listOf(GeneratedBody(100, failAt = 20), GeneratedBody(100, value = 43)) + var calls = 0 + val proxy = open { response(bodies[calls++]) } + proxy.stream.use { + assertEquals(20, it.read(ByteArray(100))) + assertThrows(IOException::class.java) { it.read(ByteArray(100)) } + } + assertEquals(2, calls) + assertTrue(bodies.all { it.closed }) + } + + @Test + fun `changed response metadata is rejected and closed`() { + val bodies = listOf(GeneratedBody(100, failAt = 20), GeneratedBody(100)) + var calls = 0 + val proxy = open { + response(bodies[calls++]).newBuilder().header("ETag", "\"$calls\"").build() + } + proxy.stream.use { + assertEquals(20, it.read(ByteArray(100))) + assertThrows(IOException::class.java) { it.read(ByteArray(100)) } + } + assertEquals(2, calls) + assertTrue(bodies.all { it.closed }) + } + + @Test + fun `retry budget includes transport errors and replay failures`() { + var calls = 0 + val bodies = mutableListOf() + val proxy = open { + calls++ + if (calls == 1) throw IOException("connect failed") + val body = GeneratedBody(100, failAt = if (calls == 2) 20 else 10) + bodies += body + response(body) + } + proxy.stream.use { + assertEquals(20, it.read(ByteArray(100))) + assertThrows(IOException::class.java) { it.read(ByteArray(100)) } + } + assertEquals(3, calls) + assertTrue(bodies.all { it.closed }) + } + + @Test + fun `close prevents further reads and retries`() { + val body = GeneratedBody(100) + var calls = 0 + val proxy = open { calls++; response(body) } + proxy.close() + proxy.close() + assertThrows(IOException::class.java) { proxy.stream.read() } + assertEquals(1, calls) + assertTrue(body.closed) + } + + @Test + fun `temporary HTTP error during replay consumes the same bounded retry budget`() { + val bodies = listOf(GeneratedBody(100, failAt = 20), GeneratedBody(0), GeneratedBody(100)) + var calls = 0 + val proxy = open { + val result = response(bodies[calls++]) + if (calls == 2) result.newBuilder().code(503).build() else result + } + proxy.stream.use { assertArrayEquals(ByteArray(100) { 42 }, it.readBytes()) } + assertEquals(3, calls) + assertTrue(bodies.all { it.closed }) + } + + @Test + fun `cancellation during replay closes both responses without retrying`() { + val bodies = listOf(GeneratedBody(100, failAt = 20), GeneratedBody(100)) + var active = true + var calls = 0 + val proxy = open(checkActive = { if (!active) throw CancellationException("cancelled") }) { + if (calls == 1) active = false + response(bodies[calls++]) + } + proxy.stream.use { + assertEquals(20, it.read(ByteArray(100))) + assertThrows(CancellationException::class.java) { it.read(ByteArray(100)) } + } + assertEquals(2, calls) + assertTrue(bodies.all { it.closed }) + } + + @Test + fun `premature EOF is retried and the replay digest survives another failure`() { + val bodies = listOf( + GeneratedBody(100, failAt = 20, prematureEof = true), + GeneratedBody(100, failAt = 40), + GeneratedBody(100), + ) + var calls = 0 + val proxy = open { response(bodies[calls++]) } + proxy.stream.use { assertArrayEquals(ByteArray(100) { 42 }, it.readBytes()) } + assertEquals(3, calls) + assertTrue(bodies.all { it.closed }) + } + + @Test + fun `unknown length supports EOF and zero length reads`() { + val body = GeneratedBody(100, advertisedLength = -1) + val proxy = open { response(body) } + assertEquals(null, proxy.contentLength) + proxy.stream.use { + assertEquals(0, it.read(ByteArray(0))) + assertArrayEquals(ByteArray(100) { 42 }, it.readBytes()) + assertEquals(-1, it.read()) + } + assertTrue(body.closed) + } + + @Test + @EnabledIfEnvironmentVariable(named = "TYPETYPE_PROXY_BENCHMARK", matches = "1") + fun `compare eager body and streaming allocations on generated media`() { + val bean = ManagementFactory.getThreadMXBean() as com.sun.management.ThreadMXBean + val thread = Thread.currentThread().threadId() + fun measure(streaming: Boolean, size: Long, strongEtag: Boolean = false): Pair { + val allocated = bean.getThreadAllocatedBytes(thread) + val start = System.nanoTime() + val body = GeneratedBody(size) + if (streaming) { + open { + val result = response(body) + if (strongEtag) result.newBuilder().header("ETag", "\"stable\"").build() else result + }.stream.use { it.copyTo(OutputStream.nullOutputStream(), 64 * 1024) } + } else { + response(body).use { it.body.bytes().inputStream().copyTo(OutputStream.nullOutputStream(), 64 * 1024) } + } + return (bean.getThreadAllocatedBytes(thread) - allocated) to (System.nanoTime() - start) + } + repeat(8) { measure(false, 1024 * 1024); measure(true, 1024 * 1024); measure(true, 1024 * 1024, true) } + repeat(3) { + val eager = measure(false, 64L * 1024 * 1024) + val streaming = measure(true, 64L * 1024 * 1024) + val validated = measure(true, 64L * 1024 * 1024, true) + println("range64MiB eagerAllocated=${eager.first} streamingAllocated=${streaming.first} " + + "etagAllocated=${validated.first} eagerNanos=${eager.second} streamingNanos=${streaming.second} " + + "etagNanos=${validated.second}") + assertTrue(streaming.first < 1024 * 1024, "Streaming must not allocate a range-sized body") + } + } + + private fun open(checkActive: () -> Unit = {}, execute: (Request) -> Response) = + (readBilibiliRangeWithRetry(execute, request, checkActive) as ExtractionResult.Success).data + + private fun response(body: GeneratedBody) = Response.Builder() + .request(request).protocol(Protocol.HTTP_1_1).code(206).message("Partial Content") + .header("Content-Type", "video/mp4") + .header("Content-Range", "bytes 0-${body.length - 1}/${body.length}") + .body(body).build() + + private class GeneratedBody( + val length: Long, + private val failAt: Long = Long.MAX_VALUE, + private val value: Byte = 42, + private val prematureEof: Boolean = false, + private val advertisedLength: Long = length, + ) : ResponseBody() { + var readBytes = 0L + var closed = false + private val bytes = ByteArray(8192) { value } + private val input = object : Source { + override fun timeout() = Timeout.NONE + override fun close() { closed = true } + override fun read(sink: Buffer, byteCount: Long): Long { + if (readBytes >= failAt) { + if (prematureEof) return -1 + throw IOException("connection reset") + } + if (readBytes == length) return -1 + val count = minOf(byteCount, bytes.size.toLong(), length - readBytes, failAt - readBytes).toInt() + sink.write(bytes, 0, count) + readBytes += count + return count.toLong() + } + }.buffer() + + override fun contentType(): MediaType? = null + override fun contentLength() = advertisedLength + override fun source() = input + } +} From db9f577d893f24ecc2f3936197b47a03faebcd61 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 9 Sep 2026 09:38:24 +0200 Subject: [PATCH 26/45] chore: update Exposed and SQLite JDBC --- build.gradle.kts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index c77bb2e5..8078cbb2 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -49,11 +49,11 @@ dependencies { implementation("org.json:json:20260814") implementation("com.squareup.okhttp3:okhttp:5.5.0") implementation("io.lettuce:lettuce-core:7.7.0.RELEASE") - implementation("org.jetbrains.exposed:exposed-core:1.4.0") - implementation("org.jetbrains.exposed:exposed-jdbc:1.4.0") + implementation("org.jetbrains.exposed:exposed-core:1.5.0") + implementation("org.jetbrains.exposed:exposed-jdbc:1.5.0") implementation("com.zaxxer:HikariCP:7.1.0") implementation("org.postgresql:postgresql:42.7.13") - implementation("org.xerial:sqlite-jdbc:3.53.2.1") + implementation("org.xerial:sqlite-jdbc:3.53.4.0") implementation("com.password4j:password4j:1.8.4") implementation("com.auth0:java-jwt:4.6.0") testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") From 76168cc5ed8de8cadb241394507e3f885738fe62 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 9 Sep 2026 09:52:28 +0200 Subject: [PATCH 27/45] perf: bound Takeout import memory caches --- .../services/YoutubeTakeoutImportCache.kt | 39 ++++++++-- .../YoutubeTakeoutImportJobService.kt | 2 + .../services/YoutubeTakeoutImportCacheTest.kt | 73 +++++++++++++++++++ 3 files changed, 107 insertions(+), 7 deletions(-) create mode 100644 src/test/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCacheTest.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCache.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCache.kt index 8095310e..45712d50 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCache.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCache.kt @@ -2,20 +2,45 @@ package dev.typetype.server.services import dev.typetype.server.models.YoutubeTakeoutParsedData import dev.typetype.server.models.YoutubeTakeoutPreviewItem +import java.time.Duration -class YoutubeTakeoutImportCache { - private val previewCache = mutableMapOf() - private val parsedCache = mutableMapOf() +class YoutubeTakeoutImportCache( + maxPreviewEntries: Int = DEFAULT_MAX_PREVIEW_ENTRIES, + maxParsedEntries: Int = DEFAULT_MAX_PARSED_ENTRIES, + ttl: Duration = DEFAULT_TTL, + clock: () -> Long = System::currentTimeMillis, +) { + private val previewCache = BoundedExpiringCache( + maxEntries = maxPreviewEntries, + ttl = ttl, + clock = clock, + ) + private val parsedCache = BoundedExpiringCache( + maxEntries = maxParsedEntries, + ttl = ttl, + clock = clock, + ) - fun getPreview(jobId: String): YoutubeTakeoutPreviewItem? = previewCache[jobId] + fun getPreview(jobId: String): YoutubeTakeoutPreviewItem? = previewCache.get(jobId) fun setPreview(jobId: String, preview: YoutubeTakeoutPreviewItem) { - previewCache[jobId] = preview + previewCache.put(jobId, preview) } - fun getParsed(jobId: String): YoutubeTakeoutParsedData? = parsedCache[jobId] + fun getParsed(jobId: String): YoutubeTakeoutParsedData? = parsedCache.get(jobId) fun setParsed(jobId: String, parsed: YoutubeTakeoutParsedData) { - parsedCache[jobId] = parsed + parsedCache.put(jobId, parsed) + } + + fun remove(jobId: String) { + previewCache.remove(jobId) + parsedCache.remove(jobId) + } + + private companion object { + const val DEFAULT_MAX_PREVIEW_ENTRIES = 256 + const val DEFAULT_MAX_PARSED_ENTRIES = 1 + val DEFAULT_TTL: Duration = Duration.ofMinutes(30) } } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt index 086d0be8..5c72b6ad 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt @@ -84,6 +84,8 @@ class YoutubeTakeoutImportJobService( privacyService.deleteArchive(archiveStore.getArchivePath(userId, jobId)) } catch (e: Exception) { statusStore.failStatus(jobId, "import_failed", e.importErrorMessage()) + } finally { + cache.remove(jobId) } } diff --git a/src/test/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCacheTest.kt b/src/test/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCacheTest.kt new file mode 100644 index 00000000..921aa169 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCacheTest.kt @@ -0,0 +1,73 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.YoutubeTakeoutCategoryCounts +import dev.typetype.server.models.YoutubeTakeoutParsedData +import dev.typetype.server.models.YoutubeTakeoutPreviewItem +import dev.typetype.server.models.YoutubeTakeoutPreviewSamples +import java.time.Duration +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Test + +class YoutubeTakeoutImportCacheTest { + @Test + fun `parsed data is bounded to one active job`() { + val cache = YoutubeTakeoutImportCache(maxParsedEntries = 1) + val first = parsedData("first") + val second = parsedData("second") + + cache.setParsed("job-1", first) + cache.setParsed("job-2", second) + + assertNull(cache.getParsed("job-1")) + assertSame(second, cache.getParsed("job-2")) + } + + @Test + fun `preview and parsed data expire`() { + var now = 100L + val cache = YoutubeTakeoutImportCache( + ttl = Duration.ofMillis(10), + clock = { now }, + ) + val preview = preview() + cache.setPreview("job", preview) + cache.setParsed("job", parsedData("job")) + + now = 110L + + assertNull(cache.getPreview("job")) + assertNull(cache.getParsed("job")) + } + + @Test + fun `remove releases both cached values`() { + val cache = YoutubeTakeoutImportCache() + cache.setPreview("job", preview()) + cache.setParsed("job", parsedData("job")) + + cache.remove("job") + + assertNull(cache.getPreview("job")) + assertNull(cache.getParsed("job")) + } + + private fun preview() = YoutubeTakeoutPreviewItem( + counts = YoutubeTakeoutCategoryCounts(0, 0, 0), + dedup = YoutubeTakeoutCategoryCounts(0, 0, 0), + samples = YoutubeTakeoutPreviewSamples(emptyList(), emptyList(), emptyList()), + warnings = emptyList(), + errors = emptyList(), + ) + + private fun parsedData(value: String) = YoutubeTakeoutParsedData( + subscriptions = emptyList(), + playlists = emptyList(), + playlistItems = mapOf(value to emptyList()), + favorites = emptyList(), + watchLater = emptyList(), + history = emptyList(), + warnings = emptyList(), + errors = emptyList(), + ) +} From f3a274c00bba737c312ba350d0a2f7d38d68b31e Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 9 Sep 2026 10:16:42 +0200 Subject: [PATCH 28/45] perf: bound auxiliary server memory caches --- .../kotlin/dev/typetype/server/AppMetrics.kt | 14 +++- .../server/services/NotificationsService.kt | 15 ++-- .../server/services/OpenMojiProxyService.kt | 64 ++++++++++------- .../services/OpenMojiProxyServiceTest.kt | 69 +++++++++++++++++++ 4 files changed, 130 insertions(+), 32 deletions(-) create mode 100644 src/test/kotlin/dev/typetype/server/services/OpenMojiProxyServiceTest.kt diff --git a/src/main/kotlin/dev/typetype/server/AppMetrics.kt b/src/main/kotlin/dev/typetype/server/AppMetrics.kt index b74be9d8..0de33b49 100644 --- a/src/main/kotlin/dev/typetype/server/AppMetrics.kt +++ b/src/main/kotlin/dev/typetype/server/AppMetrics.kt @@ -17,7 +17,7 @@ object AppMetrics { totalRequests.incrementAndGet() totalDurationMs.addAndGet(call.requestDurationMs()) statusCounts.getOrPut(status) { AtomicLong() }.incrementAndGet() - routeCounts.getOrPut("$route|$status") { AtomicLong() }.incrementAndGet() + recordRoute(route, status) } fun snapshot(): String { @@ -36,6 +36,18 @@ object AppMetrics { } } + private fun recordRoute(route: String, status: Int) { + val key = "$route|$status" + if (routeCounts.size < MAX_ROUTE_METRICS) { + routeCounts.computeIfAbsent(key) { AtomicLong() }.incrementAndGet() + return + } + routeCounts.getOrPut("$OTHER_ROUTE|$status") { AtomicLong() }.incrementAndGet() + } + + private const val MAX_ROUTE_METRICS = 512 + private const val OTHER_ROUTE = "/__other__" + } fun metricPath(path: String): String = when { diff --git a/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt b/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt index bc713ee4..f73d23eb 100644 --- a/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt @@ -8,7 +8,7 @@ import dev.typetype.server.models.NotificationItem import dev.typetype.server.models.NotificationsResponse import dev.typetype.server.models.UnreadCountResponse import dev.typetype.server.models.VideoItem -import java.util.concurrent.ConcurrentHashMap +import java.time.Duration import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.insertIgnore import org.jetbrains.exposed.v1.jdbc.insert @@ -19,7 +19,10 @@ import java.security.MessageDigest class NotificationsService( private val subscriptionFeedService: SubscriptionFeedService, ) { - private val unreadCache = ConcurrentHashMap() + private val unreadCache = BoundedExpiringCache( + maxEntries = 2048, + ttl = Duration.ofHours(1), + ) suspend fun getNotifications(userId: String, page: Int, limit: Int): NotificationsResponse { val feed = loadFeed(userId) @@ -72,7 +75,7 @@ class NotificationsService( } } } - unreadCache[userId] = CachedUnread(0) + unreadCache.put(userId, 0) return MarkNotificationsReadResponse(now, 0, true) } @@ -124,11 +127,11 @@ class NotificationsService( private suspend fun unreadCount(items: List, userId: String): Int { val value = items.count { !it.read } - unreadCache[userId] = CachedUnread(value) + unreadCache.put(userId, value) return value } - private fun cachedUnread(userId: String): Int = unreadCache[userId]?.value ?: 0 + private fun cachedUnread(userId: String): Int = unreadCache.get(userId) ?: 0 private fun notificationKey(video: VideoItem): String { val serviceId = RssVideoMetadata.serviceId(video) @@ -162,8 +165,6 @@ class NotificationsService( MessageDigest.getInstance("SHA-256").digest(notificationKey(video).toByteArray()) .joinToString("") { byte -> "%02x".format(byte) } - private data class CachedUnread(val value: Int) - private companion object { fun serviceName(serviceId: Int): String = when (serviceId) { YOUTUBE_SERVICE_ID -> "YouTube" diff --git a/src/main/kotlin/dev/typetype/server/services/OpenMojiProxyService.kt b/src/main/kotlin/dev/typetype/server/services/OpenMojiProxyService.kt index 6c191b0c..03882be1 100644 --- a/src/main/kotlin/dev/typetype/server/services/OpenMojiProxyService.kt +++ b/src/main/kotlin/dev/typetype/server/services/OpenMojiProxyService.kt @@ -5,46 +5,57 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import okhttp3.OkHttpClient import okhttp3.Request -import java.util.concurrent.ConcurrentHashMap +import java.time.Duration import java.util.concurrent.TimeUnit -class OpenMojiProxyService(private val cache: CacheService) { +class OpenMojiProxyService( + private val cache: CacheService, + private val client: OkHttpClient = defaultOpenMojiClient(), + private val clock: () -> Long = System::currentTimeMillis, +) { - private val localCache = ConcurrentHashMap() - private val failedUntilByCode = ConcurrentHashMap() - private val notFoundUntilByCode = ConcurrentHashMap() - private val client = OkHttpClient.Builder() - .connectTimeout(2, TimeUnit.SECONDS) - .readTimeout(4, TimeUnit.SECONDS) - .callTimeout(5, TimeUnit.SECONDS) - .followRedirects(true) - .build() + private val localCache = BoundedExpiringCache( + maxEntries = LOCAL_CACHE_MAX_ENTRIES, + maxWeight = LOCAL_CACHE_MAX_BYTES, + ttl = Duration.ofMinutes(10), + weigher = { it.size.toLong() }, + clock = clock, + ) + private val failedUntilByCode = BoundedExpiringCache( + maxEntries = COOLDOWN_MAX_ENTRIES, + ttl = Duration.ofMillis(FAILURE_COOLDOWN_MS), + clock = clock, + ) + private val notFoundUntilByCode = BoundedExpiringCache( + maxEntries = COOLDOWN_MAX_ENTRIES, + ttl = Duration.ofMillis(NOT_FOUND_CACHE_MS), + clock = clock, + ) suspend fun getSvg(code: String): ByteArray? { val key = cacheKey(code) - val now = System.currentTimeMillis() - localCache[code]?.takeIf { it.expiresAtMs > now }?.let { return it.bytes } - if (now < (notFoundUntilByCode[code] ?: 0L)) return null - if (now < (failedUntilByCode[code] ?: 0L)) return null + localCache.get(code)?.let { return it } + if (notFoundUntilByCode.get(code) != null) return null + if (failedUntilByCode.get(code) != null) return null runCatching { cache.get(key) }.getOrNull()?.toByteArray(Charsets.UTF_8)?.let { bytes -> - localCache[code] = LocalSvg(bytes = bytes, expiresAtMs = now + LOCAL_CACHE_TTL_MS) + localCache.put(code, bytes) return bytes } return when (val fetched = fetch(code)) { is FetchResult.Success -> { failedUntilByCode.remove(code) notFoundUntilByCode.remove(code) - localCache[code] = LocalSvg(bytes = fetched.bytes, expiresAtMs = now + LOCAL_CACHE_TTL_MS) + localCache.put(code, fetched.bytes) runCatching { cache.set(key, fetched.bytes.toString(Charsets.UTF_8), SVG_CACHE_TTL_SECONDS) } fetched.bytes } FetchResult.NotFound -> { failedUntilByCode.remove(code) - notFoundUntilByCode[code] = now + NOT_FOUND_CACHE_MS + notFoundUntilByCode.put(code, Unit) null } FetchResult.Failed -> { - failedUntilByCode[code] = now + FAILURE_COOLDOWN_MS + failedUntilByCode.put(code, Unit) null } } @@ -71,7 +82,9 @@ class OpenMojiProxyService(private val cache: CacheService) { private const val SVG_CACHE_TTL_SECONDS = 604800L private const val FAILURE_COOLDOWN_MS = 15000L private const val NOT_FOUND_CACHE_MS = 300000L - private const val LOCAL_CACHE_TTL_MS = 600000L + private const val LOCAL_CACHE_MAX_ENTRIES = 256 + private const val LOCAL_CACHE_MAX_BYTES = 16L * 1024 * 1024 + private const val COOLDOWN_MAX_ENTRIES = 1024 } private sealed interface FetchResult { @@ -80,8 +93,11 @@ class OpenMojiProxyService(private val cache: CacheService) { data object Failed : FetchResult } - private data class LocalSvg( - val bytes: ByteArray, - val expiresAtMs: Long, - ) } + +private fun defaultOpenMojiClient(): OkHttpClient = OkHttpClient.Builder() + .connectTimeout(2, TimeUnit.SECONDS) + .readTimeout(4, TimeUnit.SECONDS) + .callTimeout(5, TimeUnit.SECONDS) + .followRedirects(true) + .build() diff --git a/src/test/kotlin/dev/typetype/server/services/OpenMojiProxyServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/OpenMojiProxyServiceTest.kt new file mode 100644 index 00000000..1dac530e --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/OpenMojiProxyServiceTest.kt @@ -0,0 +1,69 @@ +package dev.typetype.server.services + +import dev.typetype.server.cache.CacheService +import kotlinx.coroutines.test.runTest +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Protocol +import okhttp3.Response +import okhttp3.ResponseBody.Companion.toResponseBody +import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test + +class OpenMojiProxyServiceTest { + @Test + fun `successful SVG is served from the bounded local cache`() = runTest { + var requests = 0 + val bytes = "".toByteArray() + val client = client { + requests++ + ResponseBody(bytes) + } + val service = OpenMojiProxyService(EmptyCache, client) + + assertArrayEquals(bytes, service.getSvg("1F60A")) + assertArrayEquals(bytes, service.getSvg("1F60A")) + assertEquals(1, requests) + } + + @Test + fun `failed fetch is cooled down and retried after expiry`() = runTest { + var now = 0L + var requests = 0 + val client = client { + requests++ + ResponseBody("unavailable".toByteArray(), code = 503) + } + val service = OpenMojiProxyService(EmptyCache, client, clock = { now }) + + assertNull(service.getSvg("missing")) + assertNull(service.getSvg("missing")) + assertEquals(1, requests) + now = 15_000L + assertNull(service.getSvg("missing")) + assertEquals(2, requests) + } + + private fun client(response: () -> ResponseBody): OkHttpClient = OkHttpClient.Builder() + .addInterceptor { chain -> + val body = response() + Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(body.code) + .message("test") + .body(body.bytes.toResponseBody("image/svg+xml".toMediaType())) + .build() + } + .build() + + private data class ResponseBody(val bytes: ByteArray, val code: Int = 200) + + private object EmptyCache : CacheService { + override suspend fun get(key: String): String? = null + override suspend fun set(key: String, value: String, ttlSeconds: Long): Unit = Unit + override suspend fun delete(key: String): Unit = Unit + } +} From 898f6df90ac7475297dc3063e0a0e173e9680e9b Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 9 Sep 2026 10:20:04 +0200 Subject: [PATCH 29/45] perf: close shared resources on shutdown --- src/main/kotlin/dev/typetype/server/Application.kt | 4 ++++ .../kotlin/dev/typetype/server/cache/DragonflyService.kt | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/dev/typetype/server/Application.kt b/src/main/kotlin/dev/typetype/server/Application.kt index a84be3ce..d339c0c0 100644 --- a/src/main/kotlin/dev/typetype/server/Application.kt +++ b/src/main/kotlin/dev/typetype/server/Application.kt @@ -104,6 +104,10 @@ fun Application.module() { CoroutineScope(SupervisorJob() + Dispatchers.IO), ) monitor.subscribe(ApplicationStopped) { portabilityEngine.close() } + monitor.subscribe(ApplicationStopped) { + svc.sabrSessionStore.release() + cache.close() + } configurePlugins(authService) installApplicationRoutes( svc = svc, diff --git a/src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt b/src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt index d8517ccd..66b2cdfe 100644 --- a/src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt +++ b/src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt @@ -9,8 +9,9 @@ import kotlinx.coroutines.future.await class DragonflyService(url: String) : CacheService { + private val client: RedisClient = RedisClient.create(url) private val connection: StatefulRedisConnection = - RedisClient.create(url).connect() + client.connect() private val async: RedisAsyncCommands = connection.async() @@ -36,6 +37,11 @@ class DragonflyService(url: String) : CacheService { suspend fun ping(): Boolean = async.ping().await() == "PONG" + fun close() { + connection.close() + client.shutdown() + } + private companion object { const val REFRESH_IF_VALUE_MATCHES = "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('expire', KEYS[1], ARGV[2]) end return 0" From 7439e165972ed57a3ca94a9fe468ae3109d66263 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 9 Sep 2026 10:26:00 +0200 Subject: [PATCH 30/45] perf: skip unnecessary BiliBili related lookups --- .../dev/typetype/server/services/BilibiliRelatedService.kt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/kotlin/dev/typetype/server/services/BilibiliRelatedService.kt b/src/main/kotlin/dev/typetype/server/services/BilibiliRelatedService.kt index 6dba7a9a..a18cb2e9 100644 --- a/src/main/kotlin/dev/typetype/server/services/BilibiliRelatedService.kt +++ b/src/main/kotlin/dev/typetype/server/services/BilibiliRelatedService.kt @@ -18,6 +18,10 @@ private val BVID_REGEX = Regex("""/(BV[0-9A-Za-z]+)""") internal class BilibiliRelatedService { suspend fun patchRelatedStreams(response: StreamResponse, videoUrl: String): StreamResponse { + val missingUploaderUrls = response.relatedStreams.filter { it.uploaderUrl.isBlank() } + if (missingUploaderUrls.isEmpty()) return response + val relatedBvids = missingUploaderUrls.mapNotNull { BVID_REGEX.find(it.url)?.groupValues?.get(1) } + if (relatedBvids.isEmpty()) return response val uploaderUrls = fetchUploaderUrls(videoUrl) return response.copy( relatedStreams = response.relatedStreams.map { item -> From 659786d671da90ed3c499f7c751dd48c42b9d8b6 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 9 Sep 2026 10:30:13 +0200 Subject: [PATCH 31/45] perf: preserve NicoNico segment caching --- .../server/routes/NicoVideoProxyRoutes.kt | 26 +------------------ .../server/services/NicoVideoProxyService.kt | 1 + 2 files changed, 2 insertions(+), 25 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/routes/NicoVideoProxyRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/NicoVideoProxyRoutes.kt index 1ec8359c..8aa904d2 100644 --- a/src/main/kotlin/dev/typetype/server/routes/NicoVideoProxyRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/NicoVideoProxyRoutes.kt @@ -1,16 +1,11 @@ package dev.typetype.server.routes import dev.typetype.server.models.ErrorResponse -import dev.typetype.server.models.ExtractionResult import dev.typetype.server.services.NicoVideoProxyService -import io.ktor.http.ContentType import io.ktor.http.HttpStatusCode import io.ktor.server.response.respond -import io.ktor.server.response.respondOutputStream import io.ktor.server.routing.Route import io.ktor.server.routing.get -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext fun Route.nicoVideoProxyRoutes(nicoVideoProxyService: NicoVideoProxyService) { get("/proxy/nicovideo") { @@ -27,25 +22,6 @@ fun Route.nicoVideoProxyRoutes(nicoVideoProxyService: NicoVideoProxyService) { nicoVideoProxyService.fetchSegment(url, rangeHeader, domandBid) } - when (result) { - is ExtractionResult.Success -> { - val proxy = result.data - try { - val status = HttpStatusCode.fromValue(proxy.status) - val contentType = ContentType.parse(proxy.contentType) - proxy.contentRange?.let { call.response.headers.append("Content-Range", it) } - proxy.acceptRanges?.let { call.response.headers.append("Accept-Ranges", it) } - call.respondOutputStream(contentType, status, proxy.contentLength) { - withContext(Dispatchers.IO) { proxy.stream.copyTo(this@respondOutputStream) } - } - } finally { - proxy.close() - } - } - is ExtractionResult.BadRequest -> - call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) - is ExtractionResult.Failure -> - call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) - } + call.respondProxyResult(result) } } diff --git a/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt b/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt index 9c475dd1..10c0365e 100644 --- a/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt +++ b/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt @@ -113,6 +113,7 @@ class NicoVideoProxyService(client: OkHttpClient = defaultNicoProxyClient()) { acceptRanges = response.header("Accept-Ranges"), stream = body.byteStream(), close = response::close, + cacheControl = response.header("Cache-Control"), )) } }, From 6f90eccabd67c66bb54dc285aef2e80d9772432d Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 9 Sep 2026 10:38:08 +0200 Subject: [PATCH 32/45] chore: update Kotlin and JWT dependencies --- build.gradle.kts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 8078cbb2..2fb66c10 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,8 +1,8 @@ import java.time.Instant plugins { - kotlin("jvm") version "2.4.10" - kotlin("plugin.serialization") version "2.4.10" + kotlin("jvm") version "2.4.20" + kotlin("plugin.serialization") version "2.4.20" id("io.ktor.plugin") version "3.5.2" id("jacoco") } @@ -29,7 +29,7 @@ dependencies { implementation("com.fasterxml.jackson.core:jackson-core") implementation(platform("io.netty:netty-bom:4.2.17.Final")) constraints { - implementation("org.jsoup:jsoup:1.23.1") { + implementation("org.jsoup:jsoup:1.23.2") { because("CVE-2026-71497 affects PipePipeExtractor's transitive jsoup version") } } @@ -55,7 +55,7 @@ dependencies { implementation("org.postgresql:postgresql:42.7.13") implementation("org.xerial:sqlite-jdbc:3.53.4.0") implementation("com.password4j:password4j:1.8.4") - implementation("com.auth0:java-jwt:4.6.0") + implementation("com.auth0:java-jwt:4.6.1") testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") testRuntimeOnly("org.junit.platform:junit-platform-launcher") testImplementation("io.mockk:mockk:1.14.11") From 23255146168ada7fcd8dd9301db494757d56b7cc Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 9 Sep 2026 10:47:37 +0200 Subject: [PATCH 33/45] perf: stop recommendation scopes on shutdown --- src/main/kotlin/dev/typetype/server/Application.kt | 1 + .../dev/typetype/server/HomeRecommendationServices.kt | 7 ++++++- src/main/kotlin/dev/typetype/server/ServiceRegistry.kt | 2 +- .../server/services/HomeRecommendationPoolResolver.kt | 7 ++++++- .../typetype/server/services/HomeRecommendationService.kt | 6 +++++- .../server/services/HomeRecommendationWarmupService.kt | 7 ++++++- 6 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/Application.kt b/src/main/kotlin/dev/typetype/server/Application.kt index d339c0c0..e58bf1e1 100644 --- a/src/main/kotlin/dev/typetype/server/Application.kt +++ b/src/main/kotlin/dev/typetype/server/Application.kt @@ -78,6 +78,7 @@ fun Application.module() { val pushNotificationScheduler = PushNotificationScheduler(svc.pushNotificationService) pushNotificationScheduler.start() monitor.subscribe(ApplicationStopped) { pushNotificationScheduler.close() } + monitor.subscribe(ApplicationStopped) { svc.homeRecommendationServices.close() } val youtubeRemoteBrowserConfig = YoutubeRemoteBrowserConfig.fromEnvironment(subtitleServiceUrl) val youtubeRemoteLoginReadinessService = YoutubeRemoteLoginReadinessService( youtubeRemoteBrowserConfig, diff --git a/src/main/kotlin/dev/typetype/server/HomeRecommendationServices.kt b/src/main/kotlin/dev/typetype/server/HomeRecommendationServices.kt index 305ccdbe..3afc8d5e 100644 --- a/src/main/kotlin/dev/typetype/server/HomeRecommendationServices.kt +++ b/src/main/kotlin/dev/typetype/server/HomeRecommendationServices.kt @@ -9,7 +9,12 @@ import dev.typetype.server.services.HomeRecommendationWarmupService data class HomeRecommendationServices( val recommendationService: HomeRecommendationService, val warmupService: HomeRecommendationWarmupService, -) +) : AutoCloseable { + override fun close() { + warmupService.close() + recommendationService.close() + } +} fun createHomeRecommendationServices( cache: DragonflyService, diff --git a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt index fbf9f139..bd37d9e7 100644 --- a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt @@ -163,7 +163,7 @@ internal class ServiceRegistry( streamService = streamService, cache = cache, ) - private val homeRecommendationServices = createHomeRecommendationServices(cache, recommendationPoolResolverDependencies) + val homeRecommendationServices = createHomeRecommendationServices(cache, recommendationPoolResolverDependencies) val homeRecommendationService = homeRecommendationServices.recommendationService val homeRecommendationWarmupService = homeRecommendationServices.warmupService } diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolver.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolver.kt index 90984e7c..8058fd5f 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolver.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolver.kt @@ -6,11 +6,12 @@ import kotlinx.coroutines.Deferred import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.async +import kotlinx.coroutines.cancel import kotlinx.coroutines.launch class HomeRecommendationPoolResolver( private val dependencies: HomeRecommendationPoolResolverDependencies, -) { +) : AutoCloseable { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) private val state = HomeRecommendationPoolResolverState() private val poolCache = HomeRecommendationPoolCache(dependencies.cache) @@ -91,4 +92,8 @@ class HomeRecommendationPoolResolver( } } + override fun close() { + scope.cancel() + } + } diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationService.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationService.kt index 5943417c..819a9dd7 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationService.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationService.kt @@ -4,7 +4,7 @@ import dev.typetype.server.models.HomeRecommendationsResponse class HomeRecommendationService( private val poolResolver: HomeRecommendationPoolResolver, -) { +) : AutoCloseable { private fun args( userId: String, serviceId: Int, @@ -46,4 +46,8 @@ class HomeRecommendationService( mode = HomeRecommendationPoolMode.SHORTS, poolResolver = poolResolver, ) + + override fun close() { + poolResolver.close() + } } diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmupService.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmupService.kt index 6d079362..af4c40fa 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmupService.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmupService.kt @@ -4,6 +4,7 @@ import dev.typetype.server.cache.CacheService import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel import kotlinx.coroutines.delay import kotlinx.coroutines.isActive import kotlinx.coroutines.launch @@ -11,7 +12,7 @@ import kotlinx.coroutines.launch class HomeRecommendationWarmupService( private val recommendationService: HomeRecommendationService, private val cache: CacheService, -) : HomeRecommendationWarmup { +) : HomeRecommendationWarmup, AutoCloseable { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) private val tracker = HomeWarmupTracker(WARMUP_THROTTLE_MS, ACTIVE_TTL_MS) private val poolCache = HomeRecommendationPoolCache(cache) @@ -72,6 +73,10 @@ class HomeRecommendationWarmupService( ), ) + override fun close() { + scope.cancel() + } + companion object { private const val WARMUP_LIMIT = 20 private const val WARMUP_THROTTLE_MS = 10 * 60 * 1000L From e55eda7530235085ad54d2d8a9dafc30342a9bb7 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 9 Sep 2026 10:52:27 +0200 Subject: [PATCH 34/45] perf: stop feed and takeout jobs on shutdown --- src/main/kotlin/dev/typetype/server/Application.kt | 2 ++ .../typetype/server/services/SubscriptionFeedService.kt | 7 +++++++ .../server/services/YoutubeTakeoutImportJobEngine.kt | 7 ++++++- .../server/services/YoutubeTakeoutImportJobService.kt | 6 +++++- 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/Application.kt b/src/main/kotlin/dev/typetype/server/Application.kt index e58bf1e1..3e382395 100644 --- a/src/main/kotlin/dev/typetype/server/Application.kt +++ b/src/main/kotlin/dev/typetype/server/Application.kt @@ -78,6 +78,8 @@ fun Application.module() { val pushNotificationScheduler = PushNotificationScheduler(svc.pushNotificationService) pushNotificationScheduler.start() monitor.subscribe(ApplicationStopped) { pushNotificationScheduler.close() } + monitor.subscribe(ApplicationStopped) { svc.subscriptionFeedService.close() } + monitor.subscribe(ApplicationStopped) { svc.youtubeTakeoutImportService.close() } monitor.subscribe(ApplicationStopped) { svc.homeRecommendationServices.close() } val youtubeRemoteBrowserConfig = YoutubeRemoteBrowserConfig.fromEnvironment(subtitleServiceUrl) val youtubeRemoteLoginReadinessService = YoutubeRemoteLoginReadinessService( diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt index 21e2627f..6966e9e4 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt @@ -10,6 +10,7 @@ import kotlinx.coroutines.CoroutineStart import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel import kotlinx.coroutines.joinAll import kotlinx.coroutines.launch import kotlinx.coroutines.withTimeoutOrNull @@ -148,6 +149,12 @@ class SubscriptionFeedService( internal fun isRefreshing(userId: String): Boolean = refreshJobs[userId]?.isActive == true + fun close() { + refreshJobs.values.forEach(Job::cancel) + refreshJobs.clear() + refreshScope.cancel() + } + private fun scheduleRefresh(userId: String, requestId: String?) { val job = refreshScope.launch(start = CoroutineStart.LAZY) { var retry = false diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobEngine.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobEngine.kt index 82b00653..b2daca0d 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobEngine.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobEngine.kt @@ -4,9 +4,10 @@ import dev.typetype.server.models.YoutubeTakeoutCommitPlan import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel import kotlinx.coroutines.launch -class YoutubeTakeoutImportJobEngine { +class YoutubeTakeoutImportJobEngine : AutoCloseable { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) fun startPreview(jobId: String, block: suspend () -> Unit): Unit { @@ -16,4 +17,8 @@ class YoutubeTakeoutImportJobEngine { fun startCommit(jobId: String, plan: YoutubeTakeoutCommitPlan, block: suspend (YoutubeTakeoutCommitPlan) -> Unit): Unit { scope.launch { block(plan) } } + + override fun close() { + scope.cancel() + } } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt index 5c72b6ad..e8a665b6 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt @@ -22,7 +22,7 @@ class YoutubeTakeoutImportJobService( private val privacyService: YoutubeTakeoutPrivacyService = YoutubeTakeoutPrivacyService(), private val cache: YoutubeTakeoutImportCache = YoutubeTakeoutImportCache(), private val engine: YoutubeTakeoutImportJobEngine = YoutubeTakeoutImportJobEngine(), -) { +) : AutoCloseable { suspend fun create(userId: String, archivePath: Path): YoutubeTakeoutImportJobStatus { val jobId = store.create(userId, archivePath) return statusStore.getStatus(userId, jobId) ?: error("Failed to create job") @@ -101,4 +101,8 @@ class YoutubeTakeoutImportJobService( } suspend fun purgeExpired() = YoutubeTakeoutImportCleanupService(privacyService).purgeExpiredJobs() + + override fun close() { + engine.close() + } } From 8cb9c876bf7a82fbbbab2eb7b009072151f1d754 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 9 Sep 2026 10:55:58 +0200 Subject: [PATCH 35/45] fix: preserve injected feed scope ownership --- .../dev/typetype/server/services/SubscriptionFeedService.kt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt index 6966e9e4..019bf267 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt @@ -24,8 +24,10 @@ class SubscriptionFeedService( channelService: ChannelService, cache: CacheService, private val clock: () -> Long = System::currentTimeMillis, - private val refreshScope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO), + refreshScope: CoroutineScope? = null, ) { + private val refreshScope = refreshScope ?: CoroutineScope(SupervisorJob() + Dispatchers.IO) + private val ownsRefreshScope = refreshScope == null private val store = SubscriptionFeedSnapshotStore(cache, clock) private val selections = SubscriptionFeedSelectionStore(cache, subscriptionsService) private val builder = SubscriptionFeedBuilder(channelService) @@ -152,7 +154,7 @@ class SubscriptionFeedService( fun close() { refreshJobs.values.forEach(Job::cancel) refreshJobs.clear() - refreshScope.cancel() + if (ownsRefreshScope) refreshScope.cancel() } private fun scheduleRefresh(userId: String, requestId: String?) { From 367fea37b15ef5c3b7845c57589c20544e9adfa4 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 9 Sep 2026 11:06:07 +0200 Subject: [PATCH 36/45] perf: cache BiliBili related lookups --- .../server/ExtractionServiceRegistry.kt | 5 +- .../server/services/BilibiliRelatedService.kt | 34 +++++++++--- .../server/services/OkHttpProxyService.kt | 6 ++- .../server/BilibiliRelatedServiceTest.kt | 54 +++++++++++++++++++ 4 files changed, 87 insertions(+), 12 deletions(-) create mode 100644 src/test/kotlin/dev/typetype/server/BilibiliRelatedServiceTest.kt diff --git a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt index 2dc6fd25..cabb823c 100644 --- a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt @@ -82,15 +82,16 @@ internal class ExtractionServiceRegistry( .build() val sabrSessionStore = SabrSessionStore(subtitleServiceUrl, initCache = cache) val youtubeSubtitleService = YouTubeSubtitleService(httpClient, subtitleServiceUrl) + private val bilibiliRelatedService = BilibiliRelatedService() private val directPipePipeStreamService = PipePipeStreamService( cache, youtubeSubtitleService, - BilibiliRelatedService(), + bilibiliRelatedService, ) private val sabrPipePipeStreamService = PipePipeStreamService( cache, youtubeSubtitleService, - BilibiliRelatedService(), + bilibiliRelatedService, sabrSessionStore::rememberExtractedInfo, ) private val publicStreamService = YoutubePlayerClientStreamService( diff --git a/src/main/kotlin/dev/typetype/server/services/BilibiliRelatedService.kt b/src/main/kotlin/dev/typetype/server/services/BilibiliRelatedService.kt index a18cb2e9..38a55888 100644 --- a/src/main/kotlin/dev/typetype/server/services/BilibiliRelatedService.kt +++ b/src/main/kotlin/dev/typetype/server/services/BilibiliRelatedService.kt @@ -10,30 +10,48 @@ import kotlinx.serialization.json.jsonPrimitive import kotlinx.serialization.json.longOrNull import org.schabi.newpipe.extractor.NewPipe import org.schabi.newpipe.extractor.services.bilibili.BilibiliService +import java.time.Duration private const val RELATED_BASE_URL = "https://api.bilibili.com/x/web-interface/archive/related?bvid=" private const val SPACE_BASE_URL = "https://space.bilibili.com/" private val BVID_REGEX = Regex("""/(BV[0-9A-Za-z]+)""") -internal class BilibiliRelatedService { +internal class BilibiliRelatedService( + private val relatedLookupOverride: (suspend (String) -> Map)? = null, +) { + private val uploaderUrlCache = BoundedExpiringCache>( + maxEntries = 256, + ttl = Duration.ofMinutes(5), + weigher = { it.size.toLong().coerceAtLeast(1L) }, + ) suspend fun patchRelatedStreams(response: StreamResponse, videoUrl: String): StreamResponse { - val missingUploaderUrls = response.relatedStreams.filter { it.uploaderUrl.isBlank() } - if (missingUploaderUrls.isEmpty()) return response - val relatedBvids = missingUploaderUrls.mapNotNull { BVID_REGEX.find(it.url)?.groupValues?.get(1) } - if (relatedBvids.isEmpty()) return response - val uploaderUrls = fetchUploaderUrls(videoUrl) + val sourceBvid = BVID_REGEX.find(videoUrl)?.groupValues?.get(1) ?: return response + val missingRelatedBvids = response.relatedStreams.asSequence() + .filter { it.uploaderUrl.isBlank() } + .mapNotNull { BVID_REGEX.find(it.url)?.groupValues?.get(1) } + .toSet() + if (missingRelatedBvids.isEmpty()) return response + val uploaderUrls = uploaderUrlCache.get(sourceBvid) ?: resolveUploaderUrls(videoUrl).also { fetched -> + if (fetched.isNotEmpty()) uploaderUrlCache.put(sourceBvid, fetched) + } + if (uploaderUrls.isEmpty()) return response return response.copy( relatedStreams = response.relatedStreams.map { item -> if (item.uploaderUrl.isNotBlank()) item else { - val bvid = BVID_REGEX.find(item.url)?.groupValues?.get(1) ?: "" - uploaderUrls[bvid]?.let { item.copy(uploaderUrl = it) } ?: item + val bvid = BVID_REGEX.find(item.url)?.groupValues?.get(1) + if (bvid in missingRelatedBvids) { + uploaderUrls[bvid]?.let { item.copy(uploaderUrl = it) } ?: item + } else item } } ) } + private suspend fun resolveUploaderUrls(videoUrl: String): Map = + relatedLookupOverride?.invoke(videoUrl) ?: fetchUploaderUrls(videoUrl) + private suspend fun fetchUploaderUrls(videoUrl: String): Map = withContext(Dispatchers.IO) { runCatching { diff --git a/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt b/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt index 24a120c7..18b5d656 100644 --- a/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt +++ b/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt @@ -13,10 +13,12 @@ import java.net.URLEncoder import java.nio.charset.StandardCharsets internal val GOOGLEVIDEO_URL_REGEX = Regex("""https://[a-z0-9.\-]+\.googlevideo\.com/\S+""") +private val CPN_TRACKING_PARAM_REGEX = Regex("[&?]cpn=[^&]*") +private val PPPID_TRACKING_PARAM_REGEX = Regex("[&?]pppid=[^&]*") internal fun stripTrackingParams(url: String): String = - url.replace(Regex("[&?]cpn=[^&]*"), "") - .replace(Regex("[&?]pppid=[^&]*"), "") + url.replace(CPN_TRACKING_PARAM_REGEX, "") + .replace(PPPID_TRACKING_PARAM_REGEX, "") internal fun rewriteHlsManifest(manifest: String): String = manifest.replace(GOOGLEVIDEO_URL_REGEX) { match -> diff --git a/src/test/kotlin/dev/typetype/server/BilibiliRelatedServiceTest.kt b/src/test/kotlin/dev/typetype/server/BilibiliRelatedServiceTest.kt new file mode 100644 index 00000000..a72bec0c --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/BilibiliRelatedServiceTest.kt @@ -0,0 +1,54 @@ +package dev.typetype.server + +import dev.typetype.server.services.BilibiliRelatedService +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class BilibiliRelatedServiceTest { + + @Test + fun `caches uploader lookups shared by repeated stream extraction`() = runTest { + var lookups = 0 + val service = BilibiliRelatedService { _ -> + lookups++ + mapOf("BV1Related" to "https://space.bilibili.com/42") + } + val response = testStreamResponse().copy( + relatedStreams = listOf( + testVideoItem().copy( + url = "https://www.bilibili.com/video/BV1Related", + uploaderUrl = "", + ), + ), + ) + + val first = service.patchRelatedStreams(response, SOURCE_URL) + val second = service.patchRelatedStreams(response, SOURCE_URL) + + assertEquals("https://space.bilibili.com/42", first.relatedStreams.single().uploaderUrl) + assertEquals("https://space.bilibili.com/42", second.relatedStreams.single().uploaderUrl) + assertEquals(1, lookups) + } + + @Test + fun `skips lookup when source or related item has no BVID`() = runTest { + var lookups = 0 + val service = BilibiliRelatedService { _ -> + lookups++ + emptyMap() + } + val response = testStreamResponse().copy( + relatedStreams = listOf(testVideoItem().copy(url = "https://example.com/video")), + ) + + val result = service.patchRelatedStreams(response, "https://example.com/source") + + assertEquals(response, result) + assertEquals(0, lookups) + } + + private companion object { + const val SOURCE_URL = "https://www.bilibili.com/video/BV1Source" + } +} From 501a31a354fff50297052a4d2999f223f849ecd3 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 9 Sep 2026 23:29:40 +0200 Subject: [PATCH 37/45] fix: hide provider media behind expiring handles --- openapi.yaml | 1 + openapi/paths/media.yaml | 59 ++++++++ .../server/ApplicationStreamRoutes.kt | 3 + .../server/ExtractionServiceRegistry.kt | 9 +- .../dev/typetype/server/ServiceRegistry.kt | 1 + .../routes/ProviderMediaHandleRoutes.kt | 37 +++++ .../server/routes/StreamRouteDependencies.kt | 2 + .../services/ProviderMediaHandleService.kt | 139 ++++++++++++++++++ .../typetype/server/services/ProxyService.kt | 8 + 9 files changed, 256 insertions(+), 3 deletions(-) create mode 100644 openapi/paths/media.yaml create mode 100644 src/main/kotlin/dev/typetype/server/routes/ProviderMediaHandleRoutes.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/ProviderMediaHandleService.kt diff --git a/openapi.yaml b/openapi.yaml index 2a953e3e..72d2cc37 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -29,6 +29,7 @@ paths: /streams/audio-only/source: { $ref: ./openapi/paths/streams.yaml#/AudioOnlySource } /subtitles/youtube/{videoId}: { $ref: ./openapi/paths/subtitles.yaml#/YoutubeSubtitle } /proxy: { $ref: ./openapi/paths/proxy.yaml#/Proxy } + /media/{handle}: { $ref: ./openapi/paths/media.yaml#/ProviderMedia } /sabr/download/{videoId}: { $ref: ./openapi/paths/sabr-download.yaml#/SabrDownload } /sabr/playback/{sessionId}/position: { $ref: ./openapi/paths/sabr-playback.yaml#/SabrPlaybackPosition } /sabr/playback/{sessionId}/window: { $ref: ./openapi/paths/sabr-playback.yaml#/SabrPlaybackWindow } diff --git a/openapi/paths/media.yaml b/openapi/paths/media.yaml new file mode 100644 index 00000000..efe7b3e1 --- /dev/null +++ b/openapi/paths/media.yaml @@ -0,0 +1,59 @@ +ProviderMedia: + get: + tags: [playback] + summary: Retrieve provider media through an opaque temporary handle + description: >- + Streams BiliBili or NicoNico media referenced by a handle returned from a provider stream + endpoint. The signed provider URL and NicoNico cookie remain server-side; clients must not + construct or proxy the provider URL themselves. Handles expire with the upstream media + signature and support byte ranges for progressive and segmented playback. + parameters: + - name: handle + in: path + required: true + schema: + type: string + pattern: '^m1_[A-Za-z0-9_-]{24}$' + - name: Range + in: header + required: false + schema: { type: string } + responses: + '200': + description: Complete provider media response. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/octet-stream: + schema: { type: string, format: binary } + video/mp4: + schema: { type: string, format: binary } + audio/mp4: + schema: { type: string, format: binary } + application/vnd.apple.mpegurl: + schema: { type: string } + '206': + description: Partial provider media response. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + Content-Range: + schema: { type: string } + Accept-Ranges: + schema: { type: string } + content: + application/octet-stream: + schema: { type: string, format: binary } + video/mp4: + schema: { type: string, format: binary } + audio/mp4: + schema: { type: string, format: binary } + '404': + $ref: ../components/common.yaml#/JsonError + '422': + $ref: ../components/common.yaml#/JsonError + '429': + $ref: ../components/common.yaml#/JsonError + '503': + $ref: ../components/common.yaml#/JsonError diff --git a/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt b/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt index 1c3fd0e5..2e729658 100644 --- a/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt @@ -5,6 +5,7 @@ import dev.typetype.server.routes.audioOnlySourceRoutes import dev.typetype.server.routes.manifestRoutes import dev.typetype.server.routes.nicoVideoProxyRoutes import dev.typetype.server.routes.proxyRoutes +import dev.typetype.server.routes.providerMediaHandleRoutes import dev.typetype.server.routes.storyboardProxyRoutes import dev.typetype.server.routes.streamRoutes import dev.typetype.server.routes.withPlayableSabrStreams @@ -30,6 +31,7 @@ internal fun Route.installStreamRoutes( adminSettingsService = adminSettingsService, blockedService = svc.blockedService, publicHlsManifestTokenService = svc.publicHlsManifestTokenService, + providerMediaHandleService = svc.providerMediaHandleService, sabrStreamContractFilter = { url, data -> data.withPlayableSabrStreams(url, svc.sabrSessionStore) }, youtubeSessionSabrStreamInfo = svc.youtubeSessionSabrStreamService?.let { service -> { userId, url -> service.getStreamInfo(userId, url) } @@ -62,6 +64,7 @@ internal fun Route.installStreamRoutes( internal fun Route.installProxyRoutes(svc: ServiceRegistry) { rateLimit(PROXY_ZONE) { proxyRoutes(svc.proxyService, svc.youtubeSubtitleDeliveryService) + providerMediaHandleRoutes(svc.providerMediaHandleService, svc.proxyService) youtubeSubtitleRoutes(svc.youtubeSubtitleDeliveryService) audioOnlySourceRoutes( streamService = svc.streamService, diff --git a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt index cabb823c..d2f72dc5 100644 --- a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt @@ -29,6 +29,7 @@ import dev.typetype.server.services.PipePipeSearchService import dev.typetype.server.services.PipePipeStreamService import dev.typetype.server.services.PipePipeSuggestionService import dev.typetype.server.services.PipePipeTrendingService +import dev.typetype.server.services.ProviderMediaHandleService import dev.typetype.server.services.SabrFallbackStreamService import dev.typetype.server.services.SabrBootstrapStreamService import dev.typetype.server.services.SabrSessionStore @@ -154,9 +155,10 @@ internal class ExtractionServiceRegistry( YoutubeScopedPublicPlaylistService(PipePipePublicPlaylistService()), cache, ) - val proxyService = OkHttpProxyService(proxyHttpClient) - val nicoVideoProxyService = NicoVideoProxyService() - val manifestService = CachedManifestService(ManifestService(streamService), cache) + val providerMediaHandleService = ProviderMediaHandleService(cache) + val proxyService = OkHttpProxyService(proxyHttpClient, providerMediaHandleService) + val nicoVideoProxyService = NicoVideoProxyService(mediaHandleService = providerMediaHandleService) + val manifestService = CachedManifestService(ManifestService(streamService, providerMediaHandleService), cache) val nativeManifestService = CachedNativeManifestService(NativeManifestService(), cache) val hlsManifestService = HlsManifestService( streamService, @@ -164,6 +166,7 @@ internal class ExtractionServiceRegistry( cache, hlsManifestUrlSigner, tokenYoutubeSessionClient::fetchHlsManifestUrl, + providerMediaHandleService, ) val youtubeSessionHlsManifestService = hlsTokenService?.let { tokenService -> youtubeSessionStreamService?.let { diff --git a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt index bd37d9e7..e775af42 100644 --- a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt @@ -80,6 +80,7 @@ internal class ServiceRegistry( val podcastService = extraction.podcastService val publicPlaylistService = extraction.publicPlaylistService val proxyService = extraction.proxyService + val providerMediaHandleService = extraction.providerMediaHandleService val youtubeSubtitleDeliveryService = extraction.youtubeSubtitleDeliveryService val nicoVideoProxyService = extraction.nicoVideoProxyService val manifestService = extraction.manifestService diff --git a/src/main/kotlin/dev/typetype/server/routes/ProviderMediaHandleRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/ProviderMediaHandleRoutes.kt new file mode 100644 index 00000000..9e8f6fa1 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/ProviderMediaHandleRoutes.kt @@ -0,0 +1,37 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.services.ProviderMediaHandleService +import dev.typetype.server.services.ProviderMediaAwareProxyService +import dev.typetype.server.services.ProxyService +import io.ktor.http.HttpStatusCode +import io.ktor.server.response.respond +import io.ktor.server.routing.Route +import io.ktor.server.routing.get + +internal fun Route.providerMediaHandleRoutes( + handleService: ProviderMediaHandleService, + proxyService: ProxyService, +) { + get("/media/{handle}") { + val handle = call.parameters["handle"] + ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing media handle")) + val target = runCatching { handleService.resolve(handle) }.getOrElse { + return@get call.respond( + HttpStatusCode.ServiceUnavailable, + ErrorResponse("Media handle service is unavailable", "media_handle_unavailable"), + ) + } ?: return@get call.respond( + HttpStatusCode.NotFound, + ErrorResponse("Media handle has expired or is unknown", "media_handle_not_found"), + ) + + val rangeHeader = call.request.headers["Range"] + val result = if (proxyService is ProviderMediaAwareProxyService) { + proxyService.pipeProviderMedia(target.url, rangeHeader, target.domandBid) + } else { + proxyService.pipe(target.url, rangeHeader, target.domandBid) + } + call.respondProxyResult(result) + } +} diff --git a/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt b/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt index 177533c7..24addc64 100644 --- a/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt +++ b/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt @@ -7,6 +7,7 @@ import dev.typetype.server.services.AdminSettingsService import dev.typetype.server.services.AuthService import dev.typetype.server.services.BlockedService import dev.typetype.server.services.PublicHlsManifestTokenService +import dev.typetype.server.services.ProviderMediaHandleService internal data class StreamRouteDependencies( val authService: AuthService?, @@ -14,6 +15,7 @@ internal data class StreamRouteDependencies( val adminSettingsService: AdminSettingsService?, val blockedService: BlockedService?, val publicHlsManifestTokenService: PublicHlsManifestTokenService?, + val providerMediaHandleService: ProviderMediaHandleService?, val sabrStreamContractFilter: (suspend (String, StreamResponse) -> StreamResponse)?, val youtubeSessionSabrStreamInfo: (suspend (String, String) -> ExtractionResult?)?, ) diff --git a/src/main/kotlin/dev/typetype/server/services/ProviderMediaHandleService.kt b/src/main/kotlin/dev/typetype/server/services/ProviderMediaHandleService.kt new file mode 100644 index 00000000..fe1889b7 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/ProviderMediaHandleService.kt @@ -0,0 +1,139 @@ +package dev.typetype.server.services + +import dev.typetype.server.cache.CacheJson +import dev.typetype.server.cache.CacheService +import dev.typetype.server.models.AudioStreamItem +import dev.typetype.server.models.StreamResponse +import dev.typetype.server.models.VideoStreamItem +import kotlinx.serialization.Serializable +import kotlinx.serialization.encodeToString +import java.net.URLDecoder +import java.nio.charset.StandardCharsets +import java.security.MessageDigest +import java.util.Base64 + +internal enum class ProviderMediaType { + BILIBILI, + NICONICO, +} + +@Serializable +internal data class ProviderMediaTarget( + val url: String, + val domandBid: String? = null, +) + +class ProviderMediaHandleService( + private val cache: CacheService, + private val nowSeconds: () -> Long = { System.currentTimeMillis() / 1000 }, +) { + internal suspend fun materialize(response: StreamResponse, provider: ProviderMediaType): StreamResponse = response.copy( + hlsUrl = response.hlsUrl.handleIfRemote(provider), + dashMpdUrl = response.dashMpdUrl.handleIfRemote(provider), + videoStreams = response.videoStreams.withHandledVideoUrls(provider), + audioStreams = response.audioStreams.withHandledAudioUrls(provider), + videoOnlyStreams = response.videoOnlyStreams.withHandledVideoUrls(provider), + ) + + internal suspend fun createPath(rawUrl: String, domandBid: String? = null): String { + val (cleanUrl, fragment) = rawUrl.splitFragment() + val resolvedBid = domandBid?.takeIf { it.isNotBlank() } + ?: fragment.takeIf { it.isNotBlank() }?.let(::parseNicoCookie) + val target = requireProxyTarget(stripTrackingParams(cleanUrl)) + require(target.provider == ProxyProvider.BILIBILI || target.provider == ProxyProvider.NICONICO) { + "Unsupported provider media URL" + } + val normalizedUrl = stripTrackingParams(target.url.toString()) + val handle = handleId(normalizedUrl, resolvedBid) + val ttl = mediaHandleTtlSeconds(normalizedUrl) + val value = CacheJson.encodeToString(ProviderMediaTarget(normalizedUrl, resolvedBid)) + cache.set(cacheKey(handle), value, ttl) + return "/media/$handle" + } + + internal suspend fun resolve(handle: String): ProviderMediaTarget? { + if (!HANDLE_PATTERN.matches(handle)) return null + return cache.get(cacheKey(handle))?.let { encoded -> + runCatching { CacheJson.decodeFromString(encoded) }.getOrNull() + } + } + + internal fun relativeManifestPath(path: String): String = + "../media/${path.substringAfterLast('/')}" + + private suspend fun String.handleIfRemote(provider: ProviderMediaType): String { + if (isBlank() || startsWith("/")) return this + return if (supportsProvider(this, provider)) createPath(this) else this + } + + private suspend fun List.withHandledVideoUrls(provider: ProviderMediaType): List { + val handled = ArrayList(size) + for (item in this) { + handled += item.copy( + url = item.url.handleIfRemote(provider), + manifestUrl = item.manifestUrl?.handleIfRemote(provider), + sabrSessionUrl = item.sabrSessionUrl?.handleIfRemote(provider), + ) + } + return handled + } + + private suspend fun List.withHandledAudioUrls(provider: ProviderMediaType): List { + val handled = ArrayList(size) + for (item in this) { + handled += item.copy( + url = item.url.handleIfRemote(provider), + manifestUrl = item.manifestUrl?.handleIfRemote(provider), + sabrSessionUrl = item.sabrSessionUrl?.handleIfRemote(provider), + ) + } + return handled + } + + private fun supportsProvider(url: String, provider: ProviderMediaType): Boolean = + runCatching { requireProxyTarget(url).provider }.getOrNull() == provider.toProxyProvider() + + private fun handleId(url: String, domandBid: String?): String { + val input = buildString { + append("provider-media:v1:") + append(url) + append('\u0000') + append(domandBid.orEmpty()) + } + val digest = MessageDigest.getInstance("SHA-256").digest(input.toByteArray(StandardCharsets.UTF_8)) + val encoded = Base64.getUrlEncoder().withoutPadding().encodeToString(digest.copyOf(HANDLE_BYTES)) + return "m1_$encoded" + } + + private fun mediaHandleTtlSeconds(url: String): Long { + val decoded = runCatching { URLDecoder.decode(url, StandardCharsets.UTF_8) }.getOrDefault(url) + val expiry = EXPIRY_PATTERN.findAll(decoded) + .mapNotNull { it.groupValues[1].toLongOrNull() } + .minOrNull() + val ttl = expiry?.minus(nowSeconds())?.minus(EXPIRY_SAFETY_SECONDS) + ?: DEFAULT_TTL_SECONDS + if (ttl <= 0L) throw IllegalArgumentException("Provider media URL has expired") + return ttl.coerceAtMost(MAX_TTL_SECONDS) + } + + private fun String.splitFragment(): Pair { + val index = indexOf('#') + return if (index < 0) this to "" else substring(0, index) to substring(index + 1) + } + + private companion object { + const val HANDLE_BYTES = 18 + const val DEFAULT_TTL_SECONDS = 1_800L + const val MAX_TTL_SECONDS = 3_600L + const val EXPIRY_SAFETY_SECONDS = 30L + val HANDLE_PATTERN = Regex("m1_[A-Za-z0-9_-]{24}") + val EXPIRY_PATTERN = Regex("(?:^|[?&#=])(?:deadline|Expires|exp)=(\\d+)", RegexOption.IGNORE_CASE) + + fun cacheKey(handle: String): String = "provider-media:v1:$handle" + + fun ProviderMediaType.toProxyProvider(): ProxyProvider = when (this) { + ProviderMediaType.BILIBILI -> ProxyProvider.BILIBILI + ProviderMediaType.NICONICO -> ProxyProvider.NICONICO + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/ProxyService.kt b/src/main/kotlin/dev/typetype/server/services/ProxyService.kt index 73d59be2..1286bc5e 100644 --- a/src/main/kotlin/dev/typetype/server/services/ProxyService.kt +++ b/src/main/kotlin/dev/typetype/server/services/ProxyService.kt @@ -6,3 +6,11 @@ import dev.typetype.server.models.ProxyResponse interface ProxyService { suspend fun pipe(url: String, rangeHeader: String?, domandBid: String? = null): ExtractionResult } + +internal interface ProviderMediaAwareProxyService { + suspend fun pipeProviderMedia( + url: String, + rangeHeader: String?, + domandBid: String? = null, + ): ExtractionResult +} From d0e736bf7f915548f507725078ab99cc9852a83e Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 9 Sep 2026 23:29:51 +0200 Subject: [PATCH 38/45] fix: rewrite provider manifests to local handles --- .../typetype/server/routes/StreamRoutes.kt | 33 +++++++++++- .../server/services/CachedManifestService.kt | 9 ++++ .../server/services/HlsManifestService.kt | 32 ++++++++++-- .../server/services/ManifestService.kt | 27 +++++++--- .../server/services/NicoVideoProxyService.kt | 50 ++++++++++++++++++- .../server/services/OkHttpProxyService.kt | 44 +++++++++++++--- .../services/ProviderHlsManifestRewrite.kt | 43 ++++++++++++++++ .../server/services/StreamCacheTtlResolver.kt | 26 ++++++---- 8 files changed, 232 insertions(+), 32 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/services/ProviderHlsManifestRewrite.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt index 9f315c08..6b6b25f5 100644 --- a/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt @@ -9,6 +9,8 @@ import dev.typetype.server.services.AuthService import dev.typetype.server.services.BlockedContentProfile import dev.typetype.server.services.BlockedService import dev.typetype.server.services.PublicHlsManifestTokenService +import dev.typetype.server.services.ProviderMediaHandleService +import dev.typetype.server.services.ProviderMediaType import dev.typetype.server.services.StreamService import dev.typetype.server.services.YOUTUBE_SESSION_REQUIRED_CODE import dev.typetype.server.services.YOUTUBE_SESSION_REQUIRED_ERROR @@ -24,6 +26,7 @@ import io.ktor.server.routing.get private const val STREAMS_CACHE_CONTROL = "public, max-age=21600, stale-while-revalidate=3600" private const val AUTHENTICATED_STREAMS_CACHE_CONTROL = "no-store" +private const val PROVIDER_STREAMS_CACHE_CONTROL = "no-store" fun Route.streamRoutes( streamService: StreamService, @@ -32,6 +35,7 @@ fun Route.streamRoutes( adminSettingsService: AdminSettingsService? = null, blockedService: BlockedService? = null, publicHlsManifestTokenService: PublicHlsManifestTokenService? = null, + providerMediaHandleService: ProviderMediaHandleService? = null, nicoNicoStreamService: StreamService = streamService, bilibiliStreamService: StreamService = streamService, sabrBootstrapStreamService: StreamService = streamService, @@ -44,6 +48,7 @@ fun Route.streamRoutes( adminSettingsService = adminSettingsService, blockedService = blockedService, publicHlsManifestTokenService = publicHlsManifestTokenService, + providerMediaHandleService = providerMediaHandleService, sabrStreamContractFilter = sabrStreamContractFilter, youtubeSessionSabrStreamInfo = youtubeSessionSabrStreamInfo, ) @@ -128,11 +133,29 @@ private fun Route.streamRoute( ErrorResponse("No playable streams available", "no_playable_streams"), ) } + val publicData = try { + dependencies.providerMediaHandleService?.let { service -> + providerMediaType(deliveryMode)?.let { service.materialize(data, it) } + } ?: data + } catch (error: Exception) { + return@get call.respond( + HttpStatusCode.BadGateway, + ErrorResponse( + error.message ?: "Provider media handle service failed", + "media_handle_unavailable", + ), + ) + } call.response.headers.append( HttpHeaders.CacheControl, - if (access.userId != null) AUTHENTICATED_STREAMS_CACHE_CONTROL else STREAMS_CACHE_CONTROL, + when { + access.userId != null -> AUTHENTICATED_STREAMS_CACHE_CONTROL + deliveryMode == StreamDeliveryMode.NicoNico || + deliveryMode == StreamDeliveryMode.BiliBili -> PROVIDER_STREAMS_CACHE_CONTROL + else -> STREAMS_CACHE_CONTROL + }, ) - call.respond(data) + call.respond(publicData) } is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message, result.code)) @@ -142,6 +165,12 @@ private fun Route.streamRoute( } } +private fun providerMediaType(deliveryMode: StreamDeliveryMode): ProviderMediaType? = when (deliveryMode) { + StreamDeliveryMode.NicoNico -> ProviderMediaType.NICONICO + StreamDeliveryMode.BiliBili -> ProviderMediaType.BILIBILI + StreamDeliveryMode.YoutubeSabr -> null +} + private data class StreamResolution( val result: ExtractionResult, val authenticated: Boolean = false, diff --git a/src/main/kotlin/dev/typetype/server/services/CachedManifestService.kt b/src/main/kotlin/dev/typetype/server/services/CachedManifestService.kt index c4afa3db..5a37951f 100644 --- a/src/main/kotlin/dev/typetype/server/services/CachedManifestService.kt +++ b/src/main/kotlin/dev/typetype/server/services/CachedManifestService.kt @@ -2,6 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.cache.CacheService import dev.typetype.server.models.ExtractionResult +import java.net.URI class CachedManifestService( private val delegate: ManifestService, @@ -9,6 +10,7 @@ class CachedManifestService( ) { suspend fun dashManifest(videoUrl: String): ExtractionResult { + if (!isCacheable(videoUrl)) return delegate.dashManifest(videoUrl) val key = "dash-manifest-v2:${CachedStreamService.cacheKey(videoUrl)}" runCatching { cache.get(key) }.getOrNull()?.let { cached -> return ExtractionResult.Success(cached) @@ -22,5 +24,12 @@ class CachedManifestService( private companion object { const val DASH_MANIFEST_TTL_SECONDS = 21600L + + fun isCacheable(videoUrl: String): Boolean { + val host = runCatching { URI(videoUrl).host.orEmpty().lowercase().trimEnd('.') }.getOrDefault("") + return host != "b23.tv" && host != "nico.ms" && + !host.endsWith(".bilibili.com") && host != "bilibili.com" && + !host.endsWith(".nicovideo.jp") && host != "nicovideo.jp" + } } } diff --git a/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt b/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt index cf12d6a9..3289b36e 100644 --- a/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt +++ b/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt @@ -17,6 +17,7 @@ class HlsManifestService( cache: CacheService? = null, private val signManifestUrl: ((String) -> String)? = null, private val attestedYoutubeHls: suspend (String) -> String? = { null }, + private val providerMediaHandleService: ProviderMediaHandleService? = null, ) { private val proxyHttp = ProxyHttpExecutor(httpClient) private val manifestCache = cache?.let(::HlsManifestCache) @@ -48,19 +49,20 @@ class HlsManifestService( } private suspend fun cachedOrFetch(manifestUrl: String, signManifestLinks: Boolean): ExtractionResult { + val cache = manifestCache.takeUnless { isProviderManifestUrl(manifestUrl) } val cacheKey = if (signManifestLinks) "signed:$manifestUrl" else manifestUrl - manifestCache?.get(cacheKey)?.let { return ExtractionResult.Success(it) } + cache?.get(cacheKey)?.let { return ExtractionResult.Success(it) } val pending = CompletableDeferred>() val existing = inFlight.putIfAbsent(cacheKey, pending) if (existing != null) return existing.await() return try { - manifestCache?.get(cacheKey)?.let { + cache?.get(cacheKey)?.let { val result = ExtractionResult.Success(it) pending.complete(result) return result } val result = fetchAndRewrite(manifestUrl, signManifestLinks) - if (result is ExtractionResult.Success) manifestCache?.set(cacheKey, result.data) + if (result is ExtractionResult.Success) cache?.set(cacheKey, result.data) pending.complete(result) result } catch (error: Throwable) { @@ -112,7 +114,19 @@ class HlsManifestService( } else { val text = body.string() response.close() - val rewritten = if (isNicoNicoManifest(fetchUrl)) { + val rewritten = if (providerMediaHandleService != null && isNicoNicoManifest(fetchUrl)) { + rewriteNicoManifestWith(text, fetchUrl) { target -> + providerMediaHandleService.relativeManifestPath( + providerMediaHandleService.createPath(target, domandBid), + ) + } + } else if (providerMediaHandleService != null && isBilibiliManifest(fetchUrl)) { + rewriteProviderHlsManifest(text, fetchUrl) { target -> + providerMediaHandleService.relativeManifestPath( + providerMediaHandleService.createPath(target), + ) + } + } else if (isNicoNicoManifest(fetchUrl)) { rewriteNicoManifest(text, fetchUrl, domandBid, "../proxy") } else { rewriteYouTubeHlsManifest(text) { target -> @@ -128,5 +142,13 @@ class HlsManifestService( } private fun isNicoNicoManifest(url: String): Boolean = - runCatching { URI(url).host.orEmpty().endsWith("nicovideo.jp") }.getOrDefault(false) + runCatching { providerForProxyHost(URI(url).host.orEmpty()) == ProxyProvider.NICONICO } + .getOrDefault(false) + + private fun isBilibiliManifest(url: String): Boolean = + runCatching { providerForProxyHost(URI(url).host.orEmpty()) == ProxyProvider.BILIBILI } + .getOrDefault(false) + + private fun isProviderManifestUrl(url: String): Boolean = + isNicoNicoManifest(url) || isBilibiliManifest(url) } diff --git a/src/main/kotlin/dev/typetype/server/services/ManifestService.kt b/src/main/kotlin/dev/typetype/server/services/ManifestService.kt index bce04f99..81c2ed36 100644 --- a/src/main/kotlin/dev/typetype/server/services/ManifestService.kt +++ b/src/main/kotlin/dev/typetype/server/services/ManifestService.kt @@ -6,7 +6,10 @@ import dev.typetype.server.models.VideoStreamItem import java.net.URLEncoder import java.nio.charset.StandardCharsets -class ManifestService(private val streamService: StreamService) { +class ManifestService( + private val streamService: StreamService, + private val providerMediaHandleService: ProviderMediaHandleService? = null, +) { suspend fun dashManifest(videoUrl: String): ExtractionResult { val result = streamService.getStreamInfo(videoUrl) if (result !is ExtractionResult.Success) return result.recast() @@ -32,7 +35,7 @@ class ManifestService(private val streamService: StreamService) { else -> 2 } - private fun buildMpd(videos: List, audios: List, duration: Long): String { + private suspend fun buildMpd(videos: List, audios: List, duration: Long): String { val sb = StringBuilder() sb.appendLine("") sb.appendLine(") { + private suspend fun appendVideoAdaptationSet(sb: StringBuilder, mimeType: String, streams: List) { sb.appendLine(" ") streams.forEachIndexed { i, s -> val height = if (s.height > 0) s.height else resolutionHeight(s.resolution) @@ -62,7 +65,7 @@ class ManifestService(private val streamService: StreamService) { val bandwidth = (s.bitrate ?: bwFromUrl(s.url) ?: (height * 1000)).coerceAtLeast(1) val sizeAttr = if (width > 0 && height > 0) " width=\"$width\" height=\"$height\"" else "" sb.appendLine(" ") - sb.appendLine(" ../proxy?url=${encode(s.url)}") + sb.appendLine(" ${mediaUrl(s.url)}") if (s.indexStart > 0L && s.indexEnd > 0L) { sb.appendLine(" ") sb.appendLine(" ") @@ -73,12 +76,12 @@ class ManifestService(private val streamService: StreamService) { sb.appendLine(" ") } - private fun appendAudioAdaptationSet(sb: StringBuilder, mimeType: String, lang: String?, label: String?, streams: List) { + private suspend fun appendAudioAdaptationSet(sb: StringBuilder, mimeType: String, lang: String?, label: String?, streams: List) { val attrs = "${if (lang != null) " lang=\"$lang\"" else ""}${if (label != null) " label=\"$label\"" else ""}" sb.appendLine(" ") streams.forEachIndexed { i, a -> sb.appendLine(" ") - sb.appendLine(" ../proxy?url=${encode(a.url)}") + sb.appendLine(" ${mediaUrl(a.url)}") if (a.indexStart > 0L && a.indexEnd > 0L) { sb.appendLine(" ") sb.appendLine(" ") @@ -110,6 +113,18 @@ class ManifestService(private val streamService: StreamService) { private fun bwFromUrl(url: String): Int? = Regex("[?&]bw=(\\d+)").find(url)?.groupValues?.get(1)?.toIntOrNull() + private suspend fun mediaUrl(url: String): String { + val service = providerMediaHandleService + if (service == null) return "../proxy?url=${encode(url)}" + if (url.startsWith("/media/")) return service.relativeManifestPath(url) + val provider = runCatching { requireProxyTarget(url).provider }.getOrNull() + return if (provider == ProxyProvider.BILIBILI || provider == ProxyProvider.NICONICO) { + service.relativeManifestPath(service.createPath(url)) + } else { + "../proxy?url=${encode(url)}" + } + } + private fun encode(url: String): String = URLEncoder.encode(url, StandardCharsets.UTF_8) private fun ExtractionResult.recast(): ExtractionResult = when (this) { diff --git a/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt b/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt index 10c0365e..3d14a286 100644 --- a/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt +++ b/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt @@ -40,7 +40,45 @@ internal fun rewriteNicoManifest(manifest: String, baseUrl: String, domandBid: S } } -class NicoVideoProxyService(client: OkHttpClient = defaultNicoProxyClient()) { +internal suspend fun rewriteNicoManifestWith( + manifest: String, + baseUrl: String, + mapUrl: suspend (String) -> String, +): String { + val base = URI(baseUrl) + val uriAttr = Regex("""URI="([^"]+)"""") + suspend fun toMapped(url: String): String { + val resolved = if (url.startsWith("http://", ignoreCase = true) || + url.startsWith("https://", ignoreCase = true) + ) url else base.resolve(url).toString() + return if (resolved.startsWith("http://", ignoreCase = true) || + resolved.startsWith("https://", ignoreCase = true) + ) mapUrl(resolved) else resolved + } + val rewritten = ArrayList() + for (line in manifest.lines()) { + val trimmed = line.trim() + when { + trimmed.isBlank() -> rewritten += line + trimmed.startsWith("#") -> { + val builder = StringBuilder(trimmed) + val matches = uriAttr.findAll(trimmed).toList() + for (match in matches.asReversed()) { + val mapped = toMapped(match.groupValues[1]) + builder.replace(match.range.first, match.range.last + 1, "URI=\"$mapped\"") + } + rewritten += builder.toString() + } + else -> rewritten += toMapped(trimmed) + } + } + return rewritten.joinToString("\n") +} + +class NicoVideoProxyService( + client: OkHttpClient = defaultNicoProxyClient(), + private val mediaHandleService: ProviderMediaHandleService? = null, +) { private val executor = ProxyHttpExecutor(client) companion object { @@ -72,7 +110,15 @@ class NicoVideoProxyService(client: OkHttpClient = defaultNicoProxyClient()) { } else { val text = body.string() response.close() - val rewritten = rewriteNicoManifest(text, manifestUrl, resolvedBid) + val rewritten = if (mediaHandleService == null) { + rewriteNicoManifest(text, manifestUrl, resolvedBid) + } else { + rewriteNicoManifestWith(text, manifestUrl) { target -> + mediaHandleService.relativeManifestPath( + mediaHandleService.createPath(target, resolvedBid), + ) + } + } ExtractionResult.Success(ProxyResponse( status = 200, contentType = "application/vnd.apple.mpegurl", diff --git a/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt b/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt index 18b5d656..53a62831 100644 --- a/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt +++ b/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt @@ -25,10 +25,27 @@ internal fun rewriteHlsManifest(manifest: String): String = "/proxy?url=" + URLEncoder.encode(match.value, StandardCharsets.UTF_8) } -class OkHttpProxyService(client: OkHttpClient) : ProxyService { +class OkHttpProxyService( + client: OkHttpClient, + private val mediaHandleService: ProviderMediaHandleService? = null, +) : ProxyService, ProviderMediaAwareProxyService { private val executor = ProxyHttpExecutor(client) - override suspend fun pipe(url: String, rangeHeader: String?, domandBid: String?): ExtractionResult { + override suspend fun pipe(url: String, rangeHeader: String?, domandBid: String?): ExtractionResult = + pipeInternal(url, rangeHeader, domandBid, providerMediaManifest = false) + + override suspend fun pipeProviderMedia( + url: String, + rangeHeader: String?, + domandBid: String?, + ): ExtractionResult = pipeInternal(url, rangeHeader, domandBid, providerMediaManifest = true) + + private suspend fun pipeInternal( + url: String, + rangeHeader: String?, + domandBid: String?, + providerMediaManifest: Boolean, + ): ExtractionResult { val requestContext = currentCoroutineContext() return withContext(Dispatchers.IO) { val hashIdx = url.indexOf('#') @@ -72,9 +89,20 @@ class OkHttpProxyService(client: OkHttpClient) : ProxyService { val acceptRanges = response.header("Accept-Ranges") val cacheControl = response.header("Cache-Control") val contentLength = response.header("Content-Length")?.toLongOrNull() - if (isHls(contentType)) { - val rewritten = if (isNicoNico(stripTrackingParams(fetchUrl))) { - rewriteNicoManifest(body.string(), stripTrackingParams(fetchUrl), resolvedDomandBid, PROXY_PATH) + val cleanFetchUrl = stripTrackingParams(fetchUrl) + if (isHls(contentType, cleanFetchUrl)) { + val rewritten = if (providerMediaManifest && mediaHandleService != null && isNicoNico(cleanFetchUrl)) { + rewriteNicoManifestWith(body.string(), cleanFetchUrl) { target -> + mediaHandleService.relativeManifestPath( + mediaHandleService.createPath(target, resolvedDomandBid), + ) + } + } else if (providerMediaManifest && mediaHandleService != null && isBilibili(cleanFetchUrl)) { + rewriteProviderHlsManifest(body.string(), cleanFetchUrl) { target -> + mediaHandleService.relativeManifestPath(mediaHandleService.createPath(target)) + } + } else if (isNicoNico(cleanFetchUrl)) { + rewriteNicoManifest(body.string(), cleanFetchUrl, resolvedDomandBid, PROXY_PATH) } else { rewriteHlsManifest(body.string()) } @@ -118,11 +146,11 @@ class OkHttpProxyService(client: OkHttpClient) : ProxyService { private fun isNicoNico(url: String): Boolean { val host = runCatching { java.net.URI(url).host ?: "" }.getOrElse { "" } - return host.endsWith("nicovideo.jp") + return providerForProxyHost(host) == ProxyProvider.NICONICO } - private fun isHls(contentType: String): Boolean = - contentType.contains("mpegurl", ignoreCase = true) + private fun isHls(contentType: String, url: String): Boolean = + contentType.contains("mpegurl", ignoreCase = true) || url.contains(".m3u8", ignoreCase = true) companion object { private const val BILIBILI_REFERER = "https://www.bilibili.com" diff --git a/src/main/kotlin/dev/typetype/server/services/ProviderHlsManifestRewrite.kt b/src/main/kotlin/dev/typetype/server/services/ProviderHlsManifestRewrite.kt new file mode 100644 index 00000000..890715e0 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/ProviderHlsManifestRewrite.kt @@ -0,0 +1,43 @@ +package dev.typetype.server.services + +import java.net.URI + +internal suspend fun rewriteProviderHlsManifest( + manifest: String, + baseUrl: String, + mapUrl: suspend (String) -> String, +): String { + val base = URI(baseUrl) + val uriAttr = Regex("""URI="([^"]+)"""") + suspend fun mapResolved(raw: String): String { + val resolved = runCatching { + if (raw.startsWith("http://", ignoreCase = true) || raw.startsWith("https://", ignoreCase = true)) { + raw + } else { + base.resolve(raw).toString() + } + }.getOrDefault(raw) + return if (resolved.startsWith("http://", ignoreCase = true) || + resolved.startsWith("https://", ignoreCase = true) + ) mapUrl(resolved) else resolved + } + + val rewritten = ArrayList() + for (line in manifest.lines()) { + val trimmed = line.trim() + when { + trimmed.isBlank() -> rewritten += line + trimmed.startsWith("#") -> { + val builder = StringBuilder(trimmed) + val matches = uriAttr.findAll(trimmed).toList() + for (match in matches.asReversed()) { + val mapped = mapResolved(match.groupValues[1]) + builder.replace(match.range.first, match.range.last + 1, "URI=\"$mapped\"") + } + rewritten += builder.toString() + } + else -> rewritten += mapResolved(trimmed) + } + } + return rewritten.joinToString("\n") +} diff --git a/src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt b/src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt index b8347c78..69cd3304 100644 --- a/src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt +++ b/src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt @@ -1,6 +1,9 @@ package dev.typetype.server.services import dev.typetype.server.models.StreamResponse +import java.net.URI +import java.net.URLDecoder +import java.nio.charset.StandardCharsets private const val DEFAULT_STREAM_TTL_SECONDS = 21_600L private const val DISLIKE_UNAVAILABLE_STREAM_TTL_SECONDS = 300L @@ -20,19 +23,24 @@ private fun StreamResponse.stableMetadataTtlSeconds(): Long = if (dislikeCount < 0L) DISLIKE_UNAVAILABLE_STREAM_TTL_SECONDS else DEFAULT_STREAM_TTL_SECONDS private fun StreamResponse.signedMediaUrls(): Sequence = sequence { + yield(hlsUrl) + yield(dashMpdUrl) videoStreams.forEach { yield(it.url) } videoOnlyStreams.forEach { yield(it.url) } audioStreams.forEach { yield(it.url) } } -private fun String.bilibiliDeadline(): Long? { - if (!isBilibiliSignedMediaUrl()) return null - return Regex("""[?&]deadline=(\d+)""").find(this)?.groupValues?.get(1)?.toLongOrNull() - ?: Regex("""[?&]hdnts=exp=(\d+)""").find(this)?.groupValues?.get(1)?.toLongOrNull() +private fun String.bilibiliDeadline(): Long? = providerMediaExpiry() + +private fun String.providerMediaExpiry(): Long? { + val host = runCatching { URI(this).host.orEmpty() }.getOrDefault("") + val provider = providerForProxyHost(host) + if (provider != ProxyProvider.BILIBILI && provider != ProxyProvider.NICONICO) return null + val decoded = runCatching { URLDecoder.decode(this, StandardCharsets.UTF_8) }.getOrDefault(this) + return PROVIDER_EXPIRY_REGEX.findAll(decoded) + .mapNotNull { it.groupValues[1].toLongOrNull() } + .minOrNull() } -private fun String.isBilibiliSignedMediaUrl(): Boolean = - contains("bilibili", ignoreCase = true) || - contains("bilivideo", ignoreCase = true) || - contains("hdslb.com", ignoreCase = true) || - contains("akamaized", ignoreCase = true) +private val PROVIDER_EXPIRY_REGEX = + Regex("(?:^|[?&#=])(?:deadline|expires|exp)=(\\d+)", RegexOption.IGNORE_CASE) From ff4f52b7354bfcac1f674478cec94dd4fd011dda Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 9 Sep 2026 23:30:03 +0200 Subject: [PATCH 39/45] test: cover provider media transport --- .../server/HlsManifestServiceCacheTest.kt | 22 +++ .../server/ProviderMediaHandleRoutesTest.kt | 63 +++++++++ .../server/ProviderMediaHandleServiceTest.kt | 128 ++++++++++++++++++ .../server/StreamCacheTtlResolverTest.kt | 6 + .../server/StreamRoutesDeliveryModeTest.kt | 29 ++++ 5 files changed, 248 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/ProviderMediaHandleRoutesTest.kt create mode 100644 src/test/kotlin/dev/typetype/server/ProviderMediaHandleServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/HlsManifestServiceCacheTest.kt b/src/test/kotlin/dev/typetype/server/HlsManifestServiceCacheTest.kt index ada3a2a6..2a54a629 100644 --- a/src/test/kotlin/dev/typetype/server/HlsManifestServiceCacheTest.kt +++ b/src/test/kotlin/dev/typetype/server/HlsManifestServiceCacheTest.kt @@ -41,6 +41,28 @@ class HlsManifestServiceCacheTest { assertEquals(1, calls) } + @Test + fun `provider manifests are fetched again instead of serving signed cache entries`() = runTest { + var calls = 0 + val client = proxyTestClient(Interceptor { chain -> + calls += 1 + Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body("#EXTM3U\nsegment.m4s".toResponseBody("application/vnd.apple.mpegurl".toMediaType())) + .build() + }) + val service = HlsManifestService(NoopStreamService, client, InMemoryCache()) + val url = "https://upos-hz-mirrorakam.akamaized.net/master.m3u8?deadline=2000000000" + + service.hlsManifest(url) + service.hlsManifest(url) + + assertEquals(2, calls) + } + @Test fun `attested manifest is scoped to youtube live`() = runTest { val requestedUrls = mutableListOf() diff --git a/src/test/kotlin/dev/typetype/server/ProviderMediaHandleRoutesTest.kt b/src/test/kotlin/dev/typetype/server/ProviderMediaHandleRoutesTest.kt new file mode 100644 index 00000000..959227f7 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/ProviderMediaHandleRoutesTest.kt @@ -0,0 +1,63 @@ +package dev.typetype.server + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.ProxyResponse +import dev.typetype.server.routes.providerMediaHandleRoutes +import dev.typetype.server.services.ProviderMediaHandleService +import dev.typetype.server.services.ProxyService +import io.ktor.client.request.get +import io.ktor.client.request.header +import io.ktor.client.statement.bodyAsText +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.testApplication +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.io.ByteArrayInputStream + +class ProviderMediaHandleRoutesTest { + private val proxyService: ProxyService = mockk() + + @Test + fun `media route forwards opaque handles and ranges`() = runTest { + val service = ProviderMediaHandleService(FakeCacheService()) + val raw = "https://upos-hz-mirrorakam.akamaized.net/video.m4s?deadline=9999999999" + val path = service.createPath(raw) + coEvery { proxyService.pipe(raw, "bytes=0-3", null) } returns ExtractionResult.Success( + ProxyResponse(206, "video/mp4", 4, "bytes 0-3/4", "bytes", ByteArrayInputStream(byteArrayOf(1, 2, 3, 4)), {}), + ) + + testApplication { + application { + install(ContentNegotiation) { json() } + routing { providerMediaHandleRoutes(service, proxyService) } + } + val response = client.get(path) { header("Range", "bytes=0-3") } + assertEquals(HttpStatusCode.PartialContent, response.status) + assertEquals("\u0001\u0002\u0003\u0004", response.bodyAsText()) + } + + coVerify(exactly = 1) { proxyService.pipe(raw, "bytes=0-3", null) } + } + + @Test + fun `expired or unknown handles return not found`() = testApplication { + val service = ProviderMediaHandleService(FakeCacheService()) + application { + install(ContentNegotiation) { json() } + routing { providerMediaHandleRoutes(service, proxyService) } + } + + val response = client.get("/media/m1_123456789012345678901234") + assertEquals(HttpStatusCode.NotFound, response.status) + assertTrue(response.bodyAsText().contains("media_handle_not_found")) + } +} diff --git a/src/test/kotlin/dev/typetype/server/ProviderMediaHandleServiceTest.kt b/src/test/kotlin/dev/typetype/server/ProviderMediaHandleServiceTest.kt new file mode 100644 index 00000000..01ad834f --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/ProviderMediaHandleServiceTest.kt @@ -0,0 +1,128 @@ +package dev.typetype.server + +import dev.typetype.server.cache.CacheService +import dev.typetype.server.services.ProviderMediaHandleService +import dev.typetype.server.services.ProviderMediaType +import dev.typetype.server.services.rewriteProviderHlsManifest +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class ProviderMediaHandleServiceTest { + @Test + fun `provider URLs become short cache-backed paths`() = runTest { + val service = ProviderMediaHandleService(FakeCacheService(), nowSeconds = { 1_000L }) + val raw = "https://upos-hz-mirrorakam.akamaized.net/video.m4s?deadline=2000000000&cpn=tracking" + + val path = service.createPath(raw) + val handle = path.removePrefix("/media/") + val target = service.resolve(handle) + + assertTrue(path.matches(Regex("/media/m1_[A-Za-z0-9_-]{24}"))) + assertFalse(path.contains("akamaized")) + assertEquals("https://upos-hz-mirrorakam.akamaized.net/video.m4s?deadline=2000000000", target?.url) + assertEquals(path, service.createPath(raw)) + } + + @Test + fun `Nico cookie fragment is stored beside the opaque target`() = runTest { + val service = ProviderMediaHandleService(FakeCacheService(), nowSeconds = { 1_000L }) + val path = service.createPath( + "https://asset.domand.nicovideo.jp/video/01.cmfv?Expires=2000000000#cookie=domand_bid%3Dcookie123&length=10", + ) + val target = service.resolve(path.removePrefix("/media/")) + + assertEquals("cookie123", target?.domandBid) + assertFalse(target?.url.orEmpty().contains("#cookie")) + assertEquals("../media/${path.substringAfterLast('/')}", service.relativeManifestPath(path)) + } + + @Test + fun `unknown handles do not resolve`() = runTest { + val service = ProviderMediaHandleService(FakeCacheService()) + assertNull(service.resolve("m1_invalid")) + } + + @Test + fun `expired provider URLs are not materialized`() = runTest { + val service = ProviderMediaHandleService(FakeCacheService(), nowSeconds = { 2_000_000_000L }) + + assertThrows(IllegalArgumentException::class.java) { + kotlinx.coroutines.runBlocking { + service.createPath("https://upos-hz-mirrorakam.akamaized.net/video.m4s?deadline=2000000000") + } + } + } + + @Test + fun `near expiry handles keep the remaining upstream lifetime`() = runTest { + val cache = RecordingProviderCacheService() + val service = ProviderMediaHandleService(cache, nowSeconds = { 1_000L }) + + service.createPath("https://upos-hz-mirrorakam.akamaized.net/video.m4s?deadline=1050") + + assertEquals(20L, cache.ttlSeconds) + } + + @Test + fun `provider HLS rewrite can map every media line`() = runTest { + val manifest = "#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\nsegment.m4s" + val result = rewriteProviderHlsManifest( + manifest, + "https://upos-hz-mirrorakam.akamaized.net/path/master.m3u8", + ) { "../media/handle" } + + assertTrue(result.contains("URI=\"../media/handle\"")) + assertTrue(result.endsWith("../media/handle")) + } + + @Test + fun `media materialization only handles the selected provider`() = runTest { + val service = ProviderMediaHandleService(FakeCacheService()) + val response = testStreamResponse( + videoOnlyStreams = listOf( + testVideoStream("https://upos-hz-mirrorakam.akamaized.net/video.m4s?deadline=9999999999"), + ), + ) + + val handled = service.materialize(response, ProviderMediaType.BILIBILI) + assertTrue(handled.videoOnlyStreams.single().url.startsWith("/media/m1_")) + } + + @Test + fun `materialization covers optional stream media paths`() = runTest { + val service = ProviderMediaHandleService(FakeCacheService()) + val video = testVideoStream( + "https://upos-hz-mirrorakam.akamaized.net/video.m4s?deadline=9999999999", + ).copy( + manifestUrl = "https://upos-hz-mirrorakam.akamaized.net/video.m3u8?deadline=9999999999", + sabrSessionUrl = "https://upos-hz-mirrorakam.akamaized.net/session?deadline=9999999999", + ) + + val handled = service.materialize( + testStreamResponse(videoOnlyStreams = listOf(video)), + ProviderMediaType.BILIBILI, + ) + + assertTrue(handled.videoOnlyStreams.single().manifestUrl?.startsWith("/media/m1_") == true) + assertTrue(handled.videoOnlyStreams.single().sabrSessionUrl?.startsWith("/media/m1_") == true) + } +} + +private class RecordingProviderCacheService : CacheService { + private val values = mutableMapOf() + var ttlSeconds: Long = 0 + + override suspend fun set(key: String, value: String, ttlSeconds: Long) { + this.ttlSeconds = ttlSeconds + values[key] = value + } + + override suspend fun get(key: String): String? = values[key] + + override suspend fun delete(key: String) { values.remove(key) } +} diff --git a/src/test/kotlin/dev/typetype/server/StreamCacheTtlResolverTest.kt b/src/test/kotlin/dev/typetype/server/StreamCacheTtlResolverTest.kt index bb0f14bb..72ebc0aa 100644 --- a/src/test/kotlin/dev/typetype/server/StreamCacheTtlResolverTest.kt +++ b/src/test/kotlin/dev/typetype/server/StreamCacheTtlResolverTest.kt @@ -36,6 +36,12 @@ class StreamCacheTtlResolverTest { assertEquals(0L, response(url).streamCacheTtlSeconds(nowEpochSeconds = 8_000L)) } + @Test + fun `niconico stream ttl follows signed expiry`() { + val url = "https://asset.domand.nicovideo.jp/video/01.cmfv?Expires=10000" + assertEquals(1_700L, response(url).streamCacheTtlSeconds(nowEpochSeconds = 8_000L)) + } + private fun response(url: String, dislikeCount: Long = 0L): StreamResponse = StreamResponse( id = "id", title = "title", diff --git a/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt b/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt index a719ef4c..143ff781 100644 --- a/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt +++ b/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt @@ -4,8 +4,10 @@ import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.StreamResponse import dev.typetype.server.routes.streamRoutes import dev.typetype.server.services.StreamService +import dev.typetype.server.services.ProviderMediaHandleService import io.ktor.client.request.get import io.ktor.client.statement.bodyAsText +import io.ktor.http.HttpHeaders import io.ktor.http.HttpStatusCode import io.ktor.serialization.kotlinx.json.json import io.ktor.server.application.Application @@ -106,6 +108,31 @@ class StreamRoutesDeliveryModeTest { coVerify(exactly = 0) { sabrService.getStreamInfo(any()) } } + @Test + fun `provider endpoints expose opaque media handles`() = testApplication { + val bilibiliVideo = "https://upos-hz-mirrorakam.akamaized.net/video.m4s?deadline=9999999999" + val nicoVideo = "https://asset.domand.nicovideo.jp/video/01.cmfv?Expires=9999999999" + coEvery { nicoNicoService.getStreamInfo(any()) } returns ExtractionResult.Success( + testStreamResponse(videoOnlyStreams = listOf(testVideoStream(nicoVideo))), + ) + coEvery { bilibiliService.getStreamInfo(any()) } returns ExtractionResult.Success( + testStreamResponse(videoOnlyStreams = listOf(testVideoStream(bilibiliVideo))), + ) + application { installRoutes(providerMediaHandleService = ProviderMediaHandleService(FakeCacheService())) } + + val nicoResponse = client.get("/streams/niconico?url=$NICONICO_URL") + val bilibiliResponse = client.get("/streams/bilibili?url=$BILIBILI_URL") + val nicoBody = nicoResponse.bodyAsText() + val bilibiliBody = bilibiliResponse.bodyAsText() + + assertTrue(nicoBody.contains("\"/media/m1_")) + assertTrue(bilibiliBody.contains("\"/media/m1_")) + assertFalse(nicoBody.contains("domand.nicovideo.jp")) + assertFalse(bilibiliBody.contains("akamaized.net")) + assertEquals("no-store", nicoResponse.headers[HttpHeaders.CacheControl]) + assertEquals("no-store", bilibiliResponse.headers[HttpHeaders.CacheControl]) + } + @Test fun `provider endpoints reject mismatched urls`() = testApplication { application { installRoutes() } @@ -119,6 +146,7 @@ class StreamRoutesDeliveryModeTest { } private fun Application.installRoutes( + providerMediaHandleService: ProviderMediaHandleService? = null, sabrFilter: suspend (String, StreamResponse) -> StreamResponse = { _, data -> data }, ) { install(ContentNegotiation) { json() } @@ -127,6 +155,7 @@ class StreamRoutesDeliveryModeTest { streamService = sabrService, nicoNicoStreamService = nicoNicoService, bilibiliStreamService = bilibiliService, + providerMediaHandleService = providerMediaHandleService, sabrStreamContractFilter = sabrFilter, ) } From b65be35c03b46599026bbc13683dfb16e5b6ec8d Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 10 Sep 2026 00:04:22 +0200 Subject: [PATCH 40/45] fix: preserve stream request cancellation --- src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt index 6b6b25f5..cc1070f3 100644 --- a/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt @@ -23,6 +23,7 @@ import io.ktor.http.HttpStatusCode import io.ktor.server.response.respond import io.ktor.server.routing.Route import io.ktor.server.routing.get +import kotlinx.coroutines.CancellationException private const val STREAMS_CACHE_CONTROL = "public, max-age=21600, stale-while-revalidate=3600" private const val AUTHENTICATED_STREAMS_CACHE_CONTROL = "no-store" @@ -137,6 +138,8 @@ private fun Route.streamRoute( dependencies.providerMediaHandleService?.let { service -> providerMediaType(deliveryMode)?.let { service.materialize(data, it) } } ?: data + } catch (error: CancellationException) { + throw error } catch (error: Exception) { return@get call.respond( HttpStatusCode.BadGateway, From 811e44f4e8f691d72778622de2150b8a147c6e17 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 10 Sep 2026 00:29:00 +0200 Subject: [PATCH 41/45] perf: parallelize provider media handle mapping --- .../server/services/HlsManifestService.kt | 2 +- .../server/services/NicoVideoProxyService.kt | 37 +------ .../server/services/OkHttpProxyService.kt | 2 +- .../services/ProviderHlsManifestRewrite.kt | 53 +++++++--- .../services/ProviderMediaHandleService.kt | 96 ++++++++++++------- 5 files changed, 106 insertions(+), 84 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt b/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt index 3289b36e..5a07b108 100644 --- a/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt +++ b/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt @@ -115,7 +115,7 @@ class HlsManifestService( val text = body.string() response.close() val rewritten = if (providerMediaHandleService != null && isNicoNicoManifest(fetchUrl)) { - rewriteNicoManifestWith(text, fetchUrl) { target -> + rewriteProviderHlsManifest(text, fetchUrl) { target -> providerMediaHandleService.relativeManifestPath( providerMediaHandleService.createPath(target, domandBid), ) diff --git a/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt b/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt index 3d14a286..8bb0c0f9 100644 --- a/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt +++ b/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt @@ -40,41 +40,6 @@ internal fun rewriteNicoManifest(manifest: String, baseUrl: String, domandBid: S } } -internal suspend fun rewriteNicoManifestWith( - manifest: String, - baseUrl: String, - mapUrl: suspend (String) -> String, -): String { - val base = URI(baseUrl) - val uriAttr = Regex("""URI="([^"]+)"""") - suspend fun toMapped(url: String): String { - val resolved = if (url.startsWith("http://", ignoreCase = true) || - url.startsWith("https://", ignoreCase = true) - ) url else base.resolve(url).toString() - return if (resolved.startsWith("http://", ignoreCase = true) || - resolved.startsWith("https://", ignoreCase = true) - ) mapUrl(resolved) else resolved - } - val rewritten = ArrayList() - for (line in manifest.lines()) { - val trimmed = line.trim() - when { - trimmed.isBlank() -> rewritten += line - trimmed.startsWith("#") -> { - val builder = StringBuilder(trimmed) - val matches = uriAttr.findAll(trimmed).toList() - for (match in matches.asReversed()) { - val mapped = toMapped(match.groupValues[1]) - builder.replace(match.range.first, match.range.last + 1, "URI=\"$mapped\"") - } - rewritten += builder.toString() - } - else -> rewritten += toMapped(trimmed) - } - } - return rewritten.joinToString("\n") -} - class NicoVideoProxyService( client: OkHttpClient = defaultNicoProxyClient(), private val mediaHandleService: ProviderMediaHandleService? = null, @@ -113,7 +78,7 @@ class NicoVideoProxyService( val rewritten = if (mediaHandleService == null) { rewriteNicoManifest(text, manifestUrl, resolvedBid) } else { - rewriteNicoManifestWith(text, manifestUrl) { target -> + rewriteProviderHlsManifest(text, manifestUrl) { target -> mediaHandleService.relativeManifestPath( mediaHandleService.createPath(target, resolvedBid), ) diff --git a/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt b/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt index 53a62831..8b8e6dbb 100644 --- a/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt +++ b/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt @@ -92,7 +92,7 @@ class OkHttpProxyService( val cleanFetchUrl = stripTrackingParams(fetchUrl) if (isHls(contentType, cleanFetchUrl)) { val rewritten = if (providerMediaManifest && mediaHandleService != null && isNicoNico(cleanFetchUrl)) { - rewriteNicoManifestWith(body.string(), cleanFetchUrl) { target -> + rewriteProviderHlsManifest(body.string(), cleanFetchUrl) { target -> mediaHandleService.relativeManifestPath( mediaHandleService.createPath(target, resolvedDomandBid), ) diff --git a/src/main/kotlin/dev/typetype/server/services/ProviderHlsManifestRewrite.kt b/src/main/kotlin/dev/typetype/server/services/ProviderHlsManifestRewrite.kt index 890715e0..e7f2a361 100644 --- a/src/main/kotlin/dev/typetype/server/services/ProviderHlsManifestRewrite.kt +++ b/src/main/kotlin/dev/typetype/server/services/ProviderHlsManifestRewrite.kt @@ -1,29 +1,41 @@ package dev.typetype.server.services +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope import java.net.URI internal suspend fun rewriteProviderHlsManifest( manifest: String, baseUrl: String, mapUrl: suspend (String) -> String, -): String { +): String = coroutineScope { val base = URI(baseUrl) val uriAttr = Regex("""URI="([^"]+)"""") - suspend fun mapResolved(raw: String): String { - val resolved = runCatching { - if (raw.startsWith("http://", ignoreCase = true) || raw.startsWith("https://", ignoreCase = true)) { - raw - } else { - base.resolve(raw).toString() - } - }.getOrDefault(raw) - return if (resolved.startsWith("http://", ignoreCase = true) || - resolved.startsWith("https://", ignoreCase = true) - ) mapUrl(resolved) else resolved + val lines = manifest.lines() + val references = lines.flatMap { line -> + val trimmed = line.trim() + when { + trimmed.isBlank() -> emptyList() + trimmed.startsWith("#") -> uriAttr.findAll(trimmed).map { it.groupValues[1] }.toList() + else -> listOf(trimmed) + } + } + val resolvedReferences = references.map { raw -> resolveProviderManifestUrl(base, raw) } + .filter { it.isHttpUrl() } + .distinct() + val mappedReferences = resolvedReferences + .chunked(MAX_CONCURRENT_MAPPINGS) + .flatMap { batch -> batch.map { target -> async { target to mapUrl(target) } }.awaitAll() } + .toMap() + + fun mapResolved(raw: String): String { + val resolved = resolveProviderManifestUrl(base, raw) + return if (resolved.isHttpUrl()) mappedReferences[resolved] ?: resolved else resolved } val rewritten = ArrayList() - for (line in manifest.lines()) { + for (line in lines) { val trimmed = line.trim() when { trimmed.isBlank() -> rewritten += line @@ -39,5 +51,18 @@ internal suspend fun rewriteProviderHlsManifest( else -> rewritten += mapResolved(trimmed) } } - return rewritten.joinToString("\n") + rewritten.joinToString("\n") } + +private fun resolveProviderManifestUrl(base: URI, raw: String): String = runCatching { + if (raw.startsWith("http://", ignoreCase = true) || raw.startsWith("https://", ignoreCase = true)) { + raw + } else { + base.resolve(raw).toString() + } +}.getOrDefault(raw) + +private fun String.isHttpUrl(): Boolean = startsWith("http://", ignoreCase = true) || + startsWith("https://", ignoreCase = true) + +private const val MAX_CONCURRENT_MAPPINGS = 16 diff --git a/src/main/kotlin/dev/typetype/server/services/ProviderMediaHandleService.kt b/src/main/kotlin/dev/typetype/server/services/ProviderMediaHandleService.kt index fe1889b7..04d46dff 100644 --- a/src/main/kotlin/dev/typetype/server/services/ProviderMediaHandleService.kt +++ b/src/main/kotlin/dev/typetype/server/services/ProviderMediaHandleService.kt @@ -5,12 +5,19 @@ import dev.typetype.server.cache.CacheService import dev.typetype.server.models.AudioStreamItem import dev.typetype.server.models.StreamResponse import dev.typetype.server.models.VideoStreamItem +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit import kotlinx.serialization.Serializable import kotlinx.serialization.encodeToString import java.net.URLDecoder import java.nio.charset.StandardCharsets import java.security.MessageDigest import java.util.Base64 +import java.util.concurrent.ConcurrentHashMap internal enum class ProviderMediaType { BILIBILI, @@ -27,13 +34,23 @@ class ProviderMediaHandleService( private val cache: CacheService, private val nowSeconds: () -> Long = { System.currentTimeMillis() / 1000 }, ) { - internal suspend fun materialize(response: StreamResponse, provider: ProviderMediaType): StreamResponse = response.copy( - hlsUrl = response.hlsUrl.handleIfRemote(provider), - dashMpdUrl = response.dashMpdUrl.handleIfRemote(provider), - videoStreams = response.videoStreams.withHandledVideoUrls(provider), - audioStreams = response.audioStreams.withHandledAudioUrls(provider), - videoOnlyStreams = response.videoOnlyStreams.withHandledVideoUrls(provider), - ) + private val writeLimiter = Semaphore(MAX_CONCURRENT_CACHE_WRITES) + private val inFlightWrites = ConcurrentHashMap>() + + internal suspend fun materialize(response: StreamResponse, provider: ProviderMediaType): StreamResponse = coroutineScope { + val hls = async { response.hlsUrl.handleIfRemote(provider) } + val dash = async { response.dashMpdUrl.handleIfRemote(provider) } + val video = async { response.videoStreams.withHandledVideoUrls(provider) } + val audio = async { response.audioStreams.withHandledAudioUrls(provider) } + val videoOnly = async { response.videoOnlyStreams.withHandledVideoUrls(provider) } + response.copy( + hlsUrl = hls.await(), + dashMpdUrl = dash.await(), + videoStreams = video.await(), + audioStreams = audio.await(), + videoOnlyStreams = videoOnly.await(), + ) + } internal suspend fun createPath(rawUrl: String, domandBid: String? = null): String { val (cleanUrl, fragment) = rawUrl.splitFragment() @@ -46,9 +63,22 @@ class ProviderMediaHandleService( val normalizedUrl = stripTrackingParams(target.url.toString()) val handle = handleId(normalizedUrl, resolvedBid) val ttl = mediaHandleTtlSeconds(normalizedUrl) - val value = CacheJson.encodeToString(ProviderMediaTarget(normalizedUrl, resolvedBid)) - cache.set(cacheKey(handle), value, ttl) - return "/media/$handle" + val key = cacheKey(handle) + val path = "/media/$handle" + val pending = CompletableDeferred() + val existing = inFlightWrites.putIfAbsent(key, pending) + if (existing != null) return existing.await() + return try { + val value = CacheJson.encodeToString(ProviderMediaTarget(normalizedUrl, resolvedBid)) + writeLimiter.withPermit { cache.set(key, value, ttl) } + pending.complete(path) + path + } catch (error: Throwable) { + pending.completeExceptionally(error) + throw error + } finally { + inFlightWrites.remove(key, pending) + } } internal suspend fun resolve(handle: String): ProviderMediaTarget? { @@ -62,32 +92,33 @@ class ProviderMediaHandleService( "../media/${path.substringAfterLast('/')}" private suspend fun String.handleIfRemote(provider: ProviderMediaType): String { - if (isBlank() || startsWith("/")) return this - return if (supportsProvider(this, provider)) createPath(this) else this + if (isBlank() || (startsWith("/") && !startsWith("//"))) return this + val candidate = if (startsWith("//")) "https:$this" else this + return if (supportsProvider(candidate, provider)) createPath(candidate) else this } - private suspend fun List.withHandledVideoUrls(provider: ProviderMediaType): List { - val handled = ArrayList(size) - for (item in this) { - handled += item.copy( - url = item.url.handleIfRemote(provider), - manifestUrl = item.manifestUrl?.handleIfRemote(provider), - sabrSessionUrl = item.sabrSessionUrl?.handleIfRemote(provider), - ) - } - return handled + private suspend fun List.withHandledVideoUrls(provider: ProviderMediaType): List = coroutineScope { + map { item -> + async { + item.copy( + url = item.url.handleIfRemote(provider), + manifestUrl = item.manifestUrl?.handleIfRemote(provider), + sabrSessionUrl = item.sabrSessionUrl?.handleIfRemote(provider), + ) + } + }.awaitAll() } - private suspend fun List.withHandledAudioUrls(provider: ProviderMediaType): List { - val handled = ArrayList(size) - for (item in this) { - handled += item.copy( - url = item.url.handleIfRemote(provider), - manifestUrl = item.manifestUrl?.handleIfRemote(provider), - sabrSessionUrl = item.sabrSessionUrl?.handleIfRemote(provider), - ) - } - return handled + private suspend fun List.withHandledAudioUrls(provider: ProviderMediaType): List = coroutineScope { + map { item -> + async { + item.copy( + url = item.url.handleIfRemote(provider), + manifestUrl = item.manifestUrl?.handleIfRemote(provider), + sabrSessionUrl = item.sabrSessionUrl?.handleIfRemote(provider), + ) + } + }.awaitAll() } private fun supportsProvider(url: String, provider: ProviderMediaType): Boolean = @@ -126,6 +157,7 @@ class ProviderMediaHandleService( const val DEFAULT_TTL_SECONDS = 1_800L const val MAX_TTL_SECONDS = 3_600L const val EXPIRY_SAFETY_SECONDS = 30L + const val MAX_CONCURRENT_CACHE_WRITES = 32 val HANDLE_PATTERN = Regex("m1_[A-Za-z0-9_-]{24}") val EXPIRY_PATTERN = Regex("(?:^|[?&#=])(?:deadline|Expires|exp)=(\\d+)", RegexOption.IGNORE_CASE) From 15dc17e96ff4269ce6e9f3a9959b991195c60423 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 10 Sep 2026 00:29:04 +0200 Subject: [PATCH 42/45] test: cover concurrent provider media mapping --- .../server/ProviderMediaHandleServiceTest.kt | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/src/test/kotlin/dev/typetype/server/ProviderMediaHandleServiceTest.kt b/src/test/kotlin/dev/typetype/server/ProviderMediaHandleServiceTest.kt index 01ad834f..d0289acd 100644 --- a/src/test/kotlin/dev/typetype/server/ProviderMediaHandleServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/ProviderMediaHandleServiceTest.kt @@ -4,6 +4,9 @@ import dev.typetype.server.cache.CacheService import dev.typetype.server.services.ProviderMediaHandleService import dev.typetype.server.services.ProviderMediaType import dev.typetype.server.services.rewriteProviderHlsManifest +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.async +import kotlinx.coroutines.delay import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse @@ -11,6 +14,8 @@ import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertThrows import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger class ProviderMediaHandleServiceTest { @Test @@ -80,6 +85,49 @@ class ProviderMediaHandleServiceTest { assertTrue(result.endsWith("../media/handle")) } + @Test + fun `provider HLS rewrite maps unique references concurrently`() = runTest { + val active = AtomicInteger() + val maximum = AtomicInteger() + val calls = AtomicInteger() + val manifest = buildString { + repeat(32) { + append("segment-$it.m4s\n") + } + append("segment-0.m4s") + } + + val result = rewriteProviderHlsManifest( + manifest, + "https://upos-hz-mirrorakam.akamaized.net/path/master.m3u8", + ) { target -> + calls.incrementAndGet() + maximum.accumulateAndGet(active.incrementAndGet()) { left, right -> maxOf(left, right) } + delay(1) + active.decrementAndGet() + "../media/${target.substringAfterLast('/')}" + } + + assertEquals(32, calls.get()) + assertTrue(maximum.get() > 1) + assertTrue(result.contains("../media/segment-31.m4s")) + } + + @Test + fun `concurrent handle creation shares one cache write`() = runTest { + val cache = BlockingProviderCacheService() + val service = ProviderMediaHandleService(cache, nowSeconds = { 1_000L }) + val raw = "https://upos-hz-mirrorakam.akamaized.net/video.m4s?deadline=2000000000" + + val first = async { service.createPath(raw) } + cache.started.await() + val second = async { service.createPath(raw) } + cache.release.complete(Unit) + + assertEquals(first.await(), second.await()) + assertEquals(1, cache.setCalls.get()) + } + @Test fun `media materialization only handles the selected provider`() = runTest { val service = ProviderMediaHandleService(FakeCacheService()) @@ -111,6 +159,20 @@ class ProviderMediaHandleServiceTest { assertTrue(handled.videoOnlyStreams.single().manifestUrl?.startsWith("/media/m1_") == true) assertTrue(handled.videoOnlyStreams.single().sabrSessionUrl?.startsWith("/media/m1_") == true) } + + @Test + fun `protocol relative provider URLs are materialized`() = runTest { + val service = ProviderMediaHandleService(FakeCacheService()) + val response = testStreamResponse( + videoOnlyStreams = listOf( + testVideoStream("//upos-hz-mirrorakam.akamaized.net/video.m4s?deadline=9999999999"), + ), + ) + + val handled = service.materialize(response, ProviderMediaType.BILIBILI) + + assertTrue(handled.videoOnlyStreams.single().url.startsWith("/media/m1_")) + } } private class RecordingProviderCacheService : CacheService { @@ -126,3 +188,23 @@ private class RecordingProviderCacheService : CacheService { override suspend fun delete(key: String) { values.remove(key) } } + +private class BlockingProviderCacheService : CacheService { + val started = CompletableDeferred() + val release = CompletableDeferred() + val setCalls = AtomicInteger() + private val values = ConcurrentHashMap() + + override suspend fun set(key: String, value: String, ttlSeconds: Long) { + setCalls.incrementAndGet() + values[key] = value + started.complete(Unit) + release.await() + } + + override suspend fun get(key: String): String? = values[key] + + override suspend fun delete(key: String) { + values.remove(key) + } +} From 5eecb9ded0bc4aaeef25491065a3585aecbe21d7 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 10 Sep 2026 01:26:29 +0200 Subject: [PATCH 43/45] perf: reuse BiliBili range connections --- .../kotlin/dev/typetype/server/services/OkHttpProxyService.kt | 1 - src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt b/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt index 8b8e6dbb..9b906a23 100644 --- a/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt +++ b/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt @@ -62,7 +62,6 @@ class OkHttpProxyService( if (bilibili) { builder.header("Referer", BILIBILI_REFERER) builder.header("Accept", ACCEPT_ANY) - if (rangeHeader != null) builder.header("Connection", "close") } if (resolvedDomandBid != null && isNicoNico(cleanUrl)) builder.header("Cookie", "domand_bid=$resolvedDomandBid") if (rangeHeader != null) builder.header("Range", rangeHeader) diff --git a/src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt b/src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt index 8e5e463b..c1910e04 100644 --- a/src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt +++ b/src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt @@ -10,6 +10,7 @@ import okhttp3.Response import okhttp3.ResponseBody.Companion.toResponseBody import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Test import java.io.IOException import java.net.InetAddress @@ -28,7 +29,7 @@ class BilibiliRangeProxyTest { assertEquals(OkHttpProxyService.BILIBILI_USER_AGENT, request.header("User-Agent")) assertEquals("https://www.bilibili.com", request.header("Referer")) assertEquals("*/*", request.header("Accept")) - assertEquals("close", request.header("Connection")) + assertNull(request.header("Connection")) assertEquals("bytes=0-3", request.header("Range")) if (calls == 1) throw IOException("unexpected end of stream") Response.Builder() From 0d0e7c5290769ff06f8b557621ea77f349240ea5 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 10 Sep 2026 10:52:59 +0200 Subject: [PATCH 44/45] fix: isolate recommendations by service --- .../dev/typetype/server/ServiceRegistry.kt | 1 + .../services/HomeRecommendationBuilder.kt | 2 ++ .../HomeRecommendationCandidateService.kt | 22 +++++++++--- .../services/HomeRecommendationPoolCache.kt | 2 +- .../HomeRecommendationPoolResolver.kt | 1 + ...eRecommendationPoolResolverDependencies.kt | 1 + .../services/RecommendationServiceId.kt | 12 +++++++ .../HomeRecommendationCandidateServiceTest.kt | 35 +++++++++++++++++-- .../HomeRecommendationServiceFastPathTest.kt | 1 + .../server/HomeRecommendationTestFixtures.kt | 1 + 10 files changed, 70 insertions(+), 8 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/services/RecommendationServiceId.kt diff --git a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt index e775af42..5ca5063c 100644 --- a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt @@ -162,6 +162,7 @@ internal class ServiceRegistry( watchLaterService = watchLaterService, blockedService = blockedService, streamService = streamService, + trendingService = trendingService, cache = cache, ) val homeRecommendationServices = createHomeRecommendationServices(cache, recommendationPoolResolverDependencies) diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationBuilder.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationBuilder.kt index 5537eac5..e345b130 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationBuilder.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationBuilder.kt @@ -11,6 +11,7 @@ class HomeRecommendationBuilder( private val watchLaterService: WatchLaterService, private val blockedService: BlockedService, private val streamService: StreamService, + private val trendingService: TrendingService, ) { suspend fun build( userId: String, @@ -30,6 +31,7 @@ class HomeRecommendationBuilder( subscriptionFeedService = subscriptionFeedService, subscriptionShortsFeedService = subscriptionShortsFeedService, streamService = streamService, + trendingService = trendingService, ) val candidatePool = candidates.fetchCandidates( userId = userId, diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidateService.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidateService.kt index 48aa4625..f75d249f 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidateService.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidateService.kt @@ -1,11 +1,14 @@ package dev.typetype.server.services +import dev.typetype.server.models.ExtractionResult + import dev.typetype.server.models.VideoItem class HomeRecommendationCandidateService( private val subscriptionFeedService: SubscriptionFeedService, private val subscriptionShortsFeedService: SubscriptionShortsFeedService, private val streamService: StreamService, + private val trendingService: TrendingService, private val discoveryAssembler: HomeRecommendationDiscoveryAssembler = HomeRecommendationDiscoveryAssembler(), private val shortsCandidateService: HomeRecommendationShortsCandidateService = HomeRecommendationShortsCandidateService(), ) { @@ -36,9 +39,11 @@ class HomeRecommendationCandidateService( return shortsCandidateService.fetch(userId, profile, signalContext, this) } val subscriptions = fetchSubscriptionCandidates(userId, mode) + .filter { recommendationServiceId(it.url) == serviceId } .map { HomeRecommendationTaggedVideo(it, HomeRecommendationSourceTag.SUBSCRIPTION) } if (mode == HomeRecommendationPoolMode.FAST) { - return HomeRecommendationCandidatePool(subscriptions = subscriptions, discovery = emptyList()) + val discovery = if (subscriptions.isEmpty()) fetchTrending(serviceId) else emptyList() + return HomeRecommendationCandidatePool(subscriptions = subscriptions, discovery = discovery) } val subscriptionSeeds = subscriptions.map { it.video.url } val relatedFromSubscriptions = relatedCandidateService.fetch( @@ -48,16 +53,16 @@ class HomeRecommendationCandidateService( relatedPerSeedLimit = HomeRecommendationCandidateLimits.RELATED_PER_SEED_LIMIT, ) val relatedFromFavorites = relatedCandidateService.fetch( - seedUrls = signalContext.favoriteUrls, + seedUrls = signalContext.favoriteUrls.filter { recommendationServiceId(it) == serviceId }, source = HomeRecommendationSourceTag.DISCOVERY_EXPLORATION, seedLimit = HomeRecommendationCandidateLimits.FAVORITE_SEED_LIMIT, relatedPerSeedLimit = HomeRecommendationCandidateLimits.RELATED_PER_SEED_LIMIT, ) val discovery = discoveryAssembler.build( profile = profile, - candidates = relatedFromSubscriptions + relatedFromFavorites, + candidates = relatedFromSubscriptions + relatedFromFavorites + fetchTrending(serviceId), explorationCap = HomeRecommendationCandidateLimits.RELATED_DISCOVERY_CAP, - ) + ).filter { recommendationServiceId(it.video.url) == serviceId } return HomeRecommendationCandidatePool(subscriptions = subscriptions, discovery = discovery) } @@ -88,4 +93,13 @@ class HomeRecommendationCandidateService( relatedPerSeedLimit: Int, ): List = relatedCandidateService.fetch(seedUrls, source, seedLimit, relatedPerSeedLimit) + + private suspend fun fetchTrending(serviceId: Int): List = + when (val result = trendingService.getTrending(serviceId)) { + is ExtractionResult.Success -> result.data + .filter { recommendationServiceId(it.url) == serviceId } + .map { HomeRecommendationTaggedVideo(it, HomeRecommendationSourceTag.DISCOVERY_TRENDING) } + is ExtractionResult.BadRequest -> emptyList() + is ExtractionResult.Failure -> emptyList() + } } diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolCache.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolCache.kt index 17525b84..1c6fc516 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolCache.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolCache.kt @@ -41,6 +41,6 @@ class HomeRecommendationPoolCache(private val cache: dev.typetype.server.cache.C companion object { private const val CACHE_TTL_SECONDS = 3_600L private const val STALE_TTL_SECONDS = 86_400L - private const val CACHE_VERSION = 9 + private const val CACHE_VERSION = 10 } } diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolver.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolver.kt index 8058fd5f..5a26dba5 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolver.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolver.kt @@ -77,6 +77,7 @@ class HomeRecommendationPoolResolver( watchLaterService = dependencies.watchLaterService, blockedService = dependencies.blockedService, streamService = dependencies.streamService, + trendingService = dependencies.trendingService, ).build( userId = userId, serviceId = serviceId, diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolverDependencies.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolverDependencies.kt index e256e9f5..8a6a4ee5 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolverDependencies.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolverDependencies.kt @@ -11,5 +11,6 @@ data class HomeRecommendationPoolResolverDependencies( val watchLaterService: WatchLaterService, val blockedService: BlockedService, val streamService: StreamService = HomeRecommendationNoopStreamService, + val trendingService: TrendingService, val cache: CacheService, ) diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationServiceId.kt b/src/main/kotlin/dev/typetype/server/services/RecommendationServiceId.kt new file mode 100644 index 00000000..63bcdbac --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/RecommendationServiceId.kt @@ -0,0 +1,12 @@ +package dev.typetype.server.services + +import java.net.URI + +internal fun recommendationServiceId(url: String): Int { + val host = runCatching { URI(url).host.orEmpty().lowercase() }.getOrDefault("") + return when { + host == "b23.tv" || host == "bilibili.com" || host.endsWith(".bilibili.com") -> BILIBILI_SERVICE_ID + host == "nico.ms" || host == "nicovideo.jp" || host.endsWith(".nicovideo.jp") -> NICONICO_SERVICE_ID + else -> YOUTUBE_SERVICE_ID + } +} diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationCandidateServiceTest.kt b/src/test/kotlin/dev/typetype/server/HomeRecommendationCandidateServiceTest.kt index ccf32409..63872e33 100644 --- a/src/test/kotlin/dev/typetype/server/HomeRecommendationCandidateServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/HomeRecommendationCandidateServiceTest.kt @@ -12,6 +12,7 @@ import dev.typetype.server.services.HomeRecommendationSourceTag import dev.typetype.server.services.StreamService import dev.typetype.server.services.SubscriptionFeedService import dev.typetype.server.services.SubscriptionShortsFeedService +import dev.typetype.server.services.TrendingService import io.mockk.coEvery import io.mockk.mockk import kotlinx.coroutines.test.runTest @@ -23,8 +24,9 @@ class HomeRecommendationCandidateServiceTest { private val subscriptionFeedService: SubscriptionFeedService = mockk() private val subscriptionShortsFeedService: SubscriptionShortsFeedService = mockk() private val streamService: StreamService = mockk() + private val trendingService: TrendingService = mockk() private val service = HomeRecommendationCandidateService( - subscriptionFeedService, subscriptionShortsFeedService, streamService, + subscriptionFeedService, subscriptionShortsFeedService, streamService, trendingService, ) @BeforeEach @@ -33,6 +35,7 @@ class HomeRecommendationCandidateServiceTest { coEvery { subscriptionFeedService.getFeed(any(), any(), any()) } returns SubscriptionFeedResponse(emptyList(), null) coEvery { subscriptionShortsFeedService.getBlendedFeed(any(), any(), any(), any()) } returns SubscriptionFeedResponse(emptyList(), null) coEvery { streamService.getStreamInfo(any()) } returns ExtractionResult.Failure("none") + coEvery { trendingService.getTrending(any()) } returns ExtractionResult.Success(emptyList()) } @Test @@ -60,6 +63,32 @@ class HomeRecommendationCandidateServiceTest { assertTrue(pool.discovery.isEmpty()) } + @Test + fun `bilibili mode excludes subscriptions from other services`() = runTest { + val youtube = video("yt", "YouTube") + val bilibili = video("bili", "BiliBili", "https://www.bilibili.com/video/BV1234567890") + coEvery { subscriptionFeedService.getCachedFeed(any(), any(), any()) } returns + SubscriptionFeedResponse(listOf(youtube, bilibili), null) + + val pool = service.fetchCandidates("u", 5, profile(), HomeRecommendationPoolMode.FAST) + + assertTrue(pool.subscriptions.map { it.video.id } == listOf("bili")) + } + + @Test + fun `niconico mode falls back to niconico trending videos`() = runTest { + val youtube = video("yt", "YouTube") + val niconico = video("nico", "NicoNico", "https://www.nicovideo.jp/watch/sm123") + coEvery { subscriptionFeedService.getCachedFeed(any(), any(), any()) } returns + SubscriptionFeedResponse(listOf(youtube), null) + coEvery { trendingService.getTrending(6) } returns ExtractionResult.Success(listOf(youtube, niconico)) + + val pool = service.fetchCandidates("u", 6, profile(), HomeRecommendationPoolMode.FAST) + + assertTrue(pool.subscriptions.isEmpty()) + assertTrue(pool.discovery.map { it.video.id } == listOf("nico")) + } + private fun profile(): HomeRecommendationProfile = HomeRecommendationProfile( seenUrls = emptySet(), blockedVideos = emptySet(), blockedChannels = emptySet(), feedbackBlockedVideos = emptySet(), feedbackBlockedChannels = emptySet(), @@ -79,8 +108,8 @@ class HomeRecommendationCandidateServiceTest { sponsorBlockSegments = emptyList(), relatedStreams = related, publishedAt = 0, ) - private fun video(id: String, title: String): VideoItem = VideoItem( - id = id, title = title, url = "https://yt.com/v/$id", thumbnailUrl = "", uploaderName = "channel", + private fun video(id: String, title: String, url: String = "https://yt.com/v/$id"): VideoItem = VideoItem( + id = id, title = title, url = url, thumbnailUrl = "", uploaderName = "channel", uploaderUrl = "https://yt.com/c/channel", uploaderAvatarUrl = "", duration = 60, viewCount = 0, uploadDate = "", uploaded = System.currentTimeMillis(), streamType = "video_stream", isShortFormContent = false, uploaderVerified = false, shortDescription = null, diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationServiceFastPathTest.kt b/src/test/kotlin/dev/typetype/server/HomeRecommendationServiceFastPathTest.kt index 0ddf66a9..08e48e41 100644 --- a/src/test/kotlin/dev/typetype/server/HomeRecommendationServiceFastPathTest.kt +++ b/src/test/kotlin/dev/typetype/server/HomeRecommendationServiceFastPathTest.kt @@ -46,6 +46,7 @@ class HomeRecommendationServiceFastPathTest { TestDatabase.truncateAll() coEvery { cache.get(any()) } returns null coEvery { cache.set(any(), any(), any()) } returns Unit + coEvery { trendingService.getTrending(any()) } returns ExtractionResult.Success(emptyList()) coEvery { searchService.search(any(), any(), any(), any(), any()) } returns ExtractionResult.Success(SearchPageResponse(emptyList(), null, null, false)) } diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationTestFixtures.kt b/src/test/kotlin/dev/typetype/server/HomeRecommendationTestFixtures.kt index 8e6e7992..e4e95c43 100644 --- a/src/test/kotlin/dev/typetype/server/HomeRecommendationTestFixtures.kt +++ b/src/test/kotlin/dev/typetype/server/HomeRecommendationTestFixtures.kt @@ -36,6 +36,7 @@ fun homeResolverDependencies( favoritesService = FavoritesService(), watchLaterService = WatchLaterService(), blockedService = BlockedService(), + trendingService = trendingService, cache = cache, ) From fe782d824cc4d36a5f5fae6dabcf5985e43e85ed Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 10 Sep 2026 11:26:09 +0200 Subject: [PATCH 45/45] chore: update PipePipeExtractor --- build.gradle.kts | 2 +- .../dev/typetype/server/services/PipePipeCommentService.kt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 2fb66c10..fe90f2be 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -44,7 +44,7 @@ dependencies { implementation("io.ktor:ktor-server-call-logging-jvm") implementation("io.ktor:ktor-server-rate-limit-jvm") implementation("ch.qos.logback:logback-classic:1.6.3") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:ca3280f28f3aa0b980b63a2b2d23c362f7616620") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:a395a9ba16ae75987969ed9e7d330c928ad3bc20") compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") implementation("org.json:json:20260814") implementation("com.squareup.okhttp3:okhttp:5.5.0") diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeCommentService.kt b/src/main/kotlin/dev/typetype/server/services/PipePipeCommentService.kt index 1721e219..422b215d 100644 --- a/src/main/kotlin/dev/typetype/server/services/PipePipeCommentService.kt +++ b/src/main/kotlin/dev/typetype/server/services/PipePipeCommentService.kt @@ -54,7 +54,7 @@ class PipePipeCommentService : CommentService { private fun CommentsInfoItem.toCommentItem(): CommentItem = CommentItem( id = commentId ?: "", - text = commentText ?: "", + text = commentText.content, author = uploaderName ?: "", authorUrl = uploaderUrl ?: "", authorAvatarUrl = (uploaderAvatarUrl ?: "").normalizeHttpSchema(),