diff --git a/.env.example b/.env.example index 9f061443..0c395a83 100644 --- a/.env.example +++ b/.env.example @@ -12,4 +12,8 @@ YOUTUBE_REMOTE_LOGIN_INTERNAL_TOKEN=replace-with-shared-internal-token YOUTUBE_REMOTE_LOGIN_TTL_MS=480000 YOUTUBE_REMOTE_LOGIN_MAX_SESSIONS=2 +TYPE_TYPE_INSTANCE_ID=typetype +TYPE_TYPE_PUSH_NOTIFICATIONS_ENABLED=true +TYPE_TYPE_PUSH_NOTIFICATIONS_INTERVAL_SECONDS=300 + ALLOWED_ORIGINS=http://localhost:5173 diff --git a/build.gradle.kts b/build.gradle.kts index c77bb2e5..fe90f2be 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,8 +1,8 @@ import java.time.Instant plugins { - kotlin("jvm") version "2.4.10" - kotlin("plugin.serialization") version "2.4.10" + kotlin("jvm") version "2.4.20" + kotlin("plugin.serialization") version "2.4.20" id("io.ktor.plugin") version "3.5.2" id("jacoco") } @@ -29,7 +29,7 @@ dependencies { implementation("com.fasterxml.jackson.core:jackson-core") implementation(platform("io.netty:netty-bom:4.2.17.Final")) constraints { - implementation("org.jsoup:jsoup:1.23.1") { + implementation("org.jsoup:jsoup:1.23.2") { because("CVE-2026-71497 affects PipePipeExtractor's transitive jsoup version") } } @@ -44,18 +44,18 @@ dependencies { implementation("io.ktor:ktor-server-call-logging-jvm") implementation("io.ktor:ktor-server-rate-limit-jvm") implementation("ch.qos.logback:logback-classic:1.6.3") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:ca3280f28f3aa0b980b63a2b2d23c362f7616620") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:a395a9ba16ae75987969ed9e7d330c928ad3bc20") compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") implementation("org.json:json:20260814") implementation("com.squareup.okhttp3:okhttp:5.5.0") implementation("io.lettuce:lettuce-core:7.7.0.RELEASE") - implementation("org.jetbrains.exposed:exposed-core:1.4.0") - implementation("org.jetbrains.exposed:exposed-jdbc:1.4.0") + implementation("org.jetbrains.exposed:exposed-core:1.5.0") + implementation("org.jetbrains.exposed:exposed-jdbc:1.5.0") implementation("com.zaxxer:HikariCP:7.1.0") implementation("org.postgresql:postgresql:42.7.13") - implementation("org.xerial:sqlite-jdbc:3.53.2.1") + implementation("org.xerial:sqlite-jdbc:3.53.4.0") implementation("com.password4j:password4j:1.8.4") - implementation("com.auth0:java-jwt:4.6.0") + implementation("com.auth0:java-jwt:4.6.1") testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") testRuntimeOnly("org.junit.platform:junit-platform-launcher") testImplementation("io.mockk:mockk:1.14.11") diff --git a/gradle.properties b/gradle.properties index a73ffe8c..3e32a72b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ org.gradle.jvmargs=-Xmx2g -XX:+UseG1GC kotlin.code.style=official -appVersion=1.8.0 +appVersion=1.8.1 systemProp.sun.net.client.defaultReadTimeout=180000 systemProp.sun.net.client.defaultConnectTimeout=60000 diff --git a/openapi.yaml b/openapi.yaml index 508706c3..72d2cc37 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -29,6 +29,7 @@ paths: /streams/audio-only/source: { $ref: ./openapi/paths/streams.yaml#/AudioOnlySource } /subtitles/youtube/{videoId}: { $ref: ./openapi/paths/subtitles.yaml#/YoutubeSubtitle } /proxy: { $ref: ./openapi/paths/proxy.yaml#/Proxy } + /media/{handle}: { $ref: ./openapi/paths/media.yaml#/ProviderMedia } /sabr/download/{videoId}: { $ref: ./openapi/paths/sabr-download.yaml#/SabrDownload } /sabr/playback/{sessionId}/position: { $ref: ./openapi/paths/sabr-playback.yaml#/SabrPlaybackPosition } /sabr/playback/{sessionId}/window: { $ref: ./openapi/paths/sabr-playback.yaml#/SabrPlaybackWindow } @@ -50,6 +51,17 @@ paths: /subscriptions/groups/{groupId}: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroup } /subscriptions/groups/{groupId}/channels: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroupChannels } /subscriptions/feed: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionFeed } + /profiles: { $ref: ./openapi/paths/profiles.yaml#/AccountProfiles } + /profiles/{profileId}: { $ref: ./openapi/paths/profiles.yaml#/AccountProfile } + /profiles/{profileId}/default: { $ref: ./openapi/paths/profiles.yaml#/AccountProfileDefault } + /profiles/{profileId}/switch: { $ref: ./openapi/paths/profiles.yaml#/AccountProfileSwitch } + /notifications: { $ref: ./openapi/paths/notifications.yaml#/Notifications } + /notifications/unread-count: { $ref: ./openapi/paths/notifications.yaml#/NotificationsUnreadCount } + /notifications/read-all: { $ref: ./openapi/paths/notifications.yaml#/NotificationsReadAll } + /notifications/{notificationId}/read: { $ref: ./openapi/paths/notifications.yaml#/NotificationsRead } + /notifications/channel-preferences: { $ref: ./openapi/paths/push-notifications.yaml#/ChannelNotificationPreferences } + /notifications/push/devices: { $ref: ./openapi/paths/push-notifications.yaml#/PushDevices } + /notifications/push/devices/{deviceId}: { $ref: ./openapi/paths/push-notifications.yaml#/PushDevice } /rss/feeds: { $ref: ./openapi/paths/rss.yaml#/RssFeeds } /rss/feeds/{id}: { $ref: ./openapi/paths/rss.yaml#/RssFeed } /rss/feeds/{id}/enabled: { $ref: ./openapi/paths/rss.yaml#/RssFeedEnabled } @@ -114,6 +126,11 @@ paths: /internal/youtube-session/browser/complete: $ref: ./openapi/paths/youtube-session.yaml#/InternalBrowserComplete components: + securitySchemes: + bearerAuth: + type: http + scheme: bearer + bearerFormat: JWT schemas: ErrorResponse: $ref: ./openapi/components/common.yaml#/ErrorResponse @@ -169,6 +186,20 @@ components: SubscriptionGroupMembershipBatchRequest: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionGroupMembershipBatchRequest } SubscriptionFeedResponse: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionFeedResponse } SubscriptionFeedPreparingResponse: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionFeedPreparingResponse } + AccountProfileItem: { $ref: ./openapi/components/profiles.yaml#/AccountProfileItem } + AccountProfilesResponse: { $ref: ./openapi/components/profiles.yaml#/AccountProfilesResponse } + ProfileNameRequest: { $ref: ./openapi/components/profiles.yaml#/ProfileNameRequest } + ProfileSwitchResponse: { $ref: ./openapi/components/profiles.yaml#/ProfileSwitchResponse } + NotificationItem: { $ref: ./openapi/components/notifications.yaml#/NotificationItem } + NotificationsResponse: { $ref: ./openapi/components/notifications.yaml#/NotificationsResponse } + UnreadCountResponse: { $ref: ./openapi/components/notifications.yaml#/UnreadCountResponse } + MarkNotificationsReadResponse: { $ref: ./openapi/components/notifications.yaml#/MarkNotificationsReadResponse } + PushNotificationCapability: { $ref: ./openapi/components/instance.yaml#/PushNotificationCapability } + PushDeviceRegistrationRequest: { $ref: ./openapi/components/push-notifications.yaml#/PushDeviceRegistrationRequest } + PushDeviceRegistrationResponse: { $ref: ./openapi/components/push-notifications.yaml#/PushDeviceRegistrationResponse } + ChannelNotificationPreferenceRequest: { $ref: ./openapi/components/push-notifications.yaml#/ChannelNotificationPreferenceRequest } + ChannelNotificationPreference: { $ref: ./openapi/components/push-notifications.yaml#/ChannelNotificationPreference } + UnifiedPushNotificationPayload: { $ref: ./openapi/components/push-notifications.yaml#/UnifiedPushNotificationPayload } RssFeedRequest: { $ref: ./openapi/components/rss.yaml#/RssFeedRequest } RssFeedItem: { $ref: ./openapi/components/rss.yaml#/RssFeedItem } RssFeedSecretItem: { $ref: ./openapi/components/rss.yaml#/RssFeedSecretItem } diff --git a/openapi/components/instance.yaml b/openapi/components/instance.yaml index c40a33cc..f2bb25aa 100644 --- a/openapi/components/instance.yaml +++ b/openapi/components/instance.yaml @@ -22,6 +22,7 @@ InstanceResponse: - youtubeRemoteLoginReady - parentalControlsEnabled - rss + - pushNotifications properties: name: { type: string, example: TypeType } tagline: { type: string, nullable: true } @@ -56,6 +57,8 @@ InstanceResponse: description: True when the instance-wide allow-list policy is enabled. rss: $ref: '#/RssInstanceCapability' + pushNotifications: + $ref: '#/PushNotificationCapability' RssInstanceCapability: type: object required: [enabled, maxFeedsPerUser, maxItems, minimumPollMinutes, rateLimitPerMinute] @@ -65,3 +68,14 @@ RssInstanceCapability: maxItems: { type: integer } minimumPollMinutes: { type: integer } rateLimitPerMinute: { type: integer } + +PushNotificationCapability: + type: object + required: [enabled, provider, eventTypes, maxDevicesPerAccount] + properties: + enabled: { type: boolean } + provider: { type: string, enum: [unifiedpush] } + eventTypes: + type: array + items: { type: string } + maxDevicesPerAccount: { type: integer, minimum: 0 } diff --git a/openapi/components/notifications.yaml b/openapi/components/notifications.yaml new file mode 100644 index 00000000..30841545 --- /dev/null +++ b/openapi/components/notifications.yaml @@ -0,0 +1,39 @@ +NotificationItem: + type: object + required: [id, type, title, createdAt, publishedAt, channelUrl, channelName, channelAvatarUrl, serviceId, serviceName, read, video] + properties: + id: { type: string } + type: { type: string } + title: { type: string } + createdAt: { type: integer, format: int64 } + publishedAt: { type: integer, format: int64 } + channelUrl: { type: string } + channelName: { type: string } + channelAvatarUrl: { type: string } + serviceId: { type: integer, format: int32, minimum: 0 } + serviceName: { type: string } + read: { type: boolean } + video: { $ref: ./media.yaml#/VideoItem } +NotificationsResponse: + type: object + required: [items, unreadCount, nextpage, available] + properties: + items: + type: array + items: { $ref: '#/NotificationItem' } + unreadCount: { type: integer, minimum: 0 } + nextpage: { type: string, nullable: true } + available: { type: boolean } +UnreadCountResponse: + type: object + required: [unreadCount, available] + properties: + unreadCount: { type: integer, minimum: 0 } + available: { type: boolean } +MarkNotificationsReadResponse: + type: object + required: [readAt, unreadCount, available] + properties: + readAt: { type: integer, format: int64 } + unreadCount: { type: integer, minimum: 0 } + available: { type: boolean } diff --git a/openapi/components/profiles.yaml b/openapi/components/profiles.yaml new file mode 100644 index 00000000..695c8b0c --- /dev/null +++ b/openapi/components/profiles.yaml @@ -0,0 +1,33 @@ +AccountProfileItem: + type: object + required: [id, name, isActive, isDefault, lastUsedAt] + properties: + id: { type: string, format: uuid } + name: { type: string, minLength: 1, maxLength: 40 } + isActive: { type: boolean } + isDefault: { type: boolean } + lastUsedAt: { type: integer, format: int64 } + publicUsername: { type: string, nullable: true } + avatarUrl: { type: string, nullable: true } + avatarType: { type: string, nullable: true } + avatarCode: { type: string, nullable: true } +AccountProfilesResponse: + type: object + required: [profiles, activeProfileId, defaultProfileId] + properties: + profiles: + type: array + items: { $ref: '#/AccountProfileItem' } + activeProfileId: { type: string, format: uuid } + defaultProfileId: { type: string, format: uuid } +ProfileNameRequest: + type: object + required: [name] + properties: + name: { type: string, minLength: 1, maxLength: 40 } +ProfileSwitchResponse: + type: object + required: [accessToken, profile] + properties: + accessToken: { type: string } + profile: { $ref: '#/AccountProfileItem' } diff --git a/openapi/components/push-notifications.yaml b/openapi/components/push-notifications.yaml new file mode 100644 index 00000000..29501691 --- /dev/null +++ b/openapi/components/push-notifications.yaml @@ -0,0 +1,62 @@ +PushDeviceRegistrationRequest: + type: object + required: [deviceId, endpoint] + properties: + deviceId: { type: string, minLength: 1, maxLength: 128 } + platform: { type: string, enum: [android], default: android } + endpoint: { type: string, format: uri, minLength: 1, maxLength: 2048 } + expiresAt: { type: integer, format: int64, nullable: true } +PushDeviceRegistrationResponse: + type: object + required: [id, deviceId, platform, expiresAt, updatedAt] + properties: + id: { type: string } + deviceId: { type: string } + platform: { type: string, enum: [android] } + expiresAt: { type: integer, format: int64, nullable: true } + updatedAt: { type: integer, format: int64 } +ChannelNotificationPreferenceRequest: + type: object + required: [channelUrl, enabled] + properties: + channelUrl: { type: string, minLength: 1 } + enabled: { type: boolean } +ChannelNotificationPreference: + type: object + required: [channelUrl, enabled, updatedAt] + properties: + channelUrl: { type: string } + enabled: { type: boolean } + updatedAt: { type: integer, format: int64 } +UnifiedPushNotificationPayload: + type: object + required: + - version + - eventType + - serviceId + - serviceName + - eventId + - videoId + - videoUrl + - channelId + - channelName + - channelAvatarUrl + - instanceId + - accountId + - publishedAt + - title + properties: + version: { type: integer, minimum: 1 } + eventType: { type: string, enum: [subscription_new_video] } + serviceId: { type: integer, minimum: 0 } + serviceName: { type: string } + eventId: { type: string } + videoId: { type: string } + videoUrl: { type: string } + channelId: { type: string } + channelName: { type: string } + channelAvatarUrl: { type: string } + instanceId: { type: string } + accountId: { type: string } + publishedAt: { type: integer, format: int64 } + title: { type: string } diff --git a/openapi/paths/media.yaml b/openapi/paths/media.yaml new file mode 100644 index 00000000..efe7b3e1 --- /dev/null +++ b/openapi/paths/media.yaml @@ -0,0 +1,59 @@ +ProviderMedia: + get: + tags: [playback] + summary: Retrieve provider media through an opaque temporary handle + description: >- + Streams BiliBili or NicoNico media referenced by a handle returned from a provider stream + endpoint. The signed provider URL and NicoNico cookie remain server-side; clients must not + construct or proxy the provider URL themselves. Handles expire with the upstream media + signature and support byte ranges for progressive and segmented playback. + parameters: + - name: handle + in: path + required: true + schema: + type: string + pattern: '^m1_[A-Za-z0-9_-]{24}$' + - name: Range + in: header + required: false + schema: { type: string } + responses: + '200': + description: Complete provider media response. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/octet-stream: + schema: { type: string, format: binary } + video/mp4: + schema: { type: string, format: binary } + audio/mp4: + schema: { type: string, format: binary } + application/vnd.apple.mpegurl: + schema: { type: string } + '206': + description: Partial provider media response. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + Content-Range: + schema: { type: string } + Accept-Ranges: + schema: { type: string } + content: + application/octet-stream: + schema: { type: string, format: binary } + video/mp4: + schema: { type: string, format: binary } + audio/mp4: + schema: { type: string, format: binary } + '404': + $ref: ../components/common.yaml#/JsonError + '422': + $ref: ../components/common.yaml#/JsonError + '429': + $ref: ../components/common.yaml#/JsonError + '503': + $ref: ../components/common.yaml#/JsonError diff --git a/openapi/paths/notifications.yaml b/openapi/paths/notifications.yaml new file mode 100644 index 00000000..eda74832 --- /dev/null +++ b/openapi/paths/notifications.yaml @@ -0,0 +1,66 @@ +Notifications: + get: + tags: [user-data] + summary: List new videos from the current profile's subscriptions + security: [{ bearerAuth: [] }] + parameters: + - name: page + in: query + required: false + schema: { type: integer, minimum: 0, default: 0 } + - name: limit + in: query + required: false + schema: { type: integer, minimum: 1, maximum: 100, default: 20 } + responses: + '200': + description: Profile-scoped notifications. Cached items remain visible during refresh. If refresh fails, available is false and previously collected items are retained. + content: + application/json: + schema: { $ref: ../components/notifications.yaml#/NotificationsResponse } + '401': { $ref: ../components/common.yaml#/JsonError } + +NotificationsUnreadCount: + get: + tags: [user-data] + summary: Read the current profile's unread notification count + security: [{ bearerAuth: [] }] + responses: + '200': + description: Unread count and feed availability. + content: + application/json: + schema: { $ref: ../components/notifications.yaml#/UnreadCountResponse } + '401': { $ref: ../components/common.yaml#/JsonError } + +NotificationsReadAll: + post: + tags: [user-data] + summary: Mark all current-profile notifications as read + security: [{ bearerAuth: [] }] + responses: + '200': + description: Read marker and resulting unread count. + content: + application/json: + schema: { $ref: ../components/notifications.yaml#/MarkNotificationsReadResponse } + '401': { $ref: ../components/common.yaml#/JsonError } + +NotificationsRead: + post: + tags: [user-data] + summary: Mark one current-profile notification as read + security: [{ bearerAuth: [] }] + parameters: + - name: notificationId + in: path + required: true + schema: { type: string } + responses: + '200': + description: Read marker and resulting unread count. + content: + application/json: + schema: { $ref: ../components/notifications.yaml#/MarkNotificationsReadResponse } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } diff --git a/openapi/paths/profiles.yaml b/openapi/paths/profiles.yaml new file mode 100644 index 00000000..c562e3b8 --- /dev/null +++ b/openapi/paths/profiles.yaml @@ -0,0 +1,102 @@ +AccountProfiles: + get: + tags: [user-data] + summary: List the profiles available to the authenticated account + security: [{ bearerAuth: [] }] + responses: + '200': + description: Account profiles and the active/default profile IDs. + content: + application/json: + schema: { $ref: ../components/profiles.yaml#/AccountProfilesResponse } + '401': { $ref: ../components/common.yaml#/JsonError } + '403': { $ref: ../components/common.yaml#/JsonError } + post: + tags: [user-data] + summary: Create an isolated account profile + security: [{ bearerAuth: [] }] + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/profiles.yaml#/ProfileNameRequest } + responses: + '201': + description: The newly created profile. + content: + application/json: + schema: { $ref: ../components/profiles.yaml#/AccountProfileItem } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + +AccountProfile: + parameters: + - name: profileId + in: path + required: true + schema: { type: string, format: uuid } + put: + tags: [user-data] + summary: Rename an account profile + security: [{ bearerAuth: [] }] + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/profiles.yaml#/ProfileNameRequest } + responses: + '200': + description: Renamed profile. + content: + application/json: + schema: { $ref: ../components/profiles.yaml#/AccountProfileItem } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + delete: + tags: [user-data] + summary: Delete an account profile and its private data + security: [{ bearerAuth: [] }] + responses: + '204': { description: Profile deleted. } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + '409': { $ref: ../components/common.yaml#/JsonError } + +AccountProfileDefault: + parameters: + - name: profileId + in: path + required: true + schema: { type: string, format: uuid } + post: + tags: [user-data] + summary: Set the default profile used after sign-in + security: [{ bearerAuth: [] }] + responses: + '200': + description: The selected default profile. + content: + application/json: + schema: { $ref: ../components/profiles.yaml#/AccountProfileItem } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + +AccountProfileSwitch: + parameters: + - name: profileId + in: path + required: true + schema: { type: string, format: uuid } + post: + tags: [user-data] + summary: Switch the authenticated session to another profile + security: [{ bearerAuth: [] }] + responses: + '200': + description: A new profile-scoped access token and profile projection. + content: + application/json: + schema: { $ref: ../components/profiles.yaml#/ProfileSwitchResponse } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } diff --git a/openapi/paths/push-notifications.yaml b/openapi/paths/push-notifications.yaml new file mode 100644 index 00000000..30bd33a4 --- /dev/null +++ b/openapi/paths/push-notifications.yaml @@ -0,0 +1,85 @@ +ChannelNotificationPreferences: + get: + tags: [user-data] + summary: List notification preferences for followed channels + security: [{ bearerAuth: [] }] + responses: + '200': + description: Channel notification preferences for the current profile. + content: + application/json: + schema: + type: array + items: { $ref: ../components/push-notifications.yaml#/ChannelNotificationPreference } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + put: + tags: [user-data] + summary: Enable or disable notifications for a followed channel + security: [{ bearerAuth: [] }] + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/push-notifications.yaml#/ChannelNotificationPreferenceRequest } + responses: + '200': + description: Updated channel notification preference. + content: + application/json: + schema: { $ref: ../components/push-notifications.yaml#/ChannelNotificationPreference } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + '409': { $ref: ../components/common.yaml#/JsonError } + +PushDevices: + get: + tags: [user-data] + summary: List registered UnifiedPush devices + security: [{ bearerAuth: [] }] + responses: + '200': + description: Devices registered by the current profile. + content: + application/json: + schema: + type: array + items: { $ref: ../components/push-notifications.yaml#/PushDeviceRegistrationResponse } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + post: + tags: [user-data] + summary: Register or replace a UnifiedPush device endpoint + security: [{ bearerAuth: [] }] + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/push-notifications.yaml#/PushDeviceRegistrationRequest } + responses: + '201': + description: Registered device. + content: + application/json: + schema: { $ref: ../components/push-notifications.yaml#/PushDeviceRegistrationResponse } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + '409': { $ref: ../components/common.yaml#/JsonError } + '422': { $ref: ../components/common.yaml#/JsonError } + +PushDevice: + delete: + tags: [user-data] + summary: Unregister a UnifiedPush device + security: [{ bearerAuth: [] }] + parameters: + - name: deviceId + in: path + required: true + schema: { type: string } + responses: + '204': { description: Device unregistered. } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } diff --git a/openapi/paths/rss.yaml b/openapi/paths/rss.yaml index bd8e610c..de483019 100644 --- a/openapi/paths/rss.yaml +++ b/openapi/paths/rss.yaml @@ -108,7 +108,7 @@ RssFeedDocument: - { name: token, in: query, required: true, schema: { type: string } } responses: '200': - description: RSS document + description: RSS 2.0 document. Video items include a media:thumbnail element when a HTTP(S) thumbnail URL is available. headers: ETag: { schema: { type: string } } Last-Modified: { schema: { type: string } } diff --git a/src/main/kotlin/dev/typetype/server/AppMetrics.kt b/src/main/kotlin/dev/typetype/server/AppMetrics.kt index b74be9d8..0de33b49 100644 --- a/src/main/kotlin/dev/typetype/server/AppMetrics.kt +++ b/src/main/kotlin/dev/typetype/server/AppMetrics.kt @@ -17,7 +17,7 @@ object AppMetrics { totalRequests.incrementAndGet() totalDurationMs.addAndGet(call.requestDurationMs()) statusCounts.getOrPut(status) { AtomicLong() }.incrementAndGet() - routeCounts.getOrPut("$route|$status") { AtomicLong() }.incrementAndGet() + recordRoute(route, status) } fun snapshot(): String { @@ -36,6 +36,18 @@ object AppMetrics { } } + private fun recordRoute(route: String, status: Int) { + val key = "$route|$status" + if (routeCounts.size < MAX_ROUTE_METRICS) { + routeCounts.computeIfAbsent(key) { AtomicLong() }.incrementAndGet() + return + } + routeCounts.getOrPut("$OTHER_ROUTE|$status") { AtomicLong() }.incrementAndGet() + } + + private const val MAX_ROUTE_METRICS = 512 + private const val OTHER_ROUTE = "/__other__" + } fun metricPath(path: String): String = when { diff --git a/src/main/kotlin/dev/typetype/server/Application.kt b/src/main/kotlin/dev/typetype/server/Application.kt index 05786bc2..3e382395 100644 --- a/src/main/kotlin/dev/typetype/server/Application.kt +++ b/src/main/kotlin/dev/typetype/server/Application.kt @@ -11,6 +11,7 @@ import dev.typetype.server.services.DownloaderGatewayService import dev.typetype.server.services.GitHubIssueService import dev.typetype.server.services.PasswordResetService import dev.typetype.server.services.ProfileService +import dev.typetype.server.services.ProfileAccountService import dev.typetype.server.services.PipePipeBackupImporterService import dev.typetype.server.services.OpenMojiProxyService import dev.typetype.server.services.InstanceService @@ -24,6 +25,7 @@ import dev.typetype.server.services.UserAdminService import dev.typetype.server.services.YoutubeRemoteBrowserConfig import dev.typetype.server.services.YoutubeRemoteBrowserService import dev.typetype.server.services.YoutubeRemoteLoginReadinessService +import dev.typetype.server.services.PushNotificationScheduler import dev.typetype.server.portability.PortabilityEngineFactory import io.ktor.server.application.Application import io.ktor.server.application.ApplicationStopped @@ -44,7 +46,8 @@ fun Application.module() { DatabaseFactory.init(dbUrl, dbUser, dbPassword) val jwtSecret = System.getenv("JWT_SECRET") ?: UUID.randomUUID().toString() val authSessionConfig = AuthSessionConfig.fromEnvironment() - val authService = AuthService(jwtSecret, sessionConfig = authSessionConfig) + val profileAccountService = ProfileAccountService() + val authService = AuthService(jwtSecret, sessionConfig = authSessionConfig, profileAccountService = profileAccountService) val oidcAuthService = OidcAuthService(OidcConfigLoader.fromEnvironment(), jwtSecret, authService) val userAdminService = UserAdminService() val passwordResetService = PasswordResetService() @@ -68,7 +71,16 @@ fun Application.module() { jwtSecret, adminSettingsService, youtubeProxySelector, + profileAccountService, + instanceId = System.getenv("TYPE_TYPE_INSTANCE_ID")?.trim().takeUnless { it.isNullOrBlank() } ?: "typetype", + pushNotificationsEnabled = System.getenv("TYPE_TYPE_PUSH_NOTIFICATIONS_ENABLED")?.toBooleanStrictOrNull() ?: true, ) + val pushNotificationScheduler = PushNotificationScheduler(svc.pushNotificationService) + pushNotificationScheduler.start() + monitor.subscribe(ApplicationStopped) { pushNotificationScheduler.close() } + monitor.subscribe(ApplicationStopped) { svc.subscriptionFeedService.close() } + monitor.subscribe(ApplicationStopped) { svc.youtubeTakeoutImportService.close() } + monitor.subscribe(ApplicationStopped) { svc.homeRecommendationServices.close() } val youtubeRemoteBrowserConfig = YoutubeRemoteBrowserConfig.fromEnvironment(subtitleServiceUrl) val youtubeRemoteLoginReadinessService = YoutubeRemoteLoginReadinessService( youtubeRemoteBrowserConfig, @@ -79,6 +91,7 @@ fun Application.module() { adminSettingsService, youtubeRemoteLoginStatusProvider = youtubeRemoteLoginReadinessService::status, oidcConfigProvider = oidcAuthService::publicConfig, + pushNotificationCapabilityProvider = svc.pushNotificationService::capability, ) val youtubeRemoteBrowserService = YoutubeRemoteBrowserService( youtubeRemoteBrowserConfig, @@ -94,6 +107,10 @@ fun Application.module() { CoroutineScope(SupervisorJob() + Dispatchers.IO), ) monitor.subscribe(ApplicationStopped) { portabilityEngine.close() } + monitor.subscribe(ApplicationStopped) { + svc.sabrSessionStore.release() + cache.close() + } configurePlugins(authService) installApplicationRoutes( svc = svc, @@ -107,6 +124,7 @@ fun Application.module() { oidcAuthService = oidcAuthService, passwordResetService = passwordResetService, profileService = profileService, + profileAccountService = profileAccountService, userAdminService = userAdminService, avatarService = avatarService, openMojiProxyService = openMojiProxyService, diff --git a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt index 3a930d92..2e52330d 100644 --- a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt @@ -7,6 +7,7 @@ import dev.typetype.server.routes.adminRssRoutes import dev.typetype.server.routes.adminIdentityRoutes import dev.typetype.server.routes.adminSessionRoutes import dev.typetype.server.routes.authRoutes +import dev.typetype.server.routes.accountProfilesRoutes import dev.typetype.server.routes.avatarRoutes import dev.typetype.server.routes.bulletCommentRoutes import dev.typetype.server.routes.channelRoutes @@ -18,6 +19,7 @@ import dev.typetype.server.routes.oidcAuthRoutes import dev.typetype.server.routes.podcastRoutes import dev.typetype.server.routes.publicMetadataRoutes import dev.typetype.server.routes.publicPlaylistRoutes +import dev.typetype.server.routes.pushNotificationRoutes import dev.typetype.server.routes.rssPublicRoutes import dev.typetype.server.routes.sabrRoutes import dev.typetype.server.routes.searchRoutes @@ -40,6 +42,7 @@ import dev.typetype.server.services.OpenMojiProxyService import dev.typetype.server.services.PasswordResetService import dev.typetype.server.services.PipePipeBackupImporterService import dev.typetype.server.services.ProfileService +import dev.typetype.server.services.ProfileAccountService import dev.typetype.server.services.UserAdminService import dev.typetype.server.services.YoutubeRemoteBrowserService import dev.typetype.server.portability.PortabilityEngine @@ -59,6 +62,7 @@ internal fun Application.installApplicationRoutes( oidcAuthService: OidcAuthService, passwordResetService: PasswordResetService, profileService: ProfileService, + profileAccountService: ProfileAccountService, userAdminService: UserAdminService, avatarService: AvatarService, openMojiProxyService: OpenMojiProxyService, @@ -110,6 +114,7 @@ internal fun Application.installApplicationRoutes( svc.homeRecommendationWarmupService, authSessionConfig, ) + accountProfilesRoutes(profileAccountService, authService, authSessionConfig) adminRoutes(authService, userAdminService, passwordResetService, adminSettingsService) adminRssRoutes(svc.rssFeedManagementService, authService) adminIdentityRoutes(svc.accountIdentityService, authService) diff --git a/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt b/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt index 1c3fd0e5..2e729658 100644 --- a/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt @@ -5,6 +5,7 @@ import dev.typetype.server.routes.audioOnlySourceRoutes import dev.typetype.server.routes.manifestRoutes import dev.typetype.server.routes.nicoVideoProxyRoutes import dev.typetype.server.routes.proxyRoutes +import dev.typetype.server.routes.providerMediaHandleRoutes import dev.typetype.server.routes.storyboardProxyRoutes import dev.typetype.server.routes.streamRoutes import dev.typetype.server.routes.withPlayableSabrStreams @@ -30,6 +31,7 @@ internal fun Route.installStreamRoutes( adminSettingsService = adminSettingsService, blockedService = svc.blockedService, publicHlsManifestTokenService = svc.publicHlsManifestTokenService, + providerMediaHandleService = svc.providerMediaHandleService, sabrStreamContractFilter = { url, data -> data.withPlayableSabrStreams(url, svc.sabrSessionStore) }, youtubeSessionSabrStreamInfo = svc.youtubeSessionSabrStreamService?.let { service -> { userId, url -> service.getStreamInfo(userId, url) } @@ -62,6 +64,7 @@ internal fun Route.installStreamRoutes( internal fun Route.installProxyRoutes(svc: ServiceRegistry) { rateLimit(PROXY_ZONE) { proxyRoutes(svc.proxyService, svc.youtubeSubtitleDeliveryService) + providerMediaHandleRoutes(svc.providerMediaHandleService, svc.proxyService) youtubeSubtitleRoutes(svc.youtubeSubtitleDeliveryService) audioOnlySourceRoutes( streamService = svc.streamService, diff --git a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt index 2dc6fd25..d2f72dc5 100644 --- a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt @@ -29,6 +29,7 @@ import dev.typetype.server.services.PipePipeSearchService import dev.typetype.server.services.PipePipeStreamService import dev.typetype.server.services.PipePipeSuggestionService import dev.typetype.server.services.PipePipeTrendingService +import dev.typetype.server.services.ProviderMediaHandleService import dev.typetype.server.services.SabrFallbackStreamService import dev.typetype.server.services.SabrBootstrapStreamService import dev.typetype.server.services.SabrSessionStore @@ -82,15 +83,16 @@ internal class ExtractionServiceRegistry( .build() val sabrSessionStore = SabrSessionStore(subtitleServiceUrl, initCache = cache) val youtubeSubtitleService = YouTubeSubtitleService(httpClient, subtitleServiceUrl) + private val bilibiliRelatedService = BilibiliRelatedService() private val directPipePipeStreamService = PipePipeStreamService( cache, youtubeSubtitleService, - BilibiliRelatedService(), + bilibiliRelatedService, ) private val sabrPipePipeStreamService = PipePipeStreamService( cache, youtubeSubtitleService, - BilibiliRelatedService(), + bilibiliRelatedService, sabrSessionStore::rememberExtractedInfo, ) private val publicStreamService = YoutubePlayerClientStreamService( @@ -153,9 +155,10 @@ internal class ExtractionServiceRegistry( YoutubeScopedPublicPlaylistService(PipePipePublicPlaylistService()), cache, ) - val proxyService = OkHttpProxyService(proxyHttpClient) - val nicoVideoProxyService = NicoVideoProxyService() - val manifestService = CachedManifestService(ManifestService(streamService), cache) + val providerMediaHandleService = ProviderMediaHandleService(cache) + val proxyService = OkHttpProxyService(proxyHttpClient, providerMediaHandleService) + val nicoVideoProxyService = NicoVideoProxyService(mediaHandleService = providerMediaHandleService) + val manifestService = CachedManifestService(ManifestService(streamService, providerMediaHandleService), cache) val nativeManifestService = CachedNativeManifestService(NativeManifestService(), cache) val hlsManifestService = HlsManifestService( streamService, @@ -163,6 +166,7 @@ internal class ExtractionServiceRegistry( cache, hlsManifestUrlSigner, tokenYoutubeSessionClient::fetchHlsManifestUrl, + providerMediaHandleService, ) val youtubeSessionHlsManifestService = hlsTokenService?.let { tokenService -> youtubeSessionStreamService?.let { diff --git a/src/main/kotlin/dev/typetype/server/HomeRecommendationServices.kt b/src/main/kotlin/dev/typetype/server/HomeRecommendationServices.kt index 305ccdbe..3afc8d5e 100644 --- a/src/main/kotlin/dev/typetype/server/HomeRecommendationServices.kt +++ b/src/main/kotlin/dev/typetype/server/HomeRecommendationServices.kt @@ -9,7 +9,12 @@ import dev.typetype.server.services.HomeRecommendationWarmupService data class HomeRecommendationServices( val recommendationService: HomeRecommendationService, val warmupService: HomeRecommendationWarmupService, -) +) : AutoCloseable { + override fun close() { + warmupService.close() + recommendationService.close() + } +} fun createHomeRecommendationServices( cache: DragonflyService, diff --git a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt index de8e6a0b..5ca5063c 100644 --- a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt @@ -16,6 +16,9 @@ import dev.typetype.server.services.FavoritesService import dev.typetype.server.services.HistoryService import dev.typetype.server.services.HomeRecommendationService import dev.typetype.server.services.NotificationsService +import dev.typetype.server.services.ChannelNotificationPreferenceService +import dev.typetype.server.services.PushNotificationService +import dev.typetype.server.services.ProfileAccountService import dev.typetype.server.services.PlaylistService import dev.typetype.server.services.ProgressService import dev.typetype.server.services.RssFeedManagementService @@ -45,9 +48,12 @@ internal class ServiceRegistry( jwtSecret: String, adminSettingsService: AdminSettingsService, youtubeProxySelector: ProxySelector? = null, + profileAccountService: ProfileAccountService? = null, + private val instanceId: String = "typetype", + private val pushNotificationsEnabled: Boolean = true, ) { val publicHlsManifestTokenService = PublicHlsManifestTokenService(jwtSecret) - val accountIdentityService = AccountIdentityService() + val accountIdentityService = AccountIdentityService(profileAccountService) val customAvatarService = CustomAvatarService() val deArrowService = DeArrowService(cache) private val extraction = ExtractionServiceRegistry( @@ -74,6 +80,7 @@ internal class ServiceRegistry( val podcastService = extraction.podcastService val publicPlaylistService = extraction.publicPlaylistService val proxyService = extraction.proxyService + val providerMediaHandleService = extraction.providerMediaHandleService val youtubeSubtitleDeliveryService = extraction.youtubeSubtitleDeliveryService val nicoVideoProxyService = extraction.nicoVideoProxyService val manifestService = extraction.manifestService @@ -99,6 +106,14 @@ internal class ServiceRegistry( ) } val notificationsService = NotificationsService(subscriptionFeedService) + val channelNotificationPreferenceService = ChannelNotificationPreferenceService(subscriptionsService) + val pushNotificationService = PushNotificationService( + subscriptionsService = subscriptionsService, + subscriptionFeedService = subscriptionFeedService, + preferenceService = channelNotificationPreferenceService, + instanceId = instanceId, + enabled = pushNotificationsEnabled, + ) val playlistService = PlaylistService() val videoMetadataRepairService = UserVideoMetadataRepairService(VideoMetadataResolver(streamService)) val savedPlaylistService = SavedPlaylistService() @@ -147,9 +162,10 @@ internal class ServiceRegistry( watchLaterService = watchLaterService, blockedService = blockedService, streamService = streamService, + trendingService = trendingService, cache = cache, ) - private val homeRecommendationServices = createHomeRecommendationServices(cache, recommendationPoolResolverDependencies) + val homeRecommendationServices = createHomeRecommendationServices(cache, recommendationPoolResolverDependencies) val homeRecommendationService = homeRecommendationServices.recommendationService val homeRecommendationWarmupService = homeRecommendationServices.warmupService } diff --git a/src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt b/src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt index d8517ccd..66b2cdfe 100644 --- a/src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt +++ b/src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt @@ -9,8 +9,9 @@ import kotlinx.coroutines.future.await class DragonflyService(url: String) : CacheService { + private val client: RedisClient = RedisClient.create(url) private val connection: StatefulRedisConnection = - RedisClient.create(url).connect() + client.connect() private val async: RedisAsyncCommands = connection.async() @@ -36,6 +37,11 @@ class DragonflyService(url: String) : CacheService { suspend fun ping(): Boolean = async.ping().await() == "PONG" + fun close() { + connection.close() + client.shutdown() + } + private companion object { const val REFRESH_IF_VALUE_MATCHES = "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('expire', KEYS[1], ARGV[2]) end return 0" diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt index 0ffc8fa9..1368eb0f 100644 --- a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt +++ b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt @@ -25,10 +25,25 @@ import dev.typetype.server.db.tables.AdminSettingsTable import dev.typetype.server.db.tables.AllowedChannelsTable import dev.typetype.server.db.tables.PasswordResetTable import dev.typetype.server.db.tables.NotificationStatesTable +import dev.typetype.server.db.tables.NotificationReadItemsTable +import dev.typetype.server.db.tables.ChannelNotificationPreferencesTable +import dev.typetype.server.db.tables.PushDevicesTable +import dev.typetype.server.db.tables.PushNotificationBaselinesTable +import dev.typetype.server.db.tables.PushNotificationSeenVideosTable +import dev.typetype.server.db.tables.PushNotificationEventsTable +import dev.typetype.server.db.tables.PushNotificationDeliveriesTable +import dev.typetype.server.db.tables.ProfileAccountsTable +import dev.typetype.server.db.tables.RecommendationEventsTable +import dev.typetype.server.db.tables.RecommendationFeedHistoryTable +import dev.typetype.server.db.tables.RecommendationFeedbackTable +import dev.typetype.server.db.tables.RecommendationOnboardingPreferencesTable +import dev.typetype.server.db.tables.RecommendationOnboardingStateTable import dev.typetype.server.db.tables.RssFeedChannelsTable import dev.typetype.server.db.tables.RssFeedServicesTable import dev.typetype.server.db.tables.RssFeedsTable import dev.typetype.server.db.tables.RssUserPoliciesTable +import dev.typetype.server.db.tables.UserChannelInterestTable +import dev.typetype.server.db.tables.UserTopicInterestTable import dev.typetype.server.db.tables.YoutubeTakeoutImportJobsTable import dev.typetype.server.db.tables.YoutubeTakeoutPlaylistKeysTable import dev.typetype.server.db.tables.YoutubeSessionPairingsTable @@ -57,6 +72,7 @@ object DatabaseFactory { transaction { SchemaUtils.create( UsersTable, + ProfileAccountsTable, UserAvatarsTable, SessionsTable, AdminSettingsTable, @@ -84,6 +100,20 @@ object DatabaseFactory { YoutubeSessionPairingsTable, BugReportsTable, NotificationStatesTable, + NotificationReadItemsTable, + ChannelNotificationPreferencesTable, + PushDevicesTable, + PushNotificationBaselinesTable, + PushNotificationSeenVideosTable, + PushNotificationEventsTable, + PushNotificationDeliveriesTable, + UserChannelInterestTable, + UserTopicInterestTable, + RecommendationEventsTable, + RecommendationFeedHistoryTable, + RecommendationFeedbackTable, + RecommendationOnboardingPreferencesTable, + RecommendationOnboardingStateTable, RssFeedsTable, RssFeedChannelsTable, RssFeedServicesTable, @@ -130,6 +160,7 @@ object DatabaseFactory { exec("ALTER TABLE bug_reports ALTER COLUMN github_issue_url TYPE TEXT") DatabaseSessionAuthMigration.apply() DatabaseOidcMigration.apply() + DatabaseProfileAccountsMigration.apply() DatabaseYoutubeRemoteLoginMigration.apply() exec("CREATE UNIQUE INDEX IF NOT EXISTS users_public_username_unique ON users (public_username)") DatabasePrimaryKeyMigrations.apply() diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseProfileAccountsMigration.kt b/src/main/kotlin/dev/typetype/server/db/DatabaseProfileAccountsMigration.kt new file mode 100644 index 00000000..8ec4421b --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/DatabaseProfileAccountsMigration.kt @@ -0,0 +1,41 @@ +package dev.typetype.server.db + +import org.jetbrains.exposed.v1.jdbc.transactions.TransactionManager + +object DatabaseProfileAccountsMigration { + fun apply() { + exec( + """ + INSERT INTO profile_accounts (profile_id, owner_user_id, display_name, is_default, last_used_at, created_at, updated_at) + SELECT id, id, COALESCE(NULLIF(name, ''), 'Profile'), TRUE, 0, created_at, updated_at + FROM users + ON CONFLICT (profile_id) DO NOTHING + """.trimIndent(), + ) + exec("CREATE INDEX IF NOT EXISTS profile_accounts_owner_idx ON profile_accounts (owner_user_id)") + exec( + """ + WITH ranked AS ( + SELECT profile_id, + row_number() OVER ( + PARTITION BY owner_user_id + ORDER BY is_default DESC, last_used_at DESC, created_at ASC, profile_id ASC + ) AS profile_rank + FROM profile_accounts + ) + UPDATE profile_accounts AS profiles + SET is_default = (ranked.profile_rank = 1) + FROM ranked + WHERE profiles.profile_id = ranked.profile_id + """.trimIndent(), + ) + exec( + "CREATE UNIQUE INDEX IF NOT EXISTS profile_accounts_one_default_idx " + + "ON profile_accounts (owner_user_id) WHERE is_default", + ) + } + + private fun exec(sql: String) { + TransactionManager.current().exec(sql) + } +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/ChannelNotificationPreferencesTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/ChannelNotificationPreferencesTable.kt new file mode 100644 index 00000000..7b5b7ef9 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/ChannelNotificationPreferencesTable.kt @@ -0,0 +1,16 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object ChannelNotificationPreferencesTable : Table("channel_notification_preferences") { + val userId = text("user_id") + val channelUrl = text("channel_url") + val enabled = bool("enabled") + val updatedAt = long("updated_at") + + init { + index(false, userId) + } + + override val primaryKey = PrimaryKey(userId, channelUrl) +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/NotificationReadItemsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/NotificationReadItemsTable.kt new file mode 100644 index 00000000..dcc10129 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/NotificationReadItemsTable.kt @@ -0,0 +1,15 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object NotificationReadItemsTable : Table("notification_read_items") { + val userId = text("user_id") + val notificationId = text("notification_id") + val readAt = long("read_at") + + init { + index(false, userId, readAt) + } + + override val primaryKey = PrimaryKey(userId, notificationId) +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/ProfileAccountsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/ProfileAccountsTable.kt new file mode 100644 index 00000000..b62fc6df --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/ProfileAccountsTable.kt @@ -0,0 +1,20 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object ProfileAccountsTable : Table("profile_accounts") { + val profileId = text("profile_id") + val ownerUserId = text("owner_user_id") + val displayName = text("display_name") + val isDefault = bool("is_default") + val lastUsedAt = long("last_used_at") + val createdAt = long("created_at") + val updatedAt = long("updated_at") + + init { + index(false, ownerUserId) + index(false, ownerUserId, isDefault) + } + + override val primaryKey = PrimaryKey(profileId) +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PushDevicesTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/PushDevicesTable.kt new file mode 100644 index 00000000..b72410e3 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/PushDevicesTable.kt @@ -0,0 +1,22 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object PushDevicesTable : Table("push_devices") { + val id = text("id") + val userId = text("user_id") + val deviceId = text("device_id") + val platform = text("platform") + val endpoint = text("endpoint") + val endpointHash = text("endpoint_hash").uniqueIndex() + val expiresAt = long("expires_at").nullable() + val createdAt = long("created_at") + val updatedAt = long("updated_at") + + init { + index(false, userId) + uniqueIndex(userId, deviceId) + } + + override val primaryKey = PrimaryKey(id) +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationBaselinesTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationBaselinesTable.kt new file mode 100644 index 00000000..e3f696a7 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationBaselinesTable.kt @@ -0,0 +1,10 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object PushNotificationBaselinesTable : Table("push_notification_baselines") { + val userId = text("user_id") + val initializedAt = long("initialized_at") + + override val primaryKey = PrimaryKey(userId) +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationDeliveriesTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationDeliveriesTable.kt new file mode 100644 index 00000000..b94f3ac9 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationDeliveriesTable.kt @@ -0,0 +1,20 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object PushNotificationDeliveriesTable : Table("push_notification_deliveries") { + val eventId = text("event_id") + val userId = text("user_id") + val deviceId = text("device_id") + val status = text("status") + val attempts = integer("attempts") + val lastError = text("last_error").nullable() + val updatedAt = long("updated_at") + + init { + index(false, userId, status) + index(false, deviceId) + } + + override val primaryKey = PrimaryKey(eventId, userId, deviceId) +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationEventsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationEventsTable.kt new file mode 100644 index 00000000..7d14c8fc --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationEventsTable.kt @@ -0,0 +1,24 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object PushNotificationEventsTable : Table("push_notification_events") { + val eventId = text("event_id") + val eventType = text("event_type") + val instanceId = text("instance_id") + val serviceId = integer("service_id") + val channelId = text("channel_id") + val videoId = text("video_id") + val videoUrl = text("video_url") + val title = text("title") + val channelName = text("channel_name") + val channelAvatarUrl = text("channel_avatar_url") + val publishedAt = long("published_at") + val createdAt = long("created_at") + + init { + index(false, createdAt) + } + + override val primaryKey = PrimaryKey(eventId) +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationSeenVideosTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationSeenVideosTable.kt new file mode 100644 index 00000000..b4df2b01 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationSeenVideosTable.kt @@ -0,0 +1,17 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object PushNotificationSeenVideosTable : Table("push_notification_seen_videos") { + val userId = text("user_id") + val serviceId = integer("service_id") + val channelId = text("channel_id") + val videoId = text("video_id") + val firstSeenAt = long("first_seen_at") + + init { + index(false, userId) + } + + override val primaryKey = PrimaryKey(userId, serviceId, channelId, videoId) +} diff --git a/src/main/kotlin/dev/typetype/server/models/AccountProfileItem.kt b/src/main/kotlin/dev/typetype/server/models/AccountProfileItem.kt new file mode 100644 index 00000000..d49f82a2 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/AccountProfileItem.kt @@ -0,0 +1,16 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class AccountProfileItem( + val id: String, + val name: String, + val isActive: Boolean, + val isDefault: Boolean, + val lastUsedAt: Long, + val publicUsername: String? = null, + val avatarUrl: String? = null, + val avatarType: String? = null, + val avatarCode: String? = null, +) diff --git a/src/main/kotlin/dev/typetype/server/models/AccountProfilesResponse.kt b/src/main/kotlin/dev/typetype/server/models/AccountProfilesResponse.kt new file mode 100644 index 00000000..d6c82db4 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/AccountProfilesResponse.kt @@ -0,0 +1,10 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class AccountProfilesResponse( + val profiles: List, + val activeProfileId: String, + val defaultProfileId: String, +) diff --git a/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt b/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt index 133514e6..dd9cf4b1 100644 --- a/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt +++ b/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt @@ -26,6 +26,7 @@ data class InstanceResponse( val youtubeRemoteLoginUnavailableReason: String? = null, val parentalControlsEnabled: Boolean = false, val rss: RssInstanceCapability = RssInstanceCapability(), + val pushNotifications: PushNotificationCapability = PushNotificationCapability(), ) @Serializable diff --git a/src/main/kotlin/dev/typetype/server/models/MarkNotificationsReadResponse.kt b/src/main/kotlin/dev/typetype/server/models/MarkNotificationsReadResponse.kt index 93e0f41d..5b9cb89e 100644 --- a/src/main/kotlin/dev/typetype/server/models/MarkNotificationsReadResponse.kt +++ b/src/main/kotlin/dev/typetype/server/models/MarkNotificationsReadResponse.kt @@ -6,4 +6,5 @@ import kotlinx.serialization.Serializable data class MarkNotificationsReadResponse( val readAt: Long, val unreadCount: Int, + val available: Boolean = true, ) diff --git a/src/main/kotlin/dev/typetype/server/models/NotificationItem.kt b/src/main/kotlin/dev/typetype/server/models/NotificationItem.kt index f188f0af..eb7bcde7 100644 --- a/src/main/kotlin/dev/typetype/server/models/NotificationItem.kt +++ b/src/main/kotlin/dev/typetype/server/models/NotificationItem.kt @@ -4,6 +4,7 @@ import kotlinx.serialization.Serializable @Serializable data class NotificationItem( + val id: String = "", val type: String, val title: String, val createdAt: Long, @@ -11,5 +12,8 @@ data class NotificationItem( val channelUrl: String, val channelName: String, val channelAvatarUrl: String, + val serviceId: Int, + val serviceName: String, + val read: Boolean = false, val video: VideoItem, ) diff --git a/src/main/kotlin/dev/typetype/server/models/NotificationsResponse.kt b/src/main/kotlin/dev/typetype/server/models/NotificationsResponse.kt index 229cef5c..9cadd471 100644 --- a/src/main/kotlin/dev/typetype/server/models/NotificationsResponse.kt +++ b/src/main/kotlin/dev/typetype/server/models/NotificationsResponse.kt @@ -7,4 +7,5 @@ data class NotificationsResponse( val items: List, val unreadCount: Int, val nextpage: String?, + val available: Boolean = true, ) diff --git a/src/main/kotlin/dev/typetype/server/models/ProfileNameRequest.kt b/src/main/kotlin/dev/typetype/server/models/ProfileNameRequest.kt new file mode 100644 index 00000000..9a369431 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/ProfileNameRequest.kt @@ -0,0 +1,6 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class ProfileNameRequest(val name: String) diff --git a/src/main/kotlin/dev/typetype/server/models/ProfileSwitchResponse.kt b/src/main/kotlin/dev/typetype/server/models/ProfileSwitchResponse.kt new file mode 100644 index 00000000..27756aeb --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/ProfileSwitchResponse.kt @@ -0,0 +1,9 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class ProfileSwitchResponse( + val accessToken: String, + val profile: AccountProfileItem, +) diff --git a/src/main/kotlin/dev/typetype/server/models/PushNotificationModels.kt b/src/main/kotlin/dev/typetype/server/models/PushNotificationModels.kt new file mode 100644 index 00000000..5a2126ab --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/PushNotificationModels.kt @@ -0,0 +1,59 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class PushNotificationCapability( + val enabled: Boolean = false, + val provider: String = "unifiedpush", + val eventTypes: List = emptyList(), + val maxDevicesPerAccount: Int = 0, +) + +@Serializable +data class PushDeviceRegistrationRequest( + val deviceId: String, + val platform: String = "android", + val endpoint: String, + val expiresAt: Long? = null, +) + +@Serializable +data class PushDeviceRegistrationResponse( + val id: String, + val deviceId: String, + val platform: String, + val expiresAt: Long? = null, + val updatedAt: Long, +) + +@Serializable +data class ChannelNotificationPreferenceRequest( + val channelUrl: String, + val enabled: Boolean, +) + +@Serializable +data class ChannelNotificationPreference( + val channelUrl: String, + val enabled: Boolean, + val updatedAt: Long, +) + +@Serializable +data class UnifiedPushNotificationPayload( + val version: Int = 1, + val eventType: String, + val serviceId: Int, + val serviceName: String, + val eventId: String, + val videoId: String, + val videoUrl: String, + val channelId: String, + val channelName: String, + val channelAvatarUrl: String, + val instanceId: String, + val accountId: String, + val publishedAt: Long, + val title: String, +) diff --git a/src/main/kotlin/dev/typetype/server/models/UnreadCountResponse.kt b/src/main/kotlin/dev/typetype/server/models/UnreadCountResponse.kt index f75b59df..26b77b89 100644 --- a/src/main/kotlin/dev/typetype/server/models/UnreadCountResponse.kt +++ b/src/main/kotlin/dev/typetype/server/models/UnreadCountResponse.kt @@ -5,4 +5,5 @@ import kotlinx.serialization.Serializable @Serializable data class UnreadCountResponse( val unreadCount: Int, + val available: Boolean = true, ) diff --git a/src/main/kotlin/dev/typetype/server/routes/AccountProfilesRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/AccountProfilesRoutes.kt new file mode 100644 index 00000000..d921a688 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/AccountProfilesRoutes.kt @@ -0,0 +1,103 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.models.ProfileNameRequest +import dev.typetype.server.models.ProfileSwitchResponse +import dev.typetype.server.services.AuthCookieHelpers +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.AuthSessionConfig +import dev.typetype.server.services.ProfileAccountService +import dev.typetype.server.services.ProfileMutationResult +import io.ktor.http.HttpStatusCode +import io.ktor.server.request.receive +import io.ktor.server.response.respond +import io.ktor.server.routing.Route +import io.ktor.server.routing.delete +import io.ktor.server.routing.get +import io.ktor.server.routing.post +import io.ktor.server.routing.put + +fun Route.accountProfilesRoutes( + profileService: ProfileAccountService, + authService: AuthService, + sessionConfig: AuthSessionConfig, +) { + get("/profiles") { + call.withJwtAuth(authService) { userId -> + val profiles = profileService.list(userId) + if (profiles == null) call.respond(HttpStatusCode.Forbidden, ErrorResponse("Profiles are unavailable")) + else call.respond(profiles) + } + } + + post("/profiles") { + call.withJwtAuth(authService) { userId -> + val body = call.receiveNameOrNull() ?: return@withJwtAuth + when (val result = profileService.create(userId, body.name)) { + is ProfileMutationResult.Success -> call.respond(HttpStatusCode.Created, result.profile) + ProfileMutationResult.InvalidName -> call.respond(HttpStatusCode.BadRequest, ErrorResponse("PROFILE_NAME_INVALID")) + ProfileMutationResult.NotFound -> call.respond(HttpStatusCode.NotFound, ErrorResponse("Profile not found")) + else -> call.respond(HttpStatusCode.BadRequest, ErrorResponse("Profile operation failed")) + } + } + } + + put("/profiles/{profileId}") { + call.withJwtAuth(authService) { userId -> + val profileId = call.parameters["profileId"] ?: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing profileId")) + val body = call.receiveNameOrNull() ?: return@withJwtAuth + call.respondMutation(profileService.rename(userId, profileId, body.name)) + } + } + + post("/profiles/{profileId}/default") { + call.withJwtAuth(authService) { userId -> + val profileId = call.parameters["profileId"] ?: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing profileId")) + call.respondMutation(profileService.setDefault(userId, profileId)) + } + } + + post("/profiles/{profileId}/switch") { + call.withJwtAuth(authService) { userId -> + val profileId = call.parameters["profileId"] ?: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing profileId")) + val profile = profileService.switch(userId, profileId) + if (profile == null) { + call.respond(HttpStatusCode.NotFound, ErrorResponse("Profile not found")) + return@withJwtAuth + } + val token = authService.issueSession(profile.id) + if (token == null) { + call.respond(HttpStatusCode.InternalServerError, ErrorResponse("Failed to create session")) + return@withJwtAuth + } + AuthCookieHelpers.setRefreshCookie(call.response, token.refreshToken, sessionConfig) + call.respond(ProfileSwitchResponse(token.accessToken, profile)) + } + } + + delete("/profiles/{profileId}") { + call.withJwtAuth(authService) { userId -> + val profileId = call.parameters["profileId"] ?: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing profileId")) + when (profileService.delete(userId, profileId)) { + ProfileMutationResult.Deleted -> call.respond(HttpStatusCode.NoContent) + ProfileMutationResult.CannotDeleteOwner -> call.respond(HttpStatusCode.Conflict, ErrorResponse("The account profile cannot be deleted")) + ProfileMutationResult.CannotDeleteActive -> call.respond(HttpStatusCode.Conflict, ErrorResponse("Switch profiles before deleting the active profile")) + ProfileMutationResult.NotFound -> call.respond(HttpStatusCode.NotFound, ErrorResponse("Profile not found")) + else -> call.respond(HttpStatusCode.BadRequest, ErrorResponse("Profile deletion failed")) + } + } + } +} + +private suspend fun io.ktor.server.application.ApplicationCall.receiveNameOrNull(): ProfileNameRequest? = + runCatching { receive() }.getOrElse { + respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) + null + } + +private suspend fun io.ktor.server.application.ApplicationCall.respondMutation(result: ProfileMutationResult): Unit = when (result) { + is ProfileMutationResult.Success -> respond(result.profile) + ProfileMutationResult.InvalidName -> respond(HttpStatusCode.BadRequest, ErrorResponse("PROFILE_NAME_INVALID")) + ProfileMutationResult.NotFound -> respond(HttpStatusCode.NotFound, ErrorResponse("Profile not found")) + else -> respond(HttpStatusCode.BadRequest, ErrorResponse("Profile operation failed")) +} diff --git a/src/main/kotlin/dev/typetype/server/routes/NicoVideoProxyRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/NicoVideoProxyRoutes.kt index 1ec8359c..8aa904d2 100644 --- a/src/main/kotlin/dev/typetype/server/routes/NicoVideoProxyRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/NicoVideoProxyRoutes.kt @@ -1,16 +1,11 @@ package dev.typetype.server.routes import dev.typetype.server.models.ErrorResponse -import dev.typetype.server.models.ExtractionResult import dev.typetype.server.services.NicoVideoProxyService -import io.ktor.http.ContentType import io.ktor.http.HttpStatusCode import io.ktor.server.response.respond -import io.ktor.server.response.respondOutputStream import io.ktor.server.routing.Route import io.ktor.server.routing.get -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext fun Route.nicoVideoProxyRoutes(nicoVideoProxyService: NicoVideoProxyService) { get("/proxy/nicovideo") { @@ -27,25 +22,6 @@ fun Route.nicoVideoProxyRoutes(nicoVideoProxyService: NicoVideoProxyService) { nicoVideoProxyService.fetchSegment(url, rangeHeader, domandBid) } - when (result) { - is ExtractionResult.Success -> { - val proxy = result.data - try { - val status = HttpStatusCode.fromValue(proxy.status) - val contentType = ContentType.parse(proxy.contentType) - proxy.contentRange?.let { call.response.headers.append("Content-Range", it) } - proxy.acceptRanges?.let { call.response.headers.append("Accept-Ranges", it) } - call.respondOutputStream(contentType, status, proxy.contentLength) { - withContext(Dispatchers.IO) { proxy.stream.copyTo(this@respondOutputStream) } - } - } finally { - proxy.close() - } - } - is ExtractionResult.BadRequest -> - call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) - is ExtractionResult.Failure -> - call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) - } + call.respondProxyResult(result) } } diff --git a/src/main/kotlin/dev/typetype/server/routes/NotificationsRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/NotificationsRoutes.kt index 9c408ac1..50fe1477 100644 --- a/src/main/kotlin/dev/typetype/server/routes/NotificationsRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/NotificationsRoutes.kt @@ -29,4 +29,15 @@ fun Route.notificationsRoutes(notificationsService: NotificationsService, authSe call.respond(notificationsService.markAllRead(userId)) } } + + post("/notifications/{notificationId}/read") { + call.withJwtAuth(authService) { userId -> + val notificationId = call.parameters["notificationId"] + if (notificationId.isNullOrBlank()) { + call.respond(io.ktor.http.HttpStatusCode.BadRequest, dev.typetype.server.models.ErrorResponse("Missing notificationId")) + return@withJwtAuth + } + call.respond(notificationsService.markRead(userId, notificationId)) + } + } } diff --git a/src/main/kotlin/dev/typetype/server/routes/ProviderMediaHandleRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/ProviderMediaHandleRoutes.kt new file mode 100644 index 00000000..9e8f6fa1 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/ProviderMediaHandleRoutes.kt @@ -0,0 +1,37 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.services.ProviderMediaHandleService +import dev.typetype.server.services.ProviderMediaAwareProxyService +import dev.typetype.server.services.ProxyService +import io.ktor.http.HttpStatusCode +import io.ktor.server.response.respond +import io.ktor.server.routing.Route +import io.ktor.server.routing.get + +internal fun Route.providerMediaHandleRoutes( + handleService: ProviderMediaHandleService, + proxyService: ProxyService, +) { + get("/media/{handle}") { + val handle = call.parameters["handle"] + ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing media handle")) + val target = runCatching { handleService.resolve(handle) }.getOrElse { + return@get call.respond( + HttpStatusCode.ServiceUnavailable, + ErrorResponse("Media handle service is unavailable", "media_handle_unavailable"), + ) + } ?: return@get call.respond( + HttpStatusCode.NotFound, + ErrorResponse("Media handle has expired or is unknown", "media_handle_not_found"), + ) + + val rangeHeader = call.request.headers["Range"] + val result = if (proxyService is ProviderMediaAwareProxyService) { + proxyService.pipeProviderMedia(target.url, rangeHeader, target.domandBid) + } else { + proxyService.pipe(target.url, rangeHeader, target.domandBid) + } + call.respondProxyResult(result) + } +} diff --git a/src/main/kotlin/dev/typetype/server/routes/PushNotificationRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/PushNotificationRoutes.kt new file mode 100644 index 00000000..abd8ac9d --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/PushNotificationRoutes.kt @@ -0,0 +1,110 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ChannelNotificationPreferenceRequest +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.models.PushDeviceRegistrationRequest +import dev.typetype.server.services.DeviceRegistrationResult +import dev.typetype.server.services.PreferenceUpdateResult +import dev.typetype.server.services.PushNotificationService +import dev.typetype.server.services.AuthService +import io.ktor.http.HttpStatusCode +import io.ktor.server.request.receive +import io.ktor.server.response.respond +import io.ktor.server.routing.Route +import io.ktor.server.routing.delete +import io.ktor.server.routing.get +import io.ktor.server.routing.post +import io.ktor.server.routing.put + +internal fun Route.pushNotificationRoutes( + service: PushNotificationService, + authService: AuthService, +) { + get("/notifications/channel-preferences") { + call.withJwtAuth(authService) { userId -> + if (!call.requirePushEnabled(service)) return@withJwtAuth + call.respond(service.listPreferences(userId)) + } + } + put("/notifications/channel-preferences") { + call.withJwtAuth(authService) { userId -> + if (!call.requirePushEnabled(service)) return@withJwtAuth + val request = runCatching { call.receive() }.getOrNull() + ?: return@withJwtAuth call.respond( + HttpStatusCode.BadRequest, + ErrorResponse("Invalid channel notification preference", "push_request_invalid"), + ) + when (val result = service.setPreference(userId, request.channelUrl, request.enabled)) { + is PreferenceUpdateResult.Updated -> call.respond(result.preference) + PreferenceUpdateResult.NotSubscribed -> call.respond( + HttpStatusCode.Conflict, + ErrorResponse("Channel is not subscribed", "channel_not_subscribed"), + ) + } + } + } + get("/notifications/push/devices") { + call.withJwtAuth(authService) { userId -> + if (!call.requirePushEnabled(service)) return@withJwtAuth + call.respond(service.listDevices(userId)) + } + } + post("/notifications/push/devices") { + call.withJwtAuth(authService) { userId -> + if (!call.requirePushEnabled(service)) return@withJwtAuth + val request = runCatching { call.receive() }.getOrNull() + ?: return@withJwtAuth call.respond( + HttpStatusCode.BadRequest, + ErrorResponse("Invalid push device registration", "push_request_invalid"), + ) + when (val result = service.registerDevice(userId, request)) { + is DeviceRegistrationResult.Success -> call.respond(HttpStatusCode.Created, result.response) + is DeviceRegistrationResult.Invalid -> call.respond( + HttpStatusCode.UnprocessableEntity, + ErrorResponse("Invalid push device registration: ${result.reason}", result.errorCode()), + ) + DeviceRegistrationResult.UnsupportedPlatform -> call.respond( + HttpStatusCode.UnprocessableEntity, + ErrorResponse("Push platform is not supported", "push_platform_unsupported"), + ) + DeviceRegistrationResult.EndpointConflict -> call.respond( + HttpStatusCode.Conflict, + ErrorResponse("Push endpoint is already registered", "push_endpoint_conflict"), + ) + DeviceRegistrationResult.LimitReached -> call.respond( + HttpStatusCode.Conflict, + ErrorResponse("Push device limit reached", "push_device_limit_reached"), + ) + } + } + } + delete("/notifications/push/devices/{deviceId}") { + call.withJwtAuth(authService) { userId -> + if (!call.requirePushEnabled(service)) return@withJwtAuth + val deviceId = call.parameters["deviceId"]?.takeIf { it.isNotBlank() } + ?: return@withJwtAuth call.respond( + HttpStatusCode.BadRequest, + ErrorResponse("Missing deviceId", "push_request_invalid"), + ) + if (service.unregisterDevice(userId, deviceId)) { + call.respond(HttpStatusCode.NoContent) + } else { + call.respond(HttpStatusCode.NotFound, ErrorResponse("Push device not found", "push_device_not_found")) + } + } + } +} + +private fun DeviceRegistrationResult.Invalid.errorCode(): String = when (reason) { + "device_id" -> "push_device_id_invalid" + "expires_at" -> "push_expiry_invalid" + else -> "push_endpoint_invalid" +} + +private suspend fun io.ktor.server.application.ApplicationCall.requirePushEnabled( + service: PushNotificationService, +): Boolean { + if (service.capability.enabled) return true + respond(HttpStatusCode.NotFound, ErrorResponse("Push notifications are unavailable", "push_notifications_unavailable")) + return false +} diff --git a/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt b/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt index 177533c7..24addc64 100644 --- a/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt +++ b/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt @@ -7,6 +7,7 @@ import dev.typetype.server.services.AdminSettingsService import dev.typetype.server.services.AuthService import dev.typetype.server.services.BlockedService import dev.typetype.server.services.PublicHlsManifestTokenService +import dev.typetype.server.services.ProviderMediaHandleService internal data class StreamRouteDependencies( val authService: AuthService?, @@ -14,6 +15,7 @@ internal data class StreamRouteDependencies( val adminSettingsService: AdminSettingsService?, val blockedService: BlockedService?, val publicHlsManifestTokenService: PublicHlsManifestTokenService?, + val providerMediaHandleService: ProviderMediaHandleService?, val sabrStreamContractFilter: (suspend (String, StreamResponse) -> StreamResponse)?, val youtubeSessionSabrStreamInfo: (suspend (String, String) -> ExtractionResult?)?, ) diff --git a/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt index 9f315c08..cc1070f3 100644 --- a/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt @@ -9,6 +9,8 @@ import dev.typetype.server.services.AuthService import dev.typetype.server.services.BlockedContentProfile import dev.typetype.server.services.BlockedService import dev.typetype.server.services.PublicHlsManifestTokenService +import dev.typetype.server.services.ProviderMediaHandleService +import dev.typetype.server.services.ProviderMediaType import dev.typetype.server.services.StreamService import dev.typetype.server.services.YOUTUBE_SESSION_REQUIRED_CODE import dev.typetype.server.services.YOUTUBE_SESSION_REQUIRED_ERROR @@ -21,9 +23,11 @@ import io.ktor.http.HttpStatusCode import io.ktor.server.response.respond import io.ktor.server.routing.Route import io.ktor.server.routing.get +import kotlinx.coroutines.CancellationException private const val STREAMS_CACHE_CONTROL = "public, max-age=21600, stale-while-revalidate=3600" private const val AUTHENTICATED_STREAMS_CACHE_CONTROL = "no-store" +private const val PROVIDER_STREAMS_CACHE_CONTROL = "no-store" fun Route.streamRoutes( streamService: StreamService, @@ -32,6 +36,7 @@ fun Route.streamRoutes( adminSettingsService: AdminSettingsService? = null, blockedService: BlockedService? = null, publicHlsManifestTokenService: PublicHlsManifestTokenService? = null, + providerMediaHandleService: ProviderMediaHandleService? = null, nicoNicoStreamService: StreamService = streamService, bilibiliStreamService: StreamService = streamService, sabrBootstrapStreamService: StreamService = streamService, @@ -44,6 +49,7 @@ fun Route.streamRoutes( adminSettingsService = adminSettingsService, blockedService = blockedService, publicHlsManifestTokenService = publicHlsManifestTokenService, + providerMediaHandleService = providerMediaHandleService, sabrStreamContractFilter = sabrStreamContractFilter, youtubeSessionSabrStreamInfo = youtubeSessionSabrStreamInfo, ) @@ -128,11 +134,31 @@ private fun Route.streamRoute( ErrorResponse("No playable streams available", "no_playable_streams"), ) } + val publicData = try { + dependencies.providerMediaHandleService?.let { service -> + providerMediaType(deliveryMode)?.let { service.materialize(data, it) } + } ?: data + } catch (error: CancellationException) { + throw error + } catch (error: Exception) { + return@get call.respond( + HttpStatusCode.BadGateway, + ErrorResponse( + error.message ?: "Provider media handle service failed", + "media_handle_unavailable", + ), + ) + } call.response.headers.append( HttpHeaders.CacheControl, - if (access.userId != null) AUTHENTICATED_STREAMS_CACHE_CONTROL else STREAMS_CACHE_CONTROL, + when { + access.userId != null -> AUTHENTICATED_STREAMS_CACHE_CONTROL + deliveryMode == StreamDeliveryMode.NicoNico || + deliveryMode == StreamDeliveryMode.BiliBili -> PROVIDER_STREAMS_CACHE_CONTROL + else -> STREAMS_CACHE_CONTROL + }, ) - call.respond(data) + call.respond(publicData) } is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message, result.code)) @@ -142,6 +168,12 @@ private fun Route.streamRoute( } } +private fun providerMediaType(deliveryMode: StreamDeliveryMode): ProviderMediaType? = when (deliveryMode) { + StreamDeliveryMode.NicoNico -> ProviderMediaType.NICONICO + StreamDeliveryMode.BiliBili -> ProviderMediaType.BILIBILI + StreamDeliveryMode.YoutubeSabr -> null +} + private data class StreamResolution( val result: ExtractionResult, val authenticated: Boolean = false, diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt index 33aac871..1903cc99 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt @@ -9,6 +9,7 @@ import dev.typetype.server.services.NoopHomeRecommendationWarmup import dev.typetype.server.services.SubscriptionsService import dev.typetype.server.services.SubscriptionGroupsService import dev.typetype.server.services.SubscriptionSelection +import dev.typetype.server.services.PushNotificationService import io.ktor.http.HttpStatusCode import io.ktor.server.application.ApplicationCall import io.ktor.server.request.receive @@ -21,11 +22,12 @@ import io.ktor.server.routing.post import java.net.URLDecoder import java.nio.charset.StandardCharsets -fun Route.subscriptionsRoutes( +internal fun Route.subscriptionsRoutes( subscriptionsService: SubscriptionsService, authService: AuthService, warmupService: HomeRecommendationWarmup = NoopHomeRecommendationWarmup, groupsService: SubscriptionGroupsService = SubscriptionGroupsService(), + pushNotificationService: PushNotificationService? = null, ) { get("/subscriptions/group-memberships") { call.withJwtAuth(authService) { userId -> @@ -65,14 +67,14 @@ fun Route.subscriptionsRoutes( call.withJwtAuth(authService) { userId -> val channelUrl = call.request.queryParameters["url"]?.takeIf { it.isNotBlank() } ?: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing channelUrl")) - call.respondDeleteResult(subscriptionsService, warmupService, userId, channelUrl) + call.respondDeleteResult(subscriptionsService, warmupService, pushNotificationService, userId, channelUrl) } } delete("/subscriptions/{channelUrl...}") { call.withJwtAuth(authService) { userId -> val channelUrl = call.extractDeleteChannelUrl() ?: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing channelUrl")) - call.respondDeleteResult(subscriptionsService, warmupService, userId, channelUrl) + call.respondDeleteResult(subscriptionsService, warmupService, pushNotificationService, userId, channelUrl) } } } @@ -80,11 +82,15 @@ fun Route.subscriptionsRoutes( private suspend fun ApplicationCall.respondDeleteResult( subscriptionsService: SubscriptionsService, warmupService: HomeRecommendationWarmup, + pushNotificationService: PushNotificationService?, userId: String, channelUrl: String, ) { val deleted = subscriptionsService.delete(userId, channelUrl) - if (deleted) warmupService.invalidateAndWarm(userId) + if (deleted) { + warmupService.invalidateAndWarm(userId) + pushNotificationService?.onSubscriptionRemoved(userId, channelUrl) + } if (deleted) respond(HttpStatusCode.NoContent) else respond(HttpStatusCode.NotFound, ErrorResponse("Not found")) } diff --git a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt index 9da6de79..e5c98696 100644 --- a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt @@ -25,6 +25,7 @@ internal fun Route.userDataRoutes( authService, svc.homeRecommendationWarmupService, svc.subscriptionGroupsService, + svc.pushNotificationService, ) subscriptionFeedRoutes( svc.subscriptionFeedService, @@ -44,6 +45,7 @@ internal fun Route.userDataRoutes( allowedChannelsRoutes(svc.allowedChannelsService, authService) blockedRoutes(svc.blockedService, authService) notificationsRoutes(svc.notificationsService, authService) + pushNotificationRoutes(svc.pushNotificationService, authService) youtubeSessionRoutes(svc.youtubeSessionService, authService) youtubeTakeoutImportRoutes(svc.youtubeTakeoutImportService, authService) profileRoutes(profileService, avatarService, svc.customAvatarService, authService) diff --git a/src/main/kotlin/dev/typetype/server/services/AccountIdentityService.kt b/src/main/kotlin/dev/typetype/server/services/AccountIdentityService.kt index 98ce5bc8..4c6b6e2a 100644 --- a/src/main/kotlin/dev/typetype/server/services/AccountIdentityService.kt +++ b/src/main/kotlin/dev/typetype/server/services/AccountIdentityService.kt @@ -11,14 +11,17 @@ import org.jetbrains.exposed.v1.core.neq import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.update -class AccountIdentityService { - suspend fun get(userId: String): AccountIdentityItem? = DatabaseFactory.query { - UsersTable.selectAll().where { UsersTable.id eq userId }.singleOrNull()?.let { +class AccountIdentityService(private val profileAccountService: ProfileAccountService? = null) { + suspend fun get(userId: String): AccountIdentityItem? { + val identityUserId = profileAccountService?.ownerUserId(userId) ?: userId + return DatabaseFactory.query { + UsersTable.selectAll().where { UsersTable.id eq identityUserId }.singleOrNull()?.let { AccountIdentityItem( email = it[UsersTable.email], name = it[UsersTable.name], managedByOidc = it[UsersTable.oidcIssuer] != null, ) + } } } @@ -29,8 +32,9 @@ class AccountIdentityService { currentPassword: String, ): AccountIdentityUpdateResult { val normalized = validate(email, name) ?: return AccountIdentityUpdateResult.InvalidInput + val identityUserId = profileAccountService?.ownerUserId(userId) ?: userId val credentials = DatabaseFactory.query { - UsersTable.selectAll().where { UsersTable.id eq userId }.singleOrNull()?.let { + UsersTable.selectAll().where { UsersTable.id eq identityUserId }.singleOrNull()?.let { Credentials(it[UsersTable.passwordHash], it[UsersTable.oidcIssuer] != null) } } ?: return AccountIdentityUpdateResult.UserNotFound @@ -38,7 +42,7 @@ class AccountIdentityService { if (!Password.check(currentPassword, credentials.passwordHash).withArgon2()) { return AccountIdentityUpdateResult.InvalidPassword } - return update(userId, normalized) + return update(identityUserId, normalized) } suspend fun updateAdmin(userId: String, email: String, name: String): AccountIdentityUpdateResult { diff --git a/src/main/kotlin/dev/typetype/server/services/AuthService.kt b/src/main/kotlin/dev/typetype/server/services/AuthService.kt index 54a81a8e..9c56e7d0 100644 --- a/src/main/kotlin/dev/typetype/server/services/AuthService.kt +++ b/src/main/kotlin/dev/typetype/server/services/AuthService.kt @@ -19,6 +19,7 @@ open class AuthService( private val jwtSecret: String, private val hasUsersProbe: (() -> Boolean)? = null, sessionConfig: AuthSessionConfig = AuthSessionConfig(), + private val profileAccountService: ProfileAccountService? = null, ) { private val accessCodec = AuthAccessTokenCodec(jwtSecret) private val sessionStore = AuthSessionStore() @@ -48,9 +49,8 @@ open class AuthService( it[UsersTable.updatedAt] = now } } - return DatabaseFactory.blocking { - tokenIssuer.issue(userId) ?: throw IllegalStateException("Failed to create session") - } + val sessionUserId = profileAccountService?.resolveSignInProfile(userId) ?: userId + return DatabaseFactory.blocking { tokenIssuer.issue(sessionUserId) ?: throw IllegalStateException("Failed to create session") } } suspend fun login(identifier: String, password: String): AuthSessionTokens? { @@ -67,11 +67,16 @@ open class AuthService( query.singleOrNull() } ?: return null + if (profileAccountService != null && !profileAccountService.isOwnerProfile(user[UsersTable.id])) { + return null + } + val hashed = user[UsersTable.passwordHash] val verified = withContext(passwordDispatcher) { Password.check(password, hashed).withArgon2() } if (!verified) return null - return DatabaseFactory.blocking { tokenIssuer.issue(user[UsersTable.id]) } + val sessionUserId = profileAccountService?.resolveSignInProfile(user[UsersTable.id]) ?: user[UsersTable.id] + return DatabaseFactory.blocking { tokenIssuer.issue(sessionUserId) } } suspend fun refreshSession(refreshToken: String): AuthSessionTokens? = DatabaseFactory.blocking { @@ -82,6 +87,11 @@ open class AuthService( tokenIssuer.issue(userId) } + suspend fun issueSessionForOwner(ownerUserId: String): AuthSessionTokens? { + val sessionUserId = profileAccountService?.resolveSignInProfile(ownerUserId) ?: ownerUserId + return issueSession(sessionUserId) + } + suspend fun logout(refreshToken: String?): Unit = DatabaseFactory.blocking { sessionRevoker.revokeByRefreshToken(refreshToken) } @@ -102,8 +112,9 @@ open class AuthService( suspend fun getUserRole(userId: String): String? { if (userId.startsWith("guest:")) return "user" + val roleUserId = profileAccountService?.ownerUserId(userId) ?: userId return DatabaseFactory.query { - UsersTable.selectAll().where { UsersTable.id eq userId }.singleOrNull() + UsersTable.selectAll().where { UsersTable.id eq roleUserId }.singleOrNull() }?.get(UsersTable.role) } diff --git a/src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt b/src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt index 3268ea8e..0f04f09a 100644 --- a/src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt +++ b/src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt @@ -4,7 +4,6 @@ import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.ProxyResponse import okhttp3.Request import okhttp3.Response -import java.io.ByteArrayInputStream import java.io.IOException private const val BILIBILI_RANGE_ATTEMPTS = 3 @@ -12,40 +11,33 @@ private const val BILIBILI_RANGE_ATTEMPTS = 3 internal fun readBilibiliRangeWithRetry( execute: (Request) -> Response, request: Request, + checkActive: () -> Unit = {}, ): ExtractionResult { var lastMessage = "Proxy fetch failed" for (attempt in 1..BILIBILI_RANGE_ATTEMPTS) { - runCatching { execute(request) } - .onSuccess { response -> - response.use { - val result = it.readBilibiliRangeBytes() - when (result) { - is ExtractionResult.Success -> return result - is ExtractionResult.BadRequest -> return result - is ExtractionResult.Failure -> lastMessage = result.message - } - } - } - .onFailure { lastMessage = it.message ?: "Proxy fetch failed" } - if (attempt == BILIBILI_RANGE_ATTEMPTS) break + checkActive() + val response = try { + execute(request) + } catch (error: IOException) { + lastMessage = error.message ?: "Proxy fetch failed" + continue + } + if (!response.isSuccessful) { + lastMessage = "Upstream returned ${response.code}" + response.close() + continue + } + val stream = RetryingProxyInputStream(execute, request, response, BILIBILI_RANGE_ATTEMPTS - attempt, checkActive) + return ExtractionResult.Success(ProxyResponse( + status = response.code, + contentType = response.header("Content-Type") ?: "application/octet-stream", + contentLength = response.body.contentLength().takeIf { it >= 0 }, + contentRange = response.header("Content-Range"), + acceptRanges = response.header("Accept-Ranges"), + cacheControl = response.header("Cache-Control"), + stream = stream, + close = stream::close, + )) } return ExtractionResult.Failure(lastMessage) } - -private fun Response.readBilibiliRangeBytes(): ExtractionResult { - if (!isSuccessful && code != 206) return ExtractionResult.Failure("Upstream returned $code") - val bytes = try { - body.bytes() - } catch (e: IOException) { - return ExtractionResult.Failure(e.message ?: "Proxy fetch failed") - } - return ExtractionResult.Success(ProxyResponse( - status = code, - contentType = header("Content-Type") ?: "application/octet-stream", - contentLength = bytes.size.toLong(), - contentRange = header("Content-Range"), - acceptRanges = header("Accept-Ranges"), - stream = ByteArrayInputStream(bytes), - close = {}, - )) -} diff --git a/src/main/kotlin/dev/typetype/server/services/BilibiliRelatedService.kt b/src/main/kotlin/dev/typetype/server/services/BilibiliRelatedService.kt index 6dba7a9a..38a55888 100644 --- a/src/main/kotlin/dev/typetype/server/services/BilibiliRelatedService.kt +++ b/src/main/kotlin/dev/typetype/server/services/BilibiliRelatedService.kt @@ -10,26 +10,48 @@ import kotlinx.serialization.json.jsonPrimitive import kotlinx.serialization.json.longOrNull import org.schabi.newpipe.extractor.NewPipe import org.schabi.newpipe.extractor.services.bilibili.BilibiliService +import java.time.Duration private const val RELATED_BASE_URL = "https://api.bilibili.com/x/web-interface/archive/related?bvid=" private const val SPACE_BASE_URL = "https://space.bilibili.com/" private val BVID_REGEX = Regex("""/(BV[0-9A-Za-z]+)""") -internal class BilibiliRelatedService { +internal class BilibiliRelatedService( + private val relatedLookupOverride: (suspend (String) -> Map)? = null, +) { + private val uploaderUrlCache = BoundedExpiringCache>( + maxEntries = 256, + ttl = Duration.ofMinutes(5), + weigher = { it.size.toLong().coerceAtLeast(1L) }, + ) suspend fun patchRelatedStreams(response: StreamResponse, videoUrl: String): StreamResponse { - val uploaderUrls = fetchUploaderUrls(videoUrl) + val sourceBvid = BVID_REGEX.find(videoUrl)?.groupValues?.get(1) ?: return response + val missingRelatedBvids = response.relatedStreams.asSequence() + .filter { it.uploaderUrl.isBlank() } + .mapNotNull { BVID_REGEX.find(it.url)?.groupValues?.get(1) } + .toSet() + if (missingRelatedBvids.isEmpty()) return response + val uploaderUrls = uploaderUrlCache.get(sourceBvid) ?: resolveUploaderUrls(videoUrl).also { fetched -> + if (fetched.isNotEmpty()) uploaderUrlCache.put(sourceBvid, fetched) + } + if (uploaderUrls.isEmpty()) return response return response.copy( relatedStreams = response.relatedStreams.map { item -> if (item.uploaderUrl.isNotBlank()) item else { - val bvid = BVID_REGEX.find(item.url)?.groupValues?.get(1) ?: "" - uploaderUrls[bvid]?.let { item.copy(uploaderUrl = it) } ?: item + val bvid = BVID_REGEX.find(item.url)?.groupValues?.get(1) + if (bvid in missingRelatedBvids) { + uploaderUrls[bvid]?.let { item.copy(uploaderUrl = it) } ?: item + } else item } } ) } + private suspend fun resolveUploaderUrls(videoUrl: String): Map = + relatedLookupOverride?.invoke(videoUrl) ?: fetchUploaderUrls(videoUrl) + private suspend fun fetchUploaderUrls(videoUrl: String): Map = withContext(Dispatchers.IO) { runCatching { diff --git a/src/main/kotlin/dev/typetype/server/services/BoundedExpiringCache.kt b/src/main/kotlin/dev/typetype/server/services/BoundedExpiringCache.kt index cba0a255..7a35ef16 100644 --- a/src/main/kotlin/dev/typetype/server/services/BoundedExpiringCache.kt +++ b/src/main/kotlin/dev/typetype/server/services/BoundedExpiringCache.kt @@ -13,6 +13,8 @@ internal class BoundedExpiringCache( private val ttlMs = ttl.toMillis() private val entries = LinkedHashMap>(maxEntries.coerceAtMost(64), 0.75f, true) private var weight = 0L + // Removals may leave an earlier bound, causing one extra scan but never delaying expiry. + private var nextExpiryMs = Long.MAX_VALUE init { require(maxEntries > 0) { "maxEntries must be positive" } @@ -33,7 +35,9 @@ internal class BoundedExpiringCache( removeEntry(key) val entryWeight = weigher(value).coerceAtLeast(0L) if (entryWeight > maxWeight) return - entries[key] = Entry(value, expiresAt(now), entryWeight) + val expiry = expiresAt(now) + entries[key] = Entry(value, expiry, entryWeight) + nextExpiryMs = minOf(nextExpiryMs, expiry) weight += entryWeight trim() } @@ -61,6 +65,7 @@ internal class BoundedExpiringCache( fun clear() { entries.clear() weight = 0L + nextExpiryMs = Long.MAX_VALUE } @Synchronized @@ -73,10 +78,15 @@ internal class BoundedExpiringCache( if (Long.MAX_VALUE - now < ttlMs) Long.MAX_VALUE else now + ttlMs private fun evictExpired(now: Long) { + if (now < nextExpiryMs) return + nextExpiryMs = Long.MAX_VALUE val iterator = entries.iterator() while (iterator.hasNext()) { val entry = iterator.next().value - if (entry.expiresAtMs > now) continue + if (entry.expiresAtMs > now) { + nextExpiryMs = minOf(nextExpiryMs, entry.expiresAtMs) + continue + } weight -= entry.weight iterator.remove() } diff --git a/src/main/kotlin/dev/typetype/server/services/CachedManifestService.kt b/src/main/kotlin/dev/typetype/server/services/CachedManifestService.kt index c4afa3db..5a37951f 100644 --- a/src/main/kotlin/dev/typetype/server/services/CachedManifestService.kt +++ b/src/main/kotlin/dev/typetype/server/services/CachedManifestService.kt @@ -2,6 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.cache.CacheService import dev.typetype.server.models.ExtractionResult +import java.net.URI class CachedManifestService( private val delegate: ManifestService, @@ -9,6 +10,7 @@ class CachedManifestService( ) { suspend fun dashManifest(videoUrl: String): ExtractionResult { + if (!isCacheable(videoUrl)) return delegate.dashManifest(videoUrl) val key = "dash-manifest-v2:${CachedStreamService.cacheKey(videoUrl)}" runCatching { cache.get(key) }.getOrNull()?.let { cached -> return ExtractionResult.Success(cached) @@ -22,5 +24,12 @@ class CachedManifestService( private companion object { const val DASH_MANIFEST_TTL_SECONDS = 21600L + + fun isCacheable(videoUrl: String): Boolean { + val host = runCatching { URI(videoUrl).host.orEmpty().lowercase().trimEnd('.') }.getOrDefault("") + return host != "b23.tv" && host != "nico.ms" && + !host.endsWith(".bilibili.com") && host != "bilibili.com" && + !host.endsWith(".nicovideo.jp") && host != "nicovideo.jp" + } } } diff --git a/src/main/kotlin/dev/typetype/server/services/ChannelNotificationPreferenceService.kt b/src/main/kotlin/dev/typetype/server/services/ChannelNotificationPreferenceService.kt new file mode 100644 index 00000000..10968e4e --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/ChannelNotificationPreferenceService.kt @@ -0,0 +1,82 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.ChannelNotificationPreferencesTable +import dev.typetype.server.models.ChannelNotificationPreference +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insertIgnore +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.update + +internal class ChannelNotificationPreferenceService( + private val subscriptionsService: SubscriptionsService, +) { + suspend fun list(userId: String): List { + val subscriptions = subscriptionsService.getAll(userId) + val urls = subscriptions.map { ChannelUrlCanonicalizer.canonicalize(it.channelUrl) } + if (urls.isEmpty()) return emptyList() + val stored = DatabaseFactory.query { + ChannelNotificationPreferencesTable.selectAll().where { + ChannelNotificationPreferencesTable.userId eq userId + }.associateBy { it[ChannelNotificationPreferencesTable.channelUrl] } + } + return urls.map { channelUrl -> + val row = stored[channelUrl] + ChannelNotificationPreference( + channelUrl, + row?.get(ChannelNotificationPreferencesTable.enabled) ?: true, + row?.get(ChannelNotificationPreferencesTable.updatedAt) ?: 0L, + ) + } + } + + suspend fun set(userId: String, rawChannelUrl: String, enabled: Boolean): PreferenceUpdateResult { + val channelUrl = ChannelUrlCanonicalizer.canonicalize(rawChannelUrl) + if (!isSubscribed(userId, channelUrl)) return PreferenceUpdateResult.NotSubscribed + val now = System.currentTimeMillis() + DatabaseFactory.query { + val inserted = ChannelNotificationPreferencesTable.insertIgnore { + it[ChannelNotificationPreferencesTable.userId] = userId + it[ChannelNotificationPreferencesTable.channelUrl] = channelUrl + it[ChannelNotificationPreferencesTable.enabled] = enabled + it[ChannelNotificationPreferencesTable.updatedAt] = now + }.insertedCount + if (inserted == 0) { + ChannelNotificationPreferencesTable.update({ + (ChannelNotificationPreferencesTable.userId eq userId) and + (ChannelNotificationPreferencesTable.channelUrl eq channelUrl) + }) { + it[ChannelNotificationPreferencesTable.enabled] = enabled + it[ChannelNotificationPreferencesTable.updatedAt] = now + } + } + } + return PreferenceUpdateResult.Updated(ChannelNotificationPreference(channelUrl, enabled, now)) + } + + suspend fun remove(userId: String, rawChannelUrl: String) { + val channelUrl = ChannelUrlCanonicalizer.canonicalize(rawChannelUrl) + DatabaseFactory.query { + ChannelNotificationPreferencesTable.deleteWhere { + (ChannelNotificationPreferencesTable.userId eq userId) and + (ChannelNotificationPreferencesTable.channelUrl eq channelUrl) + } + } + } + + suspend fun enabledChannelUrls(userId: String): Set = list(userId) + .filter(ChannelNotificationPreference::enabled) + .mapTo(mutableSetOf(), ChannelNotificationPreference::channelUrl) + + private suspend fun isSubscribed(userId: String, channelUrl: String): Boolean = + subscriptionsService.getAll(userId).any { + ChannelUrlCanonicalizer.canonicalize(it.channelUrl) == channelUrl + } +} + +internal sealed interface PreferenceUpdateResult { + data class Updated(val preference: ChannelNotificationPreference) : PreferenceUpdateResult + data object NotSubscribed : PreferenceUpdateResult +} diff --git a/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt b/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt index cf12d6a9..5a07b108 100644 --- a/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt +++ b/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt @@ -17,6 +17,7 @@ class HlsManifestService( cache: CacheService? = null, private val signManifestUrl: ((String) -> String)? = null, private val attestedYoutubeHls: suspend (String) -> String? = { null }, + private val providerMediaHandleService: ProviderMediaHandleService? = null, ) { private val proxyHttp = ProxyHttpExecutor(httpClient) private val manifestCache = cache?.let(::HlsManifestCache) @@ -48,19 +49,20 @@ class HlsManifestService( } private suspend fun cachedOrFetch(manifestUrl: String, signManifestLinks: Boolean): ExtractionResult { + val cache = manifestCache.takeUnless { isProviderManifestUrl(manifestUrl) } val cacheKey = if (signManifestLinks) "signed:$manifestUrl" else manifestUrl - manifestCache?.get(cacheKey)?.let { return ExtractionResult.Success(it) } + cache?.get(cacheKey)?.let { return ExtractionResult.Success(it) } val pending = CompletableDeferred>() val existing = inFlight.putIfAbsent(cacheKey, pending) if (existing != null) return existing.await() return try { - manifestCache?.get(cacheKey)?.let { + cache?.get(cacheKey)?.let { val result = ExtractionResult.Success(it) pending.complete(result) return result } val result = fetchAndRewrite(manifestUrl, signManifestLinks) - if (result is ExtractionResult.Success) manifestCache?.set(cacheKey, result.data) + if (result is ExtractionResult.Success) cache?.set(cacheKey, result.data) pending.complete(result) result } catch (error: Throwable) { @@ -112,7 +114,19 @@ class HlsManifestService( } else { val text = body.string() response.close() - val rewritten = if (isNicoNicoManifest(fetchUrl)) { + val rewritten = if (providerMediaHandleService != null && isNicoNicoManifest(fetchUrl)) { + rewriteProviderHlsManifest(text, fetchUrl) { target -> + providerMediaHandleService.relativeManifestPath( + providerMediaHandleService.createPath(target, domandBid), + ) + } + } else if (providerMediaHandleService != null && isBilibiliManifest(fetchUrl)) { + rewriteProviderHlsManifest(text, fetchUrl) { target -> + providerMediaHandleService.relativeManifestPath( + providerMediaHandleService.createPath(target), + ) + } + } else if (isNicoNicoManifest(fetchUrl)) { rewriteNicoManifest(text, fetchUrl, domandBid, "../proxy") } else { rewriteYouTubeHlsManifest(text) { target -> @@ -128,5 +142,13 @@ class HlsManifestService( } private fun isNicoNicoManifest(url: String): Boolean = - runCatching { URI(url).host.orEmpty().endsWith("nicovideo.jp") }.getOrDefault(false) + runCatching { providerForProxyHost(URI(url).host.orEmpty()) == ProxyProvider.NICONICO } + .getOrDefault(false) + + private fun isBilibiliManifest(url: String): Boolean = + runCatching { providerForProxyHost(URI(url).host.orEmpty()) == ProxyProvider.BILIBILI } + .getOrDefault(false) + + private fun isProviderManifestUrl(url: String): Boolean = + isNicoNicoManifest(url) || isBilibiliManifest(url) } diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationBuilder.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationBuilder.kt index 5537eac5..e345b130 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationBuilder.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationBuilder.kt @@ -11,6 +11,7 @@ class HomeRecommendationBuilder( private val watchLaterService: WatchLaterService, private val blockedService: BlockedService, private val streamService: StreamService, + private val trendingService: TrendingService, ) { suspend fun build( userId: String, @@ -30,6 +31,7 @@ class HomeRecommendationBuilder( subscriptionFeedService = subscriptionFeedService, subscriptionShortsFeedService = subscriptionShortsFeedService, streamService = streamService, + trendingService = trendingService, ) val candidatePool = candidates.fetchCandidates( userId = userId, diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidateService.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidateService.kt index 48aa4625..f75d249f 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidateService.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidateService.kt @@ -1,11 +1,14 @@ package dev.typetype.server.services +import dev.typetype.server.models.ExtractionResult + import dev.typetype.server.models.VideoItem class HomeRecommendationCandidateService( private val subscriptionFeedService: SubscriptionFeedService, private val subscriptionShortsFeedService: SubscriptionShortsFeedService, private val streamService: StreamService, + private val trendingService: TrendingService, private val discoveryAssembler: HomeRecommendationDiscoveryAssembler = HomeRecommendationDiscoveryAssembler(), private val shortsCandidateService: HomeRecommendationShortsCandidateService = HomeRecommendationShortsCandidateService(), ) { @@ -36,9 +39,11 @@ class HomeRecommendationCandidateService( return shortsCandidateService.fetch(userId, profile, signalContext, this) } val subscriptions = fetchSubscriptionCandidates(userId, mode) + .filter { recommendationServiceId(it.url) == serviceId } .map { HomeRecommendationTaggedVideo(it, HomeRecommendationSourceTag.SUBSCRIPTION) } if (mode == HomeRecommendationPoolMode.FAST) { - return HomeRecommendationCandidatePool(subscriptions = subscriptions, discovery = emptyList()) + val discovery = if (subscriptions.isEmpty()) fetchTrending(serviceId) else emptyList() + return HomeRecommendationCandidatePool(subscriptions = subscriptions, discovery = discovery) } val subscriptionSeeds = subscriptions.map { it.video.url } val relatedFromSubscriptions = relatedCandidateService.fetch( @@ -48,16 +53,16 @@ class HomeRecommendationCandidateService( relatedPerSeedLimit = HomeRecommendationCandidateLimits.RELATED_PER_SEED_LIMIT, ) val relatedFromFavorites = relatedCandidateService.fetch( - seedUrls = signalContext.favoriteUrls, + seedUrls = signalContext.favoriteUrls.filter { recommendationServiceId(it) == serviceId }, source = HomeRecommendationSourceTag.DISCOVERY_EXPLORATION, seedLimit = HomeRecommendationCandidateLimits.FAVORITE_SEED_LIMIT, relatedPerSeedLimit = HomeRecommendationCandidateLimits.RELATED_PER_SEED_LIMIT, ) val discovery = discoveryAssembler.build( profile = profile, - candidates = relatedFromSubscriptions + relatedFromFavorites, + candidates = relatedFromSubscriptions + relatedFromFavorites + fetchTrending(serviceId), explorationCap = HomeRecommendationCandidateLimits.RELATED_DISCOVERY_CAP, - ) + ).filter { recommendationServiceId(it.video.url) == serviceId } return HomeRecommendationCandidatePool(subscriptions = subscriptions, discovery = discovery) } @@ -88,4 +93,13 @@ class HomeRecommendationCandidateService( relatedPerSeedLimit: Int, ): List = relatedCandidateService.fetch(seedUrls, source, seedLimit, relatedPerSeedLimit) + + private suspend fun fetchTrending(serviceId: Int): List = + when (val result = trendingService.getTrending(serviceId)) { + is ExtractionResult.Success -> result.data + .filter { recommendationServiceId(it.url) == serviceId } + .map { HomeRecommendationTaggedVideo(it, HomeRecommendationSourceTag.DISCOVERY_TRENDING) } + is ExtractionResult.BadRequest -> emptyList() + is ExtractionResult.Failure -> emptyList() + } } diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolCache.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolCache.kt index 17525b84..1c6fc516 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolCache.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolCache.kt @@ -41,6 +41,6 @@ class HomeRecommendationPoolCache(private val cache: dev.typetype.server.cache.C companion object { private const val CACHE_TTL_SECONDS = 3_600L private const val STALE_TTL_SECONDS = 86_400L - private const val CACHE_VERSION = 9 + private const val CACHE_VERSION = 10 } } diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolver.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolver.kt index 90984e7c..5a26dba5 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolver.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolver.kt @@ -6,11 +6,12 @@ import kotlinx.coroutines.Deferred import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.async +import kotlinx.coroutines.cancel import kotlinx.coroutines.launch class HomeRecommendationPoolResolver( private val dependencies: HomeRecommendationPoolResolverDependencies, -) { +) : AutoCloseable { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) private val state = HomeRecommendationPoolResolverState() private val poolCache = HomeRecommendationPoolCache(dependencies.cache) @@ -76,6 +77,7 @@ class HomeRecommendationPoolResolver( watchLaterService = dependencies.watchLaterService, blockedService = dependencies.blockedService, streamService = dependencies.streamService, + trendingService = dependencies.trendingService, ).build( userId = userId, serviceId = serviceId, @@ -91,4 +93,8 @@ class HomeRecommendationPoolResolver( } } + override fun close() { + scope.cancel() + } + } diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolverDependencies.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolverDependencies.kt index e256e9f5..8a6a4ee5 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolverDependencies.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolverDependencies.kt @@ -11,5 +11,6 @@ data class HomeRecommendationPoolResolverDependencies( val watchLaterService: WatchLaterService, val blockedService: BlockedService, val streamService: StreamService = HomeRecommendationNoopStreamService, + val trendingService: TrendingService, val cache: CacheService, ) diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationService.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationService.kt index 5943417c..819a9dd7 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationService.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationService.kt @@ -4,7 +4,7 @@ import dev.typetype.server.models.HomeRecommendationsResponse class HomeRecommendationService( private val poolResolver: HomeRecommendationPoolResolver, -) { +) : AutoCloseable { private fun args( userId: String, serviceId: Int, @@ -46,4 +46,8 @@ class HomeRecommendationService( mode = HomeRecommendationPoolMode.SHORTS, poolResolver = poolResolver, ) + + override fun close() { + poolResolver.close() + } } diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmupService.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmupService.kt index 59d9365a..af4c40fa 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmupService.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmupService.kt @@ -4,18 +4,17 @@ import dev.typetype.server.cache.CacheService import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel import kotlinx.coroutines.delay import kotlinx.coroutines.isActive import kotlinx.coroutines.launch -import java.util.concurrent.ConcurrentHashMap class HomeRecommendationWarmupService( private val recommendationService: HomeRecommendationService, private val cache: CacheService, -) : HomeRecommendationWarmup { +) : HomeRecommendationWarmup, AutoCloseable { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) - private val activeUsers = ConcurrentHashMap() - private val warmupStartedAt = ConcurrentHashMap() + private val tracker = HomeWarmupTracker(WARMUP_THROTTLE_MS, ACTIVE_TTL_MS) private val poolCache = HomeRecommendationPoolCache(cache) init { @@ -23,12 +22,12 @@ class HomeRecommendationWarmupService( } override fun markActive(userId: String) { - activeUsers[userId] = System.currentTimeMillis() + tracker.markActive(userId, System.currentTimeMillis()) schedule(userId) } override fun invalidateAndWarm(userId: String) { - activeUsers[userId] = System.currentTimeMillis() + tracker.markActive(userId, System.currentTimeMillis()) scope.launch { invalidate(userId) SubscriptionFeedCacheInvalidation.awaitRefresh(userId) @@ -38,9 +37,7 @@ class HomeRecommendationWarmupService( private fun schedule(userId: String, force: Boolean = false) { val now = System.currentTimeMillis() - val previous = warmupStartedAt[userId] - if (!force && previous != null && now - previous < WARMUP_THROTTLE_MS) return - warmupStartedAt[userId] = now + if (!tracker.trySchedule(userId, now, force)) return scope.launch { warm(userId) } } @@ -64,8 +61,7 @@ class HomeRecommendationWarmupService( while (scope.isActive) { delay(REFRESH_INTERVAL_MS) val now = System.currentTimeMillis() - activeUsers.entries.removeIf { now - it.value > ACTIVE_TTL_MS } - activeUsers.keys.forEach { schedule(it) } + tracker.activeUsers(now).forEach { schedule(it) } } } @@ -77,6 +73,10 @@ class HomeRecommendationWarmupService( ), ) + override fun close() { + scope.cancel() + } + companion object { private const val WARMUP_LIMIT = 20 private const val WARMUP_THROTTLE_MS = 10 * 60 * 1000L diff --git a/src/main/kotlin/dev/typetype/server/services/HomeWarmupTracker.kt b/src/main/kotlin/dev/typetype/server/services/HomeWarmupTracker.kt new file mode 100644 index 00000000..a78aea45 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/HomeWarmupTracker.kt @@ -0,0 +1,29 @@ +package dev.typetype.server.services + +internal class HomeWarmupTracker( + private val throttleMs: Long, + private val activeTtlMs: Long, +) { + private val active = mutableMapOf() + private val started = mutableMapOf() + + @Synchronized + fun markActive(userId: String, now: Long) { + active[userId] = now + } + + @Synchronized + fun trySchedule(userId: String, now: Long, force: Boolean): Boolean { + val previous = started[userId] + if (!force && previous != null && now - previous < throttleMs) return false + started[userId] = now + return true + } + + @Synchronized + fun activeUsers(now: Long): List { + active.entries.removeIf { now - it.value > activeTtlMs } + started.keys.retainAll(active.keys) + return active.keys.toList() + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/InstanceService.kt b/src/main/kotlin/dev/typetype/server/services/InstanceService.kt index ab42ccb1..96456634 100644 --- a/src/main/kotlin/dev/typetype/server/services/InstanceService.kt +++ b/src/main/kotlin/dev/typetype/server/services/InstanceService.kt @@ -8,6 +8,7 @@ import dev.typetype.server.models.InstanceResponse import dev.typetype.server.models.OidcPublicConfig import dev.typetype.server.models.YoutubeRemoteLoginStatus import dev.typetype.server.models.RssInstanceCapability +import dev.typetype.server.models.PushNotificationCapability class InstanceService( private val authService: AuthService, @@ -21,6 +22,7 @@ class InstanceService( else -> YoutubeRemoteLoginStatus.NotConfigured } }, + private val pushNotificationCapabilityProvider: () -> PushNotificationCapability = { PushNotificationCapability() }, ) { suspend fun getInstance(): InstanceResponse { @@ -56,6 +58,7 @@ class InstanceService( minimumPollMinutes = settings.rssMinimumPollMinutes, rateLimitPerMinute = settings.rssRateLimitPerMinute, ), + pushNotifications = pushNotificationCapabilityProvider(), ) } diff --git a/src/main/kotlin/dev/typetype/server/services/ManifestService.kt b/src/main/kotlin/dev/typetype/server/services/ManifestService.kt index bce04f99..81c2ed36 100644 --- a/src/main/kotlin/dev/typetype/server/services/ManifestService.kt +++ b/src/main/kotlin/dev/typetype/server/services/ManifestService.kt @@ -6,7 +6,10 @@ import dev.typetype.server.models.VideoStreamItem import java.net.URLEncoder import java.nio.charset.StandardCharsets -class ManifestService(private val streamService: StreamService) { +class ManifestService( + private val streamService: StreamService, + private val providerMediaHandleService: ProviderMediaHandleService? = null, +) { suspend fun dashManifest(videoUrl: String): ExtractionResult { val result = streamService.getStreamInfo(videoUrl) if (result !is ExtractionResult.Success) return result.recast() @@ -32,7 +35,7 @@ class ManifestService(private val streamService: StreamService) { else -> 2 } - private fun buildMpd(videos: List, audios: List, duration: Long): String { + private suspend fun buildMpd(videos: List, audios: List, duration: Long): String { val sb = StringBuilder() sb.appendLine("") sb.appendLine(") { + private suspend fun appendVideoAdaptationSet(sb: StringBuilder, mimeType: String, streams: List) { sb.appendLine(" ") streams.forEachIndexed { i, s -> val height = if (s.height > 0) s.height else resolutionHeight(s.resolution) @@ -62,7 +65,7 @@ class ManifestService(private val streamService: StreamService) { val bandwidth = (s.bitrate ?: bwFromUrl(s.url) ?: (height * 1000)).coerceAtLeast(1) val sizeAttr = if (width > 0 && height > 0) " width=\"$width\" height=\"$height\"" else "" sb.appendLine(" ") - sb.appendLine(" ../proxy?url=${encode(s.url)}") + sb.appendLine(" ${mediaUrl(s.url)}") if (s.indexStart > 0L && s.indexEnd > 0L) { sb.appendLine(" ") sb.appendLine(" ") @@ -73,12 +76,12 @@ class ManifestService(private val streamService: StreamService) { sb.appendLine(" ") } - private fun appendAudioAdaptationSet(sb: StringBuilder, mimeType: String, lang: String?, label: String?, streams: List) { + private suspend fun appendAudioAdaptationSet(sb: StringBuilder, mimeType: String, lang: String?, label: String?, streams: List) { val attrs = "${if (lang != null) " lang=\"$lang\"" else ""}${if (label != null) " label=\"$label\"" else ""}" sb.appendLine(" ") streams.forEachIndexed { i, a -> sb.appendLine(" ") - sb.appendLine(" ../proxy?url=${encode(a.url)}") + sb.appendLine(" ${mediaUrl(a.url)}") if (a.indexStart > 0L && a.indexEnd > 0L) { sb.appendLine(" ") sb.appendLine(" ") @@ -110,6 +113,18 @@ class ManifestService(private val streamService: StreamService) { private fun bwFromUrl(url: String): Int? = Regex("[?&]bw=(\\d+)").find(url)?.groupValues?.get(1)?.toIntOrNull() + private suspend fun mediaUrl(url: String): String { + val service = providerMediaHandleService + if (service == null) return "../proxy?url=${encode(url)}" + if (url.startsWith("/media/")) return service.relativeManifestPath(url) + val provider = runCatching { requireProxyTarget(url).provider }.getOrNull() + return if (provider == ProxyProvider.BILIBILI || provider == ProxyProvider.NICONICO) { + service.relativeManifestPath(service.createPath(url)) + } else { + "../proxy?url=${encode(url)}" + } + } + private fun encode(url: String): String = URLEncoder.encode(url, StandardCharsets.UTF_8) private fun ExtractionResult.recast(): ExtractionResult = when (this) { diff --git a/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt b/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt index 9c475dd1..8bb0c0f9 100644 --- a/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt +++ b/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt @@ -40,7 +40,10 @@ internal fun rewriteNicoManifest(manifest: String, baseUrl: String, domandBid: S } } -class NicoVideoProxyService(client: OkHttpClient = defaultNicoProxyClient()) { +class NicoVideoProxyService( + client: OkHttpClient = defaultNicoProxyClient(), + private val mediaHandleService: ProviderMediaHandleService? = null, +) { private val executor = ProxyHttpExecutor(client) companion object { @@ -72,7 +75,15 @@ class NicoVideoProxyService(client: OkHttpClient = defaultNicoProxyClient()) { } else { val text = body.string() response.close() - val rewritten = rewriteNicoManifest(text, manifestUrl, resolvedBid) + val rewritten = if (mediaHandleService == null) { + rewriteNicoManifest(text, manifestUrl, resolvedBid) + } else { + rewriteProviderHlsManifest(text, manifestUrl) { target -> + mediaHandleService.relativeManifestPath( + mediaHandleService.createPath(target, resolvedBid), + ) + } + } ExtractionResult.Success(ProxyResponse( status = 200, contentType = "application/vnd.apple.mpegurl", @@ -113,6 +124,7 @@ class NicoVideoProxyService(client: OkHttpClient = defaultNicoProxyClient()) { acceptRanges = response.header("Accept-Ranges"), stream = body.byteStream(), close = response::close, + cacheControl = response.header("Cache-Control"), )) } }, diff --git a/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt b/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt index fb50c816..f73d23eb 100644 --- a/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt @@ -1,44 +1,68 @@ package dev.typetype.server.services import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.NotificationReadItemsTable import dev.typetype.server.db.tables.NotificationStatesTable import dev.typetype.server.models.MarkNotificationsReadResponse import dev.typetype.server.models.NotificationItem import dev.typetype.server.models.NotificationsResponse import dev.typetype.server.models.UnreadCountResponse import dev.typetype.server.models.VideoItem -import java.util.concurrent.ConcurrentHashMap +import java.time.Duration import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.insertIgnore import org.jetbrains.exposed.v1.jdbc.insert import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.update +import java.security.MessageDigest class NotificationsService( private val subscriptionFeedService: SubscriptionFeedService, ) { - private val unreadCache = ConcurrentHashMap() + private val unreadCache = BoundedExpiringCache( + maxEntries = 2048, + ttl = Duration.ofHours(1), + ) suspend fun getNotifications(userId: String, page: Int, limit: Int): NotificationsResponse { - val items = buildItems(userId) - val unreadCount = unreadCount(items, userId, refresh = true) + val feed = loadFeed(userId) + if (!feed.available && feed.videos.isEmpty()) { + return NotificationsResponse(emptyList(), cachedUnread(userId), null, false) + } + val items = withReadState(buildItems(feed.videos), userId) + val unreadCount = unreadCount(items, userId) val from = page * limit - if (from >= items.size) return NotificationsResponse(items = emptyList(), unreadCount = unreadCount, nextpage = null) + if (from >= items.size) { + return NotificationsResponse(emptyList(), unreadCount, null, feed.available) + } val to = minOf(from + limit, items.size) val nextpage = if (to < items.size) (page + 1).toString() else null - return NotificationsResponse(items = items.subList(from, to), unreadCount = unreadCount, nextpage = nextpage) + return NotificationsResponse(items.subList(from, to), unreadCount, nextpage, feed.available) } suspend fun getUnreadCount(userId: String): UnreadCountResponse { - val now = System.currentTimeMillis() - unreadCache[userId]?.takeIf { it.expiresAt > now }?.let { return UnreadCountResponse(unreadCount = it.value) } - val items = buildItems(userId) - return UnreadCountResponse(unreadCount = unreadCount(items, userId, refresh = true)) + val feed = loadFeed(userId) + if (!feed.available && feed.videos.isEmpty()) return UnreadCountResponse(cachedUnread(userId), false) + val value = unreadCount(withReadState(buildItems(feed.videos), userId), userId) + return UnreadCountResponse(value, feed.available) } suspend fun markAllRead(userId: String): MarkNotificationsReadResponse { + val feed = loadFeed(userId) + if (!feed.available) { + return MarkNotificationsReadResponse(System.currentTimeMillis(), cachedUnread(userId), false) + } + val items = buildItems(feed.videos) val now = System.currentTimeMillis() - val latestUploaded = buildItems(userId).firstOrNull()?.createdAt ?: 0L DatabaseFactory.query { + items.forEach { item -> + NotificationReadItemsTable.insertIgnore { + it[NotificationReadItemsTable.userId] = userId + it[notificationId] = item.id + it[readAt] = now + } + } + val latestUploaded = items.maxOfOrNull { it.createdAt } ?: 0L val updated = NotificationStatesTable.update({ NotificationStatesTable.userId eq userId }) { it[subscriptionLastSeenUploaded] = latestUploaded it[updatedAt] = now @@ -46,62 +70,109 @@ class NotificationsService( if (updated == 0) { NotificationStatesTable.insert { it[NotificationStatesTable.userId] = userId - it[subscriptionLastSeenUploaded] = latestUploaded - it[updatedAt] = now + it[NotificationStatesTable.subscriptionLastSeenUploaded] = latestUploaded + it[NotificationStatesTable.updatedAt] = now } } } - unreadCache[userId] = CachedUnread(value = 0, expiresAt = now + UNREAD_CACHE_TTL_MS) - return MarkNotificationsReadResponse(readAt = now, unreadCount = 0) + unreadCache.put(userId, 0) + return MarkNotificationsReadResponse(now, 0, true) } - private suspend fun buildItems(userId: String): List = subscriptionFeedService.getAll(userId) - .asSequence() - .filter { it.uploaded > 0L } - .groupBy { notificationKey(it) } - .values - .mapNotNull { group -> group.maxByOrNull { it.uploaded } } - .sortedByDescending { it.uploaded } - .map { it.toNotificationItem() } - .toList() - - private suspend fun getLastSeenUploaded(userId: String): Long = DatabaseFactory.query { - NotificationStatesTable.selectAll().where { NotificationStatesTable.userId eq userId } - .singleOrNull()?.get(NotificationStatesTable.subscriptionLastSeenUploaded) ?: 0L + suspend fun markRead(userId: String, notificationId: String): MarkNotificationsReadResponse { + val feed = loadFeed(userId) + if (!feed.available) { + return MarkNotificationsReadResponse(System.currentTimeMillis(), cachedUnread(userId), false) + } + val items = buildItems(feed.videos) + val now = System.currentTimeMillis() + if (items.any { it.id == notificationId }) { + DatabaseFactory.query { + NotificationReadItemsTable.insertIgnore { + it[NotificationReadItemsTable.userId] = userId + it[NotificationReadItemsTable.notificationId] = notificationId + it[readAt] = now + } + } + } + val unread = unreadCount(withReadState(items, userId), userId) + return MarkNotificationsReadResponse(now, unread, true) } - private suspend fun unreadCount(items: List, userId: String, refresh: Boolean): Int { - if (!refresh) { - val now = System.currentTimeMillis() - unreadCache[userId]?.takeIf { it.expiresAt > now }?.let { return it.value } + private suspend fun loadFeed(userId: String): SubscriptionFeedAvailability = + runCatching { subscriptionFeedService.getAllWithAvailability(userId) } + .getOrElse { SubscriptionFeedAvailability(emptyList(), false) } + + private suspend fun withReadState(items: List, userId: String): List { + val readIds = DatabaseFactory.query { + NotificationReadItemsTable.selectAll().where { NotificationReadItemsTable.userId eq userId } + .mapTo(HashSet()) { it[NotificationReadItemsTable.notificationId] } } - val lastSeenUploaded = getLastSeenUploaded(userId) - val value = items.count { it.createdAt > lastSeenUploaded } - val now = System.currentTimeMillis() - unreadCache[userId] = CachedUnread(value = value, expiresAt = now + UNREAD_CACHE_TTL_MS) + val legacyWatermark = DatabaseFactory.query { + NotificationStatesTable.selectAll().where { NotificationStatesTable.userId eq userId } + .singleOrNull()?.get(NotificationStatesTable.subscriptionLastSeenUploaded) ?: 0L + } + return items.map { item -> + item.copy(read = item.id in readIds || item.createdAt <= legacyWatermark) + } + } + + private fun buildItems(videos: List): List = videos.asSequence() + .map { video -> video to RssVideoMetadata.publishedAtMillis(video) } + .filter { (_, createdAt) -> createdAt > 0L } + .distinctBy { (video, _) -> notificationKey(video) } + .sortedByDescending { it.second } + .map { it.toNotificationItem() } + .toList() + + private suspend fun unreadCount(items: List, userId: String): Int { + val value = items.count { !it.read } + unreadCache.put(userId, value) return value } - private fun notificationKey(video: VideoItem): String = - video.uploaderUrl.ifBlank { video.uploaderName.ifBlank { video.url } } + private fun cachedUnread(userId: String): Int = unreadCache.get(userId) ?: 0 - private fun VideoItem.toNotificationItem(): NotificationItem = NotificationItem( - type = "subscription_new_video", - title = "$uploaderName uploaded a new video", - createdAt = uploaded, - publishedAt = uploaded, - channelUrl = uploaderUrl, - channelName = uploaderName, - channelAvatarUrl = uploaderAvatarUrl, - video = this, - ) + private fun notificationKey(video: VideoItem): String { + val serviceId = RssVideoMetadata.serviceId(video) + val channelIdentity = video.uploaderUrl.trim().ifBlank { video.uploaderAvatarUrl.trim() } + .ifBlank { video.uploaderName.trim() } + val videoIdentity = video.url.trim().ifBlank { video.id.trim() } + .ifBlank { "${video.title.trim()}:${RssVideoMetadata.publishedAtMillis(video)}" } + return "$serviceId|$channelIdentity|$videoIdentity" + } - private data class CachedUnread( - val value: Int, - val expiresAt: Long, - ) + private fun Pair.toNotificationItem(): NotificationItem { + val video = first + val createdAt = second + val serviceId = RssVideoMetadata.serviceId(video) + return NotificationItem( + id = notificationId(video), + type = "subscription_new_video", + title = "${video.uploaderName} uploaded a new video", + createdAt = createdAt, + publishedAt = createdAt, + channelUrl = video.uploaderUrl, + channelName = video.uploaderName, + channelAvatarUrl = video.uploaderAvatarUrl, + serviceId = serviceId, + serviceName = serviceName(serviceId), + video = video, + ) + } + + private fun notificationId(video: VideoItem): String = + MessageDigest.getInstance("SHA-256").digest(notificationKey(video).toByteArray()) + .joinToString("") { byte -> "%02x".format(byte) } private companion object { - const val UNREAD_CACHE_TTL_MS = 30_000L + fun serviceName(serviceId: Int): String = when (serviceId) { + YOUTUBE_SERVICE_ID -> "YouTube" + BILIBILI_SERVICE_ID -> "BiliBili" + NICONICO_SERVICE_ID -> "NicoNico" + SOUNDCLOUD_SERVICE_ID -> "SoundCloud" + MEDIA_CCC_SERVICE_ID -> "MediaCCC" + else -> "Video service" + } } } diff --git a/src/main/kotlin/dev/typetype/server/services/OidcUserService.kt b/src/main/kotlin/dev/typetype/server/services/OidcUserService.kt index d6c50d06..67db26cc 100644 --- a/src/main/kotlin/dev/typetype/server/services/OidcUserService.kt +++ b/src/main/kotlin/dev/typetype/server/services/OidcUserService.kt @@ -13,7 +13,7 @@ import java.util.UUID class OidcUserService(private val authService: AuthService) { suspend fun login(identity: OidcIdentity): AuthSessionTokens { val userId = transaction { resolveUserId(identity) } - return authService.issueSession(userId) ?: throw IllegalStateException("Failed to create session") + return authService.issueSessionForOwner(userId) ?: throw IllegalStateException("Failed to create session") } private fun resolveUserId(identity: OidcIdentity): String { diff --git a/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt b/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt index ebde6bc6..9b906a23 100644 --- a/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt +++ b/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt @@ -3,6 +3,8 @@ package dev.typetype.server.services import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.ProxyResponse import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.ensureActive import kotlinx.coroutines.withContext import okhttp3.OkHttpClient import okhttp3.Request @@ -11,21 +13,41 @@ import java.net.URLEncoder import java.nio.charset.StandardCharsets internal val GOOGLEVIDEO_URL_REGEX = Regex("""https://[a-z0-9.\-]+\.googlevideo\.com/\S+""") +private val CPN_TRACKING_PARAM_REGEX = Regex("[&?]cpn=[^&]*") +private val PPPID_TRACKING_PARAM_REGEX = Regex("[&?]pppid=[^&]*") internal fun stripTrackingParams(url: String): String = - url.replace(Regex("[&?]cpn=[^&]*"), "") - .replace(Regex("[&?]pppid=[^&]*"), "") + url.replace(CPN_TRACKING_PARAM_REGEX, "") + .replace(PPPID_TRACKING_PARAM_REGEX, "") internal fun rewriteHlsManifest(manifest: String): String = manifest.replace(GOOGLEVIDEO_URL_REGEX) { match -> "/proxy?url=" + URLEncoder.encode(match.value, StandardCharsets.UTF_8) } -class OkHttpProxyService(client: OkHttpClient) : ProxyService { +class OkHttpProxyService( + client: OkHttpClient, + private val mediaHandleService: ProviderMediaHandleService? = null, +) : ProxyService, ProviderMediaAwareProxyService { private val executor = ProxyHttpExecutor(client) override suspend fun pipe(url: String, rangeHeader: String?, domandBid: String?): ExtractionResult = - withContext(Dispatchers.IO) { + pipeInternal(url, rangeHeader, domandBid, providerMediaManifest = false) + + override suspend fun pipeProviderMedia( + url: String, + rangeHeader: String?, + domandBid: String?, + ): ExtractionResult = pipeInternal(url, rangeHeader, domandBid, providerMediaManifest = true) + + private suspend fun pipeInternal( + url: String, + rangeHeader: String?, + domandBid: String?, + providerMediaManifest: Boolean, + ): ExtractionResult { + val requestContext = currentCoroutineContext() + return withContext(Dispatchers.IO) { val hashIdx = url.indexOf('#') val fetchUrl = if (hashIdx >= 0) url.substring(0, hashIdx) else url val fragment = if (hashIdx >= 0) url.substring(hashIdx + 1) else "" @@ -40,13 +62,12 @@ class OkHttpProxyService(client: OkHttpClient) : ProxyService { if (bilibili) { builder.header("Referer", BILIBILI_REFERER) builder.header("Accept", ACCEPT_ANY) - if (rangeHeader != null) builder.header("Connection", "close") } if (resolvedDomandBid != null && isNicoNico(cleanUrl)) builder.header("Cookie", "domand_bid=$resolvedDomandBid") if (rangeHeader != null) builder.header("Range", rangeHeader) val request = builder.build() if (bilibili && rangeHeader != null) { - return@withContext readBilibiliRangeWithRetry(executor::execute, request) + return@withContext readBilibiliRangeWithRetry(executor::execute, request, requestContext::ensureActive) } executor.execute(request) }.fold( @@ -67,9 +88,20 @@ class OkHttpProxyService(client: OkHttpClient) : ProxyService { val acceptRanges = response.header("Accept-Ranges") val cacheControl = response.header("Cache-Control") val contentLength = response.header("Content-Length")?.toLongOrNull() - if (isHls(contentType)) { - val rewritten = if (isNicoNico(stripTrackingParams(fetchUrl))) { - rewriteNicoManifest(body.string(), stripTrackingParams(fetchUrl), resolvedDomandBid, PROXY_PATH) + val cleanFetchUrl = stripTrackingParams(fetchUrl) + if (isHls(contentType, cleanFetchUrl)) { + val rewritten = if (providerMediaManifest && mediaHandleService != null && isNicoNico(cleanFetchUrl)) { + rewriteProviderHlsManifest(body.string(), cleanFetchUrl) { target -> + mediaHandleService.relativeManifestPath( + mediaHandleService.createPath(target, resolvedDomandBid), + ) + } + } else if (providerMediaManifest && mediaHandleService != null && isBilibili(cleanFetchUrl)) { + rewriteProviderHlsManifest(body.string(), cleanFetchUrl) { target -> + mediaHandleService.relativeManifestPath(mediaHandleService.createPath(target)) + } + } else if (isNicoNico(cleanFetchUrl)) { + rewriteNicoManifest(body.string(), cleanFetchUrl, resolvedDomandBid, PROXY_PATH) } else { rewriteHlsManifest(body.string()) } @@ -98,9 +130,13 @@ class OkHttpProxyService(client: OkHttpClient) : ProxyService { } } }, - onFailure = { ExtractionResult.Failure(it.message ?: "Proxy fetch failed") } + onFailure = { + requestContext.ensureActive() + ExtractionResult.Failure(it.message ?: "Proxy fetch failed") + } ) } + } private fun isBilibili(url: String): Boolean { val host = runCatching { java.net.URI(url).host ?: "" }.getOrElse { "" } @@ -109,11 +145,11 @@ class OkHttpProxyService(client: OkHttpClient) : ProxyService { private fun isNicoNico(url: String): Boolean { val host = runCatching { java.net.URI(url).host ?: "" }.getOrElse { "" } - return host.endsWith("nicovideo.jp") + return providerForProxyHost(host) == ProxyProvider.NICONICO } - private fun isHls(contentType: String): Boolean = - contentType.contains("mpegurl", ignoreCase = true) + private fun isHls(contentType: String, url: String): Boolean = + contentType.contains("mpegurl", ignoreCase = true) || url.contains(".m3u8", ignoreCase = true) companion object { private const val BILIBILI_REFERER = "https://www.bilibili.com" diff --git a/src/main/kotlin/dev/typetype/server/services/OpenMojiProxyService.kt b/src/main/kotlin/dev/typetype/server/services/OpenMojiProxyService.kt index 6c191b0c..03882be1 100644 --- a/src/main/kotlin/dev/typetype/server/services/OpenMojiProxyService.kt +++ b/src/main/kotlin/dev/typetype/server/services/OpenMojiProxyService.kt @@ -5,46 +5,57 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import okhttp3.OkHttpClient import okhttp3.Request -import java.util.concurrent.ConcurrentHashMap +import java.time.Duration import java.util.concurrent.TimeUnit -class OpenMojiProxyService(private val cache: CacheService) { +class OpenMojiProxyService( + private val cache: CacheService, + private val client: OkHttpClient = defaultOpenMojiClient(), + private val clock: () -> Long = System::currentTimeMillis, +) { - private val localCache = ConcurrentHashMap() - private val failedUntilByCode = ConcurrentHashMap() - private val notFoundUntilByCode = ConcurrentHashMap() - private val client = OkHttpClient.Builder() - .connectTimeout(2, TimeUnit.SECONDS) - .readTimeout(4, TimeUnit.SECONDS) - .callTimeout(5, TimeUnit.SECONDS) - .followRedirects(true) - .build() + private val localCache = BoundedExpiringCache( + maxEntries = LOCAL_CACHE_MAX_ENTRIES, + maxWeight = LOCAL_CACHE_MAX_BYTES, + ttl = Duration.ofMinutes(10), + weigher = { it.size.toLong() }, + clock = clock, + ) + private val failedUntilByCode = BoundedExpiringCache( + maxEntries = COOLDOWN_MAX_ENTRIES, + ttl = Duration.ofMillis(FAILURE_COOLDOWN_MS), + clock = clock, + ) + private val notFoundUntilByCode = BoundedExpiringCache( + maxEntries = COOLDOWN_MAX_ENTRIES, + ttl = Duration.ofMillis(NOT_FOUND_CACHE_MS), + clock = clock, + ) suspend fun getSvg(code: String): ByteArray? { val key = cacheKey(code) - val now = System.currentTimeMillis() - localCache[code]?.takeIf { it.expiresAtMs > now }?.let { return it.bytes } - if (now < (notFoundUntilByCode[code] ?: 0L)) return null - if (now < (failedUntilByCode[code] ?: 0L)) return null + localCache.get(code)?.let { return it } + if (notFoundUntilByCode.get(code) != null) return null + if (failedUntilByCode.get(code) != null) return null runCatching { cache.get(key) }.getOrNull()?.toByteArray(Charsets.UTF_8)?.let { bytes -> - localCache[code] = LocalSvg(bytes = bytes, expiresAtMs = now + LOCAL_CACHE_TTL_MS) + localCache.put(code, bytes) return bytes } return when (val fetched = fetch(code)) { is FetchResult.Success -> { failedUntilByCode.remove(code) notFoundUntilByCode.remove(code) - localCache[code] = LocalSvg(bytes = fetched.bytes, expiresAtMs = now + LOCAL_CACHE_TTL_MS) + localCache.put(code, fetched.bytes) runCatching { cache.set(key, fetched.bytes.toString(Charsets.UTF_8), SVG_CACHE_TTL_SECONDS) } fetched.bytes } FetchResult.NotFound -> { failedUntilByCode.remove(code) - notFoundUntilByCode[code] = now + NOT_FOUND_CACHE_MS + notFoundUntilByCode.put(code, Unit) null } FetchResult.Failed -> { - failedUntilByCode[code] = now + FAILURE_COOLDOWN_MS + failedUntilByCode.put(code, Unit) null } } @@ -71,7 +82,9 @@ class OpenMojiProxyService(private val cache: CacheService) { private const val SVG_CACHE_TTL_SECONDS = 604800L private const val FAILURE_COOLDOWN_MS = 15000L private const val NOT_FOUND_CACHE_MS = 300000L - private const val LOCAL_CACHE_TTL_MS = 600000L + private const val LOCAL_CACHE_MAX_ENTRIES = 256 + private const val LOCAL_CACHE_MAX_BYTES = 16L * 1024 * 1024 + private const val COOLDOWN_MAX_ENTRIES = 1024 } private sealed interface FetchResult { @@ -80,8 +93,11 @@ class OpenMojiProxyService(private val cache: CacheService) { data object Failed : FetchResult } - private data class LocalSvg( - val bytes: ByteArray, - val expiresAtMs: Long, - ) } + +private fun defaultOpenMojiClient(): OkHttpClient = OkHttpClient.Builder() + .connectTimeout(2, TimeUnit.SECONDS) + .readTimeout(4, TimeUnit.SECONDS) + .callTimeout(5, TimeUnit.SECONDS) + .followRedirects(true) + .build() diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeCommentService.kt b/src/main/kotlin/dev/typetype/server/services/PipePipeCommentService.kt index 1721e219..422b215d 100644 --- a/src/main/kotlin/dev/typetype/server/services/PipePipeCommentService.kt +++ b/src/main/kotlin/dev/typetype/server/services/PipePipeCommentService.kt @@ -54,7 +54,7 @@ class PipePipeCommentService : CommentService { private fun CommentsInfoItem.toCommentItem(): CommentItem = CommentItem( id = commentId ?: "", - text = commentText ?: "", + text = commentText.content, author = uploaderName ?: "", authorUrl = uploaderUrl ?: "", authorAvatarUrl = (uploaderAvatarUrl ?: "").normalizeHttpSchema(), diff --git a/src/main/kotlin/dev/typetype/server/services/ProfileAccountService.kt b/src/main/kotlin/dev/typetype/server/services/ProfileAccountService.kt new file mode 100644 index 00000000..bc74cd43 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/ProfileAccountService.kt @@ -0,0 +1,191 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.ProfileAccountsTable +import dev.typetype.server.db.tables.UsersTable +import dev.typetype.server.models.AccountProfileItem +import dev.typetype.server.models.AccountProfilesResponse +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insertIgnore +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.update +import java.util.UUID + +class ProfileAccountService { + suspend fun ownerUserId(profileId: String): String? = DatabaseFactory.query { + ownerIdInTransaction(profileId) + } + + suspend fun isOwnerProfile(profileId: String): Boolean = DatabaseFactory.query { + if (profileId.startsWith("guest:")) return@query false + ProfileAccountsTable.selectAll().where { ProfileAccountsTable.profileId eq profileId } + .singleOrNull()?.let { it[ProfileAccountsTable.ownerUserId] == profileId } ?: true + } + + suspend fun list(activeProfileId: String): AccountProfilesResponse? = DatabaseFactory.query { + val ownerId = ownerIdInTransaction(activeProfileId) ?: return@query null + ensureDefaultInTransaction(ownerId) + val rows = ProfileAccountsTable.selectAll().where { ProfileAccountsTable.ownerUserId eq ownerId } + .map { row -> profileRow(row, activeProfileId) } + .sortedWith(compareByDescending { it.isDefault }.thenBy { it.name.lowercase() }) + val defaultId = rows.firstOrNull { it.isDefault }?.id ?: activeProfileId + AccountProfilesResponse(rows, activeProfileId, defaultId) + } + + suspend fun resolveSignInProfile(ownerUserId: String): String = DatabaseFactory.query { + val ownerId = ownerIdInTransaction(ownerUserId) ?: return@query ownerUserId + ensureDefaultInTransaction(ownerId) + val rows = ProfileAccountsTable.selectAll().where { ProfileAccountsTable.ownerUserId eq ownerId }.toList() + val selected = rows.filter { it[ProfileAccountsTable.lastUsedAt] > 0L } + .maxByOrNull { it[ProfileAccountsTable.lastUsedAt] } + ?: rows.firstOrNull { it[ProfileAccountsTable.isDefault] } + ?: rows.firstOrNull() + val profileId = selected?.get(ProfileAccountsTable.profileId) ?: ownerId + markUsedInTransaction(profileId, System.currentTimeMillis()) + profileId + } + + suspend fun create(activeProfileId: String, name: String): ProfileMutationResult = DatabaseFactory.query { + val normalized = normalizeName(name) ?: return@query ProfileMutationResult.InvalidName + val ownerId = ownerIdInTransaction(activeProfileId) ?: return@query ProfileMutationResult.NotFound + val profileId = UUID.randomUUID().toString() + val now = System.currentTimeMillis() + UsersTable.insertIgnore { + it[id] = profileId + it[email] = "profile-$profileId@profiles.invalid" + it[passwordHash] = "profile:$profileId" + it[UsersTable.name] = normalized + it[role] = "user" + it[verified] = true + it[createdAt] = now + it[updatedAt] = now + } + ProfileAccountsTable.insertIgnore { + it[ProfileAccountsTable.profileId] = profileId + it[ownerUserId] = ownerId + it[displayName] = normalized + it[isDefault] = false + it[lastUsedAt] = 0L + it[createdAt] = now + it[updatedAt] = now + } + ProfileMutationResult.Success(profileRowById(profileId, activeProfileId)) + } + + suspend fun rename(activeProfileId: String, profileId: String, name: String): ProfileMutationResult = DatabaseFactory.query { + val normalized = normalizeName(name) ?: return@query ProfileMutationResult.InvalidName + val ownerId = ownerIdInTransaction(activeProfileId) ?: return@query ProfileMutationResult.NotFound + val target = targetInTransaction(ownerId, profileId) ?: return@query ProfileMutationResult.NotFound + ProfileAccountsTable.update({ ProfileAccountsTable.profileId eq profileId }) { + it[displayName] = normalized + it[updatedAt] = System.currentTimeMillis() + } + ProfileMutationResult.Success(profileRowById(target, activeProfileId)) + } + + suspend fun setDefault(activeProfileId: String, profileId: String): ProfileMutationResult = DatabaseFactory.query { + val ownerId = ownerIdInTransaction(activeProfileId) ?: return@query ProfileMutationResult.NotFound + val target = targetInTransaction(ownerId, profileId) ?: return@query ProfileMutationResult.NotFound + ProfileAccountsTable.update({ ProfileAccountsTable.ownerUserId eq ownerId }) { it[isDefault] = false } + ProfileAccountsTable.update({ ProfileAccountsTable.profileId eq target }) { + it[isDefault] = true + it[updatedAt] = System.currentTimeMillis() + } + ProfileMutationResult.Success(profileRowById(target, activeProfileId)) + } + + suspend fun switch(activeProfileId: String, profileId: String): AccountProfileItem? = DatabaseFactory.query { + val ownerId = ownerIdInTransaction(activeProfileId) ?: return@query null + val target = targetInTransaction(ownerId, profileId) ?: return@query null + markUsedInTransaction(target, System.currentTimeMillis()) + profileRowById(target, target) + } + + suspend fun delete(activeProfileId: String, profileId: String): ProfileMutationResult = DatabaseFactory.query { + val ownerId = ownerIdInTransaction(activeProfileId) ?: return@query ProfileMutationResult.NotFound + val target = targetInTransaction(ownerId, profileId) ?: return@query ProfileMutationResult.NotFound + if (target == ownerId) return@query ProfileMutationResult.CannotDeleteOwner + if (target == activeProfileId) return@query ProfileMutationResult.CannotDeleteActive + ProfileDataDeletionService.deleteUser(target) + ProfileAccountsTable.deleteWhere { ProfileAccountsTable.profileId eq target } + ensureDefaultInTransaction(ownerId) + ProfileMutationResult.Deleted + } + + private fun ownerIdInTransaction(profileId: String): String? { + if (profileId.startsWith("guest:")) return null + ProfileAccountsTable.selectAll().where { ProfileAccountsTable.profileId eq profileId } + .singleOrNull()?.let { return it[ProfileAccountsTable.ownerUserId] } + val userExists = UsersTable.selectAll().where { UsersTable.id eq profileId }.any() + if (!userExists) return null + val now = System.currentTimeMillis() + ProfileAccountsTable.insertIgnore { + it[ProfileAccountsTable.profileId] = profileId + it[ownerUserId] = profileId + it[displayName] = "Profile" + it[isDefault] = true + it[lastUsedAt] = 0L + it[createdAt] = now + it[updatedAt] = now + } + return profileId + } + + private fun targetInTransaction(ownerId: String, profileId: String): String? = + ProfileAccountsTable.selectAll().where { + (ProfileAccountsTable.ownerUserId eq ownerId) and (ProfileAccountsTable.profileId eq profileId) + }.singleOrNull()?.get(ProfileAccountsTable.profileId) + + private fun profileRowById(profileId: String, activeProfileId: String): AccountProfileItem = + ProfileAccountsTable.selectAll().where { ProfileAccountsTable.profileId eq profileId } + .single().let { profileRow(it, activeProfileId) } + + private fun profileRow(row: org.jetbrains.exposed.v1.core.ResultRow, activeProfileId: String): AccountProfileItem { + val profileId = row[ProfileAccountsTable.profileId] + val user = UsersTable.selectAll().where { UsersTable.id eq profileId }.single() + return AccountProfileItem( + id = profileId, + name = row[ProfileAccountsTable.displayName], + isActive = profileId == activeProfileId, + isDefault = row[ProfileAccountsTable.isDefault], + lastUsedAt = row[ProfileAccountsTable.lastUsedAt], + publicUsername = user[UsersTable.publicUsername], + avatarUrl = user[UsersTable.avatarUrl], + avatarType = user[UsersTable.avatarType], + avatarCode = user[UsersTable.avatarCode], + ) + } + + private fun markUsedInTransaction(profileId: String, now: Long) { + ProfileAccountsTable.update({ ProfileAccountsTable.profileId eq profileId }) { it[lastUsedAt] = now } + } + + private fun ensureDefaultInTransaction(ownerId: String) { + val rows = ProfileAccountsTable.selectAll().where { ProfileAccountsTable.ownerUserId eq ownerId }.toList() + if (rows.isEmpty()) return + val selected = rows.filter { it[ProfileAccountsTable.isDefault] } + .maxByOrNull { it[ProfileAccountsTable.lastUsedAt] } + ?: rows.minByOrNull { it[ProfileAccountsTable.createdAt] } + ?: return + val selectedId = selected[ProfileAccountsTable.profileId] + ProfileAccountsTable.update({ ProfileAccountsTable.ownerUserId eq ownerId }) { + it[isDefault] = false + } + ProfileAccountsTable.update({ ProfileAccountsTable.profileId eq selectedId }) { + it[isDefault] = true + } + } + + private fun normalizeName(value: String): String? = value.trim().takeIf { it.length in 1..40 } +} + +sealed interface ProfileMutationResult { + data class Success(val profile: AccountProfileItem) : ProfileMutationResult + data object Deleted : ProfileMutationResult + data object InvalidName : ProfileMutationResult + data object CannotDeleteOwner : ProfileMutationResult + data object CannotDeleteActive : ProfileMutationResult + data object NotFound : ProfileMutationResult +} diff --git a/src/main/kotlin/dev/typetype/server/services/ProfileDataDeletionService.kt b/src/main/kotlin/dev/typetype/server/services/ProfileDataDeletionService.kt new file mode 100644 index 00000000..f4f261e2 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/ProfileDataDeletionService.kt @@ -0,0 +1,115 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.tables.AllowedChannelsTable +import dev.typetype.server.db.tables.AllowedPlaylistsTable +import dev.typetype.server.db.tables.BlockedChannelsTable +import dev.typetype.server.db.tables.BlockedKeywordsTable +import dev.typetype.server.db.tables.BlockedVideosTable +import dev.typetype.server.db.tables.BugReportsTable +import dev.typetype.server.db.tables.FavoritesTable +import dev.typetype.server.db.tables.HistoryTable +import dev.typetype.server.db.tables.NotificationStatesTable +import dev.typetype.server.db.tables.NotificationReadItemsTable +import dev.typetype.server.db.tables.ChannelNotificationPreferencesTable +import dev.typetype.server.db.tables.PushDevicesTable +import dev.typetype.server.db.tables.PushNotificationBaselinesTable +import dev.typetype.server.db.tables.PushNotificationSeenVideosTable +import dev.typetype.server.db.tables.PushNotificationDeliveriesTable +import dev.typetype.server.db.tables.PasswordResetTable +import dev.typetype.server.db.tables.PlaylistVideosTable +import dev.typetype.server.db.tables.PlaylistsTable +import dev.typetype.server.db.tables.ProgressTable +import dev.typetype.server.db.tables.RecommendationEventsTable +import dev.typetype.server.db.tables.RecommendationFeedbackTable +import dev.typetype.server.db.tables.RecommendationFeedHistoryTable +import dev.typetype.server.db.tables.RecommendationOnboardingPreferencesTable +import dev.typetype.server.db.tables.RecommendationOnboardingStateTable +import dev.typetype.server.db.tables.RssFeedChannelsTable +import dev.typetype.server.db.tables.RssFeedServicesTable +import dev.typetype.server.db.tables.RssFeedsTable +import dev.typetype.server.db.tables.RssUserPoliciesTable +import dev.typetype.server.db.tables.SavedPlaylistsTable +import dev.typetype.server.db.tables.SearchHistoryTable +import dev.typetype.server.db.tables.SessionsTable +import dev.typetype.server.db.tables.SettingsTable +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable +import dev.typetype.server.db.tables.SubscriptionGroupsTable +import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.db.tables.UserAvatarsTable +import dev.typetype.server.db.tables.UserChannelInterestTable +import dev.typetype.server.db.tables.UserTopicInterestTable +import dev.typetype.server.db.tables.WatchLaterTable +import dev.typetype.server.db.tables.YoutubeSessionPairingsTable +import dev.typetype.server.db.tables.YoutubeSessionsTable +import dev.typetype.server.db.tables.YoutubeTakeoutImportJobsTable +import dev.typetype.server.db.tables.YoutubeTakeoutPlaylistKeysTable +import dev.typetype.server.db.tables.UsersTable +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.selectAll + +internal object ProfileDataDeletionService { + fun deleteUser(userId: String) { + val feedIds = RssFeedsTable.selectAll().where { RssFeedsTable.userId eq userId } + .map { it[RssFeedsTable.id] } + feedIds.forEach { feedId -> + RssFeedChannelsTable.deleteWhere { RssFeedChannelsTable.feedId eq feedId } + RssFeedServicesTable.deleteWhere { RssFeedServicesTable.feedId eq feedId } + } + RssFeedsTable.deleteWhere { RssFeedsTable.userId eq userId } + + val groupIds = SubscriptionGroupsTable.selectAll().where { SubscriptionGroupsTable.userId eq userId } + .map { it[SubscriptionGroupsTable.id] } + SubscriptionGroupMembershipsTable.deleteWhere { SubscriptionGroupMembershipsTable.userId eq userId } + groupIds.forEach { groupId -> + SubscriptionGroupMembershipsTable.deleteWhere { SubscriptionGroupMembershipsTable.groupId eq groupId } + } + + PlaylistVideosTable.deleteWhere { PlaylistVideosTable.userId eq userId } + PlaylistsTable.deleteWhere { PlaylistsTable.userId eq userId } + deleteUserRows(userId) + UsersTable.deleteWhere { UsersTable.id eq userId } + } + + private fun deleteUserRows(userId: String) { + listOf( + { HistoryTable.deleteWhere { HistoryTable.userId eq userId } }, + { FavoritesTable.deleteWhere { FavoritesTable.userId eq userId } }, + { ProgressTable.deleteWhere { ProgressTable.userId eq userId } }, + { WatchLaterTable.deleteWhere { WatchLaterTable.userId eq userId } }, + { SubscriptionsTable.deleteWhere { SubscriptionsTable.userId eq userId } }, + { SubscriptionGroupsTable.deleteWhere { SubscriptionGroupsTable.userId eq userId } }, + { SavedPlaylistsTable.deleteWhere { SavedPlaylistsTable.userId eq userId } }, + { SearchHistoryTable.deleteWhere { SearchHistoryTable.userId eq userId } }, + { SettingsTable.deleteWhere { SettingsTable.userId eq userId } }, + { AllowedChannelsTable.deleteWhere { AllowedChannelsTable.userId eq userId } }, + { AllowedPlaylistsTable.deleteWhere { AllowedPlaylistsTable.userId eq userId } }, + { BlockedChannelsTable.deleteWhere { BlockedChannelsTable.userId eq userId } }, + { BlockedKeywordsTable.deleteWhere { BlockedKeywordsTable.userId eq userId } }, + { BlockedVideosTable.deleteWhere { BlockedVideosTable.userId eq userId } }, + { BugReportsTable.deleteWhere { BugReportsTable.userId eq userId } }, + { NotificationStatesTable.deleteWhere { NotificationStatesTable.userId eq userId } }, + { NotificationReadItemsTable.deleteWhere { NotificationReadItemsTable.userId eq userId } }, + { PushNotificationDeliveriesTable.deleteWhere { PushNotificationDeliveriesTable.userId eq userId } }, + { PushNotificationSeenVideosTable.deleteWhere { PushNotificationSeenVideosTable.userId eq userId } }, + { PushNotificationBaselinesTable.deleteWhere { PushNotificationBaselinesTable.userId eq userId } }, + { PushDevicesTable.deleteWhere { PushDevicesTable.userId eq userId } }, + { ChannelNotificationPreferencesTable.deleteWhere { ChannelNotificationPreferencesTable.userId eq userId } }, + { PasswordResetTable.deleteWhere { PasswordResetTable.userId eq userId } }, + { SessionsTable.deleteWhere { SessionsTable.userId eq userId } }, + { UserAvatarsTable.deleteWhere { UserAvatarsTable.userId eq userId } }, + { UserChannelInterestTable.deleteWhere { UserChannelInterestTable.userId eq userId } }, + { UserTopicInterestTable.deleteWhere { UserTopicInterestTable.userId eq userId } }, + { YoutubeSessionsTable.deleteWhere { YoutubeSessionsTable.userId eq userId } }, + { YoutubeSessionPairingsTable.deleteWhere { YoutubeSessionPairingsTable.userId eq userId } }, + { YoutubeTakeoutImportJobsTable.deleteWhere { YoutubeTakeoutImportJobsTable.userId eq userId } }, + { YoutubeTakeoutPlaylistKeysTable.deleteWhere { YoutubeTakeoutPlaylistKeysTable.userId eq userId } }, + { RecommendationEventsTable.deleteWhere { RecommendationEventsTable.userId eq userId } }, + { RecommendationFeedHistoryTable.deleteWhere { RecommendationFeedHistoryTable.userId eq userId } }, + { RecommendationFeedbackTable.deleteWhere { RecommendationFeedbackTable.userId eq userId } }, + { RecommendationOnboardingPreferencesTable.deleteWhere { RecommendationOnboardingPreferencesTable.userId eq userId } }, + { RecommendationOnboardingStateTable.deleteWhere { RecommendationOnboardingStateTable.userId eq userId } }, + { RssUserPoliciesTable.deleteWhere { RssUserPoliciesTable.userId eq userId } }, + ).forEach { it() } + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/ProviderHlsManifestRewrite.kt b/src/main/kotlin/dev/typetype/server/services/ProviderHlsManifestRewrite.kt new file mode 100644 index 00000000..e7f2a361 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/ProviderHlsManifestRewrite.kt @@ -0,0 +1,68 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import java.net.URI + +internal suspend fun rewriteProviderHlsManifest( + manifest: String, + baseUrl: String, + mapUrl: suspend (String) -> String, +): String = coroutineScope { + val base = URI(baseUrl) + val uriAttr = Regex("""URI="([^"]+)"""") + val lines = manifest.lines() + val references = lines.flatMap { line -> + val trimmed = line.trim() + when { + trimmed.isBlank() -> emptyList() + trimmed.startsWith("#") -> uriAttr.findAll(trimmed).map { it.groupValues[1] }.toList() + else -> listOf(trimmed) + } + } + val resolvedReferences = references.map { raw -> resolveProviderManifestUrl(base, raw) } + .filter { it.isHttpUrl() } + .distinct() + val mappedReferences = resolvedReferences + .chunked(MAX_CONCURRENT_MAPPINGS) + .flatMap { batch -> batch.map { target -> async { target to mapUrl(target) } }.awaitAll() } + .toMap() + + fun mapResolved(raw: String): String { + val resolved = resolveProviderManifestUrl(base, raw) + return if (resolved.isHttpUrl()) mappedReferences[resolved] ?: resolved else resolved + } + + val rewritten = ArrayList() + for (line in lines) { + val trimmed = line.trim() + when { + trimmed.isBlank() -> rewritten += line + trimmed.startsWith("#") -> { + val builder = StringBuilder(trimmed) + val matches = uriAttr.findAll(trimmed).toList() + for (match in matches.asReversed()) { + val mapped = mapResolved(match.groupValues[1]) + builder.replace(match.range.first, match.range.last + 1, "URI=\"$mapped\"") + } + rewritten += builder.toString() + } + else -> rewritten += mapResolved(trimmed) + } + } + rewritten.joinToString("\n") +} + +private fun resolveProviderManifestUrl(base: URI, raw: String): String = runCatching { + if (raw.startsWith("http://", ignoreCase = true) || raw.startsWith("https://", ignoreCase = true)) { + raw + } else { + base.resolve(raw).toString() + } +}.getOrDefault(raw) + +private fun String.isHttpUrl(): Boolean = startsWith("http://", ignoreCase = true) || + startsWith("https://", ignoreCase = true) + +private const val MAX_CONCURRENT_MAPPINGS = 16 diff --git a/src/main/kotlin/dev/typetype/server/services/ProviderMediaHandleService.kt b/src/main/kotlin/dev/typetype/server/services/ProviderMediaHandleService.kt new file mode 100644 index 00000000..04d46dff --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/ProviderMediaHandleService.kt @@ -0,0 +1,171 @@ +package dev.typetype.server.services + +import dev.typetype.server.cache.CacheJson +import dev.typetype.server.cache.CacheService +import dev.typetype.server.models.AudioStreamItem +import dev.typetype.server.models.StreamResponse +import dev.typetype.server.models.VideoStreamItem +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit +import kotlinx.serialization.Serializable +import kotlinx.serialization.encodeToString +import java.net.URLDecoder +import java.nio.charset.StandardCharsets +import java.security.MessageDigest +import java.util.Base64 +import java.util.concurrent.ConcurrentHashMap + +internal enum class ProviderMediaType { + BILIBILI, + NICONICO, +} + +@Serializable +internal data class ProviderMediaTarget( + val url: String, + val domandBid: String? = null, +) + +class ProviderMediaHandleService( + private val cache: CacheService, + private val nowSeconds: () -> Long = { System.currentTimeMillis() / 1000 }, +) { + private val writeLimiter = Semaphore(MAX_CONCURRENT_CACHE_WRITES) + private val inFlightWrites = ConcurrentHashMap>() + + internal suspend fun materialize(response: StreamResponse, provider: ProviderMediaType): StreamResponse = coroutineScope { + val hls = async { response.hlsUrl.handleIfRemote(provider) } + val dash = async { response.dashMpdUrl.handleIfRemote(provider) } + val video = async { response.videoStreams.withHandledVideoUrls(provider) } + val audio = async { response.audioStreams.withHandledAudioUrls(provider) } + val videoOnly = async { response.videoOnlyStreams.withHandledVideoUrls(provider) } + response.copy( + hlsUrl = hls.await(), + dashMpdUrl = dash.await(), + videoStreams = video.await(), + audioStreams = audio.await(), + videoOnlyStreams = videoOnly.await(), + ) + } + + internal suspend fun createPath(rawUrl: String, domandBid: String? = null): String { + val (cleanUrl, fragment) = rawUrl.splitFragment() + val resolvedBid = domandBid?.takeIf { it.isNotBlank() } + ?: fragment.takeIf { it.isNotBlank() }?.let(::parseNicoCookie) + val target = requireProxyTarget(stripTrackingParams(cleanUrl)) + require(target.provider == ProxyProvider.BILIBILI || target.provider == ProxyProvider.NICONICO) { + "Unsupported provider media URL" + } + val normalizedUrl = stripTrackingParams(target.url.toString()) + val handle = handleId(normalizedUrl, resolvedBid) + val ttl = mediaHandleTtlSeconds(normalizedUrl) + val key = cacheKey(handle) + val path = "/media/$handle" + val pending = CompletableDeferred() + val existing = inFlightWrites.putIfAbsent(key, pending) + if (existing != null) return existing.await() + return try { + val value = CacheJson.encodeToString(ProviderMediaTarget(normalizedUrl, resolvedBid)) + writeLimiter.withPermit { cache.set(key, value, ttl) } + pending.complete(path) + path + } catch (error: Throwable) { + pending.completeExceptionally(error) + throw error + } finally { + inFlightWrites.remove(key, pending) + } + } + + internal suspend fun resolve(handle: String): ProviderMediaTarget? { + if (!HANDLE_PATTERN.matches(handle)) return null + return cache.get(cacheKey(handle))?.let { encoded -> + runCatching { CacheJson.decodeFromString(encoded) }.getOrNull() + } + } + + internal fun relativeManifestPath(path: String): String = + "../media/${path.substringAfterLast('/')}" + + private suspend fun String.handleIfRemote(provider: ProviderMediaType): String { + if (isBlank() || (startsWith("/") && !startsWith("//"))) return this + val candidate = if (startsWith("//")) "https:$this" else this + return if (supportsProvider(candidate, provider)) createPath(candidate) else this + } + + private suspend fun List.withHandledVideoUrls(provider: ProviderMediaType): List = coroutineScope { + map { item -> + async { + item.copy( + url = item.url.handleIfRemote(provider), + manifestUrl = item.manifestUrl?.handleIfRemote(provider), + sabrSessionUrl = item.sabrSessionUrl?.handleIfRemote(provider), + ) + } + }.awaitAll() + } + + private suspend fun List.withHandledAudioUrls(provider: ProviderMediaType): List = coroutineScope { + map { item -> + async { + item.copy( + url = item.url.handleIfRemote(provider), + manifestUrl = item.manifestUrl?.handleIfRemote(provider), + sabrSessionUrl = item.sabrSessionUrl?.handleIfRemote(provider), + ) + } + }.awaitAll() + } + + private fun supportsProvider(url: String, provider: ProviderMediaType): Boolean = + runCatching { requireProxyTarget(url).provider }.getOrNull() == provider.toProxyProvider() + + private fun handleId(url: String, domandBid: String?): String { + val input = buildString { + append("provider-media:v1:") + append(url) + append('\u0000') + append(domandBid.orEmpty()) + } + val digest = MessageDigest.getInstance("SHA-256").digest(input.toByteArray(StandardCharsets.UTF_8)) + val encoded = Base64.getUrlEncoder().withoutPadding().encodeToString(digest.copyOf(HANDLE_BYTES)) + return "m1_$encoded" + } + + private fun mediaHandleTtlSeconds(url: String): Long { + val decoded = runCatching { URLDecoder.decode(url, StandardCharsets.UTF_8) }.getOrDefault(url) + val expiry = EXPIRY_PATTERN.findAll(decoded) + .mapNotNull { it.groupValues[1].toLongOrNull() } + .minOrNull() + val ttl = expiry?.minus(nowSeconds())?.minus(EXPIRY_SAFETY_SECONDS) + ?: DEFAULT_TTL_SECONDS + if (ttl <= 0L) throw IllegalArgumentException("Provider media URL has expired") + return ttl.coerceAtMost(MAX_TTL_SECONDS) + } + + private fun String.splitFragment(): Pair { + val index = indexOf('#') + return if (index < 0) this to "" else substring(0, index) to substring(index + 1) + } + + private companion object { + const val HANDLE_BYTES = 18 + const val DEFAULT_TTL_SECONDS = 1_800L + const val MAX_TTL_SECONDS = 3_600L + const val EXPIRY_SAFETY_SECONDS = 30L + const val MAX_CONCURRENT_CACHE_WRITES = 32 + val HANDLE_PATTERN = Regex("m1_[A-Za-z0-9_-]{24}") + val EXPIRY_PATTERN = Regex("(?:^|[?&#=])(?:deadline|Expires|exp)=(\\d+)", RegexOption.IGNORE_CASE) + + fun cacheKey(handle: String): String = "provider-media:v1:$handle" + + fun ProviderMediaType.toProxyProvider(): ProxyProvider = when (this) { + ProviderMediaType.BILIBILI -> ProxyProvider.BILIBILI + ProviderMediaType.NICONICO -> ProxyProvider.NICONICO + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/ProxyService.kt b/src/main/kotlin/dev/typetype/server/services/ProxyService.kt index 73d59be2..1286bc5e 100644 --- a/src/main/kotlin/dev/typetype/server/services/ProxyService.kt +++ b/src/main/kotlin/dev/typetype/server/services/ProxyService.kt @@ -6,3 +6,11 @@ import dev.typetype.server.models.ProxyResponse interface ProxyService { suspend fun pipe(url: String, rangeHeader: String?, domandBid: String? = null): ExtractionResult } + +internal interface ProviderMediaAwareProxyService { + suspend fun pipeProviderMedia( + url: String, + rangeHeader: String?, + domandBid: String? = null, + ): ExtractionResult +} diff --git a/src/main/kotlin/dev/typetype/server/services/PushDeviceRegistry.kt b/src/main/kotlin/dev/typetype/server/services/PushDeviceRegistry.kt new file mode 100644 index 00000000..5ad17817 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/PushDeviceRegistry.kt @@ -0,0 +1,146 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.PushDevicesTable +import dev.typetype.server.models.PushDeviceRegistrationRequest +import dev.typetype.server.models.PushDeviceRegistrationResponse +import java.security.MessageDigest +import java.util.UUID +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.inList +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.update + +internal class PushDeviceRegistry( + private val endpointValidator: UnifiedPushEndpointValidator = UnifiedPushEndpointValidator(), + private val maxDevicesPerAccount: Int = DEFAULT_MAX_DEVICES, +) { + suspend fun register(userId: String, request: PushDeviceRegistrationRequest): DeviceRegistrationResult { + val deviceId = request.deviceId.trim() + val platform = request.platform.trim().lowercase() + val endpoint = request.endpoint.trim() + if (!DEVICE_ID.matches(deviceId)) return DeviceRegistrationResult.Invalid("device_id") + if (platform != "android") return DeviceRegistrationResult.UnsupportedPlatform + val expiresAt = request.expiresAt + val now = System.currentTimeMillis() + if (expiresAt != null && (expiresAt <= now || expiresAt > now + MAX_EXPIRY_MS)) { + return DeviceRegistrationResult.Invalid("expires_at") + } + val uri = when (val result = endpointValidator.validate(endpoint)) { + is EndpointValidationResult.Valid -> result.uri + is EndpointValidationResult.Invalid -> return DeviceRegistrationResult.Invalid(result.reason) + } + val hash = sha256(uri.toString()) + return DatabaseFactory.query { + removeExpired(userId, now) + val existingDevice = PushDevicesTable.selectAll().where { + (PushDevicesTable.userId eq userId) and (PushDevicesTable.deviceId eq deviceId) + }.singleOrNull() + val existingEndpoint = PushDevicesTable.selectAll().where { PushDevicesTable.endpointHash eq hash }.singleOrNull() + if (existingEndpoint != null && existingEndpoint[PushDevicesTable.id] != existingDevice?.get(PushDevicesTable.id)) { + return@query DeviceRegistrationResult.EndpointConflict + } + if (existingDevice == null) { + val activeCount = PushDevicesTable.selectAll().where { PushDevicesTable.userId eq userId }.count() + if (activeCount >= maxDevicesPerAccount) return@query DeviceRegistrationResult.LimitReached + val id = UUID.randomUUID().toString() + PushDevicesTable.insert { + it[PushDevicesTable.id] = id + it[PushDevicesTable.userId] = userId + it[PushDevicesTable.deviceId] = deviceId + it[PushDevicesTable.platform] = platform + it[PushDevicesTable.endpoint] = uri.toString() + it[PushDevicesTable.endpointHash] = hash + it[PushDevicesTable.expiresAt] = expiresAt + it[PushDevicesTable.createdAt] = now + it[PushDevicesTable.updatedAt] = now + } + DeviceRegistrationResult.Success(PushDeviceRegistrationResponse(id, deviceId, platform, expiresAt, now)) + } else { + val id = existingDevice[PushDevicesTable.id] + PushDevicesTable.update({ PushDevicesTable.id eq id }) { + it[PushDevicesTable.platform] = platform + it[PushDevicesTable.endpoint] = uri.toString() + it[PushDevicesTable.endpointHash] = hash + it[PushDevicesTable.expiresAt] = expiresAt + it[PushDevicesTable.updatedAt] = now + } + DeviceRegistrationResult.Success(PushDeviceRegistrationResponse(id, deviceId, platform, expiresAt, now)) + } + } + } + + suspend fun unregister(userId: String, deviceId: String): Boolean = DatabaseFactory.query { + PushDevicesTable.deleteWhere { + (PushDevicesTable.userId eq userId) and (PushDevicesTable.deviceId eq deviceId) + } > 0 + } + + suspend fun list(userId: String): List = DatabaseFactory.query { + val now = System.currentTimeMillis() + removeExpired(userId, now) + PushDevicesTable.selectAll().where { PushDevicesTable.userId eq userId } + .map { row -> + PushDeviceRegistrationResponse( + id = row[PushDevicesTable.id], + deviceId = row[PushDevicesTable.deviceId], + platform = row[PushDevicesTable.platform], + expiresAt = row[PushDevicesTable.expiresAt], + updatedAt = row[PushDevicesTable.updatedAt], + ) + } + } + + suspend fun removeById(id: String): Boolean = DatabaseFactory.query { + PushDevicesTable.deleteWhere { PushDevicesTable.id eq id } > 0 + } + + suspend fun activeDevices(userId: String, now: Long = System.currentTimeMillis()): List = DatabaseFactory.query { + removeExpired(userId, now) + PushDevicesTable.selectAll().where { PushDevicesTable.userId eq userId }.map { row -> + PushDevice( + id = row[PushDevicesTable.id], + userId = row[PushDevicesTable.userId], + endpoint = row[PushDevicesTable.endpoint], + ) + } + } + + suspend fun registeredUserIds(): List = DatabaseFactory.query { + PushDevicesTable.selectAll().map { it[PushDevicesTable.userId] }.distinct() + } + + private fun sha256(value: String): String = MessageDigest.getInstance("SHA-256") + .digest(value.toByteArray()) + .joinToString("") { byte -> "%02x".format(byte) } + + private fun removeExpired(userId: String, now: Long) { + val expiredIds = PushDevicesTable.selectAll().where { + PushDevicesTable.userId eq userId + }.mapNotNull { row -> + row[PushDevicesTable.expiresAt]?.takeIf { it <= now }?.let { row[PushDevicesTable.id] } + } + if (expiredIds.isNotEmpty()) { + PushDevicesTable.deleteWhere { PushDevicesTable.id inList expiredIds } + } + } + + companion object { + const val DEFAULT_MAX_DEVICES = 8 + private const val MAX_EXPIRY_MS = 366L * 24 * 60 * 60 * 1000 + private val DEVICE_ID = Regex("[A-Za-z0-9._:-]{1,128}") + } +} + +internal data class PushDevice(val id: String, val userId: String, val endpoint: String) + +internal sealed interface DeviceRegistrationResult { + data class Success(val response: PushDeviceRegistrationResponse) : DeviceRegistrationResult + data class Invalid(val reason: String) : DeviceRegistrationResult + data object UnsupportedPlatform : DeviceRegistrationResult + data object EndpointConflict : DeviceRegistrationResult + data object LimitReached : DeviceRegistrationResult +} diff --git a/src/main/kotlin/dev/typetype/server/services/PushNotificationDeliveryStore.kt b/src/main/kotlin/dev/typetype/server/services/PushNotificationDeliveryStore.kt new file mode 100644 index 00000000..5d5315e3 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/PushNotificationDeliveryStore.kt @@ -0,0 +1,106 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.PushNotificationDeliveriesTable +import dev.typetype.server.db.tables.PushNotificationEventsTable +import dev.typetype.server.db.tables.PushNotificationSeenVideosTable +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.inList +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.insertIgnore +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.update + +internal class PushNotificationDeliveryStore( + private val clock: () -> Long, +) { + fun markSeen(userId: String, candidate: PushCandidate, now: Long): Boolean = + PushNotificationSeenVideosTable.insertIgnore { + it[PushNotificationSeenVideosTable.userId] = userId + it[PushNotificationSeenVideosTable.serviceId] = candidate.serviceId + it[PushNotificationSeenVideosTable.channelId] = candidate.channelId + it[PushNotificationSeenVideosTable.videoId] = candidate.videoId + it[PushNotificationSeenVideosTable.firstSeenAt] = now + }.insertedCount == 1 + + fun claim(eventId: String, userId: String, deviceId: String, createIfMissing: Boolean, now: Long): Boolean { + val existing = PushNotificationDeliveriesTable.selectAll().where { + (PushNotificationDeliveriesTable.eventId eq eventId) and + (PushNotificationDeliveriesTable.userId eq userId) and + (PushNotificationDeliveriesTable.deviceId eq deviceId) + }.singleOrNull() + if (existing == null) { + if (!createIfMissing) return false + return PushNotificationDeliveriesTable.insertIgnore { + it[PushNotificationDeliveriesTable.eventId] = eventId + it[PushNotificationDeliveriesTable.userId] = userId + it[PushNotificationDeliveriesTable.deviceId] = deviceId + it[PushNotificationDeliveriesTable.status] = PushNotificationDeliveryStatus.PENDING + it[PushNotificationDeliveriesTable.attempts] = 1 + it[PushNotificationDeliveriesTable.updatedAt] = now + }.insertedCount == 1 + } + val attempts = existing[PushNotificationDeliveriesTable.attempts] + val status = existing[PushNotificationDeliveriesTable.status] + if (status !in PushNotificationDeliveryStatus.RETRYABLE || attempts >= MAX_ATTEMPTS || + existing[PushNotificationDeliveriesTable.updatedAt] > now - RETRY_DELAY_MS + ) return false + return PushNotificationDeliveriesTable.update({ + (PushNotificationDeliveriesTable.eventId eq eventId) and + (PushNotificationDeliveriesTable.userId eq userId) and + (PushNotificationDeliveriesTable.deviceId eq deviceId) and + (PushNotificationDeliveriesTable.status inList PushNotificationDeliveryStatus.RETRYABLE) and + (PushNotificationDeliveriesTable.attempts eq attempts) and + (PushNotificationDeliveriesTable.updatedAt eq existing[PushNotificationDeliveriesTable.updatedAt]) + }) { + it[PushNotificationDeliveriesTable.status] = PushNotificationDeliveryStatus.PENDING + it[PushNotificationDeliveriesTable.attempts] = attempts + 1 + it[PushNotificationDeliveriesTable.lastError] = null + it[PushNotificationDeliveriesTable.updatedAt] = now + } == 1 + } + + suspend fun update(work: DeliveryWork, status: String, error: String?) { + DatabaseFactory.query { + PushNotificationDeliveriesTable.update({ + (PushNotificationDeliveriesTable.eventId eq work.eventId) and + (PushNotificationDeliveriesTable.userId eq work.device.userId) and + (PushNotificationDeliveriesTable.deviceId eq work.device.id) + }) { + it[PushNotificationDeliveriesTable.status] = status + it[PushNotificationDeliveriesTable.lastError] = error + it[PushNotificationDeliveriesTable.updatedAt] = clock() + } + } + } + + suspend fun clearPending(userId: String, channelId: String) { + val eventIds = DatabaseFactory.query { + PushNotificationEventsTable.selectAll().where { PushNotificationEventsTable.channelId eq channelId } + .mapTo(mutableSetOf()) { it[PushNotificationEventsTable.eventId] } + } + if (eventIds.isEmpty()) return + DatabaseFactory.query { + PushNotificationDeliveriesTable.deleteWhere { + (PushNotificationDeliveriesTable.userId eq userId) and + (PushNotificationDeliveriesTable.eventId inList eventIds) and + (PushNotificationDeliveriesTable.status inList PushNotificationDeliveryStatus.RETRYABLE) + } + } + } + + companion object { + private const val MAX_ATTEMPTS = 3 + private const val RETRY_DELAY_MS = 30_000L + } +} + +internal object PushNotificationDeliveryStatus { + const val PENDING = "pending" + const val DELIVERED = "delivered" + const val INVALID = "invalid" + const val FAILED = "failed" + val RETRYABLE = listOf(PENDING, FAILED) +} diff --git a/src/main/kotlin/dev/typetype/server/services/PushNotificationScheduler.kt b/src/main/kotlin/dev/typetype/server/services/PushNotificationScheduler.kt new file mode 100644 index 00000000..e82a5285 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/PushNotificationScheduler.kt @@ -0,0 +1,40 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.delay +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch + +internal class PushNotificationScheduler( + private val service: PushNotificationService, + private val intervalMs: Long = configuredIntervalMs(), + private val initialDelayMs: Long = 10_000L, +) { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + private var job: Job? = null + + fun start() { + if (job?.isActive == true) return + job = scope.launch { + delay(initialDelayMs) + while (isActive) { + service.pollRegisteredAccounts() + delay(intervalMs) + } + } + } + + fun close() { + scope.cancel() + } + + private companion object { + fun configuredIntervalMs(): Long = + (System.getenv("TYPE_TYPE_PUSH_NOTIFICATIONS_INTERVAL_SECONDS")?.toLongOrNull() ?: 300L) + .coerceIn(30L, 86_400L) * 1_000L + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/PushNotificationService.kt b/src/main/kotlin/dev/typetype/server/services/PushNotificationService.kt new file mode 100644 index 00000000..dd0ad996 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/PushNotificationService.kt @@ -0,0 +1,164 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.PushNotificationBaselinesTable +import dev.typetype.server.db.tables.PushNotificationEventsTable +import dev.typetype.server.models.ChannelNotificationPreference +import dev.typetype.server.models.PushDeviceRegistrationRequest +import dev.typetype.server.models.PushNotificationCapability +import dev.typetype.server.models.VideoItem +import kotlinx.coroutines.CancellationException +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.insertIgnore +import org.jetbrains.exposed.v1.jdbc.selectAll + +internal class PushNotificationService( + private val subscriptionsService: SubscriptionsService, + private val subscriptionFeedService: SubscriptionFeedService, + private val preferenceService: ChannelNotificationPreferenceService, + private val instanceId: String, + private val enabled: Boolean = true, + private val deviceRegistry: PushDeviceRegistry = PushDeviceRegistry(), + private val sender: PushEndpointSender = UnifiedPushSender(), + private val clock: () -> Long = System::currentTimeMillis, +) { + private val deliveryStore = PushNotificationDeliveryStore(clock) + + val capability: PushNotificationCapability = PushNotificationCapability( + enabled = enabled, + provider = "unifiedpush", + eventTypes = listOf(EVENT_TYPE_NEW_VIDEO), + maxDevicesPerAccount = PushDeviceRegistry.DEFAULT_MAX_DEVICES, + ) + + suspend fun registerDevice(userId: String, request: PushDeviceRegistrationRequest) = + deviceRegistry.register(userId, request) + + suspend fun listDevices(userId: String) = deviceRegistry.list(userId) + + suspend fun unregisterDevice(userId: String, deviceId: String): Boolean = + deviceRegistry.unregister(userId, deviceId) + + suspend fun listPreferences(userId: String): List = + preferenceService.list(userId) + + suspend fun setPreference(userId: String, channelUrl: String, value: Boolean): PreferenceUpdateResult { + val result = preferenceService.set(userId, channelUrl, value) + if (result is PreferenceUpdateResult.Updated && !value) { + deliveryStore.clearPending(userId, result.preference.channelUrl) + } + return result + } + + suspend fun onSubscriptionRemoved(userId: String, channelUrl: String) { + val canonical = ChannelUrlCanonicalizer.canonicalize(channelUrl) + preferenceService.remove(userId, canonical) + deliveryStore.clearPending(userId, canonical) + } + + suspend fun pollRegisteredAccounts() { + if (!enabled) return + try { + deviceRegistry.registeredUserIds().forEach { userId -> + try { + pollUser(userId) + } catch (error: CancellationException) { + throw error + } catch (error: Throwable) { + logger.warn("push_notifications event=poll_failed user={} error={}", userKey(userId), error.message) + } + } + } catch (error: CancellationException) { + throw error + } catch (error: Throwable) { + logger.warn("push_notifications event=registry_failed error={}", error.message) + } + } + + internal suspend fun pollUser(userId: String) { + val subscriptions = subscriptionsService.getAll(userId) + if (subscriptions.isEmpty()) return + val devices = deviceRegistry.activeDevices(userId, clock()) + if (devices.isEmpty()) return + val feed = subscriptionFeedService.getAllWithSources(userId) + if (!feed.available) return + val followedSince = subscriptions.associate { + ChannelUrlCanonicalizer.canonicalize(it.channelUrl) to it.subscribedAt + } + val enabledChannels = preferenceService.enabledChannelUrls(userId) + val candidates = feed.videos.flatMap { video -> + PushNotificationSupport.candidates(video, feed.sourceChannelUrls, followedSince) + } + .distinctBy { it.serviceId to (it.channelId to it.videoId) } + val work = DatabaseFactory.query { + val baseline = PushNotificationBaselinesTable.selectAll() + .where { PushNotificationBaselinesTable.userId eq userId }.singleOrNull() + if (baseline == null) { + PushNotificationBaselinesTable.insert { + it[PushNotificationBaselinesTable.userId] = userId + it[PushNotificationBaselinesTable.initializedAt] = clock() + } + candidates.forEach { deliveryStore.markSeen(userId, it, clock()) } + return@query emptyList() + } + candidates.mapNotNull candidateLoop@{ candidate -> + val isNew = deliveryStore.markSeen(userId, candidate, clock()) + if (candidate.channelId !in enabledChannels) return@candidateLoop null + val eventId = PushNotificationSupport.eventId(instanceId, candidate) + PushNotificationEventsTable.insertIgnore { + it[PushNotificationEventsTable.eventId] = eventId + it[PushNotificationEventsTable.eventType] = EVENT_TYPE_NEW_VIDEO + it[PushNotificationEventsTable.instanceId] = this@PushNotificationService.instanceId + it[PushNotificationEventsTable.serviceId] = candidate.serviceId + it[PushNotificationEventsTable.channelId] = candidate.channelId + it[PushNotificationEventsTable.videoId] = candidate.videoId + it[PushNotificationEventsTable.videoUrl] = candidate.video.url + it[PushNotificationEventsTable.title] = candidate.video.title + it[PushNotificationEventsTable.channelName] = candidate.video.uploaderName + it[PushNotificationEventsTable.channelAvatarUrl] = candidate.video.uploaderAvatarUrl + it[PushNotificationEventsTable.publishedAt] = candidate.publishedAt + it[PushNotificationEventsTable.createdAt] = clock() + } + devices.mapNotNull { device -> + if (deliveryStore.claim(eventId, userId, device.id, isNew, clock())) { + DeliveryWork(eventId, device, PushNotificationSupport.payload(instanceId, eventId, candidate, userId)) + } else null + } + }.flatten() + } + work.forEach { delivery -> deliver(delivery) } + } + + private suspend fun deliver(work: DeliveryWork) { + when (val result = sender.send(work.device.endpoint, work.eventId, work.payload)) { + PushSendResult.Delivered -> deliveryStore.update(work, PushNotificationDeliveryStatus.DELIVERED, null) + PushSendResult.InvalidEndpoint -> { + deviceRegistry.removeById(work.device.id) + deliveryStore.update(work, PushNotificationDeliveryStatus.INVALID, "endpoint_invalid") + } + is PushSendResult.Retry -> deliveryStore.update( + work, + PushNotificationDeliveryStatus.FAILED, + result.statusCode?.toString() ?: "network", + ) + } + } + + private fun userKey(userId: String): String = userId.take(8) + + private companion object { + const val EVENT_TYPE_NEW_VIDEO = "subscription_new_video" + val logger = org.slf4j.LoggerFactory.getLogger(PushNotificationService::class.java) + } +} + +internal data class PushCandidate( + val serviceId: Int, + val channelId: String, + val videoId: String, + val publishedAt: Long, + val video: VideoItem, +) + +internal data class DeliveryWork(val eventId: String, val device: PushDevice, val payload: String) diff --git a/src/main/kotlin/dev/typetype/server/services/PushNotificationSupport.kt b/src/main/kotlin/dev/typetype/server/services/PushNotificationSupport.kt new file mode 100644 index 00000000..fc1d548e --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/PushNotificationSupport.kt @@ -0,0 +1,63 @@ +package dev.typetype.server.services + +import dev.typetype.server.cache.CacheJson +import dev.typetype.server.models.UnifiedPushNotificationPayload +import dev.typetype.server.models.VideoItem +import java.security.MessageDigest + +internal object PushNotificationSupport { + fun candidates( + video: VideoItem, + sources: Map>, + followedSince: Map, + ): List { + if (video.isShortFormContent || video.isLiveContent || video.isPostLive || video.isLive) return emptyList() + val channelUrls = sources[video.subscriptionFeedKey()].orEmpty().ifEmpty { listOf(video.uploaderUrl) } + val publishedAt = RssVideoMetadata.publishedAtMillis(video) + if (publishedAt <= 0L) return emptyList() + val videoId = video.id.trim().ifBlank { video.url.trim() } + if (videoId.isBlank()) return emptyList() + return channelUrls.map { ChannelUrlCanonicalizer.canonicalize(it) } + .filter { channel -> + val subscribedAt = followedSince[channel] ?: return@filter false + subscribedAt <= 0L || publishedAt >= subscribedAt + } + .distinct() + .map { channel -> PushCandidate(RssVideoMetadata.serviceId(video), channel, videoId, publishedAt, video) } + } + + fun payload(instanceId: String, eventId: String, candidate: PushCandidate, userId: String): String = CacheJson.encodeToString( + UnifiedPushNotificationPayload.serializer(), + UnifiedPushNotificationPayload( + eventType = "subscription_new_video", + serviceId = candidate.serviceId, + serviceName = serviceName(candidate.serviceId), + eventId = eventId, + videoId = candidate.videoId, + videoUrl = candidate.video.url, + channelId = candidate.channelId, + channelName = candidate.video.uploaderName, + channelAvatarUrl = candidate.video.uploaderAvatarUrl, + instanceId = instanceId, + accountId = userId, + publishedAt = candidate.publishedAt, + title = candidate.video.title, + ), + ) + + fun eventId(instanceId: String, candidate: PushCandidate): String = sha256( + "$instanceId|subscription_new_video|${candidate.serviceId}|${candidate.channelId}|${candidate.videoId}", + ) + + private fun sha256(value: String): String = MessageDigest.getInstance("SHA-256") + .digest(value.toByteArray()).joinToString("") { byte -> "%02x".format(byte) } + + private fun serviceName(serviceId: Int): String = when (serviceId) { + YOUTUBE_SERVICE_ID -> "YouTube" + BILIBILI_SERVICE_ID -> "BiliBili" + NICONICO_SERVICE_ID -> "NicoNico" + SOUNDCLOUD_SERVICE_ID -> "SoundCloud" + MEDIA_CCC_SERVICE_ID -> "MediaCCC" + else -> "Video service" + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationServiceId.kt b/src/main/kotlin/dev/typetype/server/services/RecommendationServiceId.kt new file mode 100644 index 00000000..63bcdbac --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/RecommendationServiceId.kt @@ -0,0 +1,12 @@ +package dev.typetype.server.services + +import java.net.URI + +internal fun recommendationServiceId(url: String): Int { + val host = runCatching { URI(url).host.orEmpty().lowercase() }.getOrDefault("") + return when { + host == "b23.tv" || host == "bilibili.com" || host.endsWith(".bilibili.com") -> BILIBILI_SERVICE_ID + host == "nico.ms" || host == "nicovideo.jp" || host.endsWith(".nicovideo.jp") -> NICONICO_SERVICE_ID + else -> YOUTUBE_SERVICE_ID + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/RetryingProxyInputStream.kt b/src/main/kotlin/dev/typetype/server/services/RetryingProxyInputStream.kt new file mode 100644 index 00000000..5dbe0158 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/RetryingProxyInputStream.kt @@ -0,0 +1,122 @@ +package dev.typetype.server.services + +import okhttp3.Request +import okhttp3.Response +import java.io.EOFException +import java.io.IOException +import java.io.InputStream +import java.security.MessageDigest +import java.util.Objects + +internal class RetryingProxyInputStream( + private val execute: (Request) -> Response, + private val request: Request, + initialResponse: Response, + private var attemptsRemaining: Int, + private val checkActive: () -> Unit = {}, +) : InputStream() { + private var response = initialResponse + private var source = response.body.byteStream() + private val status = response.code + private val length = response.body.contentLength() + private val headers = listOf("Content-Range", "Content-Type", "Content-Encoding", "ETag") + .associateWith(response::header) + private var digest = if (response.header("ETag")?.let { + it.length >= 2 && it.startsWith('"') && it.endsWith('"') + } == true) { + null + } else { + MessageDigest.getInstance("SHA-256") + } + private var delivered = 0L + private var closed = false + private val singleByte = ByteArray(1) + + override fun read(): Int = if (read(singleByte, 0, 1) == -1) -1 else singleByte[0].toInt() and 0xff + + override fun read(bytes: ByteArray, offset: Int, count: Int): Int { + Objects.checkFromIndexSize(offset, count, bytes.size) + checkOpen() + if (count == 0) return 0 + if (length >= 0 && delivered == length) return -1 + while (true) { + try { + val size = if (length < 0) count else minOf(count.toLong(), length - delivered).toInt() + val read = source.read(bytes, offset, size) + if (read < 0 && length >= 0 && delivered < length) throw EOFException("Incomplete proxy body") + if (read > 0) { + digest?.update(bytes, offset, read) + delivered += read + } + return read + } catch (error: IOException) { + recover(error) + } + } + } + + private fun recover(initialError: IOException) { + response.close() + val expectedPrefix = digest?.digest() + var lastError = initialError + while (attemptsRemaining > 0) { + checkOpen() + if (Thread.currentThread().isInterrupted) throw lastError + attemptsRemaining-- + var candidate: Response? = null + try { + val opened = execute(request) + candidate = opened + if (!opened.isSuccessful) throw IOException("Upstream returned ${opened.code}") + if (opened.code != status || opened.body.contentLength() != length || + headers.any { (name, value) -> opened.header(name) != value } + ) throw ChangedProxyBodyException() + val nextSource = candidate.body.byteStream() + val nextDigest = expectedPrefix?.let { MessageDigest.getInstance("SHA-256") } + verifyPrefix(nextSource, nextDigest, expectedPrefix) + response = candidate + source = nextSource + digest = nextDigest + candidate = null + return + } catch (error: IOException) { + if (error is ChangedProxyBodyException) throw error + lastError = error + } finally { + candidate?.close() + } + } + throw lastError + } + + private fun verifyPrefix(input: InputStream, nextDigest: MessageDigest?, expected: ByteArray?) { + // Replaying the original range also works when the CDN supplies no strong validator. + val buffer = ByteArray(64 * 1024) + val verificationDigest = expected?.let { MessageDigest.getInstance("SHA-256") } + var remaining = delivered + while (remaining > 0) { + checkOpen() + val count = input.read(buffer, 0, minOf(remaining, buffer.size.toLong()).toInt()) + if (count < 0) throw EOFException("Incomplete proxy retry prefix") + nextDigest?.update(buffer, 0, count) + verificationDigest?.update(buffer, 0, count) + remaining -= count + } + if (expected != null && !MessageDigest.isEqual(expected, verificationDigest?.digest())) { + throw ChangedProxyBodyException() + } + } + + private fun checkOpen() { + if (closed) throw IOException("Proxy stream is closed") + checkActive() + } + + override fun close() { + if (closed) return + closed = true + response.close() + } +} + +private class ChangedProxyBodyException : IOException("Upstream proxy body changed during retry") diff --git a/src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt b/src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt index c84e9665..169281fa 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt +++ b/src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt @@ -3,6 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.models.RssFeedItem import dev.typetype.server.models.VideoItem import java.io.ByteArrayOutputStream +import java.net.URI import java.net.URLEncoder import java.nio.charset.StandardCharsets import java.time.Instant @@ -23,6 +24,7 @@ internal object RssDocumentRenderer { writer.writeStartDocument(StandardCharsets.UTF_8.name(), "1.0") writer.writeStartElement("rss") writer.writeAttribute("version", "2.0") + writer.writeNamespace("media", MEDIA_NAMESPACE) writer.writeStartElement("channel") writer.element("title", feed.name) writer.element("link", publicBaseUrl) @@ -52,6 +54,11 @@ internal object RssDocumentRenderer { writeEndElement() element("author", video.uploaderName) video.shortDescription?.takeIf(String::isNotBlank)?.let { element("description", it) } + video.thumbnailUrl.httpUrlOrNull()?.let { thumbnailUrl -> + writeStartElement("media", "thumbnail", MEDIA_NAMESPACE) + writeAttribute("url", thumbnailUrl) + writeEndElement() + } RssVideoMetadata.publishedAtMillis(video).takeIf { it > 0 } ?.let { element("pubDate", RFC_1123.format(Instant.ofEpochMilli(it))) } writeEndElement() @@ -63,5 +70,14 @@ internal object RssDocumentRenderer { writeEndElement() } + private fun String.httpUrlOrNull(): String? = runCatching { + URI(this).takeIf { uri -> + uri.isAbsolute && uri.host != null && + (uri.scheme.equals("http", ignoreCase = true) || + uri.scheme.equals("https", ignoreCase = true)) + }?.toString() + }.getOrNull() + + private const val MEDIA_NAMESPACE = "http://search.yahoo.com/mrss/" private val RFC_1123 = DateTimeFormatter.RFC_1123_DATE_TIME.withZone(ZoneOffset.UTC) } diff --git a/src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt b/src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt index b8347c78..69cd3304 100644 --- a/src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt +++ b/src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt @@ -1,6 +1,9 @@ package dev.typetype.server.services import dev.typetype.server.models.StreamResponse +import java.net.URI +import java.net.URLDecoder +import java.nio.charset.StandardCharsets private const val DEFAULT_STREAM_TTL_SECONDS = 21_600L private const val DISLIKE_UNAVAILABLE_STREAM_TTL_SECONDS = 300L @@ -20,19 +23,24 @@ private fun StreamResponse.stableMetadataTtlSeconds(): Long = if (dislikeCount < 0L) DISLIKE_UNAVAILABLE_STREAM_TTL_SECONDS else DEFAULT_STREAM_TTL_SECONDS private fun StreamResponse.signedMediaUrls(): Sequence = sequence { + yield(hlsUrl) + yield(dashMpdUrl) videoStreams.forEach { yield(it.url) } videoOnlyStreams.forEach { yield(it.url) } audioStreams.forEach { yield(it.url) } } -private fun String.bilibiliDeadline(): Long? { - if (!isBilibiliSignedMediaUrl()) return null - return Regex("""[?&]deadline=(\d+)""").find(this)?.groupValues?.get(1)?.toLongOrNull() - ?: Regex("""[?&]hdnts=exp=(\d+)""").find(this)?.groupValues?.get(1)?.toLongOrNull() +private fun String.bilibiliDeadline(): Long? = providerMediaExpiry() + +private fun String.providerMediaExpiry(): Long? { + val host = runCatching { URI(this).host.orEmpty() }.getOrDefault("") + val provider = providerForProxyHost(host) + if (provider != ProxyProvider.BILIBILI && provider != ProxyProvider.NICONICO) return null + val decoded = runCatching { URLDecoder.decode(this, StandardCharsets.UTF_8) }.getOrDefault(this) + return PROVIDER_EXPIRY_REGEX.findAll(decoded) + .mapNotNull { it.groupValues[1].toLongOrNull() } + .minOrNull() } -private fun String.isBilibiliSignedMediaUrl(): Boolean = - contains("bilibili", ignoreCase = true) || - contains("bilivideo", ignoreCase = true) || - contains("hdslb.com", ignoreCase = true) || - contains("akamaized", ignoreCase = true) +private val PROVIDER_EXPIRY_REGEX = + Regex("(?:^|[?&#=])(?:deadline|expires|exp)=(\\d+)", RegexOption.IGNORE_CASE) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedAvailability.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedAvailability.kt new file mode 100644 index 00000000..e29e0beb --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedAvailability.kt @@ -0,0 +1,14 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.VideoItem + +data class SubscriptionFeedAvailability( + val videos: List, + val available: Boolean, +) + +internal data class SubscriptionFeedWithSources( + val videos: List, + val available: Boolean, + val sourceChannelUrls: Map>, +) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt index 7581eb66..019bf267 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt @@ -10,6 +10,7 @@ import kotlinx.coroutines.CoroutineStart import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel import kotlinx.coroutines.joinAll import kotlinx.coroutines.launch import kotlinx.coroutines.withTimeoutOrNull @@ -23,8 +24,10 @@ class SubscriptionFeedService( channelService: ChannelService, cache: CacheService, private val clock: () -> Long = System::currentTimeMillis, - private val refreshScope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO), + refreshScope: CoroutineScope? = null, ) { + private val refreshScope = refreshScope ?: CoroutineScope(SupervisorJob() + Dispatchers.IO) + private val ownsRefreshScope = refreshScope == null private val store = SubscriptionFeedSnapshotStore(cache, clock) private val selections = SubscriptionFeedSelectionStore(cache, subscriptionsService) private val builder = SubscriptionFeedBuilder(channelService) @@ -99,15 +102,31 @@ class SubscriptionFeedService( } suspend fun getAll(userId: String): List { + return getAllWithAvailability(userId).videos + } + + suspend fun getAllWithAvailability(userId: String): SubscriptionFeedAvailability { val snapshot = store.current(userId) if (snapshot == null || snapshot.stale || clock() - snapshot.generatedAt >= FRESHNESS_MS) { scheduleRefresh(userId, currentRequestId()) } - if (snapshot != null) return snapshot.videos + if (snapshot != null) { + return SubscriptionFeedAvailability( + videos = snapshot.videos, + available = !snapshot.stale, + ) + } withTimeoutOrNull(INTERNAL_COLD_WAIT_MS) { awaitRefresh(userId) } - return store.current(userId)?.videos.orEmpty() + val refreshed = store.current(userId) + return SubscriptionFeedAvailability( + videos = refreshed?.videos.orEmpty(), + available = refreshed != null && !refreshed.stale, + ) } + internal suspend fun getAllWithSources(userId: String): SubscriptionFeedWithSources = + getAllWithAvailability(userId).let { SubscriptionFeedWithSources(it.videos, it.available, store.current(userId)?.sourceChannelUrls.orEmpty()) } + suspend fun getCachedFeed(userId: String, page: Int, limit: Int): SubscriptionFeedResponse? { val snapshot = store.current(userId) ?: return null return snapshot.page(page * limit, limit, isRefreshing(userId)) @@ -132,6 +151,12 @@ class SubscriptionFeedService( internal fun isRefreshing(userId: String): Boolean = refreshJobs[userId]?.isActive == true + fun close() { + refreshJobs.values.forEach(Job::cancel) + refreshJobs.clear() + if (ownsRefreshScope) refreshScope.cancel() + } + private fun scheduleRefresh(userId: String, requestId: String?) { val job = refreshScope.launch(start = CoroutineStart.LAZY) { var retry = false @@ -160,6 +185,7 @@ class SubscriptionFeedService( if (store.invalidationToken(userId) != invalidation) return true val valid = result.successfulSources > 0 || subscriptions.isEmpty() if (!valid) { + if (previous != null) store.markStale(userId) logger.warn( "subscription_feed event=refresh_kept_previous user={} durationMs={} failedSources={}", userKey(userId), clock() - startedAt, result.failedSources, diff --git a/src/main/kotlin/dev/typetype/server/services/UnifiedPushEndpointValidator.kt b/src/main/kotlin/dev/typetype/server/services/UnifiedPushEndpointValidator.kt new file mode 100644 index 00000000..4b59038e --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/UnifiedPushEndpointValidator.kt @@ -0,0 +1,67 @@ +package dev.typetype.server.services + +import java.net.InetAddress +import java.net.URI +import java.net.UnknownHostException + +internal class UnifiedPushEndpointValidator( + private val resolver: (String) -> Array = InetAddress::getAllByName, +) { + fun validate(raw: String): EndpointValidationResult { + if (raw.length !in 1..2048) return EndpointValidationResult.Invalid("endpoint_length") + val uri = runCatching { URI(raw) }.getOrNull() + ?: return EndpointValidationResult.Invalid("endpoint_uri") + if (uri.scheme?.lowercase() != "https" || uri.userInfo != null || uri.fragment != null) { + return EndpointValidationResult.Invalid("endpoint_scheme") + } + val host = uri.host?.trim()?.trim('[', ']')?.lowercase()?.takeIf { it.isNotBlank() } + ?: return EndpointValidationResult.Invalid("endpoint_host") + if (uri.port == 0 || uri.port < -1 || uri.port > 65535) { + return EndpointValidationResult.Invalid("endpoint_port") + } + val addresses = try { + resolver(host).toList() + } catch (_: UnknownHostException) { + return EndpointValidationResult.Invalid("endpoint_unresolvable") + } catch (_: Exception) { + return EndpointValidationResult.Invalid("endpoint_unresolvable") + } + if (addresses.isEmpty() || addresses.any(::isBlockedAddress)) { + return EndpointValidationResult.Invalid("endpoint_private_address") + } + return EndpointValidationResult.Valid(uri, addresses) + } + + private fun isBlockedAddress(address: InetAddress): Boolean { + if (address.isAnyLocalAddress || address.isLoopbackAddress || address.isLinkLocalAddress || + address.isSiteLocalAddress || address.isMulticastAddress + ) return true + val bytes = address.address + if (bytes.size == 16 && bytes.take(10).all { it == 0.toByte() } && bytes[10] == 0xff.toByte() && bytes[11] == 0xff.toByte()) { + return isBlockedIpv4(bytes.copyOfRange(12, 16)) + } + if (bytes.size == 16) { + val first = bytes[0].toInt() and 0xff + return first in 0xfc..0xfd + } + return bytes.size == 4 && isBlockedIpv4(bytes) + } + + private fun isBlockedIpv4(bytes: ByteArray): Boolean { + val first = bytes[0].toInt() and 0xff + val second = bytes[1].toInt() and 0xff + return first == 0 || first == 10 || first == 127 || + (first == 100 && second in 64..127) || + (first == 169 && second == 254) || + (first == 172 && second in 16..31) || + (first == 192 && second == 0) || + (first == 192 && second == 168) || + (first == 198 && second in 18..19) || + first >= 224 + } +} + +internal sealed interface EndpointValidationResult { + data class Valid(val uri: URI, val addresses: List) : EndpointValidationResult + data class Invalid(val reason: String) : EndpointValidationResult +} diff --git a/src/main/kotlin/dev/typetype/server/services/UnifiedPushSender.kt b/src/main/kotlin/dev/typetype/server/services/UnifiedPushSender.kt new file mode 100644 index 00000000..ac0a1d74 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/UnifiedPushSender.kt @@ -0,0 +1,68 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody +import java.net.InetAddress +import java.net.UnknownHostException + +internal class UnifiedPushSender( + private val endpointValidator: UnifiedPushEndpointValidator = UnifiedPushEndpointValidator(), + client: OkHttpClient = OkHttpClient(), +) : PushEndpointSender { + private val client = client.newBuilder() + .followRedirects(false) + .followSslRedirects(false) + .build() + + override suspend fun send(endpoint: String, eventId: String, payload: String): PushSendResult = withContext(Dispatchers.IO) { + val validated = endpointValidator.validate(endpoint) + if (validated !is EndpointValidationResult.Valid) return@withContext PushSendResult.InvalidEndpoint + val request = Request.Builder() + .url(validated.uri.toString()) + .header("Content-Type", JSON_MEDIA_TYPE.toString()) + .header("X-TypeType-Event-Id", eventId) + .post(payload.toRequestBody(JSON_MEDIA_TYPE)) + .build() + val requestClient = client.newBuilder() + .dns(StaticPushDns(validated.uri.host.orEmpty(), validated.addresses)) + .build() + runCatching { requestClient.newCall(request).execute().use { response -> response.code } } + .fold( + onSuccess = { code -> + when { + code in 200..299 -> PushSendResult.Delivered + code == 404 || code == 410 -> PushSendResult.InvalidEndpoint + else -> PushSendResult.Retry(code) + } + }, + onFailure = { PushSendResult.Retry(null) }, + ) + } + + private class StaticPushDns( + private val validatedHost: String, + private val addresses: List, + ) : okhttp3.Dns { + override fun lookup(hostname: String): List = + if (hostname.equals(validatedHost, ignoreCase = true)) addresses + else throw UnknownHostException("Unexpected push endpoint host") + } + + private companion object { + val JSON_MEDIA_TYPE = "application/json".toMediaType() + } +} + +internal interface PushEndpointSender { + suspend fun send(endpoint: String, eventId: String, payload: String): PushSendResult +} + +internal sealed interface PushSendResult { + data object Delivered : PushSendResult + data object InvalidEndpoint : PushSendResult + data class Retry(val statusCode: Int?) : PushSendResult +} diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCache.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCache.kt index 8095310e..45712d50 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCache.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCache.kt @@ -2,20 +2,45 @@ package dev.typetype.server.services import dev.typetype.server.models.YoutubeTakeoutParsedData import dev.typetype.server.models.YoutubeTakeoutPreviewItem +import java.time.Duration -class YoutubeTakeoutImportCache { - private val previewCache = mutableMapOf() - private val parsedCache = mutableMapOf() +class YoutubeTakeoutImportCache( + maxPreviewEntries: Int = DEFAULT_MAX_PREVIEW_ENTRIES, + maxParsedEntries: Int = DEFAULT_MAX_PARSED_ENTRIES, + ttl: Duration = DEFAULT_TTL, + clock: () -> Long = System::currentTimeMillis, +) { + private val previewCache = BoundedExpiringCache( + maxEntries = maxPreviewEntries, + ttl = ttl, + clock = clock, + ) + private val parsedCache = BoundedExpiringCache( + maxEntries = maxParsedEntries, + ttl = ttl, + clock = clock, + ) - fun getPreview(jobId: String): YoutubeTakeoutPreviewItem? = previewCache[jobId] + fun getPreview(jobId: String): YoutubeTakeoutPreviewItem? = previewCache.get(jobId) fun setPreview(jobId: String, preview: YoutubeTakeoutPreviewItem) { - previewCache[jobId] = preview + previewCache.put(jobId, preview) } - fun getParsed(jobId: String): YoutubeTakeoutParsedData? = parsedCache[jobId] + fun getParsed(jobId: String): YoutubeTakeoutParsedData? = parsedCache.get(jobId) fun setParsed(jobId: String, parsed: YoutubeTakeoutParsedData) { - parsedCache[jobId] = parsed + parsedCache.put(jobId, parsed) + } + + fun remove(jobId: String) { + previewCache.remove(jobId) + parsedCache.remove(jobId) + } + + private companion object { + const val DEFAULT_MAX_PREVIEW_ENTRIES = 256 + const val DEFAULT_MAX_PARSED_ENTRIES = 1 + val DEFAULT_TTL: Duration = Duration.ofMinutes(30) } } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobEngine.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobEngine.kt index 82b00653..b2daca0d 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobEngine.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobEngine.kt @@ -4,9 +4,10 @@ import dev.typetype.server.models.YoutubeTakeoutCommitPlan import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel import kotlinx.coroutines.launch -class YoutubeTakeoutImportJobEngine { +class YoutubeTakeoutImportJobEngine : AutoCloseable { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) fun startPreview(jobId: String, block: suspend () -> Unit): Unit { @@ -16,4 +17,8 @@ class YoutubeTakeoutImportJobEngine { fun startCommit(jobId: String, plan: YoutubeTakeoutCommitPlan, block: suspend (YoutubeTakeoutCommitPlan) -> Unit): Unit { scope.launch { block(plan) } } + + override fun close() { + scope.cancel() + } } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt index 086d0be8..e8a665b6 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt @@ -22,7 +22,7 @@ class YoutubeTakeoutImportJobService( private val privacyService: YoutubeTakeoutPrivacyService = YoutubeTakeoutPrivacyService(), private val cache: YoutubeTakeoutImportCache = YoutubeTakeoutImportCache(), private val engine: YoutubeTakeoutImportJobEngine = YoutubeTakeoutImportJobEngine(), -) { +) : AutoCloseable { suspend fun create(userId: String, archivePath: Path): YoutubeTakeoutImportJobStatus { val jobId = store.create(userId, archivePath) return statusStore.getStatus(userId, jobId) ?: error("Failed to create job") @@ -84,6 +84,8 @@ class YoutubeTakeoutImportJobService( privacyService.deleteArchive(archiveStore.getArchivePath(userId, jobId)) } catch (e: Exception) { statusStore.failStatus(jobId, "import_failed", e.importErrorMessage()) + } finally { + cache.remove(jobId) } } @@ -99,4 +101,8 @@ class YoutubeTakeoutImportJobService( } suspend fun purgeExpired() = YoutubeTakeoutImportCleanupService(privacyService).purgeExpiredJobs() + + override fun close() { + engine.close() + } } diff --git a/src/test/kotlin/dev/typetype/server/AccountProfilesRoutesTest.kt b/src/test/kotlin/dev/typetype/server/AccountProfilesRoutesTest.kt new file mode 100644 index 00000000..53641f7f --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/AccountProfilesRoutesTest.kt @@ -0,0 +1,162 @@ +package dev.typetype.server + +import dev.typetype.server.db.tables.UsersTable +import dev.typetype.server.models.AccountProfileItem +import dev.typetype.server.routes.accountProfilesRoutes +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.AuthSessionConfig +import dev.typetype.server.services.ProfileAccountService +import dev.typetype.server.services.ProfileMutationResult +import io.ktor.client.request.delete +import io.ktor.client.request.get +import io.ktor.client.request.headers +import io.ktor.client.request.post +import io.ktor.client.request.put +import io.ktor.client.request.setBody +import io.ktor.client.statement.bodyAsText +import io.ktor.http.ContentType +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.testApplication +import kotlinx.serialization.json.Json +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.transactions.transaction +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class AccountProfilesRoutesTest { + private val profileService = ProfileAccountService() + private val auth = AuthService.fixed(TEST_USER_ID) + + companion object { + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() { + TestDatabase.truncateAll() + transaction { + UsersTable.insert { + it[id] = TEST_USER_ID + it[email] = "profiles@test.local" + it[passwordHash] = "hash" + it[name] = "Profile" + it[role] = "user" + it[createdAt] = 0L + it[updatedAt] = 0L + } + } + } + + @Test + fun `profiles can be created renamed and made default`() = withApp { + val initial = client.get("/profiles") { bearer() } + assertEquals(HttpStatusCode.OK, initial.status) + assertTrue(initial.bodyAsText().contains("\"isDefault\":true")) + + val created = client.post("/profiles") { + bearer() + contentTypeJson() + setBody("{\"name\":\"Tech\"}") + } + assertEquals(HttpStatusCode.Created, created.status) + val profile = Json.decodeFromString(created.bodyAsText()) + + val renamed = client.put("/profiles/${profile.id}") { + bearer() + contentTypeJson() + setBody("{\"name\":\"Bricolage\"}") + } + assertEquals(HttpStatusCode.OK, renamed.status) + + val defaulted = client.post("/profiles/${profile.id}/default") { bearer() } + assertEquals(HttpStatusCode.OK, defaulted.status) + val listed = client.get("/profiles") { bearer() }.bodyAsText() + assertTrue(listed.contains("Bricolage")) + assertTrue(listed.contains("\"defaultProfileId\":\"${profile.id}\"")) + } + + @Test + fun `profile routes isolate owners and protect active deletion`() = withApp { + val created = client.post("/profiles") { + bearer() + contentTypeJson() + setBody("{\"name\":\"Private\"}") + } + val profile = Json.decodeFromString(created.bodyAsText()) + + transaction { + UsersTable.insert { + it[id] = "other-user-id" + it[email] = "other@test.local" + it[passwordHash] = "hash" + it[name] = "Other" + it[role] = "user" + it[createdAt] = 0L + it[updatedAt] = 0L + } + } + val foreignProfiles = profileService.list("other-user-id") + assertTrue(foreignProfiles?.profiles?.none { it.id == profile.id } == true) + + val activeDelete = client.delete("/profiles/${profile.id}") { bearer() } + assertEquals(HttpStatusCode.NoContent, activeDelete.status) + } + + @Test + fun `switch returns a new session and active profile cannot be deleted`() = withApp { + val created = client.post("/profiles") { + bearer() + contentTypeJson() + setBody("{\"name\":\"Travel\"}") + } + val profile = Json.decodeFromString(created.bodyAsText()) + val switched = client.post("/profiles/${profile.id}/switch") { bearer() } + assertEquals(HttpStatusCode.OK, switched.status) + assertTrue(switched.bodyAsText().contains("accessToken")) + + assertEquals(ProfileMutationResult.CannotDeleteActive, profileService.delete(profile.id, profile.id)) + } + + @Test + fun `deleting the default profile promotes the owner profile`() = withApp { + val created = client.post("/profiles") { + bearer() + contentTypeJson() + setBody("{\"name\":\"Temporary\"}") + } + val profile = Json.decodeFromString(created.bodyAsText()) + assertEquals(HttpStatusCode.OK, client.post("/profiles/${profile.id}/default") { bearer() }.status) + assertEquals(HttpStatusCode.OK, client.post("/profiles/${profile.id}/switch") { bearer() }.status) + assertEquals(HttpStatusCode.OK, client.post("/profiles/$TEST_USER_ID/switch") { bearer() }.status) + + assertEquals(ProfileMutationResult.Deleted, profileService.delete(TEST_USER_ID, profile.id)) + val listed = profileService.list(TEST_USER_ID) + assertEquals(TEST_USER_ID, listed?.defaultProfileId) + } + + private fun withApp(block: suspend io.ktor.server.testing.ApplicationTestBuilder.() -> Unit) = testApplication { + application { + install(ContentNegotiation) { json() } + routing { accountProfilesRoutes(profileService, auth, AuthSessionConfig()) } + } + block() + } + + private fun io.ktor.client.request.HttpRequestBuilder.bearer() { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + } + + private fun io.ktor.client.request.HttpRequestBuilder.contentTypeJson() { + headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + } +} diff --git a/src/test/kotlin/dev/typetype/server/AuthServiceCoreTest.kt b/src/test/kotlin/dev/typetype/server/AuthServiceCoreTest.kt index 56741b55..69d64acb 100644 --- a/src/test/kotlin/dev/typetype/server/AuthServiceCoreTest.kt +++ b/src/test/kotlin/dev/typetype/server/AuthServiceCoreTest.kt @@ -4,6 +4,8 @@ import dev.typetype.server.db.tables.UsersTable import dev.typetype.server.db.tables.SessionsTable import dev.typetype.server.services.AuthService import dev.typetype.server.services.AuthSessionConfig +import dev.typetype.server.services.ProfileAccountService +import dev.typetype.server.services.ProfileMutationResult import kotlinx.coroutines.test.runTest import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.selectAll @@ -111,6 +113,18 @@ class AuthServiceCoreTest { assertEquals(userId, byUsername?.let { service.verify(it.accessToken) }) } + @Test + fun `secondary profile credentials cannot be used as a local login`() = runTest { + val profiles = ProfileAccountService() + val service = AuthService("test-secret", profileAccountService = profiles) + val ownerSession = service.register("profile-owner@test.local", "secret-1", "Owner") + val ownerId = service.verify(ownerSession.accessToken) ?: error("missing owner id") + val child = profiles.create(ownerId, "Child") + val childId = (child as ProfileMutationResult.Success).profile.id + + assertNull(service.login("profile-$childId@profiles.invalid", "profile:$childId")) + } + @Test fun `guest token verifies and has user role`() = runTest { val service = AuthService("test-secret") diff --git a/src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt b/src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt index 796c59cc..c1910e04 100644 --- a/src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt +++ b/src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt @@ -10,6 +10,7 @@ import okhttp3.Response import okhttp3.ResponseBody.Companion.toResponseBody import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Test import java.io.IOException import java.net.InetAddress @@ -28,7 +29,7 @@ class BilibiliRangeProxyTest { assertEquals(OkHttpProxyService.BILIBILI_USER_AGENT, request.header("User-Agent")) assertEquals("https://www.bilibili.com", request.header("Referer")) assertEquals("*/*", request.header("Accept")) - assertEquals("close", request.header("Connection")) + assertNull(request.header("Connection")) assertEquals("bytes=0-3", request.header("Range")) if (calls == 1) throw IOException("unexpected end of stream") Response.Builder() @@ -53,6 +54,6 @@ class BilibiliRangeProxyTest { assertEquals(2, calls) assertEquals(206, result.data.status) assertEquals("bytes 0-3/4", result.data.contentRange) - assertArrayEquals(bytes, result.data.stream.readBytes()) + result.data.stream.use { assertArrayEquals(bytes, it.readBytes()) } } } diff --git a/src/test/kotlin/dev/typetype/server/BilibiliRelatedServiceTest.kt b/src/test/kotlin/dev/typetype/server/BilibiliRelatedServiceTest.kt new file mode 100644 index 00000000..a72bec0c --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/BilibiliRelatedServiceTest.kt @@ -0,0 +1,54 @@ +package dev.typetype.server + +import dev.typetype.server.services.BilibiliRelatedService +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class BilibiliRelatedServiceTest { + + @Test + fun `caches uploader lookups shared by repeated stream extraction`() = runTest { + var lookups = 0 + val service = BilibiliRelatedService { _ -> + lookups++ + mapOf("BV1Related" to "https://space.bilibili.com/42") + } + val response = testStreamResponse().copy( + relatedStreams = listOf( + testVideoItem().copy( + url = "https://www.bilibili.com/video/BV1Related", + uploaderUrl = "", + ), + ), + ) + + val first = service.patchRelatedStreams(response, SOURCE_URL) + val second = service.patchRelatedStreams(response, SOURCE_URL) + + assertEquals("https://space.bilibili.com/42", first.relatedStreams.single().uploaderUrl) + assertEquals("https://space.bilibili.com/42", second.relatedStreams.single().uploaderUrl) + assertEquals(1, lookups) + } + + @Test + fun `skips lookup when source or related item has no BVID`() = runTest { + var lookups = 0 + val service = BilibiliRelatedService { _ -> + lookups++ + emptyMap() + } + val response = testStreamResponse().copy( + relatedStreams = listOf(testVideoItem().copy(url = "https://example.com/video")), + ) + + val result = service.patchRelatedStreams(response, "https://example.com/source") + + assertEquals(response, result) + assertEquals(0, lookups) + } + + private companion object { + const val SOURCE_URL = "https://www.bilibili.com/video/BV1Source" + } +} diff --git a/src/test/kotlin/dev/typetype/server/HlsManifestServiceCacheTest.kt b/src/test/kotlin/dev/typetype/server/HlsManifestServiceCacheTest.kt index ada3a2a6..2a54a629 100644 --- a/src/test/kotlin/dev/typetype/server/HlsManifestServiceCacheTest.kt +++ b/src/test/kotlin/dev/typetype/server/HlsManifestServiceCacheTest.kt @@ -41,6 +41,28 @@ class HlsManifestServiceCacheTest { assertEquals(1, calls) } + @Test + fun `provider manifests are fetched again instead of serving signed cache entries`() = runTest { + var calls = 0 + val client = proxyTestClient(Interceptor { chain -> + calls += 1 + Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body("#EXTM3U\nsegment.m4s".toResponseBody("application/vnd.apple.mpegurl".toMediaType())) + .build() + }) + val service = HlsManifestService(NoopStreamService, client, InMemoryCache()) + val url = "https://upos-hz-mirrorakam.akamaized.net/master.m3u8?deadline=2000000000" + + service.hlsManifest(url) + service.hlsManifest(url) + + assertEquals(2, calls) + } + @Test fun `attested manifest is scoped to youtube live`() = runTest { val requestedUrls = mutableListOf() diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationCandidateServiceTest.kt b/src/test/kotlin/dev/typetype/server/HomeRecommendationCandidateServiceTest.kt index ccf32409..63872e33 100644 --- a/src/test/kotlin/dev/typetype/server/HomeRecommendationCandidateServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/HomeRecommendationCandidateServiceTest.kt @@ -12,6 +12,7 @@ import dev.typetype.server.services.HomeRecommendationSourceTag import dev.typetype.server.services.StreamService import dev.typetype.server.services.SubscriptionFeedService import dev.typetype.server.services.SubscriptionShortsFeedService +import dev.typetype.server.services.TrendingService import io.mockk.coEvery import io.mockk.mockk import kotlinx.coroutines.test.runTest @@ -23,8 +24,9 @@ class HomeRecommendationCandidateServiceTest { private val subscriptionFeedService: SubscriptionFeedService = mockk() private val subscriptionShortsFeedService: SubscriptionShortsFeedService = mockk() private val streamService: StreamService = mockk() + private val trendingService: TrendingService = mockk() private val service = HomeRecommendationCandidateService( - subscriptionFeedService, subscriptionShortsFeedService, streamService, + subscriptionFeedService, subscriptionShortsFeedService, streamService, trendingService, ) @BeforeEach @@ -33,6 +35,7 @@ class HomeRecommendationCandidateServiceTest { coEvery { subscriptionFeedService.getFeed(any(), any(), any()) } returns SubscriptionFeedResponse(emptyList(), null) coEvery { subscriptionShortsFeedService.getBlendedFeed(any(), any(), any(), any()) } returns SubscriptionFeedResponse(emptyList(), null) coEvery { streamService.getStreamInfo(any()) } returns ExtractionResult.Failure("none") + coEvery { trendingService.getTrending(any()) } returns ExtractionResult.Success(emptyList()) } @Test @@ -60,6 +63,32 @@ class HomeRecommendationCandidateServiceTest { assertTrue(pool.discovery.isEmpty()) } + @Test + fun `bilibili mode excludes subscriptions from other services`() = runTest { + val youtube = video("yt", "YouTube") + val bilibili = video("bili", "BiliBili", "https://www.bilibili.com/video/BV1234567890") + coEvery { subscriptionFeedService.getCachedFeed(any(), any(), any()) } returns + SubscriptionFeedResponse(listOf(youtube, bilibili), null) + + val pool = service.fetchCandidates("u", 5, profile(), HomeRecommendationPoolMode.FAST) + + assertTrue(pool.subscriptions.map { it.video.id } == listOf("bili")) + } + + @Test + fun `niconico mode falls back to niconico trending videos`() = runTest { + val youtube = video("yt", "YouTube") + val niconico = video("nico", "NicoNico", "https://www.nicovideo.jp/watch/sm123") + coEvery { subscriptionFeedService.getCachedFeed(any(), any(), any()) } returns + SubscriptionFeedResponse(listOf(youtube), null) + coEvery { trendingService.getTrending(6) } returns ExtractionResult.Success(listOf(youtube, niconico)) + + val pool = service.fetchCandidates("u", 6, profile(), HomeRecommendationPoolMode.FAST) + + assertTrue(pool.subscriptions.isEmpty()) + assertTrue(pool.discovery.map { it.video.id } == listOf("nico")) + } + private fun profile(): HomeRecommendationProfile = HomeRecommendationProfile( seenUrls = emptySet(), blockedVideos = emptySet(), blockedChannels = emptySet(), feedbackBlockedVideos = emptySet(), feedbackBlockedChannels = emptySet(), @@ -79,8 +108,8 @@ class HomeRecommendationCandidateServiceTest { sponsorBlockSegments = emptyList(), relatedStreams = related, publishedAt = 0, ) - private fun video(id: String, title: String): VideoItem = VideoItem( - id = id, title = title, url = "https://yt.com/v/$id", thumbnailUrl = "", uploaderName = "channel", + private fun video(id: String, title: String, url: String = "https://yt.com/v/$id"): VideoItem = VideoItem( + id = id, title = title, url = url, thumbnailUrl = "", uploaderName = "channel", uploaderUrl = "https://yt.com/c/channel", uploaderAvatarUrl = "", duration = 60, viewCount = 0, uploadDate = "", uploaded = System.currentTimeMillis(), streamType = "video_stream", isShortFormContent = false, uploaderVerified = false, shortDescription = null, diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationServiceFastPathTest.kt b/src/test/kotlin/dev/typetype/server/HomeRecommendationServiceFastPathTest.kt index 0ddf66a9..08e48e41 100644 --- a/src/test/kotlin/dev/typetype/server/HomeRecommendationServiceFastPathTest.kt +++ b/src/test/kotlin/dev/typetype/server/HomeRecommendationServiceFastPathTest.kt @@ -46,6 +46,7 @@ class HomeRecommendationServiceFastPathTest { TestDatabase.truncateAll() coEvery { cache.get(any()) } returns null coEvery { cache.set(any(), any(), any()) } returns Unit + coEvery { trendingService.getTrending(any()) } returns ExtractionResult.Success(emptyList()) coEvery { searchService.search(any(), any(), any(), any(), any()) } returns ExtractionResult.Success(SearchPageResponse(emptyList(), null, null, false)) } diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationTestFixtures.kt b/src/test/kotlin/dev/typetype/server/HomeRecommendationTestFixtures.kt index 8e6e7992..e4e95c43 100644 --- a/src/test/kotlin/dev/typetype/server/HomeRecommendationTestFixtures.kt +++ b/src/test/kotlin/dev/typetype/server/HomeRecommendationTestFixtures.kt @@ -36,6 +36,7 @@ fun homeResolverDependencies( favoritesService = FavoritesService(), watchLaterService = WatchLaterService(), blockedService = BlockedService(), + trendingService = trendingService, cache = cache, ) diff --git a/src/test/kotlin/dev/typetype/server/NotificationItemPublishedAtSerializationTest.kt b/src/test/kotlin/dev/typetype/server/NotificationItemPublishedAtSerializationTest.kt index cbdaaefe..7c50cf17 100644 --- a/src/test/kotlin/dev/typetype/server/NotificationItemPublishedAtSerializationTest.kt +++ b/src/test/kotlin/dev/typetype/server/NotificationItemPublishedAtSerializationTest.kt @@ -19,6 +19,8 @@ class NotificationItemPublishedAtSerializationTest { channelUrl = "https://yt.com/c/a", channelName = "A", channelAvatarUrl = "", + serviceId = 0, + serviceName = "YouTube", video = VideoItem( id = "id", title = "video", diff --git a/src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt b/src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt index 92d21d32..add61cbf 100644 --- a/src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt @@ -4,9 +4,14 @@ import dev.typetype.server.SubscriptionFeedTestFixtures.channel import dev.typetype.server.SubscriptionFeedTestFixtures.subscription import dev.typetype.server.SubscriptionFeedTestFixtures.video import dev.typetype.server.routes.notificationsRoutes +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.NotificationReadItemsTable +import dev.typetype.server.db.tables.NotificationStatesTable +import dev.typetype.server.models.NotificationsResponse import dev.typetype.server.services.AuthService import dev.typetype.server.services.ChannelService import dev.typetype.server.services.NotificationsService +import dev.typetype.server.services.SubscriptionFeedAvailability import dev.typetype.server.services.SubscriptionFeedService import dev.typetype.server.services.SubscriptionsService import io.ktor.client.request.get @@ -23,6 +28,9 @@ import io.ktor.server.testing.ApplicationTestBuilder import io.ktor.server.testing.testApplication import io.mockk.coEvery import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import kotlinx.serialization.json.Json +import org.jetbrains.exposed.v1.jdbc.insert import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeAll @@ -64,17 +72,26 @@ class NotificationsRoutesTest { } @Test - fun `GET notifications returns latest per channel with unread count`() = withApp { + fun `GET notifications returns every new video with service identity`() = withApp { subscriptionsService.add(TEST_USER_ID, subscription("https://yt.com/c/a", "A")) subscriptionsService.add(TEST_USER_ID, subscription("https://yt.com/c/b", "B")) - coEvery { channelService.getChannel("https://yt.com/c/a", null) } returns channel(video(1000L, "A"), video(3000L, "A")) - coEvery { channelService.getChannel("https://yt.com/c/b", null) } returns channel(video(2000L, "B")) + coEvery { channelService.getChannel("https://yt.com/c/a", null) } returns channel( + video(1000L, "A", "https://www.youtube.com/watch?v=yt-old"), + video(3000L, "A", "https://www.youtube.com/watch?v=yt-new"), + ) + coEvery { channelService.getChannel("https://yt.com/c/b", null) } returns channel( + video(2000L, "A", "https://www.bilibili.com/video/av2000"), + video(4000L, "A", "https://www.nicovideo.jp/watch/sm4000"), + ) val body = client.get("/notifications?page=0&limit=10") { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") }.bodyAsText() - assertTrue(body.contains("\"unreadCount\":2")) - assertTrue(body.indexOf("3000") < body.indexOf("2000")) - assertTrue(!body.contains("1000")) + assertTrue(body.contains("\"unreadCount\":4")) + assertTrue(body.indexOf("4000") < body.indexOf("3000")) + assertTrue(body.contains("yt-new")) + assertTrue(body.contains("yt-old")) + assertTrue(body.contains("\"serviceName\":\"BiliBili\"")) + assertTrue(body.contains("\"serviceName\":\"NicoNico\"")) } @Test @@ -114,4 +131,64 @@ class NotificationsRoutesTest { }.bodyAsText() assertTrue(after.contains("\"unreadCount\":0")) } + + @Test + fun `read watermark remains effective when one notification has an explicit read row`() = withApp { + subscriptionsService.add(TEST_USER_ID, subscription("https://yt.com/c/a", "A")) + coEvery { channelService.getChannel("https://yt.com/c/a", null) } returns channel( + video(1000L, "A", "https://www.youtube.com/watch?v=old"), + video(2000L, "A", "https://www.youtube.com/watch?v=new"), + ) + val initial = Json.decodeFromString(client.get("/notifications?page=0&limit=10") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + }.bodyAsText()) + val latest = initial.items.maxBy { it.createdAt } + DatabaseFactory.query { + NotificationStatesTable.insert { + it[userId] = TEST_USER_ID + it[subscriptionLastSeenUploaded] = initial.items.maxOf { item -> item.createdAt } + it[updatedAt] = System.currentTimeMillis() + } + NotificationReadItemsTable.insert { + it[userId] = TEST_USER_ID + it[notificationId] = latest.id + it[readAt] = System.currentTimeMillis() + } + } + val result = Json.decodeFromString(client.get("/notifications?page=0&limit=10") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + }.bodyAsText()) + assertTrue(result.items.all { it.read }) + assertEquals(0, result.unreadCount) + } + + @Test + fun `cached notifications remain visible when refresh fails`() = runTest { + val feed = mockk() + coEvery { feed.getAllWithAvailability(TEST_USER_ID) } returns + SubscriptionFeedAvailability(listOf(video(1000L, "A")), false) + val service = NotificationsService(feed) + val response = service.getNotifications(TEST_USER_ID, 0, 20) + assertEquals(1, response.items.size) + assertEquals(1, response.unreadCount) + assertTrue(!response.available) + val count = service.getUnreadCount(TEST_USER_ID) + assertEquals(1, count.unreadCount) + assertTrue(!count.available) + } + + @Test + fun `unread count reports unavailable instead of trusting a fresh cache`() = runTest { + val feed = mockk() + val availableVideo = video(1000L, "A") + coEvery { feed.getAllWithAvailability(TEST_USER_ID) } returnsMany listOf( + SubscriptionFeedAvailability(listOf(availableVideo), true), + SubscriptionFeedAvailability(emptyList(), false), + ) + val service = NotificationsService(feed) + assertEquals(1, service.getUnreadCount(TEST_USER_ID).unreadCount) + val unavailable = service.getUnreadCount(TEST_USER_ID) + assertEquals(1, unavailable.unreadCount) + assertTrue(!unavailable.available) + } } diff --git a/src/test/kotlin/dev/typetype/server/ProviderMediaHandleRoutesTest.kt b/src/test/kotlin/dev/typetype/server/ProviderMediaHandleRoutesTest.kt new file mode 100644 index 00000000..959227f7 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/ProviderMediaHandleRoutesTest.kt @@ -0,0 +1,63 @@ +package dev.typetype.server + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.ProxyResponse +import dev.typetype.server.routes.providerMediaHandleRoutes +import dev.typetype.server.services.ProviderMediaHandleService +import dev.typetype.server.services.ProxyService +import io.ktor.client.request.get +import io.ktor.client.request.header +import io.ktor.client.statement.bodyAsText +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.testApplication +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.io.ByteArrayInputStream + +class ProviderMediaHandleRoutesTest { + private val proxyService: ProxyService = mockk() + + @Test + fun `media route forwards opaque handles and ranges`() = runTest { + val service = ProviderMediaHandleService(FakeCacheService()) + val raw = "https://upos-hz-mirrorakam.akamaized.net/video.m4s?deadline=9999999999" + val path = service.createPath(raw) + coEvery { proxyService.pipe(raw, "bytes=0-3", null) } returns ExtractionResult.Success( + ProxyResponse(206, "video/mp4", 4, "bytes 0-3/4", "bytes", ByteArrayInputStream(byteArrayOf(1, 2, 3, 4)), {}), + ) + + testApplication { + application { + install(ContentNegotiation) { json() } + routing { providerMediaHandleRoutes(service, proxyService) } + } + val response = client.get(path) { header("Range", "bytes=0-3") } + assertEquals(HttpStatusCode.PartialContent, response.status) + assertEquals("\u0001\u0002\u0003\u0004", response.bodyAsText()) + } + + coVerify(exactly = 1) { proxyService.pipe(raw, "bytes=0-3", null) } + } + + @Test + fun `expired or unknown handles return not found`() = testApplication { + val service = ProviderMediaHandleService(FakeCacheService()) + application { + install(ContentNegotiation) { json() } + routing { providerMediaHandleRoutes(service, proxyService) } + } + + val response = client.get("/media/m1_123456789012345678901234") + assertEquals(HttpStatusCode.NotFound, response.status) + assertTrue(response.bodyAsText().contains("media_handle_not_found")) + } +} diff --git a/src/test/kotlin/dev/typetype/server/ProviderMediaHandleServiceTest.kt b/src/test/kotlin/dev/typetype/server/ProviderMediaHandleServiceTest.kt new file mode 100644 index 00000000..d0289acd --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/ProviderMediaHandleServiceTest.kt @@ -0,0 +1,210 @@ +package dev.typetype.server + +import dev.typetype.server.cache.CacheService +import dev.typetype.server.services.ProviderMediaHandleService +import dev.typetype.server.services.ProviderMediaType +import dev.typetype.server.services.rewriteProviderHlsManifest +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.async +import kotlinx.coroutines.delay +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger + +class ProviderMediaHandleServiceTest { + @Test + fun `provider URLs become short cache-backed paths`() = runTest { + val service = ProviderMediaHandleService(FakeCacheService(), nowSeconds = { 1_000L }) + val raw = "https://upos-hz-mirrorakam.akamaized.net/video.m4s?deadline=2000000000&cpn=tracking" + + val path = service.createPath(raw) + val handle = path.removePrefix("/media/") + val target = service.resolve(handle) + + assertTrue(path.matches(Regex("/media/m1_[A-Za-z0-9_-]{24}"))) + assertFalse(path.contains("akamaized")) + assertEquals("https://upos-hz-mirrorakam.akamaized.net/video.m4s?deadline=2000000000", target?.url) + assertEquals(path, service.createPath(raw)) + } + + @Test + fun `Nico cookie fragment is stored beside the opaque target`() = runTest { + val service = ProviderMediaHandleService(FakeCacheService(), nowSeconds = { 1_000L }) + val path = service.createPath( + "https://asset.domand.nicovideo.jp/video/01.cmfv?Expires=2000000000#cookie=domand_bid%3Dcookie123&length=10", + ) + val target = service.resolve(path.removePrefix("/media/")) + + assertEquals("cookie123", target?.domandBid) + assertFalse(target?.url.orEmpty().contains("#cookie")) + assertEquals("../media/${path.substringAfterLast('/')}", service.relativeManifestPath(path)) + } + + @Test + fun `unknown handles do not resolve`() = runTest { + val service = ProviderMediaHandleService(FakeCacheService()) + assertNull(service.resolve("m1_invalid")) + } + + @Test + fun `expired provider URLs are not materialized`() = runTest { + val service = ProviderMediaHandleService(FakeCacheService(), nowSeconds = { 2_000_000_000L }) + + assertThrows(IllegalArgumentException::class.java) { + kotlinx.coroutines.runBlocking { + service.createPath("https://upos-hz-mirrorakam.akamaized.net/video.m4s?deadline=2000000000") + } + } + } + + @Test + fun `near expiry handles keep the remaining upstream lifetime`() = runTest { + val cache = RecordingProviderCacheService() + val service = ProviderMediaHandleService(cache, nowSeconds = { 1_000L }) + + service.createPath("https://upos-hz-mirrorakam.akamaized.net/video.m4s?deadline=1050") + + assertEquals(20L, cache.ttlSeconds) + } + + @Test + fun `provider HLS rewrite can map every media line`() = runTest { + val manifest = "#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\nsegment.m4s" + val result = rewriteProviderHlsManifest( + manifest, + "https://upos-hz-mirrorakam.akamaized.net/path/master.m3u8", + ) { "../media/handle" } + + assertTrue(result.contains("URI=\"../media/handle\"")) + assertTrue(result.endsWith("../media/handle")) + } + + @Test + fun `provider HLS rewrite maps unique references concurrently`() = runTest { + val active = AtomicInteger() + val maximum = AtomicInteger() + val calls = AtomicInteger() + val manifest = buildString { + repeat(32) { + append("segment-$it.m4s\n") + } + append("segment-0.m4s") + } + + val result = rewriteProviderHlsManifest( + manifest, + "https://upos-hz-mirrorakam.akamaized.net/path/master.m3u8", + ) { target -> + calls.incrementAndGet() + maximum.accumulateAndGet(active.incrementAndGet()) { left, right -> maxOf(left, right) } + delay(1) + active.decrementAndGet() + "../media/${target.substringAfterLast('/')}" + } + + assertEquals(32, calls.get()) + assertTrue(maximum.get() > 1) + assertTrue(result.contains("../media/segment-31.m4s")) + } + + @Test + fun `concurrent handle creation shares one cache write`() = runTest { + val cache = BlockingProviderCacheService() + val service = ProviderMediaHandleService(cache, nowSeconds = { 1_000L }) + val raw = "https://upos-hz-mirrorakam.akamaized.net/video.m4s?deadline=2000000000" + + val first = async { service.createPath(raw) } + cache.started.await() + val second = async { service.createPath(raw) } + cache.release.complete(Unit) + + assertEquals(first.await(), second.await()) + assertEquals(1, cache.setCalls.get()) + } + + @Test + fun `media materialization only handles the selected provider`() = runTest { + val service = ProviderMediaHandleService(FakeCacheService()) + val response = testStreamResponse( + videoOnlyStreams = listOf( + testVideoStream("https://upos-hz-mirrorakam.akamaized.net/video.m4s?deadline=9999999999"), + ), + ) + + val handled = service.materialize(response, ProviderMediaType.BILIBILI) + assertTrue(handled.videoOnlyStreams.single().url.startsWith("/media/m1_")) + } + + @Test + fun `materialization covers optional stream media paths`() = runTest { + val service = ProviderMediaHandleService(FakeCacheService()) + val video = testVideoStream( + "https://upos-hz-mirrorakam.akamaized.net/video.m4s?deadline=9999999999", + ).copy( + manifestUrl = "https://upos-hz-mirrorakam.akamaized.net/video.m3u8?deadline=9999999999", + sabrSessionUrl = "https://upos-hz-mirrorakam.akamaized.net/session?deadline=9999999999", + ) + + val handled = service.materialize( + testStreamResponse(videoOnlyStreams = listOf(video)), + ProviderMediaType.BILIBILI, + ) + + assertTrue(handled.videoOnlyStreams.single().manifestUrl?.startsWith("/media/m1_") == true) + assertTrue(handled.videoOnlyStreams.single().sabrSessionUrl?.startsWith("/media/m1_") == true) + } + + @Test + fun `protocol relative provider URLs are materialized`() = runTest { + val service = ProviderMediaHandleService(FakeCacheService()) + val response = testStreamResponse( + videoOnlyStreams = listOf( + testVideoStream("//upos-hz-mirrorakam.akamaized.net/video.m4s?deadline=9999999999"), + ), + ) + + val handled = service.materialize(response, ProviderMediaType.BILIBILI) + + assertTrue(handled.videoOnlyStreams.single().url.startsWith("/media/m1_")) + } +} + +private class RecordingProviderCacheService : CacheService { + private val values = mutableMapOf() + var ttlSeconds: Long = 0 + + override suspend fun set(key: String, value: String, ttlSeconds: Long) { + this.ttlSeconds = ttlSeconds + values[key] = value + } + + override suspend fun get(key: String): String? = values[key] + + override suspend fun delete(key: String) { values.remove(key) } +} + +private class BlockingProviderCacheService : CacheService { + val started = CompletableDeferred() + val release = CompletableDeferred() + val setCalls = AtomicInteger() + private val values = ConcurrentHashMap() + + override suspend fun set(key: String, value: String, ttlSeconds: Long) { + setCalls.incrementAndGet() + values[key] = value + started.complete(Unit) + release.await() + } + + override suspend fun get(key: String): String? = values[key] + + override suspend fun delete(key: String) { + values.remove(key) + } +} diff --git a/src/test/kotlin/dev/typetype/server/PushDeviceRegistryTest.kt b/src/test/kotlin/dev/typetype/server/PushDeviceRegistryTest.kt new file mode 100644 index 00000000..0e51f6a9 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/PushDeviceRegistryTest.kt @@ -0,0 +1,45 @@ +package dev.typetype.server + +import dev.typetype.server.models.PushDeviceRegistrationRequest +import dev.typetype.server.services.DeviceRegistrationResult +import dev.typetype.server.services.EndpointValidationResult +import dev.typetype.server.services.PushDeviceRegistry +import dev.typetype.server.services.UnifiedPushEndpointValidator +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import java.net.InetAddress + +class PushDeviceRegistryTest { + private val validator = UnifiedPushEndpointValidator { + arrayOf(InetAddress.getByAddress(byteArrayOf(93.toByte(), 184.toByte(), 216.toByte(), 34))) + } + private val registry = PushDeviceRegistry(validator) + + companion object { + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() = TestDatabase.truncateAll() + + @Test + fun `registration replaces one device and prevents endpoint sharing`() = runTest { + val first = registry.register("user-a", request("device-a", "https://push.example/a")) + assertTrue(first is DeviceRegistrationResult.Success) + val replacement = registry.register("user-a", request("device-a", "https://push.example/b")) + assertTrue(replacement is DeviceRegistrationResult.Success) + assertEquals(1, registry.list("user-a").size) + assertEquals( + DeviceRegistrationResult.EndpointConflict, + registry.register("user-b", request("device-b", "https://push.example/b")), + ) + } + + private fun request(deviceId: String, endpoint: String) = PushDeviceRegistrationRequest(deviceId, endpoint = endpoint) +} diff --git a/src/test/kotlin/dev/typetype/server/PushNotificationDeliveryStoreTest.kt b/src/test/kotlin/dev/typetype/server/PushNotificationDeliveryStoreTest.kt new file mode 100644 index 00000000..a5038b89 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/PushNotificationDeliveryStoreTest.kt @@ -0,0 +1,31 @@ +package dev.typetype.server + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.services.PushNotificationDeliveryStore +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class PushNotificationDeliveryStoreTest { + private val store = PushNotificationDeliveryStore { 1_000L } + + companion object { + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() = TestDatabase.truncateAll() + + @Test + fun `does not create historical delivery and retries an existing pending delivery`() = runTest { + assertFalse(DatabaseFactory.query { store.claim("event", "user", "device", false, 1_000L) }) + assertTrue(DatabaseFactory.query { store.claim("event", "user", "device", true, 1_000L) }) + assertFalse(DatabaseFactory.query { store.claim("event", "user", "device", false, 1_001L) }) + assertTrue(DatabaseFactory.query { store.claim("event", "user", "device", false, 31_001L) }) + } +} diff --git a/src/test/kotlin/dev/typetype/server/PushNotificationSupportTest.kt b/src/test/kotlin/dev/typetype/server/PushNotificationSupportTest.kt new file mode 100644 index 00000000..3b6a18fe --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/PushNotificationSupportTest.kt @@ -0,0 +1,33 @@ +package dev.typetype.server + +import dev.typetype.server.models.UnifiedPushNotificationPayload +import dev.typetype.server.services.PushCandidate +import dev.typetype.server.services.PushNotificationSupport +import kotlinx.serialization.json.Json +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class PushNotificationSupportTest { + @Test + fun `payload identifies the provider independently of channel names`() { + val candidate = PushCandidate( + serviceId = 5, + channelId = "https://www.bilibili.com/space/42", + videoId = "BV1", + publishedAt = 1_000L, + video = SubscriptionFeedTestFixtures.video( + uploaded = 1L, + channel = "Shared name", + url = "https://www.bilibili.com/video/BV1", + ), + ) + + val payload = Json.decodeFromString( + PushNotificationSupport.payload("instance", "event", candidate, "profile"), + ) + assertEquals(5, payload.serviceId) + assertEquals("BiliBili", payload.serviceName) + assertEquals("event", payload.eventId) + assertEquals("profile", payload.accountId) + } +} diff --git a/src/test/kotlin/dev/typetype/server/RetryingProxyInputStreamTest.kt b/src/test/kotlin/dev/typetype/server/RetryingProxyInputStreamTest.kt new file mode 100644 index 00000000..5f6a17e1 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/RetryingProxyInputStreamTest.kt @@ -0,0 +1,264 @@ +package dev.typetype.server + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.services.readBilibiliRangeWithRetry +import kotlinx.coroutines.CancellationException +import okhttp3.MediaType +import okhttp3.Protocol +import okhttp3.Request +import okhttp3.Response +import okhttp3.ResponseBody +import okio.Buffer +import okio.Source +import okio.Timeout +import okio.buffer +import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable +import java.io.IOException +import java.io.OutputStream +import java.lang.management.ManagementFactory + +class RetryingProxyInputStreamTest { + private val request = Request.Builder().url("https://video.bilivideo.com/test.m4s") + .header("Range", "bytes=0-").build() + + @Test + fun `large range is read on demand without a body-sized allocation`() { + val body = GeneratedBody(3L * 1024 * 1024 * 1024) + val proxy = open { response(body) } + assertEquals(0L, body.readBytes) + assertEquals(body.length, proxy.contentLength) + val bytes = ByteArray(64 * 1024) + assertTrue(proxy.stream.read(bytes) > 0) + assertTrue(body.readBytes <= bytes.size) + proxy.close() + assertTrue(body.closed) + } + + @Test + fun `truncated response replays verified prefix without duplicate bytes`() { + val bodies = listOf(GeneratedBody(100_000, failAt = 20_000), GeneratedBody(100_000)) + var calls = 0 + val proxy = open { next -> + assertEquals(request.header("Range"), next.header("Range")) + response(bodies[calls++]) + } + proxy.stream.use { assertArrayEquals(ByteArray(100_000) { 42 }, it.readBytes()) } + assertEquals(2, calls) + assertTrue(bodies.all { it.closed }) + } + + @Test + fun `strong etag permits replay without hashing all delivered media`() { + val bodies = listOf(GeneratedBody(100, failAt = 20), GeneratedBody(100)) + var calls = 0 + val proxy = open { response(bodies[calls++]).newBuilder().header("ETag", "\"stable\"").build() } + proxy.stream.use { assertArrayEquals(ByteArray(100) { 42 }, it.readBytes()) } + assertEquals(2, calls) + assertTrue(bodies.all { it.closed }) + } + + @Test + fun `weak etag still requires identical replay bytes`() { + val bodies = listOf(GeneratedBody(100, failAt = 20), GeneratedBody(100, value = 43)) + var calls = 0 + val proxy = open { response(bodies[calls++]).newBuilder().header("ETag", "W/\"stable\"").build() } + proxy.stream.use { + assertEquals(20, it.read(ByteArray(100))) + assertThrows(IOException::class.java) { it.read(ByteArray(100)) } + } + assertEquals(2, calls) + assertTrue(bodies.all { it.closed }) + } + + @Test + fun `changed replay prefix is rejected before any new bytes are sent`() { + val bodies = listOf(GeneratedBody(100, failAt = 20), GeneratedBody(100, value = 43)) + var calls = 0 + val proxy = open { response(bodies[calls++]) } + proxy.stream.use { + assertEquals(20, it.read(ByteArray(100))) + assertThrows(IOException::class.java) { it.read(ByteArray(100)) } + } + assertEquals(2, calls) + assertTrue(bodies.all { it.closed }) + } + + @Test + fun `changed response metadata is rejected and closed`() { + val bodies = listOf(GeneratedBody(100, failAt = 20), GeneratedBody(100)) + var calls = 0 + val proxy = open { + response(bodies[calls++]).newBuilder().header("ETag", "\"$calls\"").build() + } + proxy.stream.use { + assertEquals(20, it.read(ByteArray(100))) + assertThrows(IOException::class.java) { it.read(ByteArray(100)) } + } + assertEquals(2, calls) + assertTrue(bodies.all { it.closed }) + } + + @Test + fun `retry budget includes transport errors and replay failures`() { + var calls = 0 + val bodies = mutableListOf() + val proxy = open { + calls++ + if (calls == 1) throw IOException("connect failed") + val body = GeneratedBody(100, failAt = if (calls == 2) 20 else 10) + bodies += body + response(body) + } + proxy.stream.use { + assertEquals(20, it.read(ByteArray(100))) + assertThrows(IOException::class.java) { it.read(ByteArray(100)) } + } + assertEquals(3, calls) + assertTrue(bodies.all { it.closed }) + } + + @Test + fun `close prevents further reads and retries`() { + val body = GeneratedBody(100) + var calls = 0 + val proxy = open { calls++; response(body) } + proxy.close() + proxy.close() + assertThrows(IOException::class.java) { proxy.stream.read() } + assertEquals(1, calls) + assertTrue(body.closed) + } + + @Test + fun `temporary HTTP error during replay consumes the same bounded retry budget`() { + val bodies = listOf(GeneratedBody(100, failAt = 20), GeneratedBody(0), GeneratedBody(100)) + var calls = 0 + val proxy = open { + val result = response(bodies[calls++]) + if (calls == 2) result.newBuilder().code(503).build() else result + } + proxy.stream.use { assertArrayEquals(ByteArray(100) { 42 }, it.readBytes()) } + assertEquals(3, calls) + assertTrue(bodies.all { it.closed }) + } + + @Test + fun `cancellation during replay closes both responses without retrying`() { + val bodies = listOf(GeneratedBody(100, failAt = 20), GeneratedBody(100)) + var active = true + var calls = 0 + val proxy = open(checkActive = { if (!active) throw CancellationException("cancelled") }) { + if (calls == 1) active = false + response(bodies[calls++]) + } + proxy.stream.use { + assertEquals(20, it.read(ByteArray(100))) + assertThrows(CancellationException::class.java) { it.read(ByteArray(100)) } + } + assertEquals(2, calls) + assertTrue(bodies.all { it.closed }) + } + + @Test + fun `premature EOF is retried and the replay digest survives another failure`() { + val bodies = listOf( + GeneratedBody(100, failAt = 20, prematureEof = true), + GeneratedBody(100, failAt = 40), + GeneratedBody(100), + ) + var calls = 0 + val proxy = open { response(bodies[calls++]) } + proxy.stream.use { assertArrayEquals(ByteArray(100) { 42 }, it.readBytes()) } + assertEquals(3, calls) + assertTrue(bodies.all { it.closed }) + } + + @Test + fun `unknown length supports EOF and zero length reads`() { + val body = GeneratedBody(100, advertisedLength = -1) + val proxy = open { response(body) } + assertEquals(null, proxy.contentLength) + proxy.stream.use { + assertEquals(0, it.read(ByteArray(0))) + assertArrayEquals(ByteArray(100) { 42 }, it.readBytes()) + assertEquals(-1, it.read()) + } + assertTrue(body.closed) + } + + @Test + @EnabledIfEnvironmentVariable(named = "TYPETYPE_PROXY_BENCHMARK", matches = "1") + fun `compare eager body and streaming allocations on generated media`() { + val bean = ManagementFactory.getThreadMXBean() as com.sun.management.ThreadMXBean + val thread = Thread.currentThread().threadId() + fun measure(streaming: Boolean, size: Long, strongEtag: Boolean = false): Pair { + val allocated = bean.getThreadAllocatedBytes(thread) + val start = System.nanoTime() + val body = GeneratedBody(size) + if (streaming) { + open { + val result = response(body) + if (strongEtag) result.newBuilder().header("ETag", "\"stable\"").build() else result + }.stream.use { it.copyTo(OutputStream.nullOutputStream(), 64 * 1024) } + } else { + response(body).use { it.body.bytes().inputStream().copyTo(OutputStream.nullOutputStream(), 64 * 1024) } + } + return (bean.getThreadAllocatedBytes(thread) - allocated) to (System.nanoTime() - start) + } + repeat(8) { measure(false, 1024 * 1024); measure(true, 1024 * 1024); measure(true, 1024 * 1024, true) } + repeat(3) { + val eager = measure(false, 64L * 1024 * 1024) + val streaming = measure(true, 64L * 1024 * 1024) + val validated = measure(true, 64L * 1024 * 1024, true) + println("range64MiB eagerAllocated=${eager.first} streamingAllocated=${streaming.first} " + + "etagAllocated=${validated.first} eagerNanos=${eager.second} streamingNanos=${streaming.second} " + + "etagNanos=${validated.second}") + assertTrue(streaming.first < 1024 * 1024, "Streaming must not allocate a range-sized body") + } + } + + private fun open(checkActive: () -> Unit = {}, execute: (Request) -> Response) = + (readBilibiliRangeWithRetry(execute, request, checkActive) as ExtractionResult.Success).data + + private fun response(body: GeneratedBody) = Response.Builder() + .request(request).protocol(Protocol.HTTP_1_1).code(206).message("Partial Content") + .header("Content-Type", "video/mp4") + .header("Content-Range", "bytes 0-${body.length - 1}/${body.length}") + .body(body).build() + + private class GeneratedBody( + val length: Long, + private val failAt: Long = Long.MAX_VALUE, + private val value: Byte = 42, + private val prematureEof: Boolean = false, + private val advertisedLength: Long = length, + ) : ResponseBody() { + var readBytes = 0L + var closed = false + private val bytes = ByteArray(8192) { value } + private val input = object : Source { + override fun timeout() = Timeout.NONE + override fun close() { closed = true } + override fun read(sink: Buffer, byteCount: Long): Long { + if (readBytes >= failAt) { + if (prematureEof) return -1 + throw IOException("connection reset") + } + if (readBytes == length) return -1 + val count = minOf(byteCount, bytes.size.toLong(), length - readBytes, failAt - readBytes).toInt() + sink.write(bytes, 0, count) + readBytes += count + return count.toLong() + } + }.buffer() + + override fun contentType(): MediaType? = null + override fun contentLength() = advertisedLength + override fun source() = input + } +} diff --git a/src/test/kotlin/dev/typetype/server/RssDocumentRendererTest.kt b/src/test/kotlin/dev/typetype/server/RssDocumentRendererTest.kt new file mode 100644 index 00000000..ea4148a5 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/RssDocumentRendererTest.kt @@ -0,0 +1,60 @@ +package dev.typetype.server + +import dev.typetype.server.models.RssFeedItem +import dev.typetype.server.services.RssDocumentRenderer +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class RssDocumentRendererTest { + @Test + fun `renders a media thumbnail for HTTP(S) video thumbnails`() { + val video = testVideoItem().copy(thumbnailUrl = "https://img.example/thumb.jpg") + + val xml = RssDocumentRenderer.render(feed(), listOf(video), "https://video.example", 1_000L) + .toString(Charsets.UTF_8) + + assertTrue(xml.contains("xmlns:media=\"http://search.yahoo.com/mrss/\"")) + assertTrue(xml.contains("")) + assertTrue(first.bodyAsText().contains(" StreamResponse = { _, data -> data }, ) { install(ContentNegotiation) { json() } @@ -127,6 +155,7 @@ class StreamRoutesDeliveryModeTest { streamService = sabrService, nicoNicoStreamService = nicoNicoService, bilibiliStreamService = bilibiliService, + providerMediaHandleService = providerMediaHandleService, sabrStreamContractFilter = sabrFilter, ) } diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedAvailabilityTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedAvailabilityTest.kt new file mode 100644 index 00000000..56726d94 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedAvailabilityTest.kt @@ -0,0 +1,56 @@ +package dev.typetype.server + +import dev.typetype.server.SubscriptionFeedTestFixtures.channel +import dev.typetype.server.SubscriptionFeedTestFixtures.subscription +import dev.typetype.server.SubscriptionFeedTestFixtures.video +import dev.typetype.server.services.ChannelService +import dev.typetype.server.services.SubscriptionFeedService +import dev.typetype.server.services.SubscriptionsService +import io.mockk.coEvery +import io.mockk.mockk +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test + +class SubscriptionFeedAvailabilityTest { + companion object { @BeforeAll @JvmStatic fun initDb() = TestDatabase.setup() } + + @Test + fun `aged snapshot stays available during refresh and survives refresh failure`() = runTest { + TestDatabase.truncateAll() + val subscriptions = SubscriptionsService() + subscriptions.add(TEST_USER_ID, subscription("https://yt.com/c/a", "A")) + val channels = mockk() + val gate = CompletableDeferred() + var now = 1000L + var failRefresh = false + coEvery { channels.getChannel(any(), null) } coAnswers { + if (failRefresh) { + gate.await() + error("source unavailable") + } + channel(video(1000L, "A")) + } + val feed = SubscriptionFeedService(subscriptions, channels, FakeCacheService(), clock = { now }) + feed.getAllWithAvailability(TEST_USER_ID) + feed.awaitRefresh(TEST_USER_ID) + failRefresh = true + now += 61_000 + try { + val refreshing = feed.getAllWithAvailability(TEST_USER_ID) + assertTrue(refreshing.available) + assertEquals(1, refreshing.videos.size) + } finally { + gate.complete(Unit) + feed.awaitRefresh(TEST_USER_ID) + } + val failed = feed.getAllWithAvailability(TEST_USER_ID) + feed.awaitRefresh(TEST_USER_ID) + assertFalse(failed.available) + assertEquals(1, failed.videos.size) + } +} diff --git a/src/test/kotlin/dev/typetype/server/TestDatabase.kt b/src/test/kotlin/dev/typetype/server/TestDatabase.kt index be1dc356..de24cde7 100644 --- a/src/test/kotlin/dev/typetype/server/TestDatabase.kt +++ b/src/test/kotlin/dev/typetype/server/TestDatabase.kt @@ -20,6 +20,13 @@ import dev.typetype.server.db.tables.RssFeedsTable import dev.typetype.server.db.tables.RssUserPoliciesTable import dev.typetype.server.db.tables.SavedPlaylistsTable import dev.typetype.server.db.tables.NotificationStatesTable +import dev.typetype.server.db.tables.NotificationReadItemsTable +import dev.typetype.server.db.tables.ChannelNotificationPreferencesTable +import dev.typetype.server.db.tables.PushDevicesTable +import dev.typetype.server.db.tables.PushNotificationBaselinesTable +import dev.typetype.server.db.tables.PushNotificationSeenVideosTable +import dev.typetype.server.db.tables.PushNotificationEventsTable +import dev.typetype.server.db.tables.PushNotificationDeliveriesTable import dev.typetype.server.db.tables.SearchHistoryTable import dev.typetype.server.db.tables.SettingsTable import dev.typetype.server.db.tables.SessionsTable @@ -32,7 +39,15 @@ import dev.typetype.server.db.tables.YoutubeSessionPairingsTable import dev.typetype.server.db.tables.YoutubeSessionsTable import dev.typetype.server.db.tables.UsersTable import dev.typetype.server.db.tables.UserAvatarsTable +import dev.typetype.server.db.tables.UserChannelInterestTable +import dev.typetype.server.db.tables.UserTopicInterestTable +import dev.typetype.server.db.tables.RecommendationEventsTable +import dev.typetype.server.db.tables.RecommendationFeedHistoryTable +import dev.typetype.server.db.tables.RecommendationFeedbackTable +import dev.typetype.server.db.tables.RecommendationOnboardingPreferencesTable +import dev.typetype.server.db.tables.RecommendationOnboardingStateTable import dev.typetype.server.db.tables.WatchLaterTable +import dev.typetype.server.db.tables.ProfileAccountsTable import dev.typetype.server.services.AdminSettingsService import org.jetbrains.exposed.v1.jdbc.deleteAll import org.jetbrains.exposed.v1.jdbc.transactions.transaction @@ -126,6 +141,21 @@ object TestDatabase { YoutubeSessionsTable.deleteAll() BugReportsTable.deleteAll() NotificationStatesTable.deleteAll() + NotificationReadItemsTable.deleteAll() + PushNotificationDeliveriesTable.deleteAll() + PushNotificationEventsTable.deleteAll() + PushNotificationSeenVideosTable.deleteAll() + PushNotificationBaselinesTable.deleteAll() + PushDevicesTable.deleteAll() + ChannelNotificationPreferencesTable.deleteAll() + UserChannelInterestTable.deleteAll() + UserTopicInterestTable.deleteAll() + RecommendationEventsTable.deleteAll() + RecommendationFeedHistoryTable.deleteAll() + RecommendationFeedbackTable.deleteAll() + RecommendationOnboardingPreferencesTable.deleteAll() + RecommendationOnboardingStateTable.deleteAll() + ProfileAccountsTable.deleteAll() AdminSettingsService.clearCache() } } diff --git a/src/test/kotlin/dev/typetype/server/UnifiedPushEndpointValidatorTest.kt b/src/test/kotlin/dev/typetype/server/UnifiedPushEndpointValidatorTest.kt new file mode 100644 index 00000000..a09b8279 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/UnifiedPushEndpointValidatorTest.kt @@ -0,0 +1,32 @@ +package dev.typetype.server + +import dev.typetype.server.services.EndpointValidationResult +import dev.typetype.server.services.UnifiedPushEndpointValidator +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.net.InetAddress + +class UnifiedPushEndpointValidatorTest { + private val publicAddress = InetAddress.getByAddress(byteArrayOf(93.toByte(), 184.toByte(), 216.toByte(), 34)) + private val resolver = UnifiedPushEndpointValidator { arrayOf(publicAddress) } + + @Test + fun `accepts https endpoint and rejects non https forms`() { + assertTrue(resolver.validate("https://push.example.invalid/endpoint?id=1") is EndpointValidationResult.Valid) + assertEquals("endpoint_scheme", reason(resolver.validate("http://push.example.invalid/endpoint"))) + assertEquals("endpoint_scheme", reason(resolver.validate("https://user:pass@push.example.invalid/endpoint"))) + assertEquals("endpoint_scheme", reason(resolver.validate("https://push.example.invalid/endpoint#fragment"))) + } + + @Test + fun `rejects an address that resolves into a private network`() { + val privateValidator = UnifiedPushEndpointValidator { + arrayOf(InetAddress.getByAddress(byteArrayOf(10, 0, 0, 8))) + } + assertEquals("endpoint_private_address", reason(privateValidator.validate("https://push.example.invalid/endpoint"))) + } + + private fun reason(result: EndpointValidationResult): String = + (result as EndpointValidationResult.Invalid).reason +} diff --git a/src/test/kotlin/dev/typetype/server/services/BoundedExpiringCacheTest.kt b/src/test/kotlin/dev/typetype/server/services/BoundedExpiringCacheTest.kt index bf593670..f7c658fe 100644 --- a/src/test/kotlin/dev/typetype/server/services/BoundedExpiringCacheTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/BoundedExpiringCacheTest.kt @@ -6,6 +6,54 @@ import org.junit.jupiter.api.Test import java.time.Duration class BoundedExpiringCacheTest { + @Test + fun `reads expire entries independently of access order`() { + var now = 0L + val cache = BoundedExpiringCache(10, ttl = Duration.ofMillis(10), clock = { now }) + cache.put("first", "1") + now = 5L + cache.put("second", "2") + assertEquals("1", cache.get("first")) + now = 10L + + assertEquals("2", cache.get("second")) + assertNull(cache.get("first")) + assertEquals(1, cache.size()) + assertEquals(1L, cache.weight()) + now = 15L + assertNull(cache.get("second")) + assertEquals(0L, cache.weight()) + } + + @Test + fun `replacement keeps its new expiry when the original expiry passes`() { + var now = 0L + val cache = BoundedExpiringCache(10, ttl = Duration.ofMillis(10), clock = { now }) + cache.put("key", "old") + now = 5L + cache.put("key", "new") + now = 10L + cache.evictExpired() + assertEquals("new", cache.get("key")) + now = 15L + assertNull(cache.get("key")) + } + + @Test + fun `new entries honor their expiry after clear and a backward clock change`() { + var now = 100L + val cache = BoundedExpiringCache(10, ttl = Duration.ofMillis(10), clock = { now }) + cache.put("old", "1") + cache.clear() + cache.put("later", "2") + now = 50L + cache.put("earlier", "3") + now = 60L + assertNull(cache.get("earlier")) + assertEquals("2", cache.get("later")) + assertEquals(1, cache.size()) + } + @Test fun `least recently used entry is removed at capacity`() { val cache = BoundedExpiringCache( diff --git a/src/test/kotlin/dev/typetype/server/services/HomeWarmupTrackerTest.kt b/src/test/kotlin/dev/typetype/server/services/HomeWarmupTrackerTest.kt new file mode 100644 index 00000000..996e349e --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/HomeWarmupTrackerTest.kt @@ -0,0 +1,48 @@ +package dev.typetype.server.services + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.util.concurrent.Callable +import java.util.concurrent.Executors + +class HomeWarmupTrackerTest { + @Test + fun `ordinary concurrent requests start only one warmup`() { + val tracker = HomeWarmupTracker(100L, 1_000L) + tracker.markActive("user", 0L) + Executors.newFixedThreadPool(8).use { executor -> + val results = executor.invokeAll(List(100) { + Callable { tracker.trySchedule("user", 0L, false) } + }) + assertEquals(1, results.count { it.get() }) + } + assertFalse(tracker.trySchedule("user", 99L, false)) + assertTrue(tracker.trySchedule("user", 100L, false)) + } + + @Test + fun `forced invalidation retains its immediate warmup behavior`() { + val tracker = HomeWarmupTracker(100L, 1_000L) + assertTrue(tracker.trySchedule("user", 0L, false)) + assertTrue(tracker.trySchedule("user", 1L, true)) + assertFalse(tracker.trySchedule("user", 2L, false)) + } + + @Test + fun `expiry removes both activity and throttle entries`() { + val tracker = HomeWarmupTracker(10_000L, 1_000L) + repeat(10_000) { index -> + tracker.markActive("user-$index", 0L) + tracker.trySchedule("user-$index", 0L, false) + } + tracker.markActive("retained", 1_000L) + assertEquals(listOf("retained"), tracker.activeUsers(1_001L)) + repeat(10_000) { index -> + assertTrue(tracker.trySchedule("user-$index", 1_001L, false)) + } + assertEquals(listOf("retained"), tracker.activeUsers(2_000L)) + assertTrue(tracker.activeUsers(2_001L).isEmpty()) + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/OpenMojiProxyServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/OpenMojiProxyServiceTest.kt new file mode 100644 index 00000000..1dac530e --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/OpenMojiProxyServiceTest.kt @@ -0,0 +1,69 @@ +package dev.typetype.server.services + +import dev.typetype.server.cache.CacheService +import kotlinx.coroutines.test.runTest +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Protocol +import okhttp3.Response +import okhttp3.ResponseBody.Companion.toResponseBody +import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test + +class OpenMojiProxyServiceTest { + @Test + fun `successful SVG is served from the bounded local cache`() = runTest { + var requests = 0 + val bytes = "".toByteArray() + val client = client { + requests++ + ResponseBody(bytes) + } + val service = OpenMojiProxyService(EmptyCache, client) + + assertArrayEquals(bytes, service.getSvg("1F60A")) + assertArrayEquals(bytes, service.getSvg("1F60A")) + assertEquals(1, requests) + } + + @Test + fun `failed fetch is cooled down and retried after expiry`() = runTest { + var now = 0L + var requests = 0 + val client = client { + requests++ + ResponseBody("unavailable".toByteArray(), code = 503) + } + val service = OpenMojiProxyService(EmptyCache, client, clock = { now }) + + assertNull(service.getSvg("missing")) + assertNull(service.getSvg("missing")) + assertEquals(1, requests) + now = 15_000L + assertNull(service.getSvg("missing")) + assertEquals(2, requests) + } + + private fun client(response: () -> ResponseBody): OkHttpClient = OkHttpClient.Builder() + .addInterceptor { chain -> + val body = response() + Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(body.code) + .message("test") + .body(body.bytes.toResponseBody("image/svg+xml".toMediaType())) + .build() + } + .build() + + private data class ResponseBody(val bytes: ByteArray, val code: Int = 200) + + private object EmptyCache : CacheService { + override suspend fun get(key: String): String? = null + override suspend fun set(key: String, value: String, ttlSeconds: Long): Unit = Unit + override suspend fun delete(key: String): Unit = Unit + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCacheTest.kt b/src/test/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCacheTest.kt new file mode 100644 index 00000000..921aa169 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCacheTest.kt @@ -0,0 +1,73 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.YoutubeTakeoutCategoryCounts +import dev.typetype.server.models.YoutubeTakeoutParsedData +import dev.typetype.server.models.YoutubeTakeoutPreviewItem +import dev.typetype.server.models.YoutubeTakeoutPreviewSamples +import java.time.Duration +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Test + +class YoutubeTakeoutImportCacheTest { + @Test + fun `parsed data is bounded to one active job`() { + val cache = YoutubeTakeoutImportCache(maxParsedEntries = 1) + val first = parsedData("first") + val second = parsedData("second") + + cache.setParsed("job-1", first) + cache.setParsed("job-2", second) + + assertNull(cache.getParsed("job-1")) + assertSame(second, cache.getParsed("job-2")) + } + + @Test + fun `preview and parsed data expire`() { + var now = 100L + val cache = YoutubeTakeoutImportCache( + ttl = Duration.ofMillis(10), + clock = { now }, + ) + val preview = preview() + cache.setPreview("job", preview) + cache.setParsed("job", parsedData("job")) + + now = 110L + + assertNull(cache.getPreview("job")) + assertNull(cache.getParsed("job")) + } + + @Test + fun `remove releases both cached values`() { + val cache = YoutubeTakeoutImportCache() + cache.setPreview("job", preview()) + cache.setParsed("job", parsedData("job")) + + cache.remove("job") + + assertNull(cache.getPreview("job")) + assertNull(cache.getParsed("job")) + } + + private fun preview() = YoutubeTakeoutPreviewItem( + counts = YoutubeTakeoutCategoryCounts(0, 0, 0), + dedup = YoutubeTakeoutCategoryCounts(0, 0, 0), + samples = YoutubeTakeoutPreviewSamples(emptyList(), emptyList(), emptyList()), + warnings = emptyList(), + errors = emptyList(), + ) + + private fun parsedData(value: String) = YoutubeTakeoutParsedData( + subscriptions = emptyList(), + playlists = emptyList(), + playlistItems = mapOf(value to emptyList()), + favorites = emptyList(), + watchLater = emptyList(), + history = emptyList(), + warnings = emptyList(), + errors = emptyList(), + ) +}