From 38a6a39ab8d45af3303d1e749e17d4c9b37b749b Mon Sep 17 00:00:00 2001 From: Vincent Herbst Date: Thu, 17 Sep 2026 22:33:50 +0200 Subject: [PATCH 1/4] feat(live): the DVR ring can name the packets a still needs (#544) The software path holds the whole timeshift window as demuxed packets, each with its pts and a keyframe flag, but nothing in the engine has ever turned packets into a picture. This is the first half: which packets a still at a given time needs, as a pure function of the index. Two shapes decide its rules. Packets are stored in decode order, so with B-frames the frame at the target sits behind the first packet that reaches it, which is what the reorder tail pays for. And a live scrub routinely aims a fraction past the newest packet, so a target beyond the end clamps to it instead of answering nil, which would blink the preview out at exactly the edge the viewer sits on most. Bounded in packets and in seconds, so a stream whose keyframes are minutes apart refuses rather than decoding for minutes. Only the window the span can cover is copied out under the lock: a 30 minute ring holds ~150k entries and a held scrub asks for a still every 80 ms. Part of #544. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015PM3xUJB6ZQyqnmGK1fp6F --- .../AetherEngine/Video/PacketRingBuffer.swift | 82 ++++++++++++++++ .../PacketRingBufferTests.swift | 96 +++++++++++++++++++ 2 files changed, 178 insertions(+) diff --git a/Sources/AetherEngine/Video/PacketRingBuffer.swift b/Sources/AetherEngine/Video/PacketRingBuffer.swift index 45ce28f39..cf892b807 100644 --- a/Sources/AetherEngine/Video/PacketRingBuffer.swift +++ b/Sources/AetherEngine/Video/PacketRingBuffer.swift @@ -157,6 +157,88 @@ final class PacketRingBuffer: @unchecked Sendable { .map { firstSeq + $0 } } + // MARK: - Still runs (#544) + + /// One retained packet as the still planner sees it: no bytes, no file, just the three fields + /// the decision needs. + struct IndexEntry: Equatable { + let pts: Double + let isKeyframe: Bool + let isVideo: Bool + } + + /// The sequence span a still at `target` needs: from the newest video keyframe at or before it + /// forward to the first video packet that reaches it, plus `reorderTail` further video packets. + /// Nil when no keyframe at or before the target is retained, when the index holds no video, or + /// when the span exceeds either bound. + /// + /// Two shapes decide the rules here. Packets are stored in DECODE order, so with B-frames the + /// frame at the target can sit behind the first packet that reaches it, which is what the tail + /// pays for. And a live scrub routinely aims a fraction past the newest packet, so a target + /// beyond the end clamps to it rather than answering nil, which would blink the card out at + /// exactly the edge the viewer sits on most. + static func stillRunSpan(target: Double, + index: [IndexEntry], + firstSeq: Int, + maxPackets: Int, + maxSpanSeconds: Double, + reorderTail: Int) -> ClosedRange? { + guard index.contains(where: \.isVideo) else { return nil } + guard let start = index.indices.last(where: { index[$0].isKeyframe && index[$0].pts <= target }) + else { return nil } + guard target - index[start].pts <= maxSpanSeconds else { return nil } + + let reached = index.indices[start...].first(where: { index[$0].isVideo && index[$0].pts >= target }) + guard var end = reached ?? index.indices.last(where: { index[$0].isVideo }) else { return nil } + + if reached != nil, reorderTail > 0 { + var remaining = reorderTail + var i = end + 1 + while i < index.count, remaining > 0 { + if index[i].isVideo { + remaining -= 1 + end = i + } + i += 1 + } + } + + guard end >= start, end - start + 1 <= maxPackets else { return nil } + return (firstSeq + start)...(firstSeq + end) + } + + /// The video packets a still at `target` needs, keyframe-first. Nil when the target is not + /// decodable from what the ring retains. Only the window the span can possibly cover is copied + /// out under the lock: a 30 minute window holds ~150k entries and a still is asked for every + /// 80 ms while a viewer holds the scrub. + func stillRun(target: Double, + maxPackets: Int, + maxSpanSeconds: Double, + reorderTail: Int) -> [Packet]? { + lock.lock() + guard let startIdx = entries.indices + .last(where: { entries[$0].isKeyframe && entries[$0].pts <= target }) else { + lock.unlock() + return nil + } + let upper = min(entries.count, startIdx + maxPackets + reorderTail + 1) + let window = entries[startIdx.. PacketRingBuffer.IndexEntry { + PacketRingBuffer.IndexEntry(pts: pts, isKeyframe: key, isVideo: true) + } + private func audio(_ pts: Double) -> PacketRingBuffer.IndexEntry { + PacketRingBuffer.IndexEntry(pts: pts, isKeyframe: false, isVideo: false) + } + + private func span(_ index: [PacketRingBuffer.IndexEntry], + target: Double, + firstSeq: Int = 0, + maxPackets: Int = 1000, + maxSpanSeconds: Double = 30, + reorderTail: Int = 0) -> ClosedRange? { + PacketRingBuffer.stillRunSpan(target: target, index: index, firstSeq: firstSeq, + maxPackets: maxPackets, maxSpanSeconds: maxSpanSeconds, + reorderTail: reorderTail) + } + + /// Starts at the newest keyframe at or before the target, ends at the first video packet reaching it. + func testStartsAtKeyframeBeforeTargetAndEndsWhenReached() { + let index = [video(0, key: true), video(1), video(2, key: true), video(3), video(4)] + XCTAssertEqual(span(index, target: 3), 2...3) + } + + /// A later keyframe wins: the run never decodes more of the GOP than it has to. + func testPicksTheNewestKeyframeNotTheOldest() { + let index = [video(0, key: true), video(2, key: true), video(4, key: true), video(6)] + XCTAssertEqual(span(index, target: 6), 2...3) + } + + /// Sequence numbers are absolute, so an evicted ring still addresses its packets. + func testSpanIsInAbsoluteSequenceNumbers() { + let index = [video(10, key: true), video(11), video(12)] + XCTAssertEqual(span(index, target: 12, firstSeq: 900), 900...902) + } + + /// At the live edge the target routinely overshoots the newest packet by a fraction. Clamping + /// there rather than returning nil is what keeps the card from blinking out at the edge. + func testTargetPastNewestPacketClampsToIt() { + let index = [video(0, key: true), video(1), video(2)] + XCTAssertEqual(span(index, target: 9.5), 0...2) + } + + /// Scrubbed off the back of the window: nothing decodable, and saying so is the honest answer. + func testTargetBeforeOldestKeyframeIsNil() { + let index = [video(5, key: true), video(6)] + XCTAssertNil(span(index, target: 1)) + } + + /// A stream whose keyframes are minutes apart must not hold a still request hostage. + func testRefusesAGopLongerThanTheSpanBound() { + let index = [video(0, key: true)] + (1...40).map { video(Double($0)) } + XCTAssertNil(span(index, target: 40, maxSpanSeconds: 10)) + XCTAssertNotNil(span(index, target: 40, maxSpanSeconds: 60)) + } + + /// The same guard in packets, because the cost is one file read each. + func testRefusesARunLongerThanThePacketBound() { + let index = [video(0, key: true)] + (1...40).map { video(Double($0) * 0.04) } + XCTAssertNil(span(index, target: 1.6, maxPackets: 20)) + XCTAssertNotNil(span(index, target: 1.6, maxPackets: 200)) + } + + /// Audio rides along inside the span (the caller skips it) but never ends the run. + func testAudioDoesNotEndTheRun() { + let index = [video(0, key: true), audio(0.5), audio(1.5), video(2)] + XCTAssertEqual(span(index, target: 2), 0...3) + } + + /// Packets arrive in decode order, so with B-frames the first packet reaching the target is not + /// the last one the decoder needs to emit it. The tail is what a reordered stream costs. + func testReorderTailExtendsPastTheFirstPacketReachingTheTarget() { + // decode order I P B B, presentation 0 3 1 2 + let index = [video(0, key: true), video(3), video(1), video(2)] + XCTAssertEqual(span(index, target: 2, reorderTail: 0), 0...1) + XCTAssertEqual(span(index, target: 2, reorderTail: 2), 0...3) + } + + /// The tail cannot walk past the newest packet the ring holds. + func testReorderTailStopsAtTheNewestPacket() { + let index = [video(0, key: true), video(1), video(2)] + XCTAssertEqual(span(index, target: 2, reorderTail: 8), 0...2) + } + + /// An index with no video at all (audio-only stretch) has no still in it. + func testNoVideoIsNil() { + XCTAssertNil(span([audio(0), audio(1)], target: 1)) + } +} From ca15c2b8b14935300cfe25bc2a5f5f97c6b43d9e Mon Sep 17 00:00:00 2001 From: Vincent Herbst Date: Thu, 17 Sep 2026 22:51:14 +0200 Subject: [PATCH 2/4] feat(live): a software live session decodes its scrub still from the packet ring (#544) The live scrub preview has always been a SegmentCache feature: it is gated on `nativeVideoSession != nil`, which a software session never has. So every channel the box cannot decode in hardware scrubbed against an empty card, and on an ATSC tuner that is all of them, because MPEG-2 has no hardware decoder on Apple TV. There was no fallback either: a host can pair the VOD arm with a second demuxer, but a live source is forward-only and cannot be seeked twice. What the path did have is the whole timeshift window as demuxed packets, each with its pts and a keyframe flag, and no image consumer anywhere in the engine. `liveScrubThumbnail` grows a second arm that decodes out of it: the same buffer the scrubber seeks within, so a still and a commit cannot name two different moments, and the same `sessionStartPts` conversion the DVR rewind uses. `SoftwareStillExtractor` drives a real `SoftwareVideoDecoder` rather than a minimal one of its own, because the still should be the picture the renderer would show. Broadcast is where that matters: interlaced MPEG-2 at a non-square sample aspect is the normal case on a tuner, and the deinterlace and the SAR resolution behind it are hardened here already. It decodes single-threaded (new `decodesSingleThreaded`, off everywhere else): a still run is one short GOP decoded once, where frame-level threading only adds output delay and a second worker pool. It runs on its own queue, off the demux and feed loops, so a preview frame never costs playback a packet. Measured on the harness, `play --live --sw --dvr-window 120 --host-calls still` against the paced raw-TS origin, three aims per run: before after stills hit 0 of 3 3 of 3 decode MISS in 0 ms 20 to 61 ms picture none the second it was asked for The seed burns its own second into the frame, so the file is the verdict and not the count: asked for 14.85 s it returns the frame marked 14, asked for 25.11 s the one marked 25, while playback stayed at the edge throughout. Paired with `seekback` in one run, the rewind still lands and the run verdicts OK. Part of #544. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015PM3xUJB6ZQyqnmGK1fp6F --- Sources/AetherEngine/AetherEngine+Live.swift | 19 +- .../Decoder/SoftwareVideoDecoder.swift | 15 +- .../Native/SoftwarePlaybackHost.swift | 44 +++++ .../Native/SoftwareStillExtractor.swift | 177 ++++++++++++++++++ Sources/aetherctl/PlaybackCmd.swift | 52 ++++- docs/api.md | 6 +- docs/cli.md | 2 +- 7 files changed, 305 insertions(+), 10 deletions(-) create mode 100644 Sources/AetherEngine/Native/SoftwareStillExtractor.swift diff --git a/Sources/AetherEngine/AetherEngine+Live.swift b/Sources/AetherEngine/AetherEngine+Live.swift index db37393dd..0f04094be 100644 --- a/Sources/AetherEngine/AetherEngine+Live.swift +++ b/Sources/AetherEngine/AetherEngine+Live.swift @@ -3,9 +3,24 @@ import CoreGraphics extension AetherEngine { - /// Frame from the DVR segment cache at `atSessionSeconds` (seekableLiveRange axis). No network: converts session time to raw output via seam history, then decodes locally. nil when no native live session, time outside resident window, or decode fails. + /// Frame from the live DVR window at `atSessionSeconds` (seekableLiveRange axis), decoded + /// locally with no network. + /// + /// Two sessions can answer, and both read a buffer the session already holds rather than opening + /// a second connection (a live source is forward-only, so a second demuxer could not seek it). + /// A native session decodes from its DVR segment cache after converting session time to raw + /// output via seam history. A software session has no such cache, so it decodes out of its own + /// packet ring (#544), which is the same buffer the scrubber seeks within. nil when neither is + /// live, when the time is outside the resident window, or when the decode fails. public func liveScrubThumbnail(atSessionSeconds seconds: Double, maxWidth: Int = 320) async -> CGImage? { - guard isLive, let session = nativeVideoSession else { return nil } + guard isLive else { return nil } + guard let session = nativeVideoSession else { + guard let host = softwareHost else { return nil } + let gen = loadGeneration + let image = await host.liveScrubStill(atSessionSeconds: seconds, maxWidth: maxWidth) + // A zap between the request and the frame would hand the new channel the old one's picture. + return loadGeneration == gen ? image : nil + } // seekableLiveRange is output-time + seam shift; segment table and tfdt live on raw output. Resolve newest seam (inverts $currentTime fold). let outputSeconds: Double outputSeconds = presentationAxis.itemSeconds(forSourceSeconds: seconds) diff --git a/Sources/AetherEngine/Decoder/SoftwareVideoDecoder.swift b/Sources/AetherEngine/Decoder/SoftwareVideoDecoder.swift index 7b79557ab..fb6311836 100644 --- a/Sources/AetherEngine/Decoder/SoftwareVideoDecoder.swift +++ b/Sources/AetherEngine/Decoder/SoftwareVideoDecoder.swift @@ -78,6 +78,10 @@ final class SoftwareVideoDecoder: VideoDecodingPipeline, @unchecked Sendable { /// applied to the filter there (mutating it mid-stream would need a graph rebuild). var deinterlaceConfig = DeinterlaceConfig() + /// #544: decode on the calling thread with no frame-level threading. Set before `open`; the + /// still extractor is the only caller, everything on a playback path wants the parallel default. + var decodesSingleThreaded = false + /// AE#499: what the container declared about colour, captured at `open` before a single frame /// exists. A decoded frame carries the VUI alone, and a remux whose VUI is empty would otherwise /// reach `attachColorSpace` as an untagged picture, so an HDR10 file decoded in software lost its @@ -153,8 +157,15 @@ final class SoftwareVideoDecoder: VideoDecodingPipeline, @unchecked Sendable { return AV_PIX_FMT_YUV420P } - ctx.pointee.thread_count = Int32(ProcessInfo.processInfo.activeProcessorCount) - ctx.pointee.thread_type = FF_THREAD_FRAME | FF_THREAD_SLICE + if decodesSingleThreaded { + // #544: a still run is one short GOP decoded once. Frame-level threading buys throughput + // nobody is waiting for and costs output delay plus a second worker pool. + ctx.pointee.thread_count = 1 + ctx.pointee.thread_type = 0 + } else { + ctx.pointee.thread_count = Int32(ProcessInfo.processInfo.activeProcessorCount) + ctx.pointee.thread_type = FF_THREAD_FRAME | FF_THREAD_SLICE + } // Belt-and-suspenders hwaccel=none: some decoders ignore get_format. var opts: OpaquePointer? diff --git a/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift b/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift index e7019110f..93114a46f 100644 --- a/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift +++ b/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift @@ -186,6 +186,12 @@ final class SoftwarePlaybackHost { // MARK: - Live / DVR + /// #544: decodes a scrub still out of `dvrRing`. Built beside the playback decoder so it never + /// holds a stream pointer of its own, and driven only from `stillQueue`, off the demux and feed + /// loops, so a preview frame never costs playback a packet. + private var stillExtractor: SoftwareStillExtractor? + private let stillQueue = DispatchQueue(label: "engine.sw.still", qos: .userInitiated) + /// Disk-spooled DVR rewind ring; non-nil for live sessions with dvrWindowSeconds set. Demux-thread appended (internally locked). nonisolated(unsafe) private var dvrRing: PacketRingBuffer? @@ -786,6 +792,20 @@ final class SoftwarePlaybackHost { ) } } + if isLive, dvrRing != nil { + do { + stillExtractor = try SoftwareStillExtractor( + stream: vStream, + videoStreamIndex: videoStreamIndex, + timeBaseSeconds: videoTimeBaseSeconds, + deinterlace: deinterlaceConfig) + } catch { + // A session without scrub stills still plays; the preview just stays empty. + EngineLog.emit("[SWHost] #544 still extractor unavailable (\(error))", category: .swPlayback) + stillExtractor = nil + } + } + videoDecoder.onFirstHDR10PlusDetected = { [weak self] in self?.onFirstHDR10PlusDetected?() } @@ -1268,6 +1288,28 @@ final class SoftwarePlaybackHost { return outcome } + /// #544: a scrub still for the live DVR window, decoded out of the packet ring. + /// + /// Takes the session axis, exactly as `seek` does, and converts it with the same `sessionStartPts` + /// the rewind uses, so the still and the commit can never name two different moments. Runs on its + /// own queue: the ring is internally locked and safe to read alongside the feeder, but decoding on + /// the demux or feed loop would make the viewer pay for the preview in dropped packets. + func liveScrubStill(atSessionSeconds seconds: Double, maxWidth: Int) async -> CGImage? { + guard isLive, let ring = dvrRing, let extractor = stillExtractor else { return nil } + let startPts: Double = { + liveEdgeLock.lock() + defer { liveEdgeLock.unlock() } + return sessionStartPts.isFinite ? sessionStartPts : 0 + }() + let targetSource = startPts + seconds + return await withCheckedContinuation { continuation in + stillQueue.async { + continuation.resume( + returning: extractor.still(from: ring, targetPts: targetSource, maxWidth: maxWidth)) + } + } + } + /// Live DVR rewind: reseeds decoder from the ring (source PTS axis; maps via sessionStartPts) without touching the live demuxer. After return, the loop reads new packets forward and plays back to live. private func seekLiveDVR(to targetSession: Double, ring: PacketRingBuffer, wasPlaying: Bool) async { let startPts: Double = { @@ -1371,6 +1413,8 @@ final class SoftwarePlaybackHost { vodPacketReadAhead = nil renderer.subtitleCompositor.reset() + stillExtractor?.close() + stillExtractor = nil dvrRing?.close() dvrRing = nil liveEdgeLock.lock() diff --git a/Sources/AetherEngine/Native/SoftwareStillExtractor.swift b/Sources/AetherEngine/Native/SoftwareStillExtractor.swift new file mode 100644 index 000000000..aed830988 --- /dev/null +++ b/Sources/AetherEngine/Native/SoftwareStillExtractor.swift @@ -0,0 +1,177 @@ +import CoreGraphics +import CoreMedia +import CoreVideo +import Foundation +import VideoToolbox +import AetherLibavcodec +import AetherLibavformat + +/// Decodes one still out of a run of demuxed packets (#544). No demuxer and no container: the +/// software live path already holds its whole timeshift window as packets, it only ever lacked an +/// image consumer. +/// +/// It drives a real `SoftwareVideoDecoder` rather than a minimal one of its own, so the still is the +/// picture the renderer would show. Broadcast is where that matters: interlaced MPEG-2 at a +/// non-square sample aspect is the normal case on a tuner, and both the deinterlace and the SAR +/// resolution behind it are hardened here already. A second decoder would have to re-derive them and +/// would get a combed, stretched frame wrong in exactly the cases the preview exists for. +/// +/// Not thread-safe by itself: the host owns one and serialises requests onto its own queue, off the +/// demux and feed loops, so a still never costs playback a packet. +final class SoftwareStillExtractor: @unchecked Sendable { + + /// Bounds on one run. A broadcast GOP is well under a second; these refuse the pathological + /// stream rather than letting it hold a request. + struct Limits { + var maxPackets: Int = 900 + var maxSpanSeconds: Double = 12 + /// Packets are stored in decode order, so the frame at the target can sit behind the first + /// packet that reaches it. Two B-frames is the common broadcast shape; four covers the rest. + var reorderTail: Int = 4 + } + + private let decoder = SoftwareVideoDecoder() + private let videoStreamIndex: Int32 + private let timeBaseSeconds: Double + private let limits: Limits + private var isOpen = false + + init(stream: UnsafeMutablePointer, + videoStreamIndex: Int32, + timeBaseSeconds: Double, + deinterlace: DeinterlaceConfig, + limits: Limits = Limits()) throws { + self.videoStreamIndex = videoStreamIndex + self.timeBaseSeconds = timeBaseSeconds + self.limits = limits + decoder.deinterlaceConfig = deinterlace + decoder.decodesSingleThreaded = true + try decoder.open(stream: stream) { _, _, _ in } + isOpen = true + } + + deinit { + decoder.close() + } + + func close() { + guard isOpen else { return } + isOpen = false + decoder.close() + } + + /// The frame at `targetPts` (source axis), or nil when the ring cannot serve it. + /// + /// Every request is an independent landing, so the decoder is flushed first: a still is a seek, + /// and carrying references across two unrelated positions is what produces a smeared picture. + func still(from ring: PacketRingBuffer, targetPts: Double, maxWidth: Int) -> CGImage? { + guard isOpen, timeBaseSeconds > 0, maxWidth > 0 else { return nil } + guard let run = ring.stillRun(target: targetPts, + maxPackets: limits.maxPackets, + maxSpanSeconds: limits.maxSpanSeconds, + reorderTail: limits.reorderTail), + !run.isEmpty else { return nil } + + let collector = FrameCollector() + decoder.onFrame = { pixelBuffer, pts, _ in + collector.append(pixelBuffer: pixelBuffer, seconds: pts.seconds) + } + decoder.flush() + defer { decoder.onFrame = nil } + + for packet in run { + feed(packet) + } + + guard let best = collector.best(for: targetPts) else { return nil } + return Self.image(from: best, maxWidth: maxWidth) + } + + // MARK: - Feeding + + private func feed(_ packet: PacketRingBuffer.Packet) { + guard !packet.bytes.isEmpty else { return } + guard let p = av_packet_alloc() else { return } + var pkt: UnsafeMutablePointer? = p + defer { av_packet_free(&pkt) } + + guard av_new_packet(p, Int32(packet.bytes.count)) >= 0 else { return } + packet.bytes.withUnsafeBytes { raw in + if let base = raw.baseAddress, let dst = p.pointee.data { + memcpy(dst, base, packet.bytes.count) + } + } + p.pointee.pts = Int64((packet.pts / timeBaseSeconds).rounded()) + p.pointee.dts = p.pointee.pts + p.pointee.flags = packet.isKeyframe ? AV_PKT_FLAG_KEY : 0 + p.pointee.stream_index = videoStreamIndex + decoder.decode(packet: p, epoch: nil) + } + + // MARK: - Frame selection + + /// Collects what the decoder emits during one run. `onFrame` is `@Sendable` and the decoder may + /// call it from its own drain, so the box is locked even though the run itself is serial. + private final class FrameCollector: @unchecked Sendable { + private let lock = NSLock() + private var frames: [(pixelBuffer: CVPixelBuffer, seconds: Double)] = [] + + func append(pixelBuffer: CVPixelBuffer, seconds: Double) { + lock.lock() + defer { lock.unlock() } + frames.append((pixelBuffer, seconds)) + } + + /// The newest frame at or before the target. Falling back to the oldest rather than to + /// nothing matters at the live edge, where the target can sit a fraction past every frame + /// the run produced. + func best(for target: Double) -> CVPixelBuffer? { + lock.lock() + defer { lock.unlock() } + guard !frames.isEmpty else { return nil } + let atOrBefore = frames + .filter { $0.seconds.isFinite && $0.seconds <= target } + .max(by: { $0.seconds < $1.seconds }) + return (atOrBefore ?? frames.min(by: { $0.seconds < $1.seconds }))?.pixelBuffer + } + } + + // MARK: - Image + + /// The decoder attaches the resolved sample aspect to the buffer (`#177`), so the still reads + /// its answer rather than resolving one of its own, and a 704x480 4:3 broadcast frame draws 4:3. + static func image(from pixelBuffer: CVPixelBuffer, maxWidth: Int) -> CGImage? { + var cgImage: CGImage? + guard VTCreateCGImageFromCVPixelBuffer(pixelBuffer, options: nil, imageOut: &cgImage) == noErr, + let source = cgImage else { return nil } + + let srcW = source.width + let srcH = source.height + guard srcW > 0, srcH > 0 else { return nil } + + let (dstW, dstH) = FrameDecodeContext.displayDimensions( + srcW: srcW, srcH: srcH, sar: sampleAspect(of: pixelBuffer), targetWidth: maxWidth) + if dstW == srcW && dstH == srcH { return source } + + guard let space = CGColorSpace(name: CGColorSpace.sRGB), + let ctx = CGContext(data: nil, width: dstW, height: dstH, + bitsPerComponent: 8, bytesPerRow: 0, space: space, + bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue) else { + return source + } + ctx.interpolationQuality = .high + ctx.draw(source, in: CGRect(x: 0, y: 0, width: dstW, height: dstH)) + return ctx.makeImage() ?? source + } + + static func sampleAspect(of pixelBuffer: CVPixelBuffer) -> AVRational { + guard let attachment = CVBufferCopyAttachment( + pixelBuffer, kCVImageBufferPixelAspectRatioKey, nil) as? [CFString: Any], + let h = attachment[kCVImageBufferPixelAspectRatioHorizontalSpacingKey] as? Int, + let v = attachment[kCVImageBufferPixelAspectRatioVerticalSpacingKey] as? Int, + h > 0, v > 0 else { + return AVRational(num: 1, den: 1) + } + return AVRational(num: Int32(h), den: Int32(v)) + } +} diff --git a/Sources/aetherctl/PlaybackCmd.swift b/Sources/aetherctl/PlaybackCmd.swift index 30dad5194..30853c9ca 100644 --- a/Sources/aetherctl/PlaybackCmd.swift +++ b/Sources/aetherctl/PlaybackCmd.swift @@ -1,9 +1,22 @@ import Foundation import Combine +import CoreGraphics import CoreMedia import AVFoundation +import ImageIO +import UniformTypeIdentifiers import AetherEngine +/// #544: the still drill writes what it decoded, because a hit count says the call returned an image +/// and only the file says it is the right picture. +private func writeStillPNG(_ image: CGImage, to path: String) -> Bool { + guard let dest = CGImageDestinationCreateWithURL( + URL(fileURLWithPath: path) as CFURL, UTType.png.identifier as CFString, 1, nil + ) else { return false } + CGImageDestinationAddImage(dest, image, nil) + return CGImageDestinationFinalize(dest) +} + // MARK: - play /// A host's post-play audio-track pick, replayed on the CLI (#337). The delay is the whole @@ -627,12 +640,12 @@ private func playSmokeTest(url: URL, seconds: Double, live: Bool, forceSoftware: // exists (the foreground retune's hold-paused policy). Resumed at tick 8. print(" HOSTCALL pause() right after load") engine.pause() - case "reloadlive", "seekback", "overlapseek", "ratehold-tail", "pauseseek", "pausehold": + case "reloadlive", "seekback", "overlapseek", "ratehold-tail", "pauseseek", "pausehold", "still": break // reloadlive handled at load time, seekback/overlapseek/pauseseek in the telemetry loop case let call where call.hasPrefix("seekfar"): break // #433, in the telemetry loop; `seekfar@N` picks the tick default: - print(" HOSTCALL unknown '\(call)' (use play,extractor,setrate,ratehold,pausestart,reloadlive,seekback,seekfar,overlapseek,pauseseek,pausehold)") + print(" HOSTCALL unknown '\(call)' (use play,extractor,setrate,ratehold,pausestart,reloadlive,seekback,seekfar,overlapseek,pauseseek,pausehold,still)") } } defer { if let frameExtractor { Task { await frameExtractor.shutdown() } } } @@ -795,6 +808,12 @@ private func playSmokeTest(url: URL, seconds: Double, live: Bool, forceSoftware: return parts.count == 2 ? (Int(parts[1]) ?? 15) : 15 } + // #544: scrub stills on the software live path, decoded out of the DVR packet ring. Three aims + // per run (deep in the window, just behind the playhead, and at the edge), because the edge is the + // one a live viewer sits on and the one a clamp has to cover. + var stillAttempts = 0 + var stillHits = 0 + let ticks = max(1, Int(seconds)) // Sodalite#104: where the playhead stood when the pause-and-hold drill parked it, so the run can // say whether it moved. @@ -894,6 +913,24 @@ private func playSmokeTest(url: URL, seconds: Double, live: Bool, forceSoftware: // the other half of the report. if tick >= 6 { rateHoldAtEnd = Issue436RateHold.observedRate(engine) } } + if hostCalls.contains("still"), [15, 20, 25].contains(tick) { + let offset: Double = tick == 15 ? 20 : (tick == 20 ? 5 : 0) + let label = tick == 15 ? "playhead-20" : (tick == 20 ? "playhead-5" : "edge") + let target = max(0, engine.currentTime - offset) + let started = Date() + let image = await engine.liveScrubThumbnail(atSessionSeconds: target, maxWidth: 320) + let ms = Int(Date().timeIntervalSince(started) * 1000) + stillAttempts += 1 + if let image { + stillHits += 1 + let path = "/tmp/aetherctl-still-\(tick).png" + _ = writeStillPNG(image, to: path) + print(String(format: " HOSTCALL still(at: %.2f, %@) -> %dx%d in %d ms %@", + target, label, image.width, image.height, ms, path)) + } else { + print(String(format: " HOSTCALL still(at: %.2f, %@) -> MISS in %d ms", target, label, ms)) + } + } if hostCalls.contains("seekback"), tick == 15 { let target = max(0, engine.currentTime - 20) print(String(format: " HOSTCALL seek(to: %.2f) (currentTime - 20)", target)) @@ -1104,6 +1141,17 @@ private func playSmokeTest(url: URL, seconds: Double, live: Bool, forceSoftware: print("VERDICT: session ended in error: \(message)") return 2 } + if hostCalls.contains("still") { + print("#544 scrub stills: \(stillHits) of \(stillAttempts) hit") + if stillAttempts == 0 { + print("VERDICT: #544 drill inconclusive (session ended before the first still tick)") + return 5 + } + if stillHits == 0 { + print("VERDICT: #544 reproduced (the live scrub preview has no frame to show)") + return 6 + } + } if hostCalls.contains("ratehold") { let observed = rateHoldAfterResume print(String(format: "#436 rate hold: requested %.2f, transport reported %.2f after the resume", diff --git a/docs/api.md b/docs/api.md index b883cc690..c54515d84 100644 --- a/docs/api.md +++ b/docs/api.md @@ -508,7 +508,7 @@ Time lives on `player.clock`, a separate `ObservableObject`, so ~10 Hz ticks nev | `seekToLiveEdge()` | `async`. | | `liveSourceReset` | The retune contract above. | | `liveResumeClamped`, `LiveResumeClamp` | A resume that found the playhead outside the window and moved it; see above. | -| `liveScrubThumbnail(atSessionSeconds:maxWidth:)` | Cache-backed still on the live session axis. | +| `liveScrubThumbnail(atSessionSeconds:maxWidth:)` | Still on the live session axis, decoded from what the session already holds. A native session reads its DVR segment cache; a software session reads its DVR packet ring (#544), so a tuner channel the box decodes in software has a scrub preview too. | | `$playlistShiftSeconds` | Seconds the producer subtracted from source PTS. Published values already fold it back; exposed for hosts pairing their own samples against AVPlayer's raw clock. | | `HLSLiveIngestReader(playlistURL:)`, `HLSLiveIngestReader(playlistURL:httpHeaders:)` | The ready-made `IOReader` for ingesting an upstream HLS playlist directly, with AES-128 clear-key and SSAI handling. The headers ride the playlist, every segment and every AES key, which is what a tokenized IPTV origin enforces per request. Unsupported shapes surface a typed `HLSIngestError`. | @@ -657,8 +657,8 @@ reports an intention rather than an outcome. | Symbol | Notes | | --- | --- | | `scrubThumbnail(atSeconds:maxWidth:)` | Cache-backed still for the active native session, live or VOD. Decodes bytes already produced, so it opens no second connection and works on single-connection sources (debrid / torrent links) where a second demuxer is refused. | -| `vodScrubThumbnail(atSeconds:maxWidth:)`, `liveScrubThumbnail(atSessionSeconds:maxWidth:)` | The two arms, for callers that know which axis they hold. | -| `supportsCacheBackedStills` | True while a native session exists. Gate the scrub-preview affordance on it: it reports capability, not per-frame availability, so a transient nil from `scrubThumbnail` while a segment is still being produced is expected and means "time only, no image". | +| `vodScrubThumbnail(atSeconds:maxWidth:)`, `liveScrubThumbnail(atSessionSeconds:maxWidth:)` | The two arms, for callers that know which axis they hold. The live arm also serves software sessions, out of the DVR packet ring rather than a segment cache (#544). | +| `supportsCacheBackedStills` | True while a native session exists, which is what the SEGMENT CACHE needs. Gate the scrub-preview affordance on it: it reports capability, not per-frame availability, so a transient nil from `scrubThumbnail` while a segment is still being produced is expected and means "time only, no image". It stays false on a software session, and a live one nonetheless serves stills from its packet ring, so a live caller asks `liveScrubThumbnail` rather than this flag. | ## Certificate trust diff --git a/docs/cli.md b/docs/cli.md index ffc3dfd42..d1874c004 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -106,7 +106,7 @@ the same clock as `FIRSTFRAME`. The 1 Hz tick samples the phase, which is far to signal apart from the moment the rate rolls; a healthy native join is exactly two edges, `loading` at the load and `playing` at the roll (AE#440). -`--subs ` matches against the track's libavcodec name or language and logs every overlay cue and cue trim as it lands. `--host-calls` replays host post-load behavior against the fresh session: `play`, `extractor` (`makeFrameExtractor`), `setrate` (`setRate(1.0)`), `pausestart` (Sodalite#104 round 4: `pause()` the instant load returns, before any frame exists, and `play()` at t=8; the shape of a host that holds a fresh load paused, and a software session used to answer it with eight ticks of `enq=+0 status=unknown r4d=n`, a black picture under a paused clock; now the `[SWHost] #104` lines show the first frame presented at a stopped clock and `startup 8/8 presenting` arrives while paused), `ratehold` (set 1.5, pause at tick 3, resume at tick 5, then read the rate back off the transport itself: the #436 drill, and it fails the run if the resume came back at 1.0), `reloadlive` (reload the URL on the live path when the probe flags it live, the AetherPlayer Open URL flow), `seekback` (rewind 20 s into the DVR window at t=15, return to the live edge at t=30), `overlapseek` (the #292 seek-window drills below), `pausehold` (Sodalite#104: pause at t=10 and HOLD until ten seconds before the end, printing the playhead, the edge and the resident depth every second, which is how a session paused for longer than its own DVR window is measured without waiting out a real one: pair it with `--dvr-window 30` and `--seconds 90` and the ninety minute question becomes a ninety second run), and `pauseseek` (pause at t=12, seek at t=15 while paused, resume at t=20; with `--sw` the five paused ticks between landing and resume show what the `[SWDiag]` line reports while the pump is parked and has not heard of the seek, the AE#479 shape); this is how the pre-arming `setRate` wedge was isolated. +`--subs ` matches against the track's libavcodec name or language and logs every overlay cue and cue trim as it lands. `--host-calls` replays host post-load behavior against the fresh session: `play`, `extractor` (`makeFrameExtractor`), `setrate` (`setRate(1.0)`), `pausestart` (Sodalite#104 round 4: `pause()` the instant load returns, before any frame exists, and `play()` at t=8; the shape of a host that holds a fresh load paused, and a software session used to answer it with eight ticks of `enq=+0 status=unknown r4d=n`, a black picture under a paused clock; now the `[SWHost] #104` lines show the first frame presented at a stopped clock and `startup 8/8 presenting` arrives while paused), `ratehold` (set 1.5, pause at tick 3, resume at tick 5, then read the rate back off the transport itself: the #436 drill, and it fails the run if the resume came back at 1.0), `reloadlive` (reload the URL on the live path when the probe flags it live, the AetherPlayer Open URL flow), `seekback` (rewind 20 s into the DVR window at t=15, return to the live edge at t=30), `overlapseek` (the #292 seek-window drills below), `pausehold` (Sodalite#104: pause at t=10 and HOLD until ten seconds before the end, printing the playhead, the edge and the resident depth every second, which is how a session paused for longer than its own DVR window is measured without waiting out a real one: pair it with `--dvr-window 30` and `--seconds 90` and the ninety minute question becomes a ninety second run), `still` (#544: asks for a scrub still at three aims, 20 s behind the playhead at t=15, 5 s behind at t=20 and at the edge at t=25, writing each to `/tmp/aetherctl-still-.png` and reporting hit or MISS with the decode time; pair it with `--sw --dvr-window N`, where the picture comes out of the DVR packet ring rather than a segment cache, and read the FILE as well as the count, because the bundled seed burns its own second into the frame so a still asked for 14.85 s showing `14` is the verdict that it decoded the right moment and not merely an image; exit 6 when nothing hit, 5 when the session ended before the first aim), and `pauseseek` (pause at t=12, seek at t=15 while paused, resume at t=20; with `--sw` the five paused ticks between landing and resume show what the `[SWDiag]` line reports while the pump is parked and has not heard of the seek, the AE#479 shape); this is how the pre-arming `setRate` wedge was isolated. `--seek-every N` seeks once every N ticks past tick 10, walking `--seek-pattern ` if one is given (a short backward hop otherwise), and `--seek-count K` stops after K seeks so a run can be a BURST and then play. Both halves are needed for anything about what a seek sequence leaves behind: the burst puts the store in the state under test, and only the playing half shows what the overlay carries through it. That pairing is what made AE#362's second mechanism reproducible (a hole between a restarted pump and the island the previous run left ahead of it, decoded across and then never re-read). From fb047c32a6704a980599fcec8bc601c61e67fcc5 Mon Sep 17 00:00:00 2001 From: Vincent Herbst Date: Thu, 17 Sep 2026 22:54:40 +0200 Subject: [PATCH 3/4] fix(live): a still run holds two pool buffers, not a whole GOP (#544) Two hazards from the first cut, both found reading it back rather than from a failing run, which is why they are worth naming. The collector kept every frame the run decoded so it could pick afterwards, and those buffers come out of the decoder's own pool. A long GOP would therefore hold the pool empty against the next run, and the bound that allows 900 packets is exactly the bound that makes it possible. It now keeps the two candidates it can actually return: the newest frame at or before the target, and the oldest as the fallback the live edge needs. Teardown ran on the main actor while a decode could be in flight on the still queue. The decoder's lock makes that survivable rather than safe, so the close now goes onto the still queue itself and lands after whatever was running. Re-measured unchanged: 3 of 3 hit, 16 to 52 ms, and the frame asked for at 15.09 s still comes back marked 15. Full suite green. Part of #544. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015PM3xUJB6ZQyqnmGK1fp6F --- .../Native/SoftwarePlaybackHost.swift | 8 +++- .../Native/SoftwareStillExtractor.swift | 38 +++++++++++++------ 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift b/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift index 93114a46f..3459c50b5 100644 --- a/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift +++ b/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift @@ -1413,8 +1413,12 @@ final class SoftwarePlaybackHost { vodPacketReadAhead = nil renderer.subtitleCompositor.reset() - stillExtractor?.close() - stillExtractor = nil + if let extractor = stillExtractor { + stillExtractor = nil + // Torn down ON the still queue, so a decode already in flight finishes against a codec + // context that is still open rather than one freed out from under it. + stillQueue.async { extractor.close() } + } dvrRing?.close() dvrRing = nil liveEdgeLock.lock() diff --git a/Sources/AetherEngine/Native/SoftwareStillExtractor.swift b/Sources/AetherEngine/Native/SoftwareStillExtractor.swift index aed830988..26ef6d549 100644 --- a/Sources/AetherEngine/Native/SoftwareStillExtractor.swift +++ b/Sources/AetherEngine/Native/SoftwareStillExtractor.swift @@ -72,7 +72,7 @@ final class SoftwareStillExtractor: @unchecked Sendable { reorderTail: limits.reorderTail), !run.isEmpty else { return nil } - let collector = FrameCollector() + let collector = FrameCollector(target: targetPts) decoder.onFrame = { pixelBuffer, pts, _ in collector.append(pixelBuffer: pixelBuffer, seconds: pts.seconds) } @@ -83,7 +83,7 @@ final class SoftwareStillExtractor: @unchecked Sendable { feed(packet) } - guard let best = collector.best(for: targetPts) else { return nil } + guard let best = collector.best else { return nil } return Self.image(from: best, maxWidth: maxWidth) } @@ -110,29 +110,43 @@ final class SoftwareStillExtractor: @unchecked Sendable { // MARK: - Frame selection - /// Collects what the decoder emits during one run. `onFrame` is `@Sendable` and the decoder may - /// call it from its own drain, so the box is locked even though the run itself is serial. + /// Keeps the best candidate as the run decodes rather than every frame it produced. The buffers + /// come out of the decoder's own pool, so holding a whole GOP of them would starve the pool the + /// next run has to draw from. Two are enough: the one the target asks for, and the oldest as the + /// fallback below. + /// + /// `onFrame` is `@Sendable` and the decoder calls it from its own drain, so the box is locked + /// even though the run itself is serial. private final class FrameCollector: @unchecked Sendable { private let lock = NSLock() - private var frames: [(pixelBuffer: CVPixelBuffer, seconds: Double)] = [] + private let target: Double + private var atOrBefore: (pixelBuffer: CVPixelBuffer, seconds: Double)? + private var earliest: (pixelBuffer: CVPixelBuffer, seconds: Double)? + + init(target: Double) { + self.target = target + } func append(pixelBuffer: CVPixelBuffer, seconds: Double) { + guard seconds.isFinite else { return } lock.lock() defer { lock.unlock() } - frames.append((pixelBuffer, seconds)) + if earliest == nil || seconds < earliest!.seconds { + earliest = (pixelBuffer, seconds) + } + guard seconds <= target else { return } + if atOrBefore == nil || seconds > atOrBefore!.seconds { + atOrBefore = (pixelBuffer, seconds) + } } /// The newest frame at or before the target. Falling back to the oldest rather than to /// nothing matters at the live edge, where the target can sit a fraction past every frame /// the run produced. - func best(for target: Double) -> CVPixelBuffer? { + var best: CVPixelBuffer? { lock.lock() defer { lock.unlock() } - guard !frames.isEmpty else { return nil } - let atOrBefore = frames - .filter { $0.seconds.isFinite && $0.seconds <= target } - .max(by: { $0.seconds < $1.seconds }) - return (atOrBefore ?? frames.min(by: { $0.seconds < $1.seconds }))?.pixelBuffer + return (atOrBefore ?? earliest)?.pixelBuffer } } From 7a25eb5f212fee08ea09a064dfe8788496147e69 Mon Sep 17 00:00:00 2001 From: Vincent Herbst Date: Thu, 17 Sep 2026 23:17:36 +0200 Subject: [PATCH 4/4] fix(live): a still stops rebuilding the deinterlace graph, and four review findings (#544) The worst one was not a crash, it was the diagnostic log. `flush()` tears down the deinterlace filter graph, and the still extractor flushes before every run, so on interlaced content every single still rebuilt a Metal pipeline and a full-resolution hwframes pool AND emitted an unconditional `[Deinterlace] engaged` line. A held scrub asks about sixteen times a second, and a host's ring buffer is 300 lines, so half a minute of scrubbing would overwrite the entire log a live playback report depends on, on App Store builds. `flush` grows a `resetFilterGraph` parameter and the still path keeps the graph: a run decodes a full GOP and returns the frame at its target, so the filter has context from this position by the time that frame is made. Measured on an interlaced MPEG-2 fixture, three stills: three graph builds before, one per session after. Three more, each with a failure it actually has: - **Superseded requests ran to completion.** The queue is serial and nothing cancelled, so it took work faster than it retired it: the card fell further behind the thumb with every request and kept decoding past the commit. A newest-wins ticket drops a request that was superseded while it waited. - **A truncated window read as the end of the ring.** `stillRunSpan` could not tell "the ring ran out" from "the caller's window ran out", so a long-GOP high-frame-rate channel could get a still from up to a GOP before the time it asked for, presented as the answer. It now takes `indexReachesEnd` and refuses rather than clamping, with two tests on the distinction. - **The image could alias a pool buffer.** At 1:1 the CGImage was handed back as VideoToolbox made it, and VideoToolbox documents it as backed by the pixel buffer, which returns to the decoder's pool straight afterwards. It is always drawn into an owned bitmap now. Plus: the still path allocates through `trackedPacketAlloc`, so it is visible to the leak instrument like every other packet path; the drill's third aim really is the live edge rather than the playhead; and the collector's fallback comment now names the case it actually covers (the eviction race, not the edge). Verified against a broadcast-shaped fixture this time, MPEG-2 704x480 SAR 10:11 interlaced tff with AC-3, which is what a tuner serves: 3 of 3 stills, 37 to 70 ms, returned 320x240 (so the sample aspect is honoured, not stretched 320x218) and showing the second each one asked for. Full suite green on both runners. Part of #544. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015PM3xUJB6ZQyqnmGK1fp6F --- .../Decoder/SoftwareVideoDecoder.swift | 12 +++++++- .../Native/SoftwarePlaybackHost.swift | 29 +++++++++++++++++++ .../Native/SoftwareStillExtractor.swift | 22 +++++++++----- .../AetherEngine/Video/PacketRingBuffer.swift | 12 ++++++-- Sources/aetherctl/PlaybackCmd.swift | 23 +++++++++++---- .../PacketRingBufferTests.swift | 19 ++++++++++-- 6 files changed, 100 insertions(+), 17 deletions(-) diff --git a/Sources/AetherEngine/Decoder/SoftwareVideoDecoder.swift b/Sources/AetherEngine/Decoder/SoftwareVideoDecoder.swift index fb6311836..37c421432 100644 --- a/Sources/AetherEngine/Decoder/SoftwareVideoDecoder.swift +++ b/Sources/AetherEngine/Decoder/SoftwareVideoDecoder.swift @@ -431,14 +431,24 @@ final class SoftwareVideoDecoder: VideoDecodingPipeline, @unchecked Sendable { onFrame?(pixelBuffer, cmPTS, hdr10PlusData) } + /// #544: `resetFilterGraph: false` keeps the deinterlace graph across the flush. The still + /// extractor flushes before every run, and rebuilding the graph means a fresh Metal pipeline, a + /// fresh full-resolution hwframes pool AND an unconditional `[Deinterlace] engaged` line, about + /// sixteen times a second while a viewer holds the scrub. That line alone overwrites a host's + /// whole diagnostic ring in half a minute. A still run decodes a full GOP and returns the frame + /// at its target, so the filter has context from this position by the time that frame is made. func flush() { + flush(resetFilterGraph: true) + } + + func flush(resetFilterGraph: Bool) { lock.lock() defer { lock.unlock() } // AE#492: retires every packet a caller had already decided to send. Bumped under the lock, // so a feed that has not reached `avcodec_send_packet` yet is refused from here on. _feedEpoch &+= 1 // Deinterlacer temporal references are stale across seeks; drop the graph (lazily rebuilt on next interlaced frame). - deinterlacer.teardown() + if resetFilterGraph { deinterlacer.teardown() } guard let ctx = codecContext else { return } avcodec_flush_buffers(ctx) } diff --git a/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift b/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift index 3459c50b5..4dbaabf84 100644 --- a/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift +++ b/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift @@ -191,6 +191,28 @@ final class SoftwarePlaybackHost { /// loops, so a preview frame never costs playback a packet. private var stillExtractor: SoftwareStillExtractor? private let stillQueue = DispatchQueue(label: "engine.sw.still", qos: .userInitiated) + private let stillRequests = StillRequestCounter() + + /// Newest-wins ticket for still requests. A held scrub asks about sixteen times a second and the + /// queue is serial, so without a ticket the queue takes work faster than it retires it: the card + /// falls further behind the thumb with every request, and the decoding outlives the commit. + final class StillRequestCounter: @unchecked Sendable { + private let lock = NSLock() + private var value: UInt64 = 0 + + func next() -> UInt64 { + lock.lock() + defer { lock.unlock() } + value &+= 1 + return value + } + + var latest: UInt64 { + lock.lock() + defer { lock.unlock() } + return value + } + } /// Disk-spooled DVR rewind ring; non-nil for live sessions with dvrWindowSeconds set. Demux-thread appended (internally locked). nonisolated(unsafe) private var dvrRing: PacketRingBuffer? @@ -1302,8 +1324,15 @@ final class SoftwarePlaybackHost { return sessionStartPts.isFinite ? sessionStartPts : 0 }() let targetSource = startPts + seconds + let requests = stillRequests + let ticket = requests.next() return await withCheckedContinuation { continuation in stillQueue.async { + guard ticket == requests.latest else { + // Superseded while it waited: decoding it would only push the newer one later. + continuation.resume(returning: nil) + return + } continuation.resume( returning: extractor.still(from: ring, targetPts: targetSource, maxWidth: maxWidth)) } diff --git a/Sources/AetherEngine/Native/SoftwareStillExtractor.swift b/Sources/AetherEngine/Native/SoftwareStillExtractor.swift index 26ef6d549..2234cc82d 100644 --- a/Sources/AetherEngine/Native/SoftwareStillExtractor.swift +++ b/Sources/AetherEngine/Native/SoftwareStillExtractor.swift @@ -76,7 +76,7 @@ final class SoftwareStillExtractor: @unchecked Sendable { decoder.onFrame = { pixelBuffer, pts, _ in collector.append(pixelBuffer: pixelBuffer, seconds: pts.seconds) } - decoder.flush() + decoder.flush(resetFilterGraph: false) defer { decoder.onFrame = nil } for packet in run { @@ -91,9 +91,10 @@ final class SoftwareStillExtractor: @unchecked Sendable { private func feed(_ packet: PacketRingBuffer.Packet) { guard !packet.bytes.isEmpty else { return } - guard let p = av_packet_alloc() else { return } + // Through the tracked pair, so the still path stays visible to PacketBalanceTracker. + guard let p = trackedPacketAlloc() else { return } var pkt: UnsafeMutablePointer? = p - defer { av_packet_free(&pkt) } + defer { trackedPacketFree(&pkt) } guard av_new_packet(p, Int32(packet.bytes.count)) >= 0 else { return } packet.bytes.withUnsafeBytes { raw in @@ -140,9 +141,13 @@ final class SoftwareStillExtractor: @unchecked Sendable { } } - /// The newest frame at or before the target. Falling back to the oldest rather than to - /// nothing matters at the live edge, where the target can sit a fraction past every frame - /// the run produced. + /// The newest frame at or before the target. + /// + /// The fallback is not the live edge: a run opens on a keyframe at or before the target, so + /// a frame at or before it normally exists. It covers the eviction race, where that opening + /// keyframe was dropped between the index snapshot and the disk read and the run therefore + /// begins AFTER the target. Returning the earliest frame there is a picture from up to one + /// GOP late, which is a better answer than an empty card. var best: CVPixelBuffer? { lock.lock() defer { lock.unlock() } @@ -165,8 +170,11 @@ final class SoftwareStillExtractor: @unchecked Sendable { let (dstW, dstH) = FrameDecodeContext.displayDimensions( srcW: srcW, srcH: srcH, sar: sampleAspect(of: pixelBuffer), targetWidth: maxWidth) - if dstW == srcW && dstH == srcH { return source } + // Always drawn into an owned bitmap, even at 1:1. VideoToolbox documents the CGImage as + // backed by the CVPixelBuffer it was made from, and that buffer goes back to the decoder's + // pool the moment this run lets go of it, so handing the source out would let the next run + // repaint a picture the host is still showing. guard let space = CGColorSpace(name: CGColorSpace.sRGB), let ctx = CGContext(data: nil, width: dstW, height: dstH, bitsPerComponent: 8, bytesPerRow: 0, space: space, diff --git a/Sources/AetherEngine/Video/PacketRingBuffer.swift b/Sources/AetherEngine/Video/PacketRingBuffer.swift index cf892b807..ea7f4872b 100644 --- a/Sources/AetherEngine/Video/PacketRingBuffer.swift +++ b/Sources/AetherEngine/Video/PacketRingBuffer.swift @@ -177,18 +177,24 @@ final class PacketRingBuffer: @unchecked Sendable { /// pays for. And a live scrub routinely aims a fraction past the newest packet, so a target /// beyond the end clamps to it rather than answering nil, which would blink the card out at /// exactly the edge the viewer sits on most. + /// `indexReachesEnd` says whether `index` runs to the ring's newest entry. It is what separates + /// the two ways the walk can run out of packets: the ring genuinely ending (clamp to it) from a + /// caller's bounded window ending (refuse). Without it a truncated window silently returns a + /// picture from before the requested time and calls it the answer. static func stillRunSpan(target: Double, index: [IndexEntry], firstSeq: Int, maxPackets: Int, maxSpanSeconds: Double, - reorderTail: Int) -> ClosedRange? { + reorderTail: Int, + indexReachesEnd: Bool) -> ClosedRange? { guard index.contains(where: \.isVideo) else { return nil } guard let start = index.indices.last(where: { index[$0].isKeyframe && index[$0].pts <= target }) else { return nil } guard target - index[start].pts <= maxSpanSeconds else { return nil } let reached = index.indices[start...].first(where: { index[$0].isVideo && index[$0].pts >= target }) + guard reached != nil || indexReachesEnd else { return nil } guard var end = reached ?? index.indices.last(where: { index[$0].isVideo }) else { return nil } if reached != nil, reorderTail > 0 { @@ -222,6 +228,7 @@ final class PacketRingBuffer: @unchecked Sendable { return nil } let upper = min(entries.count, startIdx + maxPackets + reorderTail + 1) + let reachesEnd = upper == entries.count let window = entries[startIdx..= 6 { rateHoldAtEnd = Issue436RateHold.observedRate(engine) } } if hostCalls.contains("still"), [15, 20, 25].contains(tick) { - let offset: Double = tick == 15 ? 20 : (tick == 20 ? 5 : 0) - let label = tick == 15 ? "playhead-20" : (tick == 20 ? "playhead-5" : "edge") - let target = max(0, engine.currentTime - offset) + // The third aim is the live EDGE itself, not the playhead: a target a fraction past the + // newest packet is the clamp case, and it is where a live viewer sits most. + let target: Double + let label: String + switch tick { + case 15: + target = max(0, engine.currentTime - 20) + label = "playhead-20" + case 20: + target = max(0, engine.currentTime - 5) + label = "playhead-5" + default: + target = engine.seekableLiveRange?.upperBound ?? engine.currentTime + label = "edge" + } let started = Date() let image = await engine.liveScrubThumbnail(atSessionSeconds: target, maxWidth: 320) let ms = Int(Date().timeIntervalSince(started) * 1000) @@ -924,9 +936,10 @@ private func playSmokeTest(url: URL, seconds: Double, live: Bool, forceSoftware: if let image { stillHits += 1 let path = "/tmp/aetherctl-still-\(tick).png" - _ = writeStillPNG(image, to: path) + let written = writeStillPNG(image, to: path) print(String(format: " HOSTCALL still(at: %.2f, %@) -> %dx%d in %d ms %@", - target, label, image.width, image.height, ms, path)) + target, label, image.width, image.height, ms, + written ? path : "(png write failed)")) } else { print(String(format: " HOSTCALL still(at: %.2f, %@) -> MISS in %d ms", target, label, ms)) } diff --git a/Tests/AetherEngineTests/PacketRingBufferTests.swift b/Tests/AetherEngineTests/PacketRingBufferTests.swift index 39131039e..cb2df5a70 100644 --- a/Tests/AetherEngineTests/PacketRingBufferTests.swift +++ b/Tests/AetherEngineTests/PacketRingBufferTests.swift @@ -117,10 +117,11 @@ final class PacketRingStillRunTests: XCTestCase { firstSeq: Int = 0, maxPackets: Int = 1000, maxSpanSeconds: Double = 30, - reorderTail: Int = 0) -> ClosedRange? { + reorderTail: Int = 0, + indexReachesEnd: Bool = true) -> ClosedRange? { PacketRingBuffer.stillRunSpan(target: target, index: index, firstSeq: firstSeq, maxPackets: maxPackets, maxSpanSeconds: maxSpanSeconds, - reorderTail: reorderTail) + reorderTail: reorderTail, indexReachesEnd: indexReachesEnd) } /// Starts at the newest keyframe at or before the target, ends at the first video packet reaching it. @@ -193,4 +194,18 @@ final class PacketRingStillRunTests: XCTestCase { func testNoVideoIsNil() { XCTAssertNil(span([audio(0), audio(1)], target: 1)) } + + /// A window the CALLER truncated must not be read as the ring ending. Clamping there would + /// return a picture from before the requested time and present it as the answer. + func testTruncatedWindowRefusesInsteadOfClamping() { + let index = [video(0, key: true), video(1), video(2)] + XCTAssertNil(span(index, target: 9.5, indexReachesEnd: false)) + XCTAssertEqual(span(index, target: 9.5, indexReachesEnd: true), 0...2) + } + + /// A target the window DOES reach is unaffected by where the window ends. + func testTruncatedWindowStillAnswersATargetItCovers() { + let index = [video(0, key: true), video(1), video(2)] + XCTAssertEqual(span(index, target: 2, indexReachesEnd: false), 0...2) + } }