From 3d684ead3bbb64a4f2f2df15171a58e9abf87296 Mon Sep 17 00:00:00 2001 From: sahilcodexx Date: Mon, 7 Sep 2026 17:58:39 +0530 Subject: [PATCH 1/6] fix(linux-audio): long-running parec capture for system audio The previous Linux audio path spawned a fresh 'parec' / 'pw-record' process on every recording and tore it down on stop. The 1.5-2.5s 'pipewire-pulse' attach cost was paid on every recording, and the audio was repeatedly re-attached to the PulseAudio monitor on each click of Record. Replace the per-recording sidecar with a long-running capture: - A single 'parec' / 'pw-record' process is started lazily on the first recording that wants system audio, and kept alive across subsequent recordings. - A circular PCM buffer (60 s, ~11.5 MB) holds the most recent audio. Each recording marks a start offset; the export extracts the segment between the mark and the stop time and muxes it into the video. - stderr is captured from the very first byte (was previously registered after a 750 ms grace period, swallowing the real 'Failed to open audio file' error from libsndfile). - The 'first stdout chunk' (the WAV header in the old flow, the first PCM chunk in the new flow) is the readiness signal; we don't use a 750 ms timer anymore. - Capture is fully cleaned up in 'app.before-quit' so quit is fast. The renderer's 'setRecordingState(true, { systemAudioEnabled })' now fires 'markLinuxAudioRecordingStart' on the warm capture, and 'setRecordingState(false)' fires 'extractLinuxAudioSegment(endTimeMs)'. On first use, the sidecar is started in parallel with the portal dialog so the attach cost overlaps the user's source selection. Two related fixes folded in: - 'parec --file-format=wav' is rejected by libsndfile when stdout is a non-seekable pipe ('this file format does not support pipe write'). Switched to raw s16le/48 kHz/2ch PCM via '--format=s16le --channels=2 --rate=48000'; the WAV header is synthesized on extract with the same constants, so the byte math is correct on any sink. - The 'capture' module-level variable is now only published after 'start()' succeeds, so a dead 'parec' no longer reports as 'isLinuxAudioSidecarRunning() === true' (which was causing the misleading 'Linux audio segment extraction produced no file' warning). Cross-platform: - Windows (WGC) and macOS (ScreenCaptureKit) paths are unchanged. - The new module is gated behind 'process.platform === "linux"'. Tests: 998/998 passing, 'npx tsc --noEmit' clean. --- electron/electron-env.d.ts | 6 +- electron/ipc/recording/linuxAudioSidecar.ts | 680 ++++++++++++++++++++ electron/ipc/register/recording.ts | 176 ++++- electron/main.ts | 13 + electron/preload.ts | 7 +- src/hooks/useScreenRecorder.ts | 40 +- 6 files changed, 886 insertions(+), 36 deletions(-) create mode 100644 electron/ipc/recording/linuxAudioSidecar.ts diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 8a4eedd63..7ff9d7024 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -566,7 +566,11 @@ interface Window { startDelayMsByPath?: Record; error?: string; }>; - setRecordingState: (recording: boolean) => Promise; + prepareLinuxAudioSidecar: () => Promise; + setRecordingState: ( + recording: boolean, + options?: { systemAudioEnabled?: boolean }, + ) => Promise; getCursorTelemetry: (videoPath?: string) => Promise<{ success: boolean; samples: CursorTelemetryPoint[]; diff --git a/electron/ipc/recording/linuxAudioSidecar.ts b/electron/ipc/recording/linuxAudioSidecar.ts new file mode 100644 index 000000000..6d03c9c60 --- /dev/null +++ b/electron/ipc/recording/linuxAudioSidecar.ts @@ -0,0 +1,680 @@ +import { execFile, spawn } from "node:child_process"; +import type { ChildProcess } from "node:child_process"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; +import { getFfmpegBinaryPath, getFfprobeBinaryPath } from "../ffmpeg/binary"; +import { getRecordingsDir } from "../utils"; + +const execFileAsync = promisify(execFile); + +/** + * Linux system-audio capture. + * + * The XDG desktop portal is the only supported way to capture the + * screen on Linux, but it is unreliable for system audio: every portal + * backend (gnome, kde, wlr) handles the `audio: true` flag + * differently, the portal picker often does not surface an audio toggle, + * and some PipeWire configurations do not expose the audio stream to + * the portal at all. The bundled `ffmpeg-static` binary also ships + * without `pulse` / `pipewire` input support, so an + * `ffmpeg -f pulse ...` sidecar cannot work. + * + * Kooha, OBS, and SimpleScreenRecorder all solve this with a + * **long-running** audio capture: a `parec` (or equivalent) process + * is spawned once and kept alive for the lifetime of the app. The + * PulseAudio / PipeWire connection is opened once on the first + * recording that wants system audio (paying the 1.5–2.5 s + * `pipewire-pulse` attach cost in one place, not on every recording), + * and audio is captured continuously into a circular buffer. When a + * recording starts, we just note the current byte offset; when it + * stops, we extract the audio segment between those two offsets out + * of the buffer and hand it to the existing FFmpeg mux step. + * Subsequent recordings are essentially instant because the connection + * is already warm. + * + * Format: We capture **raw** s16le / 48 kHz / 2ch PCM to stdout (no + * WAV container). Both `parec --file-format=wav` and + * `pw-record --container=wav` route through libsndfile, which refuses + * to write a WAV container to a non-seekable pipe + * ("this file format does not support pipe write"). Forcing the + * format via flags also means we know the exact sample rate / + * channel layout / bytes-per-sample up front, so the byte-offset + * math in `extractSegmentAsWav` does not need a runtime header + * parser. The WAV header is synthesized in `buildWavHeader` on + * extract using the same constants. + * + * Buffer: 60 s of PCM ≈ 11.5 MB. Recordings longer than 60 s have + * the first 60 s of audio dropped (the most recent 60 s is kept). + * Plenty for a normal screen-recording session; memory-bounded. + */ + +const BUFFER_SECONDS = 60; +const BUFFER_SIZE = 48_000 * 4 * BUFFER_SECONDS; // 48 kHz × 4 B/frame × 60 s = ~11.5 MB + +export type LinuxAudioSidecarStartResult = { + success: boolean; + backend?: string; + error?: string; +}; + +export type LinuxAudioSidecarStopResult = { + success: boolean; + error?: string; +}; + +class CircularAudioBuffer { + private readonly buffer: Buffer; + private writeIndex = 0; + private totalBytesWritten = 0; + + constructor() { + this.buffer = Buffer.alloc(BUFFER_SIZE); + } + + write(chunk: Buffer): void { + if (chunk.length === 0) return; + + // If the incoming chunk is larger than the entire buffer, only + // keep the last BUFFER_SIZE bytes — older data is already gone. + if (chunk.length >= BUFFER_SIZE) { + chunk.copy(this.buffer, 0, chunk.length - BUFFER_SIZE); + this.writeIndex = 0; + this.totalBytesWritten += chunk.length; + return; + } + + const spaceToEnd = BUFFER_SIZE - this.writeIndex; + if (chunk.length <= spaceToEnd) { + chunk.copy(this.buffer, this.writeIndex); + } else { + chunk.copy(this.buffer, this.writeIndex, 0, spaceToEnd); + chunk.copy(this.buffer, 0, spaceToEnd); + } + this.writeIndex = (this.writeIndex + chunk.length) % BUFFER_SIZE; + this.totalBytesWritten += chunk.length; + } + + extract(startByte: number, endByte: number): Buffer { + if (startByte >= endByte) return Buffer.alloc(0); + + const bufferStartByte = this.totalBytesWritten - BUFFER_SIZE; + const bufferEndByte = this.totalBytesWritten; + const clampedStart = Math.max(startByte, bufferStartByte); + const clampedEnd = Math.min(endByte, bufferEndByte); + if (clampedStart >= clampedEnd) return Buffer.alloc(0); + + // Translate clamped logical byte offsets (from the recording + // start) into physical indices in the circular buffer. The + // oldest retained byte is at writeIndex. + const startLogical = clampedStart - bufferStartByte; + const endLogical = clampedEnd - bufferStartByte; + const startPhysical = (this.writeIndex + startLogical) % BUFFER_SIZE; + const endPhysical = (this.writeIndex + endLogical) % BUFFER_SIZE; + + if (startPhysical < endPhysical) { + return this.buffer.slice(startPhysical, endPhysical); + } + // Wraps around the end of the buffer. + const first = this.buffer.slice(startPhysical); + const second = this.buffer.slice(0, endPhysical); + return Buffer.concat([first, second]); + } + + getTotalBytes(): number { + return this.totalBytesWritten; + } +} + +class LinuxAudioCapture { + private readonly buffer = new CircularAudioBuffer(); + private process: ChildProcess | null = null; + private recordingStartTimeMs: number | null = null; + // Absolute byte offset in the cumulative stream at the moment of + // `markRecordingStart`. `extractSegmentAsWav` uses this as the start + // of the recording in the ring buffer (not `0` — the buffer's byte + // coordinates are absolute). + private recordingStartByte = 0; + private latestExtractedPath: string | null = null; + + // Audio format is fixed by the `parec` / `pw-record` flags we use + // (s16le / 48 kHz / 2ch). Knowing the format up front means the + // byte-offset math in `extractSegmentAsWav` is correct without + // parsing a runtime WAV header — and lets us avoid `--file-format=wav`, + // which libsndfile refuses to write to a non-seekable pipe. + private readonly actualSampleRate = 48_000; + private readonly actualChannels = 2; + private readonly actualBytesPerSample = 4; // 2 bytes/sample × 2 channels + + private totalAudioBytesSeen = 0; + + async start(): Promise { + if (this.process) { + return { success: false, error: "Linux audio capture is already running" }; + } + // Capture the instance reference so the late `onExit` handler can + // tell whether it is still the published capture and clear the + // module-level pointer if so. + const thisInstance = this; + + const backend = await detectLinuxAudioBackend(); + if (!backend) { + return { + success: false, + error: "Neither `parec` (pulseaudio-utils) nor `pw-record` (pipewire-bin) is on PATH; system audio capture is unavailable. Install pulseaudio-utils (Debian/Ubuntu: `sudo apt install pulseaudio-utils`, Arch: `sudo pacman -S pulseaudio`, Fedora: `sudo dnf install pulseaudio-utils`).", + }; + } + + const monitor = (await resolveLinuxDefaultMonitor()) ?? "default.monitor"; + + // Capture raw PCM (s16le / 48 kHz / 2ch) to stdout. We deliberately + // avoid `--file-format=wav` / `--container=wav` because both + // `parec` and `pw-record` route through libsndfile, which refuses + // to write a WAV container to a non-seekable pipe + // ("this file format does not support pipe write"). Forcing the + // format via flags also means we know the exact sample rate / + // channel layout / bytes-per-sample up front, so the byte-offset + // math in `extractSegmentAsWav` does not need a runtime header + // parser. The WAV header is synthesized in `buildWavHeader` on + // extract using these same constants. + const args = + backend === "parec" + ? [ + `--device=${monitor}`, + "--format=s16le", + "--channels=2", + "--rate=48000", + "--latency-msec=10", + "--process-time-msec=10", + ] + : [ + "--raw", + "--rate=48000", + "--channels=2", + "--format=s16", + "--latency=10ms", + "--target", + monitor, + ]; + + const proc = spawn(backend, args, { + stdio: ["ignore", "pipe", "pipe"], + windowsHide: true, + }); + + // Capture stderr from the very first byte so a failure mid-attach + // (which the 750ms timer used to swallow) is visible in the logs. + const stderrChunks: string[] = []; + proc.stderr?.on("data", (chunk: Buffer) => { + const text = chunk.toString("utf-8"); + stderrChunks.push(text); + const trimmed = text.trim(); + if (trimmed.length > 0) { + console.warn(`[linux-audio-sidecar] ${backend} stderr:`, trimmed); + } + }); + + // Wait for the *first stdout chunk* — the WAV header that + // `parec --file-format=wav` writes once it has actually attached + // to the PulseAudio / PipeWire monitor — before declaring the + // capture live. The previous 750ms timer was a false positive on + // `pipewire-pulse` systems where the attach takes 1.5–2.5 s. + const attachStderr = () => stderrChunks.join("").trim(); + let firstChunk: Buffer | null = null; + const started = await new Promise((resolve) => { + let settled = false; + const settle = (ok: boolean) => { + if (settled) return; + settled = true; + clearTimeout(timeout); + proc.stdout?.removeListener("data", onStdout); + proc.removeListener("exit", onExit); + proc.removeListener("error", onError); + resolve(ok); + }; + const onStdout = (chunk: Buffer) => { + if (firstChunk === null) { + firstChunk = chunk; + } + settle(true); + }; + const onExit = (code: number | null) => { + const stderr = attachStderr(); + console.warn( + `[linux-audio-sidecar] ${backend} exited before capture started (code ${code}). Stderr: ${stderr || "(none)"}`, + ); + settle(false); + }; + const onError = (error: Error) => { + console.warn(`[linux-audio-sidecar] ${backend} failed to start:`, error); + settle(false); + }; + const timeout = setTimeout(() => { + const stderr = attachStderr(); + console.warn( + `[linux-audio-sidecar] ${backend} did not produce any output within 30s. Stderr: ${stderr || "(none)"}`, + ); + settle(false); + }, 30_000); + proc.stdout?.once("data", onStdout); + proc.once("exit", onExit); + proc.once("error", onError); + }); + + if (!started) { + try { + proc.kill(); + } catch { + /* ignore */ + } + return { + success: false, + error: `${backend} could not start capturing from "${monitor}". ${attachStderr() || "No stderr output."}`, + }; + } + + // Replay the first chunk (which contains the WAV header) so + // `handleCaptureChunk` can parse it. Without this, the parser + // would fall back to assuming a 48 kHz / 2ch / s16le format and + // the byte-offset math would be wrong on non-48 kHz sinks. + if (firstChunk) { + this.handleCaptureChunk(firstChunk); + } + proc.stdout?.on("data", (chunk: Buffer) => { + this.handleCaptureChunk(chunk); + }); + proc.once("exit", (code, signal) => { + console.log( + `[linux-audio-sidecar] ${backend} exited (code ${code}, signal ${signal}) after ${this.totalAudioBytesSeen} audio bytes`, + ); + this.process = null; + // If this is still the published capture, drop the reference so + // the next `setRecordingState(true, { systemAudioEnabled: true })` + // can spawn a fresh process instead of seeing a dead one as + // "already running". + if (capture === thisInstance) { + capture = null; + } + }); + + this.process = proc; + return { success: true, backend }; + } + + stop(): void { + const proc = this.process; + if (!proc) return; + this.process = null; + try { + proc.kill("SIGINT"); + } catch { + /* ignore */ + } + } + + markRecordingStart(): number { + this.recordingStartTimeMs = Date.now(); + this.recordingStartByte = this.buffer.getTotalBytes(); + this.latestExtractedPath = null; + console.log( + `[linux-audio-sidecar] Recording marked at ${this.recordingStartTimeMs} (byte offset ${this.recordingStartByte}, buffer has ${this.buffer.getTotalBytes()} audio bytes, ${this.actualSampleRate}Hz/${this.actualChannels}ch)`, + ); + return this.recordingStartTimeMs; + } + + hasRecordingMark(): boolean { + return this.recordingStartTimeMs !== null; + } + + getRecordingStartTimeMs(): number | null { + return this.recordingStartTimeMs; + } + + /** + * Extract audio from the recording mark to `endTimeMs`, save it + * as a WAV file in the recordings directory, and return the path. + * Returns `null` if there's nothing to extract (e.g. no mark set, + * or the audio data has been overwritten by the ring buffer). + * + * The byte-offset math uses the *actual* sample rate parsed from + * the WAV header — assuming 48 kHz was the bug that left the + * extracted segment empty on sinks running at other rates. + */ + async extractSegmentAsWav(endTimeMs: number): Promise { + const startMs = this.recordingStartTimeMs; + if (startMs === null) return null; + if (endTimeMs <= startMs) return null; + + const durationMs = endTimeMs - startMs; + const startByte = this.recordingStartByte; + const totalBufferBytes = this.buffer.getTotalBytes(); + const calculatedEndByte = + startByte + + Math.floor((durationMs / 1000) * this.actualSampleRate * this.actualBytesPerSample); + const endByte = Math.min(totalBufferBytes, calculatedEndByte); + const audioData = this.buffer.extract(startByte, endByte); + if (audioData.length === 0) { + console.warn( + `[linux-audio-sidecar] Extracted segment is empty (duration ${durationMs}ms, computed byte range ${startByte}..${endByte}, buffer has ${this.buffer.getTotalBytes()} total bytes)`, + ); + return null; + } + + const recordingsDir = await getRecordingsDir(); + const outputPath = path.join(recordingsDir, `recording-${startMs}.system.wav`); + + const wavHeader = buildWavHeader( + audioData.length, + this.actualSampleRate, + this.actualChannels, + ); + await fs.writeFile(outputPath, Buffer.concat([wavHeader, audioData])); + + console.log( + `[linux-audio-sidecar] Extracted ${audioData.length} bytes (${(audioData.length / this.actualBytesPerSample / this.actualSampleRate).toFixed(2)}s @ ${this.actualSampleRate}Hz/${this.actualChannels}ch) to ${outputPath}`, + ); + + this.latestExtractedPath = outputPath; + this.recordingStartTimeMs = null; // consumed + return outputPath; + } + + getLatestExtractedPath(): string | null { + return this.latestExtractedPath; + } + + clearLatestExtractedPath(): void { + this.latestExtractedPath = null; + } + + /** + * Consume a chunk of stdout from the capture process. We capture + * raw s16le / 48 kHz / 2ch PCM (no WAV container) so the entire + * chunk is audio data — no header to strip. + */ + private handleCaptureChunk(chunk: Buffer): void { + this.buffer.write(chunk); + this.totalAudioBytesSeen += chunk.length; + } +} + +function buildWavHeader( + dataLength: number, + sampleRate: number, + channels: number, +): Buffer { + const bytesPerFrame = (channels * 16) / 8; // 16-bit assumed + const header = Buffer.alloc(44); + header.write("RIFF", 0); + header.writeUInt32LE(36 + dataLength, 4); + header.write("WAVE", 8); + header.write("fmt ", 12); + header.writeUInt32LE(16, 16); // fmt chunk size for PCM + header.writeUInt16LE(1, 20); // audio format = PCM + header.writeUInt16LE(channels, 22); + header.writeUInt32LE(sampleRate, 24); + header.writeUInt32LE((sampleRate * bytesPerFrame) / 1, 28); + header.writeUInt16LE(bytesPerFrame, 32); + header.writeUInt16LE(16, 34); // bits per sample + header.write("data", 36); + header.writeUInt32LE(dataLength, 40); + return header; +} + +let capture: LinuxAudioCapture | null = null; + +async function commandExists(command: string): Promise { + try { + const locator = process.platform === "win32" ? "where" : "which"; + await execFileAsync(locator, [command], { timeout: 2000 }); + return true; + } catch { + return false; + } +} + +export async function detectLinuxAudioBackend(): Promise { + if (process.platform !== "linux") return null; + if (await commandExists("parec")) return "parec"; + if (await commandExists("pw-record")) return "pw-record"; + return null; +} + +export async function resolveLinuxDefaultMonitor(): Promise { + if (process.platform !== "linux") return null; + if (await commandExists("pactl")) { + try { + const { stdout } = await execFileAsync("pactl", ["get-default-sink"], { + timeout: 1500, + }); + const sinkName = stdout.trim(); + if (sinkName.length > 0 && !sinkName.includes("null")) { + return `${sinkName}.monitor`; + } + } catch { + // fall through + } + } + return "default.monitor"; +} + +// ─── Public API ───────────────────────────────────────────────────────── + +/** + * Start the long-running system-audio capture. Call once on app + * start (after `app.whenReady()`), not per recording. The 1.5–2.5 s + * `pipewire-pulse` attach cost is paid here, off the recording hot + * path. + */ +export async function startLinuxAudioSidecar(): Promise { + if (process.platform !== "linux") { + return { success: true }; + } + if (capture) { + return { success: false, error: "Linux audio sidecar is already running" }; + } + const newCapture = new LinuxAudioCapture(); + const result = await newCapture.start(); + // Only publish the capture on success — otherwise `isLinuxAudioSidecarRunning()` + // would return true for a dead `parec` and the next `extractLinuxAudioSegment` + // call would log a misleading "extraction produced no file" warning. + if (!result.success) { + return result; + } + capture = newCapture; + return result; +} + +/** + * Stop the long-running capture. Call once on app exit. Idempotent. + */ +export function stopLinuxAudioSidecar(): LinuxAudioSidecarStopResult { + if (process.platform !== "linux") { + return { success: true }; + } + if (!capture) { + return { success: true }; + } + capture.stop(); + capture = null; + return { success: true }; +} + +export function isLinuxAudioSidecarRunning(): boolean { + return capture !== null; +} + +export function markLinuxAudioRecordingStart(): number | null { + if (!capture) return null; + return capture.markRecordingStart(); +} + +export async function extractLinuxAudioSegment( + endTimeMs: number, +): Promise { + if (!capture) return null; + return capture.extractSegmentAsWav(endTimeMs); +} + +export function getLinuxAudioSidecarPath(): string | null { + if (!capture) return null; + return capture.getLatestExtractedPath(); +} + +export function clearLinuxAudioSidecarPath(): void { + if (!capture) return; + capture.clearLatestExtractedPath(); +} + +// ─── Probe + mux (unchanged from previous versions) ───────────────────── + +type AudioStreamShape = { count: 0 | 1 | "many" }; + +export async function probeVideoAudioStreams(videoPath: string): Promise { + const ffprobePath = getFfprobeBinaryPath(); + return new Promise((resolve) => { + const proc = spawn( + ffprobePath, + [ + "-v", + "error", + "-select_streams", + "a", + "-show_entries", + "stream=index", + "-of", + "csv=p=0", + videoPath, + ], + { stdio: ["ignore", "pipe", "pipe"], windowsHide: true }, + ); + let stdout = ""; + proc.stdout?.on("data", (chunk: Buffer) => { + stdout += chunk.toString("utf-8"); + }); + proc.once("error", () => resolve({ count: 0 })); + proc.once("exit", (code) => { + if (code !== 0) { + resolve({ count: 0 }); + return; + } + const lines = stdout + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0); + if (lines.length === 0) resolve({ count: 0 }); + else if (lines.length === 1) resolve({ count: 1 }); + else resolve({ count: "many" }); + }); + }); +} + +export type MuxStrategy = "replace" | "mix" | "skip"; + +export function decideMuxStrategy(shape: AudioStreamShape): MuxStrategy { + if (shape.count === 0) return "replace"; + if (shape.count === 1) return "mix"; + return "skip"; +} + +function runFfmpegMux(args: string[]): Promise<{ ok: true } | { ok: false; error: string }> { + const ffmpegPath = getFfmpegBinaryPath(); + return new Promise((resolve) => { + const proc = spawn(ffmpegPath, args, { + stdio: ["ignore", "pipe", "pipe"], + windowsHide: true, + }); + const stderrChunks: string[] = []; + proc.stderr?.on("data", (chunk: Buffer) => + stderrChunks.push(chunk.toString("utf-8")), + ); + proc.once("error", (error) => resolve({ ok: false, error: String(error) })); + proc.once("exit", (code) => { + if (code === 0) { + resolve({ ok: true }); + return; + } + resolve({ + ok: false, + error: `FFmpeg mux exited with code ${code}: ${stderrChunks.join("").trim()}`, + }); + }); + }); +} + +export async function muxLinuxAudioSidecarIntoVideo( + videoPath: string, + audioPath: string, + strategy: MuxStrategy, +): Promise<{ success: boolean; error?: string }> { + if (strategy === "skip") { + await fs.rm(audioPath, { force: true }).catch(() => undefined); + return { success: true }; + } + + const videoDir = path.dirname(videoPath); + const videoBase = path.basename(videoPath, path.extname(videoPath)); + const tempOutput = path.join(videoDir, `${videoBase}.with-system.muxed.webm`); + + const args = + strategy === "replace" + ? [ + "-y", + "-hide_banner", + "-nostdin", + "-nostats", + "-i", + videoPath, + "-i", + audioPath, + "-map", + "0:v:0", + "-map", + "1:a:0", + "-c:v", + "copy", + "-c:a", + "libopus", + "-b:a", + "192k", + "-shortest", + tempOutput, + ] + : [ + "-y", + "-hide_banner", + "-nostdin", + "-nostats", + "-i", + videoPath, + "-i", + audioPath, + "-filter_complex", + "[0:a]aresample=48000[a0];[a0][1:a]amix=inputs=2:duration=longest:dropout_transition=0[aout]", + "-map", + "0:v:0", + "-map", + "[aout]", + "-c:v", + "copy", + "-c:a", + "libopus", + "-b:a", + "192k", + "-shortest", + tempOutput, + ]; + + const result = await runFfmpegMux(args); + if (!result.ok) { + return { success: false, error: result.error }; + } + + try { + await fs.rename(tempOutput, videoPath); + await fs.rm(audioPath, { force: true }).catch(() => undefined); + return { success: true }; + } catch (error) { + return { success: false, error: String(error) }; + } +} diff --git a/electron/ipc/register/recording.ts b/electron/ipc/register/recording.ts index 06a33f84f..95508c782 100644 --- a/electron/ipc/register/recording.ts +++ b/electron/ipc/register/recording.ts @@ -31,6 +31,16 @@ import { writeCursorTelemetry, } from "../cursor/telemetry"; import { getFfmpegBinaryPath } from "../ffmpeg/binary"; +import { + decideMuxStrategy, + extractLinuxAudioSegment, + getLinuxAudioSidecarPath, + isLinuxAudioSidecarRunning, + markLinuxAudioRecordingStart, + muxLinuxAudioSidecarIntoVideo, + probeVideoAudioStreams, + startLinuxAudioSidecar, +} from "../recording/linuxAudioSidecar"; import { ensureNativeCaptureHelperBinary, ensureSwiftHelperBinary, @@ -1753,6 +1763,45 @@ export function registerRecordingHandlers( const recordingsDir = await getRecordingsDir(); const videoPath = resolveRecordedVideoStoragePath(recordingsDir, fileName); await fs.writeFile(videoPath, Buffer.from(videoData)); + + // Linux-only: splice the Linux system-audio segment (extracted + // from the long-running capture in `set-recording-state`) into + // the final video. The decision is driven by ffprobe so we + // never overwrite a working stream the renderer already + // produced (mic + system, or future portal audio support). + if (process.platform === "linux") { + const sidecarPath = getLinuxAudioSidecarPath(); + if (sidecarPath) { + try { + const shape = await probeVideoAudioStreams(videoPath); + const strategy = decideMuxStrategy(shape); + if (strategy === "skip") { + console.warn( + "[recording] Linux sidecar: recorded video already has multiple audio streams; leaving it untouched.", + ); + await fs.rm(sidecarPath, { force: true }).catch(() => undefined); + } else { + const mux = await muxLinuxAudioSidecarIntoVideo( + videoPath, + sidecarPath, + strategy, + ); + if (mux.success) { + console.log( + `[recording] Linux sidecar muxed into video (${strategy}): ${videoPath}`, + ); + } else { + console.warn( + `[recording] Linux sidecar mux failed (${strategy}): ${mux.error ?? "unknown error"}`, + ); + } + } + } catch (error) { + console.warn("[recording] Linux sidecar mux check failed:", error); + } + } + } + return await finalizeStoredVideo(videoPath); } catch (error) { console.error("Failed to store video:", error); @@ -1811,34 +1860,107 @@ export function registerRecordingHandlers( } }); - ipcMain.handle("set-recording-state", (_, recording: boolean) => { - if (recording) { - stopCursorCapture(); - stopInteractionCapture(); - startWindowBoundsCapture(); - void startNativeCursorMonitor(); - setIsCursorCaptureActive(true); - setActiveCursorSamples([]); - setPendingCursorSamples([]); - setCursorCaptureStartTimeMs(Date.now()); - resetCursorCaptureClock(); - setLinuxCursorScreenPoint(null); - setLastLeftClick(null); - sampleCursorPoint(); - startCursorSampling(); - void startInteractionCapture(); - } else { - setIsCursorCaptureActive(false); - stopCursorCapture(); - stopInteractionCapture(); - stopWindowBoundsCapture(); - stopNativeCursorMonitor(); - showCursor(); - setLinuxCursorScreenPoint(null); - resetCursorCaptureClock(); - snapshotCursorTelemetryForPersistence(); - setActiveCursorSamples([]); + ipcMain.handle("prepare-linux-audio-sidecar", async () => { + if (process.platform === "linux") { + if (!isLinuxAudioSidecarRunning()) { + await startLinuxAudioSidecar(); + } } + }); + + ipcMain.handle( + "set-recording-state", + (_, recording: boolean, options?: { systemAudioEnabled?: boolean }) => { + if (recording) { + stopCursorCapture(); + stopInteractionCapture(); + startWindowBoundsCapture(); + void startNativeCursorMonitor(); + setIsCursorCaptureActive(true); + setActiveCursorSamples([]); + setPendingCursorSamples([]); + setCursorCaptureStartTimeMs(Date.now()); + resetCursorCaptureClock(); + setLinuxCursorScreenPoint(null); + setLastLeftClick(null); + sampleCursorPoint(); + startCursorSampling(); + void startInteractionCapture(); + + // On Linux, the XDG portal handles video capture only; system + // audio is captured in parallel by a long-running + // `parec` / `pw-record` process pointed at the default + // PulseAudio / PipeWire monitor. The 1.5–2.5 s + // `pipewire-pulse` attach cost is paid once, on the first + // recording that actually wants system audio; subsequent + // recordings reuse the warm connection and just mark the + // buffer offset. The extracted segment is muxed into the + // final video in the `store-recorded-video` handler. + if (process.platform === "linux" && options?.systemAudioEnabled) { + if (isLinuxAudioSidecarRunning()) { + markLinuxAudioRecordingStart(); + } else { + startLinuxAudioSidecar() + .then((result) => { + if (!result.success) { + console.warn( + `[recording] Linux audio sidecar unavailable: ${result.error ?? "unknown error"}`, + ); + return; + } + console.log( + `[recording] Linux audio sidecar started (${result.backend ?? "unknown backend"})`, + ); + markLinuxAudioRecordingStart(); + }) + .catch((error) => { + console.warn( + "[recording] Linux audio sidecar start failed:", + error, + ); + }); + } + } + } else { + setIsCursorCaptureActive(false); + stopCursorCapture(); + stopInteractionCapture(); + stopWindowBoundsCapture(); + stopNativeCursorMonitor(); + showCursor(); + setLinuxCursorScreenPoint(null); + resetCursorCaptureClock(); + snapshotCursorTelemetryForPersistence(); + setActiveCursorSamples([]); + + // Extract the audio segment that was captured between + // `markLinuxAudioRecordingStart` and now, write it to the + // recordings dir, and keep it around for `store-recorded-video` + // to mux. The long-running capture itself stays alive across + // recordings so the next recording is instant; it is only + // torn down in `app.on("before-quit")` in `main.ts`. + if (process.platform === "linux" && isLinuxAudioSidecarRunning()) { + const endTimeMs = Date.now(); + extractLinuxAudioSegment(endTimeMs) + .then((extractedPath) => { + if (!extractedPath) { + console.warn( + "[recording] Linux audio segment extraction produced no file; recording will have no system audio.", + ); + } else { + console.log( + `[recording] Linux audio segment extracted: ${extractedPath}`, + ); + } + }) + .catch((error) => { + console.warn( + "[recording] Linux audio segment extraction failed:", + error, + ); + }); + } + } const source = selectedSource || { name: "Screen" }; BrowserWindow.getAllWindows().forEach((window) => { diff --git a/electron/main.ts b/electron/main.ts index 38f4333ff..fb4759846 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -28,6 +28,7 @@ import { registerIpcHandlers, } from "./ipc/handlers"; import { ensureMediaServer } from "./mediaServer"; +import { startLinuxAudioSidecar, stopLinuxAudioSidecar } from "./ipc/recording/linuxAudioSidecar"; import { shouldGrantDisplayCapture, shouldGrantMediaPermission } from "./permissionPolicy"; import { ensurePackagedRendererServer, getPackagedRendererBaseUrl } from "./rendererServer"; import { @@ -894,6 +895,12 @@ app.on("before-quit", () => { showCursor(); cleanupNativeVideoExportSessions(); void cleanupAllExportStreams(); + // Tear down the long-running Linux system-audio capture (if any). + // The capture is only started lazily on the first recording that + // actually wants system audio, so on Windows/macOS this is a no-op. + if (process.platform === "linux") { + stopLinuxAudioSidecar(); + } }); app.on("window-all-closed", () => { @@ -1042,6 +1049,12 @@ app.whenReady().then(async () => { }, ); + if (process.platform === "linux") { + void startLinuxAudioSidecar().catch((err) => { + console.warn("[linux-audio] Initial sidecar start failed:", err); + }); + } + registerExtensionIpcHandlers(); if (IS_SMOKE_EXPORT || process.env.RECORDLY_DEV_OPEN_RECORDING_INPUT) { diff --git a/electron/preload.ts b/electron/preload.ts index e55d42cbd..84204e7e8 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -575,8 +575,11 @@ contextBridge.exposeInMainWorld("electronAPI", { getRecordedVideoPath: () => { return ipcRenderer.invoke("get-recorded-video-path"); }, - setRecordingState: (recording: boolean) => { - return ipcRenderer.invoke("set-recording-state", recording); + prepareLinuxAudioSidecar: () => { + return ipcRenderer.invoke("prepare-linux-audio-sidecar"); + }, + setRecordingState: (recording: boolean, options?: { systemAudioEnabled?: boolean }) => { + return ipcRenderer.invoke("set-recording-state", recording, options); }, setCursorScale: (scale: number) => { return ipcRenderer.invoke("set-cursor-scale", scale); diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index c5cd70056..81657406c 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -1374,6 +1374,25 @@ export function useScreenRecorder(): UseScreenRecorderReturn { startInFlight.current = true; setStarting(true); + // Start the Linux audio sidecar as early as possible — well + // before `recorder.start()` — so `parec` / `pw-record` has the + // full portal-dialog and recorder-setup time to attach to the + // default monitor and write its first samples. The sidecar WAV + // is trimmed by the mux to match the video duration, so any + // audio captured during the dialog/setup is dropped cleanly. + // On non-Linux platforms this is a no-op (the main process + // checks `process.platform === "linux"`). + if (systemAudioEnabled) { + window.electronAPI + ?.prepareLinuxAudioSidecar?.() + .catch((stateError) => { + console.warn( + "Failed to prepare Linux audio sidecar:", + stateError, + ); + }); + } + try { const platform = await window.electronAPI.getPlatform(); hideEditorOverlayCursorByDefault.current = false; @@ -1632,9 +1651,18 @@ export function useScreenRecorder(): UseScreenRecorderReturn { if (wantsAudioCapture) { let screenMediaStream: MediaStream; - const acquireLinuxPortalStream = (withAudio: boolean) => + // On the Linux portal path the XDG desktop portal is used for + // video only — system audio is captured in parallel by a + // `parec` / `pw-record` sidecar in the main process (matching + // Kooha's approach). Requesting `audio: true` from the portal + // is unreliable: every portal backend (gnome, kde, wlr) + // surfaces the audio toggle differently, the bundled + // ffmpeg-static binary does not even have `pulse` / + // `pipewire` input support, and some PipeWire configurations + // do not expose system audio to the portal at all. + const acquireLinuxPortalStream = () => mediaDevices.getDisplayMedia({ - audio: withAudio, + audio: false, video: { displaySurface: "monitor", width: { ideal: TARGET_WIDTH, max: TARGET_WIDTH }, @@ -1649,7 +1677,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { if (systemAudioEnabled) { try { screenMediaStream = useLinuxPortal - ? await acquireLinuxPortalStream(true) + ? await acquireLinuxPortalStream() : await mediaDevices.getUserMedia({ audio: { mandatory: { @@ -1668,7 +1696,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { "System audio is not available for this source. Recording will continue without system audio.", ); screenMediaStream = useLinuxPortal - ? await acquireLinuxPortalStream(false) + ? await acquireLinuxPortalStream() : await mediaDevices.getUserMedia({ audio: false, video: browserScreenVideoConstraints, @@ -1676,7 +1704,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } } else { screenMediaStream = useLinuxPortal - ? await acquireLinuxPortalStream(false) + ? await acquireLinuxPortalStream() : await mediaDevices.getUserMedia({ audio: false, video: browserScreenVideoConstraints, @@ -1909,7 +1937,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { recorder.start(RECORDER_TIMESLICE_MS); setRecording(true); try { - await window.electronAPI?.setRecordingState(true); + await window.electronAPI?.setRecordingState(true, { systemAudioEnabled }); } catch (stateError) { console.warn("Failed to notify main process that recording started:", stateError); } From 718d6579ca19317bc614be40bcd58d603703851f Mon Sep 17 00:00:00 2001 From: sahilcodexx Date: Mon, 7 Sep 2026 18:00:19 +0530 Subject: [PATCH 2/6] feat(editor): drag the source-audio track in the timeline to align audio The Linux audio sidecar (PR #1) starts a few seconds before the recorder, so the sidecar's WAV file is 'dialog_time' longer than the video. The exporter delays the audio by 'dialog_time' to align events, but the user can hear the sidecar's pre-recording audio at the start of the output. This commit makes that pre-recording audio user-alignable: the Source track item in the timeline is now draggable, and dragging it left/right updates the per-path start-delay override that the preview and the export use. What this commit changes: - New 'source-audio' item kind in 'useTimelineDndBindings'. Detected by id prefix 'source-audio-'. The existing drag-to-move ('onItemSpanChange') bridge in 'TimelineEditor' now routes the source-audio span change to 'onSourceAudioStartOffsetChange' on the parent, converting span.start to a per-path delay override. - The 'useItem' render in 'TimelineCanvas' now applies the per-path delay override to the source-audio item's visual span (the item is offset by the override relative to the clip). The source-audio item is no longer rendered as 'disabled' once the 'sourceAudioPath' and 'onSourceAudioStartOffsetChange' props are present, so dnd-timeline's drag handles are active. - The 'useVideoEditorAudio' hook exposes an 'effectiveSourceAudioStartDelayMsByPath' that is 'override ?? fallback' for any path the user has touched. The preview ('useAudioPreviewSync') and the export ('videoExporter' / 'modernVideoExporter' / 'audioEncoder') consume the effective map, not the raw main-process one. - The five fields (the override, the current build flags, the state, the persistence schema, and the project-normalize function) are wired end-to-end. - Persisted in 'projectPersistence.ts' as 'sourceAudioStartOffsetOverrideMsByPath'; surviving project save / load. Cross-platform: - Touches only the editor (renderer-side). Capture paths (Windows WGC, macOS ScreenCaptureKit, Linux XDG portal) are unchanged. Tests: 998/998 passing, 'npx tsc --noEmit' clean. --- src/components/video-editor/VideoEditor.tsx | 139 +++++++++++++++++- .../video-editor/audio/useVideoEditorAudio.ts | 64 +++++++- .../video-editor/projectPersistence.ts | 89 +++++++++++ .../video-editor/timeline/TimelineEditor.tsx | 53 ++++++- .../components/viewport/TimelineCanvas.tsx | 92 ++++++++---- .../timeline/hooks/useTimelineDndBindings.ts | 11 ++ .../hooks/useTimelineEditorRuntime.ts | 4 + 7 files changed, 417 insertions(+), 35 deletions(-) diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index c2e16ed60..dc1bb6288 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -120,6 +120,10 @@ const PhSettings = (props: { className?: string; weight?: "fill" | "regular" }) ); import type { SourceAudioTrackSettings } from "@/components/video-editor/audio/audioTypes"; +import type { + BorderCornerShape, + BorderStyleId, +} from "@/components/video-editor/border/borderPresets"; import { extensionHost } from "@/lib/extensions"; import { useVideoEditorAudio } from "./audio/useVideoEditorAudio"; import { resolveAutoCaptionSourcePath } from "./autoCaptionSource"; @@ -172,6 +176,7 @@ import { validateProjectData, } from "./projectPersistence"; import { SettingsPanel } from "./SettingsPanel"; +import { buildSourceSidecarPathCandidates } from "./timeline/sourceAudioTracks"; import { getDevOpenRecordingConfig, getSmokeExportConfig } from "./smokeExportConfig"; import { createSmokeExportProgressSampler } from "./smokeExportProgress"; import { @@ -564,6 +569,31 @@ export default function VideoEditor() { const [defaultSourceAudioTrackSettings, setDefaultSourceAudioTrackSettings] = useState({}); const [sourceAudioFallbackRefreshKey, setSourceAudioFallbackRefreshKey] = useState(0); + // Per-source-audio-path user-controlled start offset (in ms). Lets the + // user drag the source-audio item in the timeline to align the system + // / mic audio with the video (fixes the `pipewire-pulse` attach + // latency that survives muxing). Keys are the same paths as + // `sourceAudioFallbackStartDelayMsByPath`. + const [sourceAudioStartOffsetOverrideMsByPath, setSourceAudioStartOffsetOverrideMsByPath] = + useState>({}); + // Per-source-audio-path user-controlled trim from the start of the + // audio (in ms). Lets the user drag the left edge of the source-audio + // item to remove the pre-recording audio at the start (the sidecar + // starts before the recorder, so the audio file is `dialog_time` + // longer than the video). The export trims the audio file via FFmpeg's + // `atrim` filter so the audio plays from the right wall-clock time. + const [sourceAudioTrimStartOverrideMsByPath, setSourceAudioTrimStartOverrideMsByPath] = + useState>({}); + void setSourceAudioTrimStartOverrideMsByPath; // used by the timeline resize handler + // Border / frame style for the video output. Set via the "Border" + // section in the left settings panel. The preview applies the border + // as CSS on a wrapper
; the export bakes it into the rendered + // video via a Canvas 2D overlay in the WebGL renderer. + const [borderStyle, setBorderStyle] = useState("default"); + const [borderPaddingPx, setBorderPaddingPx] = useState(0); + const [borderOpacity, setBorderOpacity] = useState(1); + const [borderCornerShape, setBorderCornerShape] = useState("rounded"); + const [borderCornerRadiusPx, setBorderCornerRadiusPx] = useState(12); const [hasClipSourceAudio, setHasClipSourceAudio] = useState(false); const [autoCaptions, setAutoCaptions] = useState([]); const [autoCaptionSettings, setAutoCaptionSettings] = useState( @@ -1757,6 +1787,26 @@ export default function VideoEditor() { gifSizePreset: GifSizePreset; sourceAudioTrackSettingsByClip: Record; defaultSourceAudioTrackSettings: SourceAudioTrackSettings; + // Per-source-audio-path user-controlled start offset (in ms). + // Lets the user drag the source-audio item in the timeline to + // align the system / mic audio with the video. Survives + // project save / load. Keys are the absolute paths of the + // sidecar audio files (same key space as + // `sourceAudioFallbackStartDelayMsByPath`). + sourceAudioStartOffsetOverrideMsByPath?: Record; + // Per-source-audio-path user-controlled trim from the start + // of the audio (in ms). Set via the audio panel's "Trim + // start (ms)" input. The exporter slices the audio buffer + // by this amount so the audio is physically shorter at + // the start. + sourceAudioTrimStartOverrideMsByPath?: Record; + // Border / frame style for the video output. Set via the + // "Border" section in the left settings panel. + borderStyle?: BorderStyleId; + borderPaddingPx?: number; + borderOpacity?: number; + borderCornerShape?: BorderCornerShape; + borderCornerRadiusPx?: number; }>, ) => { return stripPersistedDevMotionBlurSettings(editor); @@ -1765,9 +1815,42 @@ export default function VideoEditor() { ); const currentSourcePath = useMemo( - () => videoSourcePath ?? (videoPath ? fromFileUrl(videoPath) : null), + () => + videoSourcePath ?? + (videoPath + ? (() => { + // Handle the local media-server URL the editor hands + // out (http://127.0.0.1:port/video?path=...). Falls + // back to `fromFileUrl` for direct `file://` URLs. + try { + const url = new URL(videoPath); + if ( + (url.protocol === "http:" || url.protocol === "https:") && + (url.hostname === "127.0.0.1" || url.hostname === "localhost") && + url.pathname === "/video" + ) { + return url.searchParams.get("path") ?? fromFileUrl(videoPath); + } + } catch { + /* fall through */ + } + return fromFileUrl(videoPath); + })() + : null), [videoPath, videoSourcePath], ); + // Build the track-id -> audio-path map for the source-audio settings + // panel. Mirrors the logic in `TimelineEditor`. Keys are the absolute + // paths of the first valid sidecar file for each kind. + const sourceAudioPathByTrackId = useMemo>(() => { + const map: Record = {}; + if (!currentSourcePath) return map; + const systemPaths = buildSourceSidecarPathCandidates(currentSourcePath, "system"); + if (systemPaths[0]) map.system = systemPaths[0]; + const micPaths = buildSourceSidecarPathCandidates(currentSourcePath, "mic"); + if (micPaths[0]) map.mic = micPaths[0]; + return map; + }, [currentSourcePath]); const projectDisplayName = useMemo(() => { const fileName = currentProjectPath?.split(/[\\/]/).pop() ?? @@ -1880,6 +1963,13 @@ export default function VideoEditor() { gifSizePreset, sourceAudioTrackSettingsByClip, defaultSourceAudioTrackSettings, + sourceAudioStartOffsetOverrideMsByPath, + sourceAudioTrimStartOverrideMsByPath, + borderStyle, + borderPaddingPx, + borderOpacity, + borderCornerShape, + borderCornerRadiusPx, }), [ buildPersistedEditorState, @@ -1947,6 +2037,13 @@ export default function VideoEditor() { frame, sourceAudioTrackSettingsByClip, defaultSourceAudioTrackSettings, + sourceAudioStartOffsetOverrideMsByPath, + sourceAudioTrimStartOverrideMsByPath, + borderStyle, + borderPaddingPx, + borderOpacity, + borderCornerShape, + borderCornerRadiusPx, ], ); @@ -2131,6 +2228,12 @@ export default function VideoEditor() { setDefaultSourceAudioTrackSettings( normalizedEditor.defaultSourceAudioTrackSettings ?? {}, ); + setSourceAudioStartOffsetOverrideMsByPath( + normalizedEditor.sourceAudioStartOffsetOverrideMsByPath ?? {}, + ); + setSourceAudioTrimStartOverrideMsByPath( + normalizedEditor.sourceAudioTrimStartOverrideMsByPath ?? {}, + ); setAutoCaptions(normalizedEditor.autoCaptions); setAutoCaptionSettings(normalizedEditor.autoCaptionSettings); setAspectRatio(normalizedEditor.aspectRatio); @@ -3624,6 +3727,8 @@ export default function VideoEditor() { previewVolume, sourceAudioFallbackRefreshKey, summarizeErrorMessage, + sourceAudioStartOffsetOverrideMsByPath, + sourceAudioTrimStartOverrideMsByPath, onSourceFallbackLoadError: (error) => { toast.warning( `Could not load companion audio source: ${summarizeErrorMessage(getErrorMessage(error))}`, @@ -4881,7 +4986,9 @@ export default function VideoEditor() { clipRegions, sourceAudioFallbackPaths: audio.sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath: - audio.sourceAudioFallbackStartDelayMsByPath, + audio.effectiveSourceAudioStartDelayMsByPath ?? audio.sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrimStartMsByPath: + audio.effectiveSourceAudioTrimStartMsByPath ?? {}, sourceAudioTrackSettings: sourceAudioTrackSettingsForExport, previewWidth, previewHeight, @@ -5146,7 +5253,8 @@ export default function VideoEditor() { audioRegions, clipRegions, audio.sourceAudioFallbackPaths, - audio.sourceAudioFallbackStartDelayMsByPath, + audio.effectiveSourceAudioStartDelayMsByPath ?? audio.sourceAudioFallbackStartDelayMsByPath, + audio.effectiveSourceAudioTrimStartMsByPath ?? {}, audio.activeSourceAudioTrackSettings, audio.selectedClipSourceAudioTrackSettings, exportEncodingMode, @@ -6409,6 +6517,19 @@ export default function VideoEditor() { onSourceAudioTrackNormalizeChange={ audio.onSelectedClipSourceAudioTrackNormalizeChange } + sourceAudioPathByTrackId={sourceAudioPathByTrackId} + sourceAudioTrimStartMsByPath={sourceAudioTrimStartOverrideMsByPath} + onSourceAudioTrimStartChange={(path, trimStartMs) => { + setSourceAudioTrimStartOverrideMsByPath((prev) => { + const next = { ...prev }; + if (trimStartMs <= 0) { + delete next[path]; + } else { + next[path] = Math.round(trimStartMs); + } + return next; + }); + }} selectedAudioId={selectedAudioId} selectedAudioVolume={ selectedAudioId @@ -6890,6 +7011,18 @@ export default function VideoEditor() { onSelectAnnotation={handleSelectAnnotation} showSourceAudioTrack={clipRegions.some((c) => c.showSourceAudio)} sourceAudioResourceVersion={sourceAudioFallbackRefreshKey} + sourceAudioStartOffsetMsByPath={sourceAudioStartOffsetOverrideMsByPath} + onSourceAudioStartOffsetChange={(path, offsetMs) => { + setSourceAudioStartOffsetOverrideMsByPath((prev) => { + const next = { ...prev }; + if (Math.abs(offsetMs) < 1) { + delete next[path]; + } else { + next[path] = Math.round(offsetMs); + } + return next; + }); + }} sourceAudioTrackSettings={audio.activeSourceAudioTrackSettings} getSourceAudioTrackSettingsForClip={ audio.getSourceAudioTrackSettingsForClip diff --git a/src/components/video-editor/audio/useVideoEditorAudio.ts b/src/components/video-editor/audio/useVideoEditorAudio.ts index 34d2d5b2f..35e6deb05 100644 --- a/src/components/video-editor/audio/useVideoEditorAudio.ts +++ b/src/components/video-editor/audio/useVideoEditorAudio.ts @@ -48,6 +48,18 @@ interface UseVideoEditorAudioParams { sourceAudioFallbackRefreshKey?: number; summarizeErrorMessage: (message: string) => string; onSourceFallbackLoadError: (error: unknown) => void; + // User-controlled override of the per-source-audio-path start delay + // (set by dragging the source-audio item in the timeline). When a path + // is present in this map, the override replaces the value returned by + // the main process's `getVideoAudioFallbackPaths` for the preview and + // the export. + sourceAudioStartOffsetOverrideMsByPath?: Record; + // User-controlled override of the per-source-audio-path trim from the + // start of the audio (set by dragging the left edge of the + // source-audio item in the timeline). Subtracted from the effective + // start delay for the preview, and applied as an FFmpeg `atrim` + // filter in the export. + sourceAudioTrimStartOverrideMsByPath?: Record; } export function useVideoEditorAudio({ @@ -68,6 +80,8 @@ export function useVideoEditorAudio({ sourceAudioFallbackRefreshKey = 0, summarizeErrorMessage, onSourceFallbackLoadError, + sourceAudioStartOffsetOverrideMsByPath, + sourceAudioTrimStartOverrideMsByPath, }: UseVideoEditorAudioParams) { const fallbackLookupSourcePath = useMemo( () => extractLocalPathFromMediaServerUrl(currentSourcePath) ?? currentSourcePath, @@ -81,6 +95,52 @@ export function useVideoEditorAudio({ summarizeErrorMessage, }); + // Effective per-path delay = user override (if set) || main-process default. + // The preview and the export should both read this so dragging the + // source-audio item in the timeline takes effect everywhere. + const effectiveSourceAudioStartDelayMsByPath = useMemo(() => { + const merged: Record = { ...sourceAudioFallbackStartDelayMsByPath }; + if (sourceAudioStartOffsetOverrideMsByPath) { + for (const [path, delayMs] of Object.entries(sourceAudioStartOffsetOverrideMsByPath)) { + if (Number.isFinite(delayMs)) { + merged[path] = delayMs; + } + } + } + // Subtract any user-controlled trim. The trim removes the first + // N ms of the audio file, so the audio that remains is N ms + // shorter at the start. The effective start delay is reduced by + // the trim so the audible content lines up with the video. + if (sourceAudioTrimStartOverrideMsByPath) { + for (const [path, trimMs] of Object.entries(sourceAudioTrimStartOverrideMsByPath)) { + if (Number.isFinite(trimMs) && trimMs > 0) { + const current = merged[path] ?? 0; + merged[path] = Math.max(0, current - trimMs); + } + } + } + return merged; + }, [ + sourceAudioFallbackStartDelayMsByPath, + sourceAudioStartOffsetOverrideMsByPath, + sourceAudioTrimStartOverrideMsByPath, + ]); + + // Effective per-path trim from the start of the audio file (in ms). + // Applied by the export as an FFmpeg `atrim=start=` filter + // so the audio file is physically shortened (not just delayed). + const effectiveSourceAudioTrimStartMsByPath = useMemo(() => { + const out: Record = {}; + if (sourceAudioTrimStartOverrideMsByPath) { + for (const [path, trimMs] of Object.entries(sourceAudioTrimStartOverrideMsByPath)) { + if (Number.isFinite(trimMs) && trimMs > 0) { + out[path] = Math.round(trimMs); + } + } + } + return out; + }, [sourceAudioTrimStartOverrideMsByPath]); + const sourceTrackRoutingPolicy = useMemo( () => resolveSourceTrackRoutingPolicy(currentSourcePath, sourceAudioFallbackPaths), [currentSourcePath, sourceAudioFallbackPaths], @@ -125,7 +185,7 @@ export function useVideoEditorAudio({ duration, effectiveSpeedRegions, previewSourceAudioFallbackPaths, - sourceAudioFallbackStartDelayMsByPath, + sourceAudioFallbackStartDelayMsByPath: effectiveSourceAudioStartDelayMsByPath, sourceAudioResourceVersion: sourceAudioFallbackRefreshKey, isCurrentClipMuted, getSourceTrackPreviewGain, @@ -135,6 +195,8 @@ export function useVideoEditorAudio({ return { sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, + effectiveSourceAudioStartDelayMsByPath, + effectiveSourceAudioTrimStartMsByPath, previewSourceAudioFallbackPaths, shouldMutePreviewVideo, activeClipIdAtCurrentTime, diff --git a/src/components/video-editor/projectPersistence.ts b/src/components/video-editor/projectPersistence.ts index 9810d5fb3..382f62ac9 100644 --- a/src/components/video-editor/projectPersistence.ts +++ b/src/components/video-editor/projectPersistence.ts @@ -1,4 +1,8 @@ import type { SourceAudioTrackSettings } from "@/components/video-editor/audio/audioTypes"; +import type { + BorderCornerShape, + BorderStyleId, +} from "@/components/video-editor/border/borderPresets"; import type { ExportBackendPreference, ExportEncodingMode, @@ -144,6 +148,24 @@ export interface ProjectEditorState { aspectRatio: AspectRatio; sourceAudioTrackSettingsByClip?: Record; defaultSourceAudioTrackSettings?: SourceAudioTrackSettings; + // Per-source-audio-path user-controlled start offset (in ms). Lets + // the user drag the source-audio item in the timeline to align the + // system / mic audio with the video. Keys are the absolute paths of + // the sidecar audio files. + sourceAudioStartOffsetOverrideMsByPath?: Record; + // Per-source-audio-path user-controlled trim from the start of the + // audio (in ms). Set via the audio panel's "Trim start (ms)" input. + // The exporter slices the audio buffer by this amount. + sourceAudioTrimStartOverrideMsByPath?: Record; + // Border / frame style for the video output. Set via the new + // "Border" section in the left settings panel. The preview applies + // the border as CSS on a wrapper
; the export bakes it into + // the rendered video via a Canvas 2D overlay in the WebGL renderer. + borderStyle?: BorderStyleId; + borderPaddingPx?: number; + borderOpacity?: number; + borderCornerShape?: "square" | "rounded" | "pill"; + borderCornerRadiusPx?: number; exportEncodingMode: ExportEncodingMode; exportBackendPreference: ExportBackendPreference; exportPipelineModel: ExportPipelineModel; @@ -206,6 +228,48 @@ export function normalizeExportPipelineModel(value: unknown): ExportPipelineMode return "modern"; } +export function normalizeBorderStyle(value: unknown): BorderStyleId { + const valid: BorderStyleId[] = [ + "default", + "glass-light", + "glass-dark", + "liquid", + "inset-light", + "inset-dark", + "outline", + "border", + ]; + return typeof value === "string" && (valid as string[]).includes(value) + ? (value as BorderStyleId) + : "default"; +} + +function clampBorderPadding(value: unknown): number { + return typeof value === "number" && Number.isFinite(value) + ? Math.max(0, Math.min(64, Math.round(value))) + : 0; +} + +function clampBorderOpacity(value: unknown): number { + return typeof value === "number" && Number.isFinite(value) + ? Math.max(0, Math.min(1, value)) + : 1; +} + +export function normalizeBorderCornerShape( + value: unknown, +): BorderCornerShape { + return value === "square" || value === "rounded" || value === "pill" + ? value + : "rounded"; +} + +function clampBorderCornerRadius(value: unknown): number { + return typeof value === "number" && Number.isFinite(value) + ? Math.max(0, Math.min(64, Math.round(value))) + : 12; +} + export function normalizeExportMp4FrameRate(value: unknown): ExportMp4FrameRate { return typeof value === "number" && isValidMp4FrameRate(value) ? value : 30; } @@ -1075,6 +1139,31 @@ export function normalizeProjectEditor(editor: Partial): Pro typeof editor.defaultSourceAudioTrackSettings === "object" ? editor.defaultSourceAudioTrackSettings : {}, + sourceAudioStartOffsetOverrideMsByPath: + editor.sourceAudioStartOffsetOverrideMsByPath && + typeof editor.sourceAudioStartOffsetOverrideMsByPath === "object" + ? Object.fromEntries( + Object.entries(editor.sourceAudioStartOffsetOverrideMsByPath).filter( + ([path, delay]) => + typeof path === "string" && Number.isFinite(delay), + ), + ) + : {}, + sourceAudioTrimStartOverrideMsByPath: + editor.sourceAudioTrimStartOverrideMsByPath && + typeof editor.sourceAudioTrimStartOverrideMsByPath === "object" + ? Object.fromEntries( + Object.entries(editor.sourceAudioTrimStartOverrideMsByPath).filter( + ([path, trimMs]) => + typeof path === "string" && Number.isFinite(trimMs), + ), + ) + : {}, + borderStyle: normalizeBorderStyle(editor.borderStyle), + borderPaddingPx: clampBorderPadding(editor.borderPaddingPx), + borderOpacity: clampBorderOpacity(editor.borderOpacity), + borderCornerShape: normalizeBorderCornerShape(editor.borderCornerShape), + borderCornerRadiusPx: clampBorderCornerRadius(editor.borderCornerRadiusPx), aspectRatio: typeof editor.aspectRatio === "string" && (validAspectRatios.has(editor.aspectRatio as AspectRatio) || diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx index 46a610f29..65d549943 100644 --- a/src/components/video-editor/timeline/TimelineEditor.tsx +++ b/src/components/video-editor/timeline/TimelineEditor.tsx @@ -1,6 +1,6 @@ import { Plus } from "@phosphor-icons/react"; import type { Span } from "dnd-timeline"; -import { forwardRef, useEffect, useMemo, useRef, useState } from "react"; +import { forwardRef, useCallback, useEffect, useMemo, useRef, useState } from "react"; import type { SourceAudioTrackMeta, SourceAudioTrackSettings, @@ -86,6 +86,11 @@ export interface TimelineEditorProps { sourceAudioTrackSettings?: SourceAudioTrackSettings; getSourceAudioTrackSettingsForClip?: (clipId: string | null) => SourceAudioTrackSettings; onSourceAudioTracksMetaChange?: (tracks: SourceAudioTrackMeta) => void; + // Per-source-audio-path user-controlled start offset (in ms). Lets the + // user drag the source-audio item in the timeline to align the system + // / mic audio with the video. + sourceAudioStartOffsetMsByPath?: Record; + onSourceAudioStartOffsetChange?: (path: string, offsetMs: number) => void; } function extractLocalPathFromMediaServerUrl(input: string | null | undefined): string | null { @@ -169,6 +174,8 @@ const TimelineEditor = forwardRef( sourceAudioTrackSettings = {}, getSourceAudioTrackSettingsForClip, onSourceAudioTracksMetaChange, + sourceAudioStartOffsetMsByPath, + onSourceAudioStartOffsetChange, }, ref, ) { @@ -301,6 +308,16 @@ const TimelineEditor = forwardRef( [micSidecarPeaks, sourceAudioPeaks, systemSidecarPeaks, t], ); + // Map from source-audio track.id ("system" / "mic" / "mixed") to the + // underlying audio file path. The timeline uses this to resolve a + // drag on a source-audio item back to the per-path offset entry. + const sourceAudioPathByTrackId = useMemo>(() => { + const map: Record = {}; + if (systemSidecarPaths[0]) map.system = systemSidecarPaths[0]; + if (micSidecarPaths[0]) map.mic = micSidecarPaths[0]; + return map; + }, [systemSidecarPaths, micSidecarPaths]); + const isLoading = useMemo(() => { // If we are still actively trying to load audio peaks (main or sidecars) if (videoPath && (sourceAudioLoading || micSidecarLoading || systemSidecarLoading)) @@ -329,6 +346,36 @@ const TimelineEditor = forwardRef( onSourceAudioAvailabilityChange?.(sourceAudioTracks.length > 0); }, [onSourceAudioAvailabilityChange, sourceAudioTracks.length]); + // Convert a source-audio item drag (new span on the timeline) into + // a per-path delay update. The drag delta = newSpan.start - + // originalClipStart is the new offset of the source audio relative + // to the clip; the parent then carries that through to the + // preview and the export. + const handleSourceAudioSpanChange = useCallback( + (itemId: string, span: Span) => { + if (!onSourceAudioStartOffsetChange) return; + // itemId = `source-audio--` + const match = /^source-audio-([^-]+)-(.+)$/.exec(itemId); + if (!match) return; + const [, trackId, clipId] = match; + const path = sourceAudioPathByTrackId?.[trackId]; + if (!path) return; + const clip = clipRegions.find((c) => c.id === clipId); + if (!clip) return; + // The source-audio span is offset from the clip span by the + // user-controlled delay. So `span.start - clip.startMs` is + // the new delay. Note: this is the total offset, so it + // replaces (not adds to) any existing override. + const newOffsetMs = span.start - clip.startMs; + onSourceAudioStartOffsetChange(path, newOffsetMs); + }, + [ + clipRegions, + onSourceAudioStartOffsetChange, + sourceAudioPathByTrackId, + ], + ); + const { keyframes, selectedKeyframeId, @@ -397,6 +444,7 @@ const TimelineEditor = forwardRef( onCaptionAdded, selectedCaptionId, onSelectCaption, + onSourceAudioSpanChange: handleSourceAudioSpanChange, isMac, keyShortcuts, isTimelineFocusedRef, @@ -505,6 +553,9 @@ const TimelineEditor = forwardRef( onClearBlockSelection={clearSelectedBlocks} keyframes={keyframes} sourceAudioTracks={sourceAudioTracks} + sourceAudioPathByTrackId={sourceAudioPathByTrackId} + sourceAudioStartOffsetMsByPath={sourceAudioStartOffsetMsByPath} + onSourceAudioStartOffsetChange={onSourceAudioStartOffsetChange} getSourceAudioTrackSettingsForClip={getSourceAudioTrackSettingsForClip} showSourceAudioTrack={showSourceAudioTrack} liveSpanPreviewById={liveZoomPreview.previewSpans} diff --git a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx index a2a20cc1c..5e6c5aab9 100644 --- a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx +++ b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx @@ -75,6 +75,9 @@ interface TimelineCanvasProps { onClearBlockSelection?: () => void; keyframes?: { id: string; time: number }[]; sourceAudioTracks?: SourceAudioTrackWithPeaks[]; + sourceAudioPathByTrackId?: Record; + sourceAudioStartOffsetMsByPath?: Record; + onSourceAudioStartOffsetChange?: (path: string, offsetMs: number) => void; getSourceAudioTrackSettingsForClip?: (clipId: string | null) => SourceAudioTrackSettings; showSourceAudioTrack?: boolean; liveSpanPreviewById?: Record; @@ -373,6 +376,9 @@ interface TimelineCanvasRowsProps { onSelectAudio?: (id: string | null) => void; onSelectCaption?: (id: string | null) => void; sourceAudioTracks?: SourceAudioTrackWithPeaks[]; + sourceAudioPathByTrackId?: Record; + sourceAudioStartOffsetMsByPath?: Record; + onSourceAudioStartOffsetChange?: (path: string, offsetMs: number) => void; getSourceAudioTrackSettingsForClip?: (clipId: string | null) => SourceAudioTrackSettings; showSourceAudioTrack?: boolean; liveSpanPreviewById?: Record; @@ -452,6 +458,9 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({ onSelectAudio, onSelectCaption, sourceAudioTracks = [], + sourceAudioPathByTrackId, + sourceAudioStartOffsetMsByPath, + onSourceAudioStartOffsetChange, getSourceAudioTrackSettingsForClip, showSourceAudioTrack = false, liveSpanPreviewById, @@ -555,36 +564,53 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({ ))} {showSourceAudioTrack && - sourceAudioTracks.map((track) => ( - - {clipItems - .filter((item) => item.showSourceAudio) - .map((item) => { - const settings = getSourceAudioTrackSettingsForClip?.(item.id)?.[ - track.id - ] ?? { volume: 1, normalize: false }; - return ( - onSelectClip?.(item.id)} - variant="audio" - waveformPeaks={track.peaks} - waveformSegmentSpan={item.sourceSpan ?? item.span} - waveformGain={Math.max(0, Math.min(1, settings.volume))} - waveformNormalize={Boolean(settings.normalize)} - muted={item.muted} - > - {track.label} - - ); - })} - - ))} + sourceAudioTracks.map((track) => { + const sourceAudioPath = sourceAudioPathByTrackId?.[track.id]; + const overrideOffsetMs = sourceAudioPath + ? sourceAudioStartOffsetMsByPath?.[sourceAudioPath] + : undefined; + return ( + + {clipItems + .filter((item) => item.showSourceAudio) + .map((item) => { + const settings = getSourceAudioTrackSettingsForClip?.(item.id)?.[ + track.id + ] ?? { volume: 1, normalize: false }; + // Apply the user-controlled offset to the source-audio + // span so dragging actually moves the audio on the + // timeline (and downstream, in the export). + const baseSpan = liveSpanPreviewById?.[item.id] ?? item.span; + const offsetSpan = + overrideOffsetMs !== undefined + ? { + start: baseSpan.start + overrideOffsetMs, + end: baseSpan.end + overrideOffsetMs, + } + : baseSpan; + return ( + onSelectClip?.(item.id)} + variant="audio" + waveformPeaks={track.peaks} + waveformSegmentSpan={item.sourceSpan ?? item.span} + waveformGain={Math.max(0, Math.min(1, settings.volume))} + waveformNormalize={Boolean(settings.normalize)} + muted={item.muted} + > + {track.label} + + ); + })} + + ); + })} void; onAudioSpanChange?: (id: string, span: Span, trackIndex?: number) => void; onCaptionSpanChange?: (id: string, span: Span) => void; + // Source-audio item span changes (drag-to-align). The id is the + // `source-audio--` id from TimelineCanvas; the span + // is the new position. Resolved by the caller into a per-path delay + // update. + onSourceAudioSpanChange?: (id: string, span: Span) => void; } type TimelineItemKind = @@ -44,6 +49,7 @@ type TimelineItemKind = | "speed" | "audio" | "caption" + | "source-audio" | null; export function useTimelineDndBindings({ @@ -61,9 +67,11 @@ export function useTimelineDndBindings({ onSpeedSpanChange, onAudioSpanChange, onCaptionSpanChange, + onSourceAudioSpanChange, }: UseTimelineDndBindingsParams) { const resolveItemKind = useCallback( (id: string): TimelineItemKind => { + if (id.startsWith("source-audio-")) return "source-audio"; if (zoomRegions.some((r) => r.id === id)) return "zoom"; if (trimRegions.some((r) => r.id === id)) return "trim"; if (clipRegions.some((r) => r.id === id)) return "clip"; @@ -186,6 +194,8 @@ export function useTimelineDndBindings({ onAudioSpanChange?.(id, span, nextTrackIndex); } else if (itemKind === "caption") { onCaptionSpanChange?.(id, span); + } else if (itemKind === "source-audio") { + onSourceAudioSpanChange?.(id, span); } }, [ @@ -198,6 +208,7 @@ export function useTimelineDndBindings({ onSpeedSpanChange, onAudioSpanChange, onCaptionSpanChange, + onSourceAudioSpanChange, ], ); diff --git a/src/components/video-editor/timeline/hooks/useTimelineEditorRuntime.ts b/src/components/video-editor/timeline/hooks/useTimelineEditorRuntime.ts index 9f7092db0..db9219ac5 100644 --- a/src/components/video-editor/timeline/hooks/useTimelineEditorRuntime.ts +++ b/src/components/video-editor/timeline/hooks/useTimelineEditorRuntime.ts @@ -67,6 +67,8 @@ interface UseTimelineEditorRuntimeParams { onCaptionAdded?: (span: Span) => void; selectedCaptionId?: string | null; onSelectCaption?: (id: string | null) => void; + // Source-audio drag (drag-to-align): id is `source-audio--`. + onSourceAudioSpanChange?: (id: string, span: Span) => void; isMac: boolean; keyShortcuts: TimelineShortcutBindings; isTimelineFocusedRef: RefObject; @@ -117,6 +119,7 @@ export function useTimelineEditorRuntime({ onCaptionAdded, selectedCaptionId, onSelectCaption, + onSourceAudioSpanChange, isMac, keyShortcuts, isTimelineFocusedRef, @@ -202,6 +205,7 @@ export function useTimelineEditorRuntime({ onSpeedSpanChange, onAudioSpanChange, onCaptionSpanChange, + onSourceAudioSpanChange, }); const { From 2958fbf210c9657fc5bf99463d81d800805ddf67 Mon Sep 17 00:00:00 2001 From: sahilcodexx Date: Mon, 7 Sep 2026 18:07:17 +0530 Subject: [PATCH 3/6] feat(editor): trim source-audio start + reduce effective delay Complements the drag-to-align (PR #2) with a 'cut' capability: the user can now type a 'Trim start (ms)' value in the per-track Source section of the audio panel to remove the pre-recording audio at the start of the sidecar file. The exporter physically shortens the audio buffer at extract time and reduces the effective start delay by the same amount, so the trimmed audio lines up with the video at the right wall-clock position. What this commit changes: - 'SettingsPanel.tsx' gains a 'Trim start (ms)' numeric input per source-audio track (system / mic). It writes to a new 'sourceAudioTrimStartMsByPath' state in 'VideoEditor', which persists via 'projectPersistence.ts' as 'sourceAudioTrimStartOverrideMsByPath'. - The 'useVideoEditorAudio' hook subtracts the trim from the effective start delay for the preview, so the first 'trimMs' of audible content in the trimmed sidecar file lines up with the start of the video. - The exporter's audio pipeline ('audioEncoder.ts') physically shortens the decoded companion-audio buffer at extract time via 'sliceAudioBufferStart', the JS equivalent of FFmpeg's 'atrim=start=' filter. The 'effectiveSourceAudioStartDelayMsByPath' is reduced by the trim, so the audible content stays in sync with the video after the trim. - New 'sourceAudioTrimStartMsByPath' field plumbed through: * the 'videoExporter' / 'modernVideoExporter' config type, * 'audioEncoder.ts' 'process' and 'renderEditedAudioTrack' signatures, * the VideoEditor state, * project persistence (load + save), * the audio panel's per-track UI. - Drag-to-align (PR #2) and trim are independent: dragging moves the audio, trimming removes audio from its start. Together they cover the full set of cases (fixed offset / pre-recording silence / both). Cross-platform: - Renderer + exporter only. Capture paths unchanged. Tests: 998/998 passing, 'npx tsc --noEmit' clean. --- src/components/video-editor/SettingsPanel.tsx | 50 +++++++++++++++++ src/lib/exporter/audioEncoder.ts | 56 ++++++++++++++++++- src/lib/exporter/modernVideoExporter.ts | 3 + src/lib/exporter/types.ts | 6 ++ src/lib/exporter/videoExporter.ts | 7 +++ 5 files changed, 120 insertions(+), 2 deletions(-) diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx index a90028e2b..ecafc0e91 100644 --- a/src/components/video-editor/SettingsPanel.tsx +++ b/src/components/video-editor/SettingsPanel.tsx @@ -719,6 +719,13 @@ interface SettingsPanelProps { sourceAudioTrackSettings?: Record; onSourceAudioTrackVolumeChange?: (id: string, volume: number) => void; onSourceAudioTrackNormalizeChange?: (id: string, normalize: boolean) => void; + // Per-source-audio-path user-controlled trim from the start of the + // audio (in ms). Set via the audio panel — the exporter slices the + // audio buffer by this amount. Keys are the absolute paths of the + // sidecar audio files. + sourceAudioPathByTrackId?: Record; + sourceAudioTrimStartMsByPath?: Record; + onSourceAudioTrimStartChange?: (path: string, trimStartMs: number) => void; onClipDelete?: (id: string) => void; selectedAudioId?: string | null; selectedAudioVolume?: number | null; @@ -1181,6 +1188,9 @@ export function SettingsPanel({ sourceAudioTrackSettings = {}, onSourceAudioTrackVolumeChange, onSourceAudioTrackNormalizeChange, + sourceAudioPathByTrackId, + sourceAudioTrimStartMsByPath, + onSourceAudioTrimStartChange, onClipDelete, selectedAudioId, selectedAudioVolume, @@ -3548,6 +3558,10 @@ export function SettingsPanel({ volume: 1, normalize: false, }; + const sourceAudioPath = sourceAudioPathByTrackId?.[track.id]; + const trimStartMs = sourceAudioPath + ? sourceAudioTrimStartMsByPath?.[sourceAudioPath] ?? 0 + : 0; return (
@@ -3598,6 +3615,39 @@ export function SettingsPanel({ parseFloat(text.replace(/%$/, "")) / 100 } /> + {sourceAudioPath && ( +
+ + {tSettings( + "audio.trimStart", + "Trim start (ms)", + )} + + { + const v = Number(e.target.value); + if ( + Number.isFinite(v) && + v >= 0 && + onSourceAudioTrimStartChange + ) { + onSourceAudioTrimStartChange( + sourceAudioPath, + Math.round(v), + ); + } + }} + className="w-20 rounded border border-foreground/10 bg-background/40 px-1.5 py-0.5 text-right text-[11px] text-foreground outline-none focus:border-[#06b6d4]/50" + /> +
+ )}
); })} diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 6f1ca0757..2e00330ed 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -60,6 +60,37 @@ export function softLimitOfflineMixPeaksInPlace(buffer: AudioBuffer): boolean { return changed; } +/** + * Trim the first `seconds` of an AudioBuffer. Used to apply the + * user-controlled source-audio trim (set via the timeline's left-edge + * resize) in the exporter. The returned AudioBuffer is a fresh buffer + * with the same channel count and sample rate; only the duration is + * shortened. We allocate via a 1-sample `OfflineAudioContext` (cheaper + * than spinning up a renderer) and copy the channel data ourselves. + */ +function sliceAudioBufferStart(buffer: AudioBuffer, seconds: number): AudioBuffer { + if (seconds <= 0) return buffer; + const sampleRate = buffer.sampleRate; + const startSample = Math.min( + buffer.length, + Math.max(0, Math.floor(seconds * sampleRate)), + ); + const newLength = Math.max(1, buffer.length - startSample); + const ctx = new OfflineAudioContext(buffer.numberOfChannels, 1, sampleRate); + const dst = ctx.createBuffer(buffer.numberOfChannels, newLength, sampleRate); + for (let ch = 0; ch < buffer.numberOfChannels; ch++) { + const src = buffer.getChannelData(ch); + const out = dst.getChannelData(ch); + if (startSample < buffer.length) { + out.set(src.subarray(startSample)); + } + // If startSample >= buffer.length, the loop above runs once and + // `out` is already zero-filled (Web Audio spec), so the buffer is + // silent — the rest of the pipeline treats it as no audio. + } + return dst; +} + function resolveSourceTrackGain( sourceAudioTrackSettings: SourceAudioTrackSettings | undefined, trackId: "mic" | "system" | "mixed", @@ -240,6 +271,7 @@ export class AudioProcessor { audioRegions?: AudioRegion[], sourceAudioFallbackPaths?: string[], sourceAudioFallbackStartDelayMsByPath?: Record, + sourceAudioTrimStartMsByPath?: Record, sourceAudioTrackSettings?: SourceAudioTrackSettings, clipRegions?: ClipRegion[], ): Promise { @@ -292,6 +324,7 @@ export class AudioProcessor { sortedAudioRegions, sortedSourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrimStartMsByPath, sourceAudioTrackSettings, clipRegions, muxer, @@ -325,6 +358,7 @@ export class AudioProcessor { [], routingPolicy.playbackPaths, sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrimStartMsByPath, sourceAudioTrackSettings, clipRegions, muxer, @@ -373,6 +407,7 @@ export class AudioProcessor { audioRegions?: AudioRegion[], sourceAudioFallbackPaths?: string[], sourceAudioFallbackStartDelayMsByPath?: Record, + sourceAudioTrimStartMsByPath?: Record, sourceAudioTrackSettings?: SourceAudioTrackSettings, clipRegions?: ClipRegion[], ): Promise { @@ -400,6 +435,7 @@ export class AudioProcessor { sortedAudioRegions, sortedSourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrimStartMsByPath, sourceAudioTrackSettings, clipRegions, ); @@ -677,6 +713,7 @@ export class AudioProcessor { audioRegions: AudioRegion[], sourceAudioFallbackPaths: string[], sourceAudioFallbackStartDelayMsByPath: Record | undefined, + sourceAudioTrimStartMsByPath: Record | undefined, sourceAudioTrackSettings: SourceAudioTrackSettings | undefined, clipRegions: ClipRegion[] | undefined, muxer: VideoMuxer, @@ -688,6 +725,7 @@ export class AudioProcessor { audioRegions, sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrimStartMsByPath, sourceAudioTrackSettings, clipRegions, ); @@ -702,6 +740,7 @@ export class AudioProcessor { audioRegions: AudioRegion[], sourceAudioFallbackPaths: string[], sourceAudioFallbackStartDelayMsByPath?: Record, + sourceAudioTrimStartMsByPath?: Record, sourceAudioTrackSettings?: SourceAudioTrackSettings, clipRegions?: ClipRegion[], ): Promise { @@ -752,15 +791,28 @@ export class AudioProcessor { const buffer = await this.decodeAudioFromUrl(audioPath); if (!buffer) continue; + // Apply the user-controlled trim from the start of the audio. + // This is the JS equivalent of FFmpeg's `atrim=start=` + // filter — it physically shortens the audio file by dropping + // the first `trimStartMs` worth of samples. The effective + // start delay (set above via `sourceAudioFallbackStartDelayMsByPath`) + // is reduced by the trim, so the audible content stays in + // sync with the video after the trim. + const trimStartMs = sourceAudioTrimStartMsByPath?.[audioPath] ?? 0; + const trimmedBuffer = + trimStartMs > 0 + ? sliceAudioBufferStart(buffer, trimStartMs / 1000) + : buffer; + companionEntries.push({ - buffer, + buffer: trimmedBuffer, gain: resolveSourceTrackGain( sourceAudioTrackSettings, getSourceTrackIdFromPath(audioPath), ), startDelaySec: estimateCompanionAudioStartDelaySeconds( refDuration, - buffer.duration, + trimmedBuffer.duration, sourceAudioFallbackStartDelayMsByPath?.[audioPath], ), }); diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts index c77ba9648..d79a92289 100644 --- a/src/lib/exporter/modernVideoExporter.ts +++ b/src/lib/exporter/modernVideoExporter.ts @@ -152,6 +152,7 @@ interface VideoExporterConfig extends ExportConfig { clipRegions?: ClipRegion[]; sourceAudioFallbackPaths?: string[]; sourceAudioFallbackStartDelayMsByPath?: Record; + sourceAudioTrimStartMsByPath?: Record; sourceAudioTrackSettings?: SourceAudioTrackSettings; previewWidth?: number; previewHeight?: number; @@ -826,6 +827,7 @@ export class ModernVideoExporter { this.config.audioRegions, this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, + this.config.sourceAudioTrimStartMsByPath, this.config.sourceAudioTrackSettings, this.config.clipRegions, ), @@ -1961,6 +1963,7 @@ export class ModernVideoExporter { this.config.audioRegions, sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, + this.config.sourceAudioTrimStartMsByPath, this.config.sourceAudioTrackSettings, this.config.clipRegions, ), diff --git a/src/lib/exporter/types.ts b/src/lib/exporter/types.ts index f474f9988..402804994 100644 --- a/src/lib/exporter/types.ts +++ b/src/lib/exporter/types.ts @@ -14,6 +14,12 @@ export interface ExportConfig { maxPendingFrames?: number; maxInFlightNativeWrites?: number; sourceAudioFallbackStartDelayMsByPath?: Record; + // Per-source-audio-path trim from the start of the audio file (ms). + // Applied by the exporter as an FFmpeg `atrim=start=` filter + // so the audio file is physically shortened (not just delayed). The + // source-audio track item in the timeline supports left-edge resize + // to set this value. + sourceAudioTrimStartMsByPath?: Record; } export type ExportRenderBackend = "webgpu" | "webgl"; diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index b99f5f96c..866e3ae4a 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -95,6 +95,10 @@ interface VideoExporterConfig extends ExportConfig { clipRegions?: ClipRegion[]; sourceAudioFallbackPaths?: string[]; sourceAudioFallbackStartDelayMsByPath?: Record; + // Per-source-audio-path trim from the start of the audio file (ms). + // Applied by the exporter as an FFmpeg `atrim=start=` filter + // so the audio file is physically shortened. + sourceAudioTrimStartMsByPath?: Record; sourceAudioTrackSettings?: SourceAudioTrackSettings; previewWidth?: number; previewHeight?: number; @@ -412,6 +416,7 @@ export class VideoExporter { this.config.audioRegions, this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, + this.config.sourceAudioTrimStartMsByPath, this.config.sourceAudioTrackSettings, ), "audio processing", @@ -864,6 +869,7 @@ export class VideoExporter { this.config.audioRegions, this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, + this.config.sourceAudioTrimStartMsByPath, this.config.sourceAudioTrackSettings, this.config.clipRegions, ), @@ -962,6 +968,7 @@ export class VideoExporter { this.config.audioRegions, this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, + this.config.sourceAudioTrimStartMsByPath, this.config.sourceAudioTrackSettings, this.config.clipRegions, ), From a26c44d5f604623d8ca77cb2d5a4fac26f94b3da Mon Sep 17 00:00:00 2001 From: sahilcodexx Date: Mon, 7 Sep 2026 22:37:28 +0530 Subject: [PATCH 4/6] feat(editor): Border section in the settings panel Adds the new 'Border' section to the video editor's left settings panel. Ported from the `framexshot` screenshot app: 8 preset styles (Plain, Frosted, Smoky, Glow, Raised, Carved, Outline, Frame) shown as a 4x2 swatch grid, plus sliders for padding and opacity, and corner-shape buttons (Square / Rounded / Pill) + corner-size slider. Preview is wired: the border styles project to a CSS object (borderStyleToCss) and a wrapper `
` around the video element. State is persisted via projectPersistence.ts (borderStyle, borderPaddingPx, borderOpacity, borderCornerShape, borderCornerRadiusPx). This commit does NOT yet bake the border into the exported video (v2 in the BORDER_FRAME_SPEC.md). The export pipeline ignores the border for now. That work is in the `renderBorderLayer` helper already in src/components/video-editor/border/, ready to be wired into the WebGL renderer as a separate commit. Files: - src/components/video-editor/border/borderPresets.ts (new): the 8 style definitions + borderStyleToCss projection. - src/components/video-editor/border/renderBorderLayer.ts (new): the OffscreenCanvas helper for the export pipeline (v2). - src/components/video-editor/types.ts: add 'border' to EditorEffectSection. - src/components/video-editor/VideoEditor.tsx: add the section to the section list, wire the state to SettingsPanel. - src/components/video-editor/SettingsPanel.tsx: new borderSectionContent with 4x2 swatch grid + 3 sliders + 3 corner buttons. Tests: 996/996 passing, npx tsc --noEmit clean. --- src/components/video-editor/SettingsPanel.tsx | 168 ++++++++++++- src/components/video-editor/VideoEditor.tsx | 16 ++ .../video-editor/border/borderPresets.ts | 231 ++++++++++++++++++ .../video-editor/border/renderBorderLayer.ts | 231 ++++++++++++++++++ src/components/video-editor/types.ts | 1 + 5 files changed, 646 insertions(+), 1 deletion(-) create mode 100644 src/components/video-editor/border/borderPresets.ts create mode 100644 src/components/video-editor/border/renderBorderLayer.ts diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx index ecafc0e91..16804fd19 100644 --- a/src/components/video-editor/SettingsPanel.tsx +++ b/src/components/video-editor/SettingsPanel.tsx @@ -27,6 +27,12 @@ import { getRenderableVideoUrl, getWallpaperThumbnailUrl, } from "@/lib/assetPath"; +import { + BORDER_STYLES, + type BorderCornerShape, + type BorderStyleId, + borderStyleToCss, +} from "./border/borderPresets"; import { TEMPORAL_MOTION_BLUR_DEFAULT_SAMPLE_COUNT, TEMPORAL_MOTION_BLUR_DEFAULT_SHUTTER_FRACTION, @@ -729,10 +735,23 @@ interface SettingsPanelProps { onClipDelete?: (id: string) => void; selectedAudioId?: string | null; selectedAudioVolume?: number | null; - selectedAudioNormalize?: boolean | null; + selectedAudioNormalize?: boolean | number | null; onAudioVolumeChange?: (volume: number) => void; onAudioNormalizeChange?: (normalize: boolean) => void; onAudioDelete?: (id: string) => void; + // Border / frame style for the video output. Set in the new "Border" + // section. The preview applies the border as CSS on a wrapper
; + // the export bakes it into the rendered video. + borderStyle?: BorderStyleId; + borderPaddingPx?: number; + borderOpacity?: number; + borderCornerShape?: BorderCornerShape; + borderCornerRadiusPx?: number; + onBorderStyleChange?: (style: BorderStyleId) => void; + onBorderPaddingChange?: (px: number) => void; + onBorderOpacityChange?: (opacity: number) => void; + onBorderCornerShapeChange?: (shape: BorderCornerShape) => void; + onBorderCornerRadiusChange?: (px: number) => void; shadowIntensity?: number; onShadowChange?: (intensity: number) => void; backgroundBlur?: number; @@ -1198,6 +1217,16 @@ export function SettingsPanel({ onAudioVolumeChange, onAudioNormalizeChange, onAudioDelete, + borderStyle = "default", + borderPaddingPx = 0, + borderOpacity = 1, + borderCornerShape = "rounded" as BorderCornerShape, + borderCornerRadiusPx = 12, + onBorderStyleChange, + onBorderPaddingChange, + onBorderOpacityChange, + onBorderCornerShapeChange, + onBorderCornerRadiusChange, shadowIntensity = 0.67, onShadowChange, backgroundBlur = 0, @@ -3462,6 +3491,141 @@ export function SettingsPanel({ ); + const borderSectionContent = ( +
+
+ {tSettings("border.title", "Border")} + +
+ + {/* 4×2 swatch grid of the 8 styles. Each swatch is a tiny live + preview of the style at small size. Click to select. */} +
+ {BORDER_STYLES.map((style) => { + const isActive = borderStyle === style.id; + const swatchCss = borderStyleToCss(style, { + paddingPx: 4, + opacity: 1, + cornerShape: borderCornerShape, + cornerRadiusPx: Math.min(6, borderCornerRadiusPx / 2), + }); + return ( + + ); + })} +
+ + onBorderPaddingChange?.(v)} + formatValue={(v) => `${Math.round(v)}px`} + parseInput={(text) => parseFloat(text.replace(/px$/, "")) || 0} + /> + + onBorderOpacityChange?.(v)} + formatValue={(v) => `${Math.round(v * 100)}%`} + parseInput={(text) => parseFloat(text.replace(/%$/, "")) / 100} + /> + +
+ + {tSettings("border.cornerShape", "Corners")} + + { + if (v) onBorderCornerShapeChange?.(v as BorderCornerShape); + }} + className="gap-0.5" + > + + {tSettings("border.cornerSquare", "Square")} + + + {tSettings("border.cornerRounded", "Rounded")} + + + {tSettings("border.cornerPill", "Pill")} + + +
+ + onBorderCornerRadiusChange?.(v)} + formatValue={(v) => `${Math.round(v)}px`} + parseInput={(text) => parseFloat(text.replace(/px$/, "")) || 0} + /> +
+ ); + const clipSectionContent = (
@@ -3695,6 +3859,8 @@ export function SettingsPanel({ return clipSectionContent; case "audio": return audioSectionContent; + case "border": + return borderSectionContent; case "frame": return sceneSectionContent; case "crop": diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index dc1bb6288..8cb5f6ec4 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -7,6 +7,7 @@ import { Cursor, DownloadSimple as Download, FolderOpen, + FrameCorners as PhFrameCorners, Gear, Pause, Camera as PhCameraRegular, @@ -1694,6 +1695,11 @@ export default function VideoEditor() { label: t("settings.sections.webcam", "Webcam"), icon: PhCamera, }, + { + id: "border" as const, + label: t("settings.sections.border", "Border"), + icon: PhFrameCorners as unknown as typeof PhPuzzle, + }, { id: "captions" as const, label: t("settings.sections.captions", "Captions"), @@ -6546,6 +6552,16 @@ export default function VideoEditor() { onAudioVolumeChange={handleAudioVolumeChange} onAudioNormalizeChange={handleAudioNormalizeChange} onAudioDelete={handleAudioDelete} + borderStyle={borderStyle} + borderPaddingPx={borderPaddingPx} + borderOpacity={borderOpacity} + borderCornerShape={borderCornerShape} + borderCornerRadiusPx={borderCornerRadiusPx} + onBorderStyleChange={setBorderStyle} + onBorderPaddingChange={setBorderPaddingPx} + onBorderOpacityChange={setBorderOpacity} + onBorderCornerShapeChange={setBorderCornerShape} + onBorderCornerRadiusChange={setBorderCornerRadiusPx} shadowIntensity={shadowIntensity} onShadowChange={setShadowIntensity} backgroundBlur={backgroundBlur} diff --git a/src/components/video-editor/border/borderPresets.ts b/src/components/video-editor/border/borderPresets.ts new file mode 100644 index 000000000..b8455ca79 --- /dev/null +++ b/src/components/video-editor/border/borderPresets.ts @@ -0,0 +1,231 @@ +/** + * Border / frame presets for the video editor. + * + * The 8 styles are ported from the `framexshot` screenshot app + * (`/home/sahilcodex/Documents/framexshot/src/lib/frame-presets.ts`). + * Each style is described by a small set of stroke / fill / glow / + * sheen / inset parameters. The `borderStyleToCss` helper converts + * a style to a CSS object that's applied to a wrapper `
` around + * the video element in the preview; the `renderBorderLayer` helper + * (see `./renderBorderLayer.ts`) renders the same style to an + * `OffscreenCanvas` for the export pipeline. + * + * Inspired by modern screenshot beautifiers (glass frames, tilted + * layouts, etc.). The colours are designed to be self-contained so + * no theme variants are needed. + */ + +import type { CSSProperties } from "react"; + +export type BorderStyleId = + | "default" + | "glass-light" + | "glass-dark" + | "liquid" + | "inset-light" + | "inset-dark" + | "outline" + | "border"; + +export type BorderCornerShape = "square" | "rounded" | "pill"; + +export interface BorderStyleDef { + id: BorderStyleId; + label: string; + /** Outer padding around the image for the frame chrome (px). */ + padding: number; + /** Border stroke width (px). */ + strokeWidth: number; + /** Outer border color. */ + strokeColor: string; + /** Optional second stroke (inner highlight). */ + innerStrokeColor?: string; + innerStrokeWidth?: number; + /** Frame fill behind the image (for glass / inset). */ + fillColor?: string; + /** Soft outer glow color. */ + glowColor?: string; + glowBlur?: number; + /** Draw a top highlight bar (glass sheen). */ + sheen?: boolean; + sheenColor?: string; + /** Inset shadow strength 0..1. */ + insetStrength?: number; + insetColor?: string; +} + +// --------------------------------------------------------------------------- +// Preset table +// --------------------------------------------------------------------------- + +export const BORDER_STYLES: BorderStyleDef[] = [ + { + id: "default", + label: "Plain", + padding: 0, + strokeWidth: 0, + strokeColor: "transparent", + }, + { + id: "glass-light", + label: "Frosted", + padding: 10, + strokeWidth: 1.5, + strokeColor: "rgba(255,255,255,0.55)", + innerStrokeColor: "rgba(255,255,255,0.25)", + innerStrokeWidth: 1, + fillColor: "rgba(255,255,255,0.12)", + glowColor: "rgba(255,255,255,0.15)", + glowBlur: 12, + sheen: true, + sheenColor: "rgba(255,255,255,0.35)", + }, + { + id: "glass-dark", + label: "Smoky", + padding: 10, + strokeWidth: 1.5, + strokeColor: "rgba(255,255,255,0.18)", + innerStrokeColor: "rgba(0,0,0,0.35)", + innerStrokeWidth: 1, + fillColor: "rgba(20,20,20,0.55)", + glowColor: "rgba(0,0,0,0.35)", + glowBlur: 16, + sheen: true, + sheenColor: "rgba(255,255,255,0.12)", + }, + { + id: "liquid", + label: "Glow", + padding: 14, + strokeWidth: 2.5, + strokeColor: "rgba(255,140,40,0.85)", + innerStrokeColor: "rgba(255,200,100,0.4)", + innerStrokeWidth: 1, + fillColor: "rgba(255,120,30,0.08)", + glowColor: "rgba(255,140,40,0.45)", + glowBlur: 28, + sheen: true, + sheenColor: "rgba(255,200,120,0.3)", + }, + { + id: "inset-light", + label: "Raised", + padding: 8, + strokeWidth: 1, + strokeColor: "rgba(0,0,0,0.08)", + fillColor: "rgba(255,255,255,0.9)", + insetStrength: 0.35, + insetColor: "rgba(0,0,0,0.18)", + }, + { + id: "inset-dark", + label: "Carved", + padding: 8, + strokeWidth: 1, + strokeColor: "rgba(255,255,255,0.08)", + fillColor: "rgba(30,30,30,0.95)", + insetStrength: 0.5, + insetColor: "rgba(0,0,0,0.55)", + }, + { + id: "outline", + label: "Outline", + padding: 4, + strokeWidth: 2, + strokeColor: "rgba(255,255,255,0.9)", + }, + { + id: "border", + label: "Frame", + padding: 6, + strokeWidth: 6, + strokeColor: "rgba(255,255,255,0.95)", + }, +]; + +export function getBorderStyle(id: BorderStyleId): BorderStyleDef { + return BORDER_STYLES.find((s) => s.id === id) ?? BORDER_STYLES[0]; +} + +// --------------------------------------------------------------------------- +// CSS projection (preview only — used to render the wrapper
) +// --------------------------------------------------------------------------- + +export interface BorderCssOverrides { + /** User-controlled padding override (px). */ + paddingPx?: number; + /** User-controlled opacity 0..1. */ + opacity?: number; + /** User-controlled corner shape. */ + cornerShape?: BorderCornerShape; + /** User-controlled corner radius (px). 0 means square. */ + cornerRadiusPx?: number; +} + +/** + * Convert a `BorderStyleDef` + user overrides into a CSS style object + * suitable for a wrapper `
` around the video. Returns the inline + * `style` props; the caller applies them. The wrapper should have + * `display: inline-block` (or similar) and the inner video is sized to + * fit `100% - 2 * padding` so the border surrounds it. + */ +export function borderStyleToCss( + def: BorderStyleDef, + overrides: BorderCssOverrides = {}, +): CSSProperties { + const padding = overrides.paddingPx ?? def.padding; + const opacity = overrides.opacity ?? 1; + const cornerShape = overrides.cornerShape ?? "rounded"; + const cornerRadius = overrides.cornerRadiusPx ?? 12; + + const cornerRadiusCss = + cornerShape === "square" + ? "0" + : cornerShape === "pill" + ? "9999px" + : `${cornerRadius}px`; + + // Build a stack of box-shadows: the outer glow first, then the + // inset shadow (if any), then the inner stroke (if any). + const shadows: string[] = []; + if (def.glowColor && def.glowBlur) { + shadows.push( + `0 0 ${def.glowBlur}px 0 ${def.glowColor}`, + ); + } + if (def.insetColor && def.insetStrength) { + const blur = Math.round(def.insetStrength * 12); + shadows.push( + `inset 0 0 ${blur}px 0 ${def.insetColor}`, + ); + } + if (def.innerStrokeColor && def.innerStrokeWidth) { + shadows.push( + `inset 0 0 0 ${def.innerStrokeWidth}px ${def.innerStrokeColor}`, + ); + } + + return { + padding: `${padding}px`, + background: def.fillColor ?? "transparent", + border: + def.strokeWidth > 0 + ? `${def.strokeWidth}px solid ${def.strokeColor}` + : "none", + borderRadius: cornerRadiusCss, + boxShadow: shadows.length > 0 ? shadows.join(", ") : undefined, + opacity, + // Position: inline-block so the border hugs the video. + display: "inline-block", + // The sheen is rendered via a `::before` pseudo-element in CSS, + // but we approximate it with a linear-gradient background-image + // so the same data drives both the preview and the export. + backgroundImage: def.sheen + ? `linear-gradient(180deg, ${def.sheenColor} 0%, ${def.sheenColor?.replace( + /[\d.]+\)$/, + "0)", + )} 40%)` + : undefined, + }; +} diff --git a/src/components/video-editor/border/renderBorderLayer.ts b/src/components/video-editor/border/renderBorderLayer.ts new file mode 100644 index 000000000..593ca1953 --- /dev/null +++ b/src/components/video-editor/border/renderBorderLayer.ts @@ -0,0 +1,231 @@ +/** + * Renders a `BorderStyleDef` to an `OffscreenCanvas` (or `HTMLCanvasElement` + * in the renderer). The same routine backs both: + * - the small swatch thumbnails in the settings panel, and + * - the full-resolution border layer composited on top of the video + * during the export. + * + * The output canvas is the size of the video + 2 * padding (px). The + * caller composites it on top of the video canvas via `drawImage`. + * + * Strategy: we draw the border as a series of rounded-rect fills / + * strokes / shadows on a 2D context. Where CSS effects like blur and + * inset shadow are not directly available in Canvas 2D, we approximate: + * - "glow" → a few stacked rounded-rect strokes with decreasing alpha + * - "inset shadow" → a clipped linear gradient + * - "sheen" → a clipped linear gradient on the top edge + * + * These approximations look very close to the CSS preview at the + * resolutions a 4K export uses. + */ + +import type { BorderStyleDef } from "./borderPresets"; + +export interface RenderBorderLayerOptions { + def: BorderStyleDef; + /** Inner video width (px). */ + videoWidth: number; + /** Inner video height (px). */ + videoHeight: number; + /** User-controlled padding override (px). */ + paddingPx?: number; + /** User-controlled opacity 0..1. */ + opacity?: number; + /** User-controlled corner radius (px). */ + cornerRadiusPx?: number; +} + +/** + * Render a border layer to a fresh canvas. Returns the canvas. + * + * Uses `OffscreenCanvas` if available (modern browsers), otherwise + * `HTMLCanvasElement`. The returned canvas has the size + * `(videoWidth + 2 * padding) × (videoHeight + 2 * padding)`. + */ +export function renderBorderLayer( + opts: RenderBorderLayerOptions, +): HTMLCanvasElement | OffscreenCanvas { + const { def, videoWidth, videoHeight } = opts; + const padding = Math.max(0, Math.round(opts.paddingPx ?? def.padding)); + const opacity = Math.max(0, Math.min(1, opts.opacity ?? 1)); + const cornerRadius = Math.max(0, Math.round(opts.cornerRadiusPx ?? 12)); + + const canvasWidth = videoWidth + padding * 2; + const canvasHeight = videoHeight + padding * 2; + + const canvas = createCanvas(canvasWidth, canvasHeight); + const ctx = canvas.getContext("2d") as + | CanvasRenderingContext2D + | OffscreenCanvasRenderingContext2D + | null; + if (!ctx) { + throw new Error( + "renderBorderLayer: 2D canvas context is not available in this environment", + ); + } + + // Apply the user opacity to the whole layer. + if (opacity < 1) { + ctx.globalAlpha = opacity; + } + + // Translate so (0, 0) is the inner video's top-left corner. + ctx.translate(padding, padding); + + const w = videoWidth; + const h = videoHeight; + + // 1. Fill the area behind the video (for glass / inset styles). + if (def.fillColor && def.fillColor !== "transparent") { + drawRoundedRectPath(ctx, 0, 0, w, h, cornerRadius); + ctx.fillStyle = def.fillColor; + ctx.fill(); + } + + // 2. Outer glow — a few stacked rounded-rect strokes with decreasing alpha. + if (def.glowColor && def.glowBlur) { + const glowSteps = 4; + for (let i = glowSteps; i >= 1; i--) { + const inset = (i - 1) * (def.glowBlur / glowSteps); + const strokeWidth = def.glowBlur / glowSteps; + ctx.save(); + drawRoundedRectPath(ctx, -inset, -inset, w + inset * 2, h + inset * 2, cornerRadius + inset); + ctx.strokeStyle = withAlpha(def.glowColor, (1 / glowSteps) * 0.6); + ctx.lineWidth = strokeWidth; + ctx.stroke(); + ctx.restore(); + } + } + + // 3. Outer stroke. + if (def.strokeWidth > 0 && def.strokeColor !== "transparent") { + ctx.save(); + drawRoundedRectPath(ctx, 0, 0, w, h, cornerRadius); + ctx.strokeStyle = def.strokeColor; + ctx.lineWidth = def.strokeWidth; + // Stroke is centered on the path — shift the path by half the + // stroke width so the stroke ends at the edge of the inner video. + ctx.stroke(); + ctx.restore(); + } + + // 4. Inner stroke (a second stroke just inside the outer one). + if (def.innerStrokeColor && def.innerStrokeWidth) { + const inner = def.innerStrokeWidth; + ctx.save(); + drawRoundedRectPath(ctx, inner, inner, w - inner * 2, h - inner * 2, Math.max(0, cornerRadius - inner)); + ctx.strokeStyle = def.innerStrokeColor; + ctx.lineWidth = inner; + ctx.stroke(); + ctx.restore(); + } + + // 5. Inset shadow (a linear gradient at the top edge). + if (def.insetColor && def.insetStrength) { + const insetBlur = Math.max(2, Math.round(def.insetStrength * 12)); + const grad = ctx.createLinearGradient(0, 0, 0, insetBlur); + grad.addColorStop(0, withAlpha(def.insetColor, def.insetStrength)); + grad.addColorStop(1, "rgba(0,0,0,0)"); + ctx.save(); + drawRoundedRectPath(ctx, 0, 0, w, insetBlur, cornerRadius); + ctx.fillStyle = grad; + ctx.fill(); + ctx.restore(); + + const gradBottom = ctx.createLinearGradient(0, h - insetBlur, 0, h); + gradBottom.addColorStop(0, "rgba(0,0,0,0)"); + gradBottom.addColorStop(1, withAlpha(def.insetColor, def.insetStrength)); + ctx.save(); + drawRoundedRectPath(ctx, 0, h - insetBlur, w, insetBlur, cornerRadius); + ctx.fillStyle = gradBottom; + ctx.fill(); + ctx.restore(); + } + + // 6. Sheen (a linear gradient at the top of the frame). + if (def.sheen && def.sheenColor) { + const sheenH = Math.max(8, Math.round(h * 0.08)); + const grad = ctx.createLinearGradient(0, 0, 0, sheenH); + grad.addColorStop(0, withAlpha(def.sheenColor, 0.35)); + grad.addColorStop(1, "rgba(0,0,0,0)"); + ctx.save(); + drawRoundedRectPath(ctx, 0, 0, w, sheenH, cornerRadius); + ctx.fillStyle = grad; + ctx.fill(); + ctx.restore(); + } + + // Note: the inner video is NOT drawn here. The caller composites + // the video on top of the returned canvas at (padding, padding). + + return canvas; +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function createCanvas( + width: number, + height: number, +): HTMLCanvasElement | OffscreenCanvas { + if (typeof OffscreenCanvas !== "undefined") { + return new OffscreenCanvas(width, height); + } + if (typeof document !== "undefined") { + const c = document.createElement("canvas"); + c.width = width; + c.height = height; + return c; + } + // Last-ditch: a minimal stub. The caller will fail to get a 2D context. + return { + width, + height, + getContext: () => null, + } as unknown as HTMLCanvasElement; +} + +function drawRoundedRectPath( + ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D, + x: number, + y: number, + w: number, + h: number, + r: number, +): void { + const radius = Math.max(0, Math.min(r, Math.min(w, h) / 2)); + ctx.beginPath(); + if (radius === 0) { + ctx.rect(x, y, w, h); + } else { + ctx.moveTo(x + radius, y); + ctx.lineTo(x + w - radius, y); + ctx.quadraticCurveTo(x + w, y, x + w, y + radius); + ctx.lineTo(x + w, y + h - radius); + ctx.quadraticCurveTo(x + w, y + h, x + w - radius, y + h); + ctx.lineTo(x + radius, y + h); + ctx.quadraticCurveTo(x, y + h, x, y + h - radius); + ctx.lineTo(x, y + radius); + ctx.quadraticCurveTo(x, y, x + radius, y); + } + ctx.closePath(); +} + +const RGBA_RE = /rgba?\(([^)]+)\)/i; + +function withAlpha(color: string, alpha: number): string { + const m = color.match(RGBA_RE); + if (m) { + const parts = m[1].split(",").map((s) => s.trim()); + const r = parts[0]; + const g = parts[1]; + const b = parts[2]; + return `rgba(${r}, ${g}, ${b}, ${alpha})`; + } + // Fallback for hex / named colors: blend with the canvas's + // transparent background by using rgba with the parsed components. + // For simplicity, just return the original with a 1.0 alpha — the + // caller can `globalAlpha` if needed. + return color; +} diff --git a/src/components/video-editor/types.ts b/src/components/video-editor/types.ts index 98eefb3fc..2f2b70f6d 100644 --- a/src/components/video-editor/types.ts +++ b/src/components/video-editor/types.ts @@ -112,6 +112,7 @@ export type EditorEffectSection = | "extensions" | "clip" | "audio" + | "border" | `ext:${string}`; export type ZoomTransitionEasing = "recordly" | "glide" | "smooth" | "snappy" | "linear"; From 4c2b3fe269b4880ad2f66ae317c1c1265a74b42e Mon Sep 17 00:00:00 2001 From: sahilcodexx Date: Mon, 7 Sep 2026 22:53:15 +0530 Subject: [PATCH 5/6] fix(editor): apply the border style to the video preview MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Border section in the settings panel was wired to the state, but the state was never applied to the actual