diff --git a/electron/ipc/ffmpeg/filters.test.ts b/electron/ipc/ffmpeg/filters.test.ts index deab8017..bc28a3ca 100644 --- a/electron/ipc/ffmpeg/filters.test.ts +++ b/electron/ipc/ffmpeg/filters.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it } from "vitest"; -import { appendSyncedAudioFilter, getAudioSyncAdjustment } from "./filters"; +import { + appendSyncedAudioFilter, + applyRecordedAudioStartDelay, + getAudioSyncAdjustment, +} from "./filters"; describe("getAudioSyncAdjustment", () => { it("does not speed up longer audio tracks that would advance speech", () => { @@ -30,6 +34,15 @@ describe("getAudioSyncAdjustment", () => { }); }); + it("pads trailing silence instead of prepending extreme delay for very short audio tracks", () => { + expect(getAudioSyncAdjustment(600, 480)).toEqual({ + mode: "pad", + delayMs: 0, + tempoRatio: 1, + durationDeltaMs: 120000, + }); + }); + it("does not inject atempo when longer audio stays on the anchored path", () => { const filterParts: string[] = []; appendSyncedAudioFilter(filterParts, "[1:a]", "aout", getAudioSyncAdjustment(120, 122.5)); @@ -47,4 +60,51 @@ describe("getAudioSyncAdjustment", () => { "[1:a]atempo=0.975000,aresample=async=1:first_pts=0,asetpts=PTS-STARTPTS[aout]", ]); }); + + it("pads the tail for very short audio tracks", () => { + const filterParts: string[] = []; + appendSyncedAudioFilter(filterParts, "[1:a]", "aout", getAudioSyncAdjustment(600, 480)); + + expect(filterParts).toEqual([ + "[1:a]apad=pad_dur=120.000,aresample=async=1:first_pts=0,asetpts=PTS-STARTPTS[aout]", + ]); + }); +}); + +describe("applyRecordedAudioStartDelay", () => { + it("pads the tail when recorded metadata says audio started on time", () => { + expect(applyRecordedAudioStartDelay(getAudioSyncAdjustment(120, 110), 0)).toEqual({ + mode: "pad", + delayMs: 0, + tempoRatio: 1, + durationDeltaMs: 10000, + }); + }); + + it("prefers a measured start delay over a tempo-only heuristic", () => { + expect(applyRecordedAudioStartDelay(getAudioSyncAdjustment(120, 119.8), 275)).toEqual({ + mode: "delay", + delayMs: 275, + tempoRatio: 1, + durationDeltaMs: 200, + }); + }); + + it("applies a measured start delay even when durations already match", () => { + expect(applyRecordedAudioStartDelay(getAudioSyncAdjustment(120, 120), 275)).toEqual({ + mode: "delay", + delayMs: 275, + tempoRatio: 1, + durationDeltaMs: 0, + }); + }); + + it("leaves tempo correction alone when recorded metadata says there was no late start", () => { + expect(applyRecordedAudioStartDelay(getAudioSyncAdjustment(120, 117), 0)).toEqual({ + mode: "tempo", + delayMs: 0, + tempoRatio: 0.975, + durationDeltaMs: 3000, + }); + }); }); diff --git a/electron/ipc/ffmpeg/filters.ts b/electron/ipc/ffmpeg/filters.ts index ce973f16..735e3c66 100644 --- a/electron/ipc/ffmpeg/filters.ts +++ b/electron/ipc/ffmpeg/filters.ts @@ -1,5 +1,7 @@ import type { AudioSyncAdjustment, PauseSegment } from "../types"; +const MAX_AUDIO_SYNC_DELAY_MS = 15000; + export function buildAtempoFilters(tempoRatio: number): string[] { if (!Number.isFinite(tempoRatio) || tempoRatio <= 0) { return []; @@ -58,9 +60,43 @@ export function getAudioSyncAdjustment( return { mode: "tempo", delayMs: 0, tempoRatio, durationDeltaMs }; } + if (durationDeltaMs > MAX_AUDIO_SYNC_DELAY_MS) { + return { mode: "pad", delayMs: 0, tempoRatio: 1, durationDeltaMs }; + } + return { mode: "delay", delayMs: durationDeltaMs, tempoRatio: 1, durationDeltaMs }; } +export function applyRecordedAudioStartDelay( + adjustment: AudioSyncAdjustment, + recordedStartDelayMs?: number | null, +): AudioSyncAdjustment { + if (!Number.isFinite(recordedStartDelayMs) || (recordedStartDelayMs ?? 0) < 0) { + return adjustment; + } + + const delayMs = Math.max(0, Math.round(recordedStartDelayMs ?? 0)); + if (delayMs > 20) { + return { + mode: "delay", + delayMs, + tempoRatio: 1, + durationDeltaMs: adjustment.durationDeltaMs, + }; + } + + if (adjustment.mode !== "delay" && adjustment.mode !== "pad") { + return adjustment; + } + + return { + mode: "pad", + delayMs: 0, + tempoRatio: 1, + durationDeltaMs: adjustment.durationDeltaMs, + }; +} + export function appendSyncedAudioFilter( filterParts: string[], inputLabel: string, @@ -77,6 +113,10 @@ export function appendSyncedAudioFilter( filters.push(...buildAtempoFilters(adjustment.tempoRatio)); } + if (adjustment.mode === "pad" && adjustment.durationDeltaMs > 0) { + filters.push(`apad=pad_dur=${formatFfmpegSeconds(adjustment.durationDeltaMs)}`); + } + filters.push("aresample=async=1:first_pts=0", "asetpts=PTS-STARTPTS"); filterParts.push(`${inputLabel}${filters.join(",")}[${outputLabel}]`); } diff --git a/electron/ipc/recording/diagnostics.test.ts b/electron/ipc/recording/diagnostics.test.ts index 9a775339..dca8f3fc 100644 --- a/electron/ipc/recording/diagnostics.test.ts +++ b/electron/ipc/recording/diagnostics.test.ts @@ -166,6 +166,18 @@ describe("getCompanionAudioFallbackPaths", () => { }); }); + it("ignores invalid sidecar timing metadata values", async () => { + const micPath = path.join(tempRoot, "recording.mic.wav"); + await Promise.all([ + fs.writeFile(micPath, "mic"), + fs.writeFile(`${micPath}.json`, JSON.stringify({ startDelayMs: -250 })), + ]); + + const { getCompanionAudioStartDelayMs } = await import("./diagnostics"); + + await expect(getCompanionAudioStartDelayMs(micPath)).resolves.toBeNull(); + }); + it("rejects tiny MP4 container-only outputs before they reach the editor", async () => { const videoPath = path.join(tempRoot, "recording-123.mp4"); await fs.writeFile(videoPath, Buffer.alloc(261)); diff --git a/electron/ipc/recording/diagnostics.ts b/electron/ipc/recording/diagnostics.ts index 4588631e..fb7b6c4a 100644 --- a/electron/ipc/recording/diagnostics.ts +++ b/electron/ipc/recording/diagnostics.ts @@ -119,6 +119,16 @@ async function readCompanionAudioTimingMetadata( } } +export async function getCompanionAudioStartDelayMs(companionPath: string) { + const metadata = await readCompanionAudioTimingMetadata(companionPath); + const startDelayMs = metadata?.startDelayMs; + if (!Number.isFinite(startDelayMs) || (startDelayMs ?? 0) < 0) { + return null; + } + + return Math.round(startDelayMs ?? 0); +} + export async function hasEmbeddedAudioStream(videoPath: string) { const ffmpegPath = getFfmpegBinaryPath(); let stderr = ""; @@ -172,22 +182,19 @@ export async function getCompanionAudioFallbackInfo(videoPath: string) { const metadataEntries = await Promise.all( paths.map(async (audioPath) => { - const metadata = await readCompanionAudioTimingMetadata(audioPath); - const startDelayMs = metadata?.startDelayMs; - if (!Number.isFinite(startDelayMs) || (startDelayMs ?? 0) < 0) { + const startDelayMs = await getCompanionAudioStartDelayMs(audioPath); + if (!Number.isFinite(startDelayMs)) { return null; } - return [audioPath, Math.round(startDelayMs ?? 0)] as const; + return [audioPath, startDelayMs] as const; }), ); return { paths, startDelayMsByPath: Object.fromEntries( - metadataEntries.filter( - (entry): entry is readonly [string, number] => entry !== null, - ), + metadataEntries.filter((entry): entry is readonly [string, number] => entry !== null), ), }; } diff --git a/electron/ipc/recording/windows.ts b/electron/ipc/recording/windows.ts index 4a32f86c..612cc9f0 100644 --- a/electron/ipc/recording/windows.ts +++ b/electron/ipc/recording/windows.ts @@ -7,24 +7,29 @@ import { BrowserWindow } from "electron"; import { getFfmpegBinaryPath } from "../ffmpeg/binary"; import { appendSyncedAudioFilter, + applyRecordedAudioStartDelay, buildPausedAudioFilter, getAudioSyncAdjustment, normalizePauseSegments, } from "../ffmpeg/filters"; import { getWindowsCaptureExePath } from "../paths/binaries"; import { + selectedSource, setWindowsCaptureProcess, + setWindowsCaptureStopRequested, + setWindowsNativeCaptureActive, windowsCaptureOutputBuffer, + windowsCaptureStopRequested, windowsCaptureTargetPath, windowsNativeCaptureActive, - setWindowsNativeCaptureActive, - windowsCaptureStopRequested, - setWindowsCaptureStopRequested, - selectedSource, } from "../state"; import type { AudioSyncAdjustment, PauseSegment } from "../types"; import { moveFileWithOverwrite } from "../utils"; -import { probeMediaDurationSeconds, validateRecordedVideo } from "./diagnostics"; +import { + getCompanionAudioStartDelayMs, + probeMediaDurationSeconds, + validateRecordedVideo, +} from "./diagnostics"; import { emitRecordingInterrupted } from "./events"; const execFileAsync = promisify(execFile); @@ -191,9 +196,21 @@ export async function muxNativeWindowsVideoWithAudio( if (videoDuration > 0) { for (let i = 0; i < audioFilePaths.length; i++) { const audioDuration = await probeMediaDurationSeconds(audioFilePaths[i]); - const adjustment = getAudioSyncAdjustment(videoDuration, audioDuration); + const recordedStartDelayMs = await getCompanionAudioStartDelayMs(audioFilePaths[i]); + const adjustment = applyRecordedAudioStartDelay( + getAudioSyncAdjustment(videoDuration, audioDuration), + recordedStartDelayMs, + ); audioAdjustments.set(audioInputs[i], adjustment); - if (adjustment.mode === "tempo") { + if (Number.isFinite(recordedStartDelayMs) && adjustment.mode === "delay") { + console.log( + `[mux-win] ${audioInputs[i]} audio recorded a start delay of ${adjustment.delayMs}ms`, + ); + } else if (Number.isFinite(recordedStartDelayMs) && adjustment.mode === "pad") { + console.log( + `[mux-win] ${audioInputs[i]} audio started on time but ends ${adjustment.durationDeltaMs}ms early — padding trailing silence`, + ); + } else if (adjustment.mode === "tempo") { console.log( `[mux-win] ${audioInputs[i]} audio differs from video by ${adjustment.durationDeltaMs}ms — applying tempo ratio ${adjustment.tempoRatio.toFixed(6)}`, ); @@ -201,6 +218,10 @@ export async function muxNativeWindowsVideoWithAudio( console.log( `[mux-win] ${audioInputs[i]} audio appears to start late by ${adjustment.delayMs}ms — adding leading silence`, ); + } else if (adjustment.mode === "pad" && adjustment.durationDeltaMs > 0) { + console.log( + `[mux-win] ${audioInputs[i]} audio is much shorter than video by ${adjustment.durationDeltaMs}ms — padding trailing silence`, + ); } } } @@ -326,7 +347,10 @@ export async function muxNativeWindowsVideoWithAudio( for (const audioPath of [systemAudioPath, micAudioPath]) { if (audioPath) { - await fs.rm(audioPath, { force: true }).catch(() => undefined); + await Promise.all([ + fs.rm(audioPath, { force: true }).catch(() => undefined), + fs.rm(`${audioPath}.json`, { force: true }).catch(() => undefined), + ]); } } } diff --git a/electron/ipc/types.ts b/electron/ipc/types.ts index 5922fb0e..7c7de9c1 100644 --- a/electron/ipc/types.ts +++ b/electron/ipc/types.ts @@ -156,7 +156,7 @@ export type UiohookModuleNamespace = { }; export type AudioSyncAdjustment = { - mode: "none" | "tempo" | "delay"; + mode: "none" | "tempo" | "delay" | "pad"; delayMs: number; tempoRatio: number; durationDeltaMs: number; diff --git a/electron/native/bin/win32-x64/helpers-manifest.json b/electron/native/bin/win32-x64/helpers-manifest.json index dbae2bae..0ae7bb77 100644 --- a/electron/native/bin/win32-x64/helpers-manifest.json +++ b/electron/native/bin/win32-x64/helpers-manifest.json @@ -1,21 +1,21 @@ { - "version": 1, - "platform": "win32", - "arch": "x64", - "helpers": { - "wgc-capture": { - "binaryName": "wgc-capture.exe", - "binarySha256": "013164c0a1391d334e5aa2fe0ff2e47b507407e36c7e44e7333a70a84b944643", - "sourceDir": "electron/native/wgc-capture", - "sourceFingerprint": "9e9bce082266ca5968cf5f0b535b469d47c4ec3a775a93726171c0dfdbcdaa44", - "updatedAt": "2026-03-29T02:15:34.516Z" - }, - "cursor-monitor": { - "binaryName": "cursor-monitor.exe", - "binarySha256": "b0732abc06998a40c3e95078465ad750a6169901944571c39cfd7996effe39c0", - "sourceDir": "electron/native/cursor-monitor", - "sourceFingerprint": "6ad1b8b50bb336f2a48937b06f5ec56d90b6ab4a3e56a4bca278cf67a5d3e52e", - "updatedAt": "2026-03-29T02:15:38.286Z" - } - } + "version": 1, + "platform": "win32", + "arch": "x64", + "helpers": { + "wgc-capture": { + "binaryName": "wgc-capture.exe", + "binarySha256": "bb4c2aa4141e81e1a05b54bd49dbb114eb3a5ff4a666bf7d26525a9da585128b", + "sourceDir": "electron/native/wgc-capture", + "sourceFingerprint": "5bb02a69049a02909d38188cae1dfdfb94fd49465b51ed6ab78a98092b7520cc", + "updatedAt": "2026-04-25T09:17:16.237Z" + }, + "cursor-monitor": { + "binaryName": "cursor-monitor.exe", + "binarySha256": "b0732abc06998a40c3e95078465ad750a6169901944571c39cfd7996effe39c0", + "sourceDir": "electron/native/cursor-monitor", + "sourceFingerprint": "6ad1b8b50bb336f2a48937b06f5ec56d90b6ab4a3e56a4bca278cf67a5d3e52e", + "updatedAt": "2026-03-29T02:15:38.286Z" + } + } } diff --git a/electron/native/bin/win32-x64/wgc-capture.exe b/electron/native/bin/win32-x64/wgc-capture.exe index e5d0df8f..d16a0724 100644 Binary files a/electron/native/bin/win32-x64/wgc-capture.exe and b/electron/native/bin/win32-x64/wgc-capture.exe differ diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cpp index c949e8a2..7fca95b8 100644 --- a/electron/native/wgc-capture/src/main.cpp +++ b/electron/native/wgc-capture/src/main.cpp @@ -14,6 +14,7 @@ #include #include #include +#include static std::atomic g_stopRequested{false}; static std::atomic g_pauseRequested{false}; @@ -156,6 +157,43 @@ static std::wstring utf8ToWide(const std::string& str) { return wstr; } +static void writeCompanionAudioTimingMetadata( + const std::string& audioPath, + int64_t firstVideoTimestampHns, + const WasapiCapture& capture +) { + if (audioPath.empty() || firstVideoTimestampHns < 0) { + return; + } + + const int64_t firstPacketQpcHns = capture.firstPacketQpcHns(); + if (firstPacketQpcHns < 0) { + return; + } + + int64_t startDelayMs = (firstPacketQpcHns - firstVideoTimestampHns + 5000) / 10000; + if (startDelayMs < 0) { + startDelayMs = 0; + } + + std::ofstream metadataFile(audioPath + ".json", std::ios::trunc); + if (!metadataFile) { + std::cerr << "WARNING: Failed to write audio timing metadata for " << audioPath << std::endl; + return; + } + + metadataFile << "{\"startDelayMs\":" << startDelayMs; + const uint32_t discontinuityCount = capture.dataDiscontinuityCount(); + if (discontinuityCount > 0) { + metadataFile << ",\"dataDiscontinuityCount\":" << discontinuityCount; + } + const uint32_t timestampErrorCount = capture.timestampErrorCount(); + if (timestampErrorCount > 0) { + metadataFile << ",\"timestampErrorCount\":" << timestampErrorCount; + } + metadataFile << "}"; +} + static void stdinListenerThread() { std::string line; while (std::getline(std::cin, line)) { @@ -250,9 +288,12 @@ int main(int argc, char* argv[]) { // Set up frame callback std::atomic frameCount{0}; + std::atomic firstVideoTimestampHns{-1}; std::atomic recordingStartedAnnounced{false}; session.setFrameCallback([&](ID3D11Texture2D* texture, int64_t timestampHns) { g_lastFrameTimestampHns = timestampHns; + int64_t expectedFirstVideoTimestampHns = -1; + firstVideoTimestampHns.compare_exchange_strong(expectedFirstVideoTimestampHns, timestampHns); if (g_stopRequested) return; if (g_pauseRequested) return; @@ -337,6 +378,19 @@ int main(int argc, char* argv[]) { if (audioActive) loopback.stop(); if (micActive) micCapture.stop(); + if (audioActive) { + writeCompanionAudioTimingMetadata( + config.audioOutputPath, + firstVideoTimestampHns.load(), + loopback); + } + if (micActive) { + writeCompanionAudioTimingMetadata( + config.micOutputPath, + firstVideoTimestampHns.load(), + micCapture); + } + if (frameCount.load() <= 0) { std::cerr << "ERROR: No video frames were captured before stop" << std::endl; DeleteFileW(outputPathW.c_str()); @@ -350,9 +404,19 @@ int main(int argc, char* argv[]) { std::cout << "Recording stopped. Output path: " << config.outputPath << std::endl; if (audioActive) { + if (loopback.dataDiscontinuityCount() > 0 || loopback.timestampErrorCount() > 0) { + std::cerr << "WARNING: System audio timing metadata includes discontinuities=" + << loopback.dataDiscontinuityCount() + << " timestampErrors=" << loopback.timestampErrorCount() << std::endl; + } std::cout << "Audio path: " << config.audioOutputPath << std::endl; } if (micActive) { + if (micCapture.dataDiscontinuityCount() > 0 || micCapture.timestampErrorCount() > 0) { + std::cerr << "WARNING: Microphone timing metadata includes discontinuities=" + << micCapture.dataDiscontinuityCount() + << " timestampErrors=" << micCapture.timestampErrorCount() << std::endl; + } std::cout << "Mic path: " << config.micOutputPath << std::endl; } std::cout.flush(); diff --git a/electron/native/wgc-capture/src/wasapi_loopback.cpp b/electron/native/wgc-capture/src/wasapi_loopback.cpp index 1659d013..e4008def 100644 --- a/electron/native/wgc-capture/src/wasapi_loopback.cpp +++ b/electron/native/wgc-capture/src/wasapi_loopback.cpp @@ -137,6 +137,9 @@ bool WasapiCapture::start() { } totalDataBytes_ = 0; + firstPacketQpcHns_ = -1; + dataDiscontinuityCount_ = 0; + timestampErrorCount_ = 0; writeWavHeader(outputFile_, 0); HRESULT hr = audioClient_->Start(); @@ -250,13 +253,37 @@ void WasapiCapture::captureThread() { BYTE* data = nullptr; UINT32 numFrames = 0; DWORD flags = 0; + UINT64 devicePosition = 0; + UINT64 qpcPosition = 0; - hr = captureClient_->GetBuffer(&data, &numFrames, &flags, nullptr, nullptr); + hr = captureClient_->GetBuffer( + &data, + &numFrames, + &flags, + &devicePosition, + &qpcPosition); if (FAILED(hr)) { std::cerr << "WASAPI: GetBuffer failed hr=0x" << std::hex << hr << std::dec << std::endl; break; } + if ((flags & AUDCLNT_BUFFERFLAGS_DATA_DISCONTINUITY) != 0) { + dataDiscontinuityCount_.fetch_add(1); + } + if ((flags & AUDCLNT_BUFFERFLAGS_TIMESTAMP_ERROR) != 0) { + timestampErrorCount_.fetch_add(1); + } + if ( + numFrames > 0 && + qpcPosition > 0 && + (flags & AUDCLNT_BUFFERFLAGS_TIMESTAMP_ERROR) == 0 + ) { + int64_t expected = -1; + firstPacketQpcHns_.compare_exchange_strong( + expected, + static_cast(qpcPosition)); + } + UINT32 totalSamples = numFrames * channels; if (flags & AUDCLNT_BUFFERFLAGS_SILENT) { diff --git a/electron/native/wgc-capture/src/wasapi_loopback.h b/electron/native/wgc-capture/src/wasapi_loopback.h index a4facf13..1792a863 100644 --- a/electron/native/wgc-capture/src/wasapi_loopback.h +++ b/electron/native/wgc-capture/src/wasapi_loopback.h @@ -19,6 +19,9 @@ public: bool pause(); bool resume(); void stop(); + int64_t firstPacketQpcHns() const { return firstPacketQpcHns_.load(); } + uint32_t dataDiscontinuityCount() const { return dataDiscontinuityCount_.load(); } + uint32_t timestampErrorCount() const { return timestampErrorCount_.load(); } private: bool initializeCommon(); @@ -41,4 +44,7 @@ private: DWORD streamFlags_ = 0; UINT32 bufferFrameCount_ = 0; + std::atomic firstPacketQpcHns_{-1}; + std::atomic dataDiscontinuityCount_{0}; + std::atomic timestampErrorCount_{0}; };