From 1168e0edcc7e13e4554715d0cf088ecd7d6a6555 Mon Sep 17 00:00:00 2001 From: wiiiii123 Date: Wed, 6 May 2026 06:43:40 +0700 Subject: [PATCH] fix(export): preserve pitch for fallback speed audio --- electron/electron-env.d.ts | 2 + electron/ipc/recording/ffmpeg.ts | 8 +-- electron/preload.ts | 5 ++ package.json | 9 ++-- scripts/smoke-electron-main-cjs.mjs | 28 ++++++++++ src/lib/exporter/editedTrackStrategy.test.ts | 4 +- src/lib/exporter/editedTrackStrategy.ts | 6 ++- ...rnVideoExporter.nativeStaticLayout.test.ts | 54 +++++++++++++++++++ src/lib/exporter/modernVideoExporter.ts | 27 +++++++--- src/lib/exporter/videoExporter.ts | 27 +++++++--- 10 files changed, 143 insertions(+), 27 deletions(-) create mode 100644 scripts/smoke-electron-main-cjs.mjs diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index afee0f4f..05564d9b 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -427,6 +427,7 @@ interface Window { audioMode?: "none" | "copy-source" | "trim-source" | "edited-track"; audioSourcePath?: string | null; audioSourceSampleRate?: number; + outputDurationSec?: number; trimSegments?: Array<{ startMs: number; endMs: number }>; editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback"; editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>; @@ -445,6 +446,7 @@ interface Window { audioMode?: "none" | "copy-source" | "trim-source" | "edited-track"; audioSourcePath?: string | null; audioSourceSampleRate?: number; + outputDurationSec?: number; trimSegments?: Array<{ startMs: number; endMs: number }>; editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback"; editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>; diff --git a/electron/ipc/recording/ffmpeg.ts b/electron/ipc/recording/ffmpeg.ts index ab5d436c..fee481cd 100644 --- a/electron/ipc/recording/ffmpeg.ts +++ b/electron/ipc/recording/ffmpeg.ts @@ -1,12 +1,12 @@ import type { ChildProcessWithoutNullStreams } from "node:child_process"; -import { access } from "node:fs/promises"; -import type { SelectedSource } from "../types"; +import fs from "node:fs/promises"; +import { resolveLinuxWindowBounds } from "../cursor/bounds"; import { ffmpegCaptureOutputBuffer, } from "../state"; +import type { SelectedSource } from "../types"; import { getScreen, parseWindowId } from "../utils"; import { resolveWindowsCaptureDisplay } from "../windowsCaptureSelection"; -import { resolveLinuxWindowBounds } from "../cursor/bounds"; export function getDisplayBoundsForSource(source: SelectedSource) { return resolveWindowsCaptureDisplay( @@ -175,7 +175,7 @@ export function waitForFfmpegCaptureStop(process: ChildProcessWithoutNullStreams cleanup(); try { - await access(outputPath); + await fs.access(outputPath); if (code === 0 || code === null) { resolve(outputPath); return; diff --git a/electron/preload.ts b/electron/preload.ts index 082be8e7..146beacc 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -367,6 +367,7 @@ contextBridge.exposeInMainWorld("electronAPI", { audioMode?: "none" | "copy-source" | "trim-source" | "edited-track"; audioSourcePath?: string | null; audioSourceSampleRate?: number; + outputDurationSec?: number; trimSegments?: Array<{ startMs: number; endMs: number }>; editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback"; editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>; @@ -386,7 +387,11 @@ contextBridge.exposeInMainWorld("electronAPI", { options?: { audioMode?: "none" | "copy-source" | "trim-source" | "edited-track"; audioSourcePath?: string | null; + audioSourceSampleRate?: number; + outputDurationSec?: number; trimSegments?: Array<{ startMs: number; endMs: number }>; + editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback"; + editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>; editedAudioData?: ArrayBuffer; editedAudioMimeType?: string | null; }, diff --git a/package.json b/package.json index 7efc2451..75c61f9c 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "scripts": { "dev": "vite --config vite.config.ts", "postinstall": "node scripts/postinstall.mjs", - "build": "npm run build:platform-native-helpers && tsc && vite build --config vite.config.ts && electron-builder", + "build": "npm run build:platform-native-helpers && tsc && vite build --config vite.config.ts && npm run smoke:electron-main-cjs && electron-builder", "lint": "biome check .", "lint:fix": "biome check --write .", "format": "biome format --write .", @@ -30,11 +30,12 @@ "build:nvidia-cuda-compositor": "node scripts/build-nvidia-cuda-compositor.mjs", "build:windows-capture": "node scripts/build-windows-capture.mjs", "build:cursor-monitor": "node scripts/build-cursor-monitor.mjs", - "build:mac": "npm run build:platform-native-helpers && tsc && vite build --config vite.config.ts && electron-builder --mac", - "build:win": "npm run build:platform-native-helpers && tsc && vite build --config vite.config.ts && electron-builder --win", - "build:linux": "npm run build:platform-native-helpers && tsc && vite build --config vite.config.ts && electron-builder --linux", + "build:mac": "npm run build:platform-native-helpers && tsc && vite build --config vite.config.ts && npm run smoke:electron-main-cjs && electron-builder --mac", + "build:win": "npm run build:platform-native-helpers && tsc && vite build --config vite.config.ts && npm run smoke:electron-main-cjs && electron-builder --win", + "build:linux": "npm run build:platform-native-helpers && tsc && vite build --config vite.config.ts && npm run smoke:electron-main-cjs && electron-builder --linux", "i18n:check": "node scripts/i18n-check.mjs", "benchmark:export-queues": "node scripts/benchmark-export-queues.mjs", + "smoke:electron-main-cjs": "node scripts/smoke-electron-main-cjs.mjs", "smoke:packaged-binaries": "node scripts/smoke-packaged-binaries.mjs", "checksums:release": "node scripts/write-release-checksums.mjs", "release:create": "node scripts/create-release.mjs", diff --git a/scripts/smoke-electron-main-cjs.mjs b/scripts/smoke-electron-main-cjs.mjs new file mode 100644 index 00000000..0586723b --- /dev/null +++ b/scripts/smoke-electron-main-cjs.mjs @@ -0,0 +1,28 @@ +import fs from "node:fs/promises"; + +const mainBundleUrl = new URL("../dist-electron/main.cjs", import.meta.url); +const mainBundlePath = mainBundleUrl.pathname; + +let source; +try { + source = await fs.readFile(mainBundleUrl, "utf8"); +} catch (error) { + throw new Error(`Unable to read dist-electron/main.cjs: ${error}`); +} + +const esmImportPattern = /^[ \t]*import\s+(?:(?:[\w*{][^\n;]*?)\s+from\s+)?["'][^"']+["'];?/gm; +const matches = [...source.matchAll(esmImportPattern)].map((match) => ({ + line: source.slice(0, match.index).split(/\r?\n/).length, + text: match[0].trim(), +})); + +if (matches.length > 0) { + const details = matches.map((match) => `line ${match.line}: ${match.text}`).join("\n"); + throw new Error(`dist-electron/main.cjs contains ESM import syntax:\n${details}`); +} + +if (/\bimport\.meta\b/.test(source)) { + throw new Error("dist-electron/main.cjs contains import.meta syntax"); +} + +console.log(`Electron main CJS smoke passed: ${mainBundlePath}`); diff --git a/src/lib/exporter/editedTrackStrategy.test.ts b/src/lib/exporter/editedTrackStrategy.test.ts index 9201b533..3265661d 100644 --- a/src/lib/exporter/editedTrackStrategy.test.ts +++ b/src/lib/exporter/editedTrackStrategy.test.ts @@ -23,7 +23,7 @@ describe("editedTrackStrategy", () => { ).toBe("filtergraph-fast-path"); }); - it("falls back when the edited track depends on sidecar sources", () => { + it("marks a single external source audio track as a filtergraph candidate", () => { const speedRegions: SpeedRegion[] = [ { id: "speed-1", startMs: 2_000, endMs: 10_000, speed: 1.25 }, ]; @@ -37,7 +37,7 @@ describe("editedTrackStrategy", () => { audioRegions: [], sourceAudioFallbackPaths: ["mic.m4a"], }), - ).toBe("offline-render-fallback"); + ).toBe("filtergraph-fast-path"); }); it("falls back for multi-source mixes", () => { diff --git a/src/lib/exporter/editedTrackStrategy.ts b/src/lib/exporter/editedTrackStrategy.ts index 5bee89ee..fa33a3b6 100644 --- a/src/lib/exporter/editedTrackStrategy.ts +++ b/src/lib/exporter/editedTrackStrategy.ts @@ -158,7 +158,11 @@ export function classifyEditedTrackStrategy(input: EditedTrackStrategyInput): Ed return "offline-render-fallback"; } - if (input.sourceAudioFallbackPaths.length > 0) { + if ( + input.sourceAudioFallbackPaths.some( + (audioPath) => audioPath !== input.primaryAudioSourcePath, + ) + ) { return "offline-render-fallback"; } diff --git a/src/lib/exporter/modernVideoExporter.nativeStaticLayout.test.ts b/src/lib/exporter/modernVideoExporter.nativeStaticLayout.test.ts index 4ca3b0c5..4405ee31 100644 --- a/src/lib/exporter/modernVideoExporter.nativeStaticLayout.test.ts +++ b/src/lib/exporter/modernVideoExporter.nativeStaticLayout.test.ts @@ -39,6 +39,7 @@ function createExporter(overrides: Record = {}) { experimentalNativeExport: true, ...overrides, } as never) as unknown as { + buildNativeAudioPlan: (videoInfo: DecodedVideoInfo) => unknown; getNativeStaticLayoutSkipReason: ( audioPlan: unknown, videoInfo: DecodedVideoInfo, @@ -52,6 +53,59 @@ afterEach(() => { }); describe("ModernVideoExporter native static-layout eligibility", () => { + it("uses FFmpeg filtergraph audio for speed edits with a single external source track", () => { + const speedRegions: SpeedRegion[] = [ + { id: "speed-1", startMs: 1_000, endMs: 4_000, speed: 1.5 }, + ]; + const exporter = createExporter({ + speedRegions, + sourceAudioFallbackPaths: ["C:\\recordly\\recording.system.wav"], + }); + + expect( + exporter.buildNativeAudioPlan({ + ...videoInfo, + hasAudio: false, + audioCodec: undefined, + audioSampleRate: undefined, + }), + ).toMatchObject({ + audioMode: "edited-track", + strategy: "filtergraph-fast-path", + audioSourcePath: "C:\\recordly\\recording.system.wav", + audioSourceSampleRate: 48_000, + editedTrackSegments: [ + { startMs: 0, endMs: 1_000, speed: 1 }, + { startMs: 1_000, endMs: 4_000, speed: 1.5 }, + { startMs: 4_000, endMs: 60_000, speed: 1 }, + ], + }); + }); + + it("keeps timed companion audio on the offline render path", () => { + const audioPath = "C:\\recordly\\recording.system.wav"; + const speedRegions: SpeedRegion[] = [ + { id: "speed-1", startMs: 1_000, endMs: 4_000, speed: 1.5 }, + ]; + const exporter = createExporter({ + speedRegions, + sourceAudioFallbackPaths: [audioPath], + sourceAudioFallbackStartDelayMsByPath: { [audioPath]: 250 }, + }); + + expect( + exporter.buildNativeAudioPlan({ + ...videoInfo, + hasAudio: false, + audioCodec: undefined, + audioSampleRate: undefined, + }), + ).toEqual({ + audioMode: "edited-track", + strategy: "offline-render-fallback", + }); + }); + it("allows native video when only the audio track needs offline editing", () => { const audioRegions: AudioRegion[] = [ { diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts index b2cf7ab2..3ad79d22 100644 --- a/src/lib/exporter/modernVideoExporter.ts +++ b/src/lib/exporter/modernVideoExporter.ts @@ -158,6 +158,8 @@ type NativeAudioPlan = editedTrackSegments: Array<{ startMs: number; endMs: number; speed: number }>; }; +const FILTERGRAPH_FALLBACK_AUDIO_SAMPLE_RATE = 48_000; + type NativeStaticLayoutTimelineSegment = { sourceStartMs: number; sourceEndMs: number; @@ -996,6 +998,11 @@ export class ModernVideoExporter { (videoInfo.hasAudio ? localVideoSourcePath : null) ?? sourceAudioFallbackPaths[0] ?? null; + const usesEmbeddedPrimaryAudio = + Boolean(videoInfo.hasAudio) && primaryAudioSourcePath === localVideoSourcePath; + const primaryAudioSourceSampleRate = usesEmbeddedPrimaryAudio + ? videoInfo.audioSampleRate + : FILTERGRAPH_FALLBACK_AUDIO_SAMPLE_RATE; if ( !videoInfo.hasAudio && @@ -1016,13 +1023,16 @@ export class ModernVideoExporter { Math.round((videoInfo.streamDuration ?? videoInfo.duration) * 1000), ); const trimRegions = this.config.trimRegions ?? []; + const canUsePrimaryAudioFiltergraph = + Boolean(primaryAudioSourcePath) && + !hasTimedSourceAudioFallback && + (usesEmbeddedPrimaryAudio || + sourceAudioFallbackPaths.includes(primaryAudioSourcePath ?? "")) && + typeof primaryAudioSourceSampleRate === "number" && + Number.isFinite(primaryAudioSourceSampleRate) && + primaryAudioSourceSampleRate > 0; const strategy = - videoInfo.hasAudio && - localVideoSourcePath && - sourceAudioFallbackPaths.length === 0 && - typeof videoInfo.audioSampleRate === "number" && - Number.isFinite(videoInfo.audioSampleRate) && - videoInfo.audioSampleRate > 0 + canUsePrimaryAudioFiltergraph ? classifyEditedTrackStrategy({ primaryAudioSourcePath, sourceDurationMs, @@ -1034,8 +1044,8 @@ export class ModernVideoExporter { : "offline-render-fallback"; if (strategy === "filtergraph-fast-path") { - const audioSourcePath = localVideoSourcePath; - const audioSourceSampleRate = videoInfo.audioSampleRate; + const audioSourcePath = primaryAudioSourcePath; + const audioSourceSampleRate = primaryAudioSourceSampleRate; const editedTrackSegments = buildEditedTrackSourceSegments( sourceDurationMs, trimRegions, @@ -2260,6 +2270,7 @@ export class ModernVideoExporter { audioPlan.strategy === "filtergraph-fast-path" ? audioPlan.audioSourceSampleRate : undefined, + outputDurationSec: this.effectiveDurationSec, editedAudioData: editedAudioBuffer, editedAudioMimeType, }; diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index 6ba44e63..6b1bf6ae 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -113,6 +113,8 @@ type NativeAudioPlan = editedTrackSegments: Array<{ startMs: number; endMs: number; speed: number }>; }; +const FILTERGRAPH_FALLBACK_AUDIO_SAMPLE_RATE = 48_000; + export class VideoExporter { private config: VideoExporterConfig; private streamingDecoder: StreamingVideoDecoder | null = null; @@ -533,6 +535,11 @@ export class VideoExporter { (videoInfo.hasAudio ? localVideoSourcePath : null) ?? sourceAudioFallbackPaths[0] ?? null; + const usesEmbeddedPrimaryAudio = + Boolean(videoInfo.hasAudio) && primaryAudioSourcePath === localVideoSourcePath; + const primaryAudioSourceSampleRate = usesEmbeddedPrimaryAudio + ? videoInfo.audioSampleRate + : FILTERGRAPH_FALLBACK_AUDIO_SAMPLE_RATE; if ( !videoInfo.hasAudio && @@ -553,13 +560,16 @@ export class VideoExporter { Math.round((videoInfo.streamDuration ?? videoInfo.duration) * 1000), ); const trimRegions = this.config.trimRegions ?? []; + const canUsePrimaryAudioFiltergraph = + Boolean(primaryAudioSourcePath) && + !hasTimedSourceAudioFallback && + (usesEmbeddedPrimaryAudio || + sourceAudioFallbackPaths.includes(primaryAudioSourcePath ?? "")) && + typeof primaryAudioSourceSampleRate === "number" && + Number.isFinite(primaryAudioSourceSampleRate) && + primaryAudioSourceSampleRate > 0; const strategy = - videoInfo.hasAudio && - localVideoSourcePath && - sourceAudioFallbackPaths.length === 0 && - typeof videoInfo.audioSampleRate === "number" && - Number.isFinite(videoInfo.audioSampleRate) && - videoInfo.audioSampleRate > 0 + canUsePrimaryAudioFiltergraph ? classifyEditedTrackStrategy({ primaryAudioSourcePath, sourceDurationMs, @@ -571,8 +581,8 @@ export class VideoExporter { : "offline-render-fallback"; if (strategy === "filtergraph-fast-path") { - const audioSourcePath = localVideoSourcePath; - const audioSourceSampleRate = videoInfo.audioSampleRate; + const audioSourcePath = primaryAudioSourcePath; + const audioSourceSampleRate = primaryAudioSourceSampleRate; const editedTrackSegments = buildEditedTrackSourceSegments( sourceDurationMs, trimRegions, @@ -950,6 +960,7 @@ export class VideoExporter { audioPlan.strategy === "filtergraph-fast-path" ? audioPlan.audioSourceSampleRate : undefined, + outputDurationSec: this.effectiveDurationSec, editedAudioData: editedAudioBuffer, editedAudioMimeType, };