fix(export): preserve pitch for fallback speed audio

This commit is contained in:
wiiiii123 committed 2026-05-06 06:43:40 +07:00
1 parent 3ab66c9057
commit 97abaa4f4e
10 files changed
+143 -27

No files matched your search

+2 -2
View File
@@ -23,7 +23,7 @@ describe("editedTrackStrategy", () => {
).toBe("filtergraph-fast-path");
});
it("falls back when the edited track depends on sidecar sources", () => {
it("marks a single external source audio track as a filtergraph candidate", () => {
const speedRegions: SpeedRegion[] = [
{ id: "speed-1", startMs: 2_000, endMs: 10_000, speed: 1.25 },
];
@@ -37,7 +37,7 @@ describe("editedTrackStrategy", () => {
audioRegions: [],
sourceAudioFallbackPaths: ["mic.m4a"],
}),
).toBe("offline-render-fallback");
).toBe("filtergraph-fast-path");
});
it("falls back for multi-source mixes", () => {
+5 -1
View File
@@ -158,7 +158,11 @@ export function classifyEditedTrackStrategy(input: EditedTrackStrategyInput): Ed
return "offline-render-fallback";
}
if (input.sourceAudioFallbackPaths.length > 0) {
if (
input.sourceAudioFallbackPaths.some(
(audioPath) => audioPath !== input.primaryAudioSourcePath,
)
) {
return "offline-render-fallback";
}
@@ -39,6 +39,7 @@ function createExporter(overrides: Record<string, unknown> = {}) {
experimentalNativeExport: true,
...overrides,
} as never) as unknown as {
buildNativeAudioPlan: (videoInfo: DecodedVideoInfo) => unknown;
getNativeStaticLayoutSkipReason: (
audioPlan: unknown,
videoInfo: DecodedVideoInfo,
@@ -52,6 +53,59 @@ afterEach(() => {
});
describe("ModernVideoExporter native static-layout eligibility", () => {
it("uses FFmpeg filtergraph audio for speed edits with a single external source track", () => {
const speedRegions: SpeedRegion[] = [
{ id: "speed-1", startMs: 1_000, endMs: 4_000, speed: 1.5 },
];
const exporter = createExporter({
speedRegions,
sourceAudioFallbackPaths: ["C:\\recordly\\recording.system.wav"],
});
expect(
exporter.buildNativeAudioPlan({
...videoInfo,
hasAudio: false,
audioCodec: undefined,
audioSampleRate: undefined,
}),
).toMatchObject({
audioMode: "edited-track",
strategy: "filtergraph-fast-path",
audioSourcePath: "C:\\recordly\\recording.system.wav",
audioSourceSampleRate: 48_000,
editedTrackSegments: [
{ startMs: 0, endMs: 1_000, speed: 1 },
{ startMs: 1_000, endMs: 4_000, speed: 1.5 },
{ startMs: 4_000, endMs: 60_000, speed: 1 },
],
});
});
it("keeps timed companion audio on the offline render path", () => {
const audioPath = "C:\\recordly\\recording.system.wav";
const speedRegions: SpeedRegion[] = [
{ id: "speed-1", startMs: 1_000, endMs: 4_000, speed: 1.5 },
];
const exporter = createExporter({
speedRegions,
sourceAudioFallbackPaths: [audioPath],
sourceAudioFallbackStartDelayMsByPath: { [audioPath]: 250 },
});
expect(
exporter.buildNativeAudioPlan({
...videoInfo,
hasAudio: false,
audioCodec: undefined,
audioSampleRate: undefined,
}),
).toEqual({
audioMode: "edited-track",
strategy: "offline-render-fallback",
});
});
it("allows native video when only the audio track needs offline editing", () => {
const audioRegions: AudioRegion[] = [
{
+19 -8
View File
@@ -158,6 +158,8 @@ type NativeAudioPlan =
editedTrackSegments: Array<{ startMs: number; endMs: number; speed: number }>;
};
const FILTERGRAPH_FALLBACK_AUDIO_SAMPLE_RATE = 48_000;
type NativeStaticLayoutTimelineSegment = {
sourceStartMs: number;
sourceEndMs: number;
@@ -996,6 +998,11 @@ export class ModernVideoExporter {
(videoInfo.hasAudio ? localVideoSourcePath : null) ??
sourceAudioFallbackPaths[0] ??
null;
const usesEmbeddedPrimaryAudio =
Boolean(videoInfo.hasAudio) && primaryAudioSourcePath === localVideoSourcePath;
const primaryAudioSourceSampleRate = usesEmbeddedPrimaryAudio
? videoInfo.audioSampleRate
: FILTERGRAPH_FALLBACK_AUDIO_SAMPLE_RATE;
if (
!videoInfo.hasAudio &&
@@ -1016,13 +1023,16 @@ export class ModernVideoExporter {
Math.round((videoInfo.streamDuration ?? videoInfo.duration) * 1000),
);
const trimRegions = this.config.trimRegions ?? [];
const canUsePrimaryAudioFiltergraph =
Boolean(primaryAudioSourcePath) &&
!hasTimedSourceAudioFallback &&
(usesEmbeddedPrimaryAudio ||
sourceAudioFallbackPaths.includes(primaryAudioSourcePath ?? "")) &&
typeof primaryAudioSourceSampleRate === "number" &&
Number.isFinite(primaryAudioSourceSampleRate) &&
primaryAudioSourceSampleRate > 0;
const strategy =
videoInfo.hasAudio &&
localVideoSourcePath &&
sourceAudioFallbackPaths.length === 0 &&
typeof videoInfo.audioSampleRate === "number" &&
Number.isFinite(videoInfo.audioSampleRate) &&
videoInfo.audioSampleRate > 0
canUsePrimaryAudioFiltergraph
? classifyEditedTrackStrategy({
primaryAudioSourcePath,
sourceDurationMs,
@@ -1034,8 +1044,8 @@ export class ModernVideoExporter {
: "offline-render-fallback";
if (strategy === "filtergraph-fast-path") {
const audioSourcePath = localVideoSourcePath;
const audioSourceSampleRate = videoInfo.audioSampleRate;
const audioSourcePath = primaryAudioSourcePath;
const audioSourceSampleRate = primaryAudioSourceSampleRate;
const editedTrackSegments = buildEditedTrackSourceSegments(
sourceDurationMs,
trimRegions,
@@ -2260,6 +2270,7 @@ export class ModernVideoExporter {
audioPlan.strategy === "filtergraph-fast-path"
? audioPlan.audioSourceSampleRate
: undefined,
outputDurationSec: this.effectiveDurationSec,
editedAudioData: editedAudioBuffer,
editedAudioMimeType,
};
+19 -8
View File
@@ -113,6 +113,8 @@ type NativeAudioPlan =
editedTrackSegments: Array<{ startMs: number; endMs: number; speed: number }>;
};
const FILTERGRAPH_FALLBACK_AUDIO_SAMPLE_RATE = 48_000;
export class VideoExporter {
private config: VideoExporterConfig;
private streamingDecoder: StreamingVideoDecoder | null = null;
@@ -533,6 +535,11 @@ export class VideoExporter {
(videoInfo.hasAudio ? localVideoSourcePath : null) ??
sourceAudioFallbackPaths[0] ??
null;
const usesEmbeddedPrimaryAudio =
Boolean(videoInfo.hasAudio) && primaryAudioSourcePath === localVideoSourcePath;
const primaryAudioSourceSampleRate = usesEmbeddedPrimaryAudio
? videoInfo.audioSampleRate
: FILTERGRAPH_FALLBACK_AUDIO_SAMPLE_RATE;
if (
!videoInfo.hasAudio &&
@@ -553,13 +560,16 @@ export class VideoExporter {
Math.round((videoInfo.streamDuration ?? videoInfo.duration) * 1000),
);
const trimRegions = this.config.trimRegions ?? [];
const canUsePrimaryAudioFiltergraph =
Boolean(primaryAudioSourcePath) &&
!hasTimedSourceAudioFallback &&
(usesEmbeddedPrimaryAudio ||
sourceAudioFallbackPaths.includes(primaryAudioSourcePath ?? "")) &&
typeof primaryAudioSourceSampleRate === "number" &&
Number.isFinite(primaryAudioSourceSampleRate) &&
primaryAudioSourceSampleRate > 0;
const strategy =
videoInfo.hasAudio &&
localVideoSourcePath &&
sourceAudioFallbackPaths.length === 0 &&
typeof videoInfo.audioSampleRate === "number" &&
Number.isFinite(videoInfo.audioSampleRate) &&
videoInfo.audioSampleRate > 0
canUsePrimaryAudioFiltergraph
? classifyEditedTrackStrategy({
primaryAudioSourcePath,
sourceDurationMs,
@@ -571,8 +581,8 @@ export class VideoExporter {
: "offline-render-fallback";
if (strategy === "filtergraph-fast-path") {
const audioSourcePath = localVideoSourcePath;
const audioSourceSampleRate = videoInfo.audioSampleRate;
const audioSourcePath = primaryAudioSourcePath;
const audioSourceSampleRate = primaryAudioSourceSampleRate;
const editedTrackSegments = buildEditedTrackSourceSegments(
sourceDurationMs,
trimRegions,
@@ -950,6 +960,7 @@ export class VideoExporter {
audioPlan.strategy === "filtergraph-fast-path"
? audioPlan.audioSourceSampleRate
: undefined,
outputDurationSec: this.effectiveDurationSec,
editedAudioData: editedAudioBuffer,
editedAudioMimeType,
};