mirror of
https://github.com/webadderallorg/Recordly.git
synced 2026-09-24 14:55:37 +00:00
fix(export): fast-path simple edited audio tracks
This commit is contained in:
Vendored
+6
@@ -194,7 +194,10 @@ interface Window {
|
||||
options?: {
|
||||
audioMode?: "none" | "copy-source" | "trim-source" | "edited-track";
|
||||
audioSourcePath?: string | null;
|
||||
audioSourceSampleRate?: number;
|
||||
trimSegments?: Array<{ startMs: number; endMs: number }>;
|
||||
editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback";
|
||||
editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>;
|
||||
editedAudioData?: ArrayBuffer;
|
||||
editedAudioMimeType?: string | null;
|
||||
},
|
||||
@@ -213,7 +216,10 @@ interface Window {
|
||||
options?: {
|
||||
audioMode?: "none" | "copy-source" | "trim-source" | "edited-track";
|
||||
audioSourcePath?: string | null;
|
||||
audioSourceSampleRate?: number;
|
||||
trimSegments?: Array<{ startMs: number; endMs: number }>;
|
||||
editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback";
|
||||
editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>;
|
||||
editedAudioData?: ArrayBuffer;
|
||||
editedAudioMimeType?: string | null;
|
||||
},
|
||||
|
||||
@@ -14,6 +14,7 @@ import type {
|
||||
NativeVideoExportFinishOptions,
|
||||
} from "../nativeVideoExport";
|
||||
import {
|
||||
buildEditedTrackSourceAudioFilter,
|
||||
buildNativeVideoExportArgs,
|
||||
buildTrimmedSourceAudioFilter,
|
||||
getEditedAudioExtension,
|
||||
@@ -387,8 +388,10 @@ export async function muxNativeVideoExportAudio(
|
||||
const metrics: NativeVideoAudioMuxMetrics = {};
|
||||
const tempArtifacts: string[] = [];
|
||||
let audioInputPath = options.audioSourcePath ?? null;
|
||||
const useEditedTrackFiltergraph =
|
||||
audioMode === "edited-track" && options.editedTrackStrategy === "filtergraph-fast-path";
|
||||
|
||||
if (audioMode === "edited-track") {
|
||||
if (audioMode === "edited-track" && !useEditedTrackFiltergraph) {
|
||||
if (!options.editedAudioData) {
|
||||
throw new Error("Edited audio data is missing for native export");
|
||||
}
|
||||
@@ -435,6 +438,15 @@ export async function muxNativeVideoExportAudio(
|
||||
} else {
|
||||
args.push("-map", "0:v:0", "-map", "1:a:0");
|
||||
}
|
||||
} else if (useEditedTrackFiltergraph) {
|
||||
const filter = buildEditedTrackSourceAudioFilter(
|
||||
options.editedTrackSegments ?? [],
|
||||
options.audioSourceSampleRate ?? 0,
|
||||
);
|
||||
if (!filter) {
|
||||
throw new Error("Edited-track filtergraph inputs are incomplete for native export");
|
||||
}
|
||||
args.push("-filter_complex", filter, "-map", "0:v:0", "-map", "[aout]");
|
||||
} else {
|
||||
args.push("-map", "0:v:0", "-map", "1:a:0");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
buildEditedTrackSourceAudioFilter,
|
||||
buildTrimmedSourceAudioFilter,
|
||||
} from "./nativeVideoExport";
|
||||
|
||||
describe("buildTrimmedSourceAudioFilter", () => {
|
||||
it("concatenates trimmed source segments into a single output label", () => {
|
||||
expect(
|
||||
buildTrimmedSourceAudioFilter([
|
||||
{ startMs: 0, endMs: 2_000 },
|
||||
{ startMs: 4_000, endMs: 6_000 },
|
||||
]),
|
||||
).toContain("concat=n=2:v=0:a=1[aout]");
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildEditedTrackSourceAudioFilter", () => {
|
||||
it("builds a concat filtergraph that pitch-shifts via asetrate for speed changes", () => {
|
||||
const filter = buildEditedTrackSourceAudioFilter(
|
||||
[
|
||||
{ startMs: 0, endMs: 2_000, speed: 1 },
|
||||
{ startMs: 2_000, endMs: 6_000, speed: 1.5 },
|
||||
],
|
||||
44_100,
|
||||
);
|
||||
|
||||
expect(filter).toContain(
|
||||
"[1:a]atrim=start=2.000:end=6.000,asetpts=PTS-STARTPTS,asetrate=66150,aresample=44100[edited_audio_1]",
|
||||
);
|
||||
expect(filter).toContain("concat=n=2:v=0:a=1[aout]");
|
||||
});
|
||||
|
||||
it("returns null when the edited-track filtergraph inputs are incomplete", () => {
|
||||
expect(buildEditedTrackSourceAudioFilter([], 44_100)).toBeNull();
|
||||
expect(
|
||||
buildEditedTrackSourceAudioFilter(
|
||||
[{ startMs: 0, endMs: 2_000, speed: 1.5 }],
|
||||
Number.NaN,
|
||||
),
|
||||
).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -3,6 +3,9 @@ const NATIVE_EXPORT_INPUT_BYTES_PER_PIXEL = 4;
|
||||
export type NativeExportEncodingMode = "fast" | "balanced" | "quality";
|
||||
|
||||
export type NativeVideoExportAudioMode = "none" | "copy-source" | "trim-source" | "edited-track";
|
||||
export type NativeVideoExportEditedTrackStrategy =
|
||||
| "filtergraph-fast-path"
|
||||
| "offline-render-fallback";
|
||||
|
||||
export interface NativeVideoExportStartOptions {
|
||||
width: number;
|
||||
@@ -18,10 +21,17 @@ export interface NativeVideoExportAudioSegment {
|
||||
endMs: number;
|
||||
}
|
||||
|
||||
export interface NativeVideoExportEditedTrackSegment extends NativeVideoExportAudioSegment {
|
||||
speed: number;
|
||||
}
|
||||
|
||||
export interface NativeVideoExportFinishOptions {
|
||||
audioMode?: NativeVideoExportAudioMode;
|
||||
audioSourcePath?: string | null;
|
||||
audioSourceSampleRate?: number;
|
||||
trimSegments?: NativeVideoExportAudioSegment[];
|
||||
editedTrackStrategy?: NativeVideoExportEditedTrackStrategy;
|
||||
editedTrackSegments?: NativeVideoExportEditedTrackSegment[];
|
||||
editedAudioData?: ArrayBuffer;
|
||||
editedAudioMimeType?: string | null;
|
||||
}
|
||||
@@ -162,6 +172,54 @@ export function buildTrimmedSourceAudioFilter(
|
||||
return filterParts.join(";");
|
||||
}
|
||||
|
||||
export function buildEditedTrackSourceAudioFilter(
|
||||
segments: NativeVideoExportEditedTrackSegment[],
|
||||
sourceSampleRate: number,
|
||||
): string | null {
|
||||
if (segments.length === 0 || !Number.isFinite(sourceSampleRate) || sourceSampleRate <= 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const normalizedSourceSampleRate = Math.round(sourceSampleRate);
|
||||
const filterParts: string[] = [];
|
||||
const segmentLabels: string[] = [];
|
||||
|
||||
segments.forEach((segment, index) => {
|
||||
if (segment.endMs - segment.startMs <= 0.5) {
|
||||
return;
|
||||
}
|
||||
|
||||
const label = `edited_audio_${index}`;
|
||||
const speed = Number.isFinite(segment.speed) && segment.speed > 0 ? segment.speed : 1;
|
||||
const segmentFilter = [
|
||||
`[1:a]atrim=start=${formatFfmpegSeconds(segment.startMs)}:end=${formatFfmpegSeconds(segment.endMs)}`,
|
||||
"asetpts=PTS-STARTPTS",
|
||||
];
|
||||
|
||||
if (Math.abs(speed - 1) > 0.0001) {
|
||||
segmentFilter.push(
|
||||
`asetrate=${Math.max(1, Math.round(normalizedSourceSampleRate * speed))}`,
|
||||
`aresample=${normalizedSourceSampleRate}`,
|
||||
);
|
||||
}
|
||||
|
||||
filterParts.push(`${segmentFilter.join(",")}[${label}]`);
|
||||
segmentLabels.push(`[${label}]`);
|
||||
});
|
||||
|
||||
if (segmentLabels.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (segmentLabels.length === 1) {
|
||||
filterParts.push(`${segmentLabels[0]}anull[aout]`);
|
||||
} else {
|
||||
filterParts.push(`${segmentLabels.join("")}concat=n=${segmentLabels.length}:v=0:a=1[aout]`);
|
||||
}
|
||||
|
||||
return filterParts.join(";");
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds FFmpeg arguments for a zero-copy H.264 stream export.
|
||||
* FFmpeg receives a pre-encoded Annex B H.264 stream on stdin (produced by the
|
||||
|
||||
@@ -143,7 +143,10 @@ contextBridge.exposeInMainWorld("electronAPI", {
|
||||
options?: {
|
||||
audioMode?: "none" | "copy-source" | "trim-source" | "edited-track";
|
||||
audioSourcePath?: string | null;
|
||||
audioSourceSampleRate?: number;
|
||||
trimSegments?: Array<{ startMs: number; endMs: number }>;
|
||||
editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback";
|
||||
editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>;
|
||||
editedAudioData?: ArrayBuffer;
|
||||
editedAudioMimeType?: string | null;
|
||||
},
|
||||
@@ -186,7 +189,10 @@ contextBridge.exposeInMainWorld("electronAPI", {
|
||||
options?: {
|
||||
audioMode?: "none" | "copy-source" | "trim-source" | "edited-track";
|
||||
audioSourcePath?: string | null;
|
||||
audioSourceSampleRate?: number;
|
||||
trimSegments?: Array<{ startMs: number; endMs: number }>;
|
||||
editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback";
|
||||
editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>;
|
||||
editedAudioData?: ArrayBuffer;
|
||||
editedAudioMimeType?: string | null;
|
||||
},
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { AudioRegion, SpeedRegion, TrimRegion } from "@/components/video-editor/types";
|
||||
import { buildEditedTrackSourceSegments, classifyEditedTrackStrategy } from "./editedTrackStrategy";
|
||||
|
||||
const SOURCE_DURATION_MS = 20_000;
|
||||
const EMPTY_TRIMS: TrimRegion[] = [];
|
||||
|
||||
describe("editedTrackStrategy", () => {
|
||||
it("marks single-source speed-only edits as filtergraph candidates", () => {
|
||||
const speedRegions: SpeedRegion[] = [
|
||||
{ id: "speed-1", startMs: 2_000, endMs: 10_000, speed: 1.5 },
|
||||
];
|
||||
|
||||
expect(
|
||||
classifyEditedTrackStrategy({
|
||||
primaryAudioSourcePath: "recording.mp4",
|
||||
sourceDurationMs: SOURCE_DURATION_MS,
|
||||
trimRegions: EMPTY_TRIMS,
|
||||
speedRegions,
|
||||
audioRegions: [],
|
||||
sourceAudioFallbackPaths: [],
|
||||
}),
|
||||
).toBe("filtergraph-fast-path");
|
||||
});
|
||||
|
||||
it("falls back when the edited track depends on sidecar sources", () => {
|
||||
const speedRegions: SpeedRegion[] = [
|
||||
{ id: "speed-1", startMs: 2_000, endMs: 10_000, speed: 1.25 },
|
||||
];
|
||||
|
||||
expect(
|
||||
classifyEditedTrackStrategy({
|
||||
primaryAudioSourcePath: "mic.m4a",
|
||||
sourceDurationMs: SOURCE_DURATION_MS,
|
||||
trimRegions: EMPTY_TRIMS,
|
||||
speedRegions,
|
||||
audioRegions: [],
|
||||
sourceAudioFallbackPaths: ["mic.m4a"],
|
||||
}),
|
||||
).toBe("offline-render-fallback");
|
||||
});
|
||||
|
||||
it("falls back for multi-source mixes", () => {
|
||||
const speedRegions: SpeedRegion[] = [
|
||||
{ id: "speed-1", startMs: 2_000, endMs: 10_000, speed: 1.5 },
|
||||
];
|
||||
|
||||
expect(
|
||||
classifyEditedTrackStrategy({
|
||||
primaryAudioSourcePath: "recording.mp4",
|
||||
sourceDurationMs: SOURCE_DURATION_MS,
|
||||
trimRegions: EMPTY_TRIMS,
|
||||
speedRegions,
|
||||
audioRegions: [],
|
||||
sourceAudioFallbackPaths: ["mic.m4a"],
|
||||
}),
|
||||
).toBe("offline-render-fallback");
|
||||
});
|
||||
|
||||
it("falls back when audio regions require overlay mixing", () => {
|
||||
const speedRegions: SpeedRegion[] = [
|
||||
{ id: "speed-1", startMs: 2_000, endMs: 10_000, speed: 1.5 },
|
||||
];
|
||||
const audioRegions: AudioRegion[] = [
|
||||
{
|
||||
id: "audio-1",
|
||||
audioPath: "overlay.wav",
|
||||
startMs: 1_000,
|
||||
endMs: 4_000,
|
||||
volume: 0.8,
|
||||
},
|
||||
];
|
||||
|
||||
expect(
|
||||
classifyEditedTrackStrategy({
|
||||
primaryAudioSourcePath: "recording.mp4",
|
||||
sourceDurationMs: SOURCE_DURATION_MS,
|
||||
trimRegions: EMPTY_TRIMS,
|
||||
speedRegions,
|
||||
audioRegions,
|
||||
sourceAudioFallbackPaths: [],
|
||||
}),
|
||||
).toBe("offline-render-fallback");
|
||||
});
|
||||
|
||||
it("falls back for speeds outside the conservative filtergraph window", () => {
|
||||
const speedRegions = [
|
||||
{ id: "speed-1", startMs: 2_000, endMs: 10_000, speed: 2.5 },
|
||||
] as SpeedRegion[];
|
||||
|
||||
expect(
|
||||
classifyEditedTrackStrategy({
|
||||
primaryAudioSourcePath: "recording.mp4",
|
||||
sourceDurationMs: SOURCE_DURATION_MS,
|
||||
trimRegions: EMPTY_TRIMS,
|
||||
speedRegions,
|
||||
audioRegions: [],
|
||||
sourceAudioFallbackPaths: [],
|
||||
}),
|
||||
).toBe("offline-render-fallback");
|
||||
});
|
||||
|
||||
it("builds source segments that preserve trims and speed boundaries", () => {
|
||||
const trimRegions: TrimRegion[] = [{ id: "trim-1", startMs: 10_000, endMs: 12_000 }];
|
||||
const speedRegions: SpeedRegion[] = [
|
||||
{ id: "speed-1", startMs: 2_000, endMs: 6_000, speed: 1.5 },
|
||||
{ id: "speed-2", startMs: 14_000, endMs: 18_000, speed: 0.75 },
|
||||
];
|
||||
|
||||
expect(
|
||||
buildEditedTrackSourceSegments(SOURCE_DURATION_MS, trimRegions, speedRegions),
|
||||
).toEqual([
|
||||
{ startMs: 0, endMs: 2_000, speed: 1 },
|
||||
{ startMs: 2_000, endMs: 6_000, speed: 1.5 },
|
||||
{ startMs: 6_000, endMs: 10_000, speed: 1 },
|
||||
{ startMs: 12_000, endMs: 14_000, speed: 1 },
|
||||
{ startMs: 14_000, endMs: 18_000, speed: 0.75 },
|
||||
{ startMs: 18_000, endMs: 20_000, speed: 1 },
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,141 @@
|
||||
import type { AudioRegion, SpeedRegion, TrimRegion } from "@/components/video-editor/types";
|
||||
|
||||
const MIN_FILTERGRAPH_SPEED = 0.5;
|
||||
const MAX_FILTERGRAPH_SPEED = 2;
|
||||
|
||||
export type EditedTrackStrategy = "filtergraph-fast-path" | "offline-render-fallback";
|
||||
|
||||
export interface EditedTrackSourceSegment {
|
||||
startMs: number;
|
||||
endMs: number;
|
||||
speed: number;
|
||||
}
|
||||
|
||||
export interface EditedTrackStrategyInput {
|
||||
primaryAudioSourcePath: string | null;
|
||||
sourceDurationMs: number;
|
||||
trimRegions: TrimRegion[];
|
||||
speedRegions: SpeedRegion[];
|
||||
audioRegions: AudioRegion[];
|
||||
sourceAudioFallbackPaths: string[];
|
||||
}
|
||||
|
||||
function isSafeFiltergraphSpeed(speed: number): boolean {
|
||||
return (
|
||||
Number.isFinite(speed) && speed >= MIN_FILTERGRAPH_SPEED && speed <= MAX_FILTERGRAPH_SPEED
|
||||
);
|
||||
}
|
||||
|
||||
function hasSafeFiltergraphSpeedRegions(speedRegions: SpeedRegion[]): boolean {
|
||||
return speedRegions.every((region) => isSafeFiltergraphSpeed(region.speed));
|
||||
}
|
||||
|
||||
function buildKeptRanges(
|
||||
sourceDurationMs: number,
|
||||
trimRegions: TrimRegion[],
|
||||
): Array<{ startMs: number; endMs: number }> {
|
||||
const sortedTrimRegions = [...trimRegions].sort((left, right) => left.startMs - right.startMs);
|
||||
const keptRanges: Array<{ startMs: number; endMs: number }> = [];
|
||||
let cursorMs = 0;
|
||||
|
||||
for (const trimRegion of sortedTrimRegions) {
|
||||
const startMs = Math.max(0, trimRegion.startMs);
|
||||
const endMs = Math.min(sourceDurationMs, trimRegion.endMs);
|
||||
if (endMs <= startMs) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (startMs > cursorMs) {
|
||||
keptRanges.push({ startMs: cursorMs, endMs: startMs });
|
||||
}
|
||||
|
||||
cursorMs = Math.max(cursorMs, endMs);
|
||||
}
|
||||
|
||||
if (cursorMs < sourceDurationMs) {
|
||||
keptRanges.push({ startMs: cursorMs, endMs: sourceDurationMs });
|
||||
}
|
||||
|
||||
return keptRanges;
|
||||
}
|
||||
|
||||
export function buildEditedTrackSourceSegments(
|
||||
sourceDurationMs: number,
|
||||
trimRegions: TrimRegion[],
|
||||
speedRegions: SpeedRegion[],
|
||||
): EditedTrackSourceSegment[] {
|
||||
const segments: EditedTrackSourceSegment[] = [];
|
||||
const keptRanges = buildKeptRanges(sourceDurationMs, trimRegions);
|
||||
|
||||
for (const keptRange of keptRanges) {
|
||||
const boundaries = new Set<number>([keptRange.startMs, keptRange.endMs]);
|
||||
for (const speedRegion of speedRegions) {
|
||||
const startMs = Math.max(keptRange.startMs, speedRegion.startMs);
|
||||
const endMs = Math.min(keptRange.endMs, speedRegion.endMs);
|
||||
if (endMs > startMs) {
|
||||
boundaries.add(startMs);
|
||||
boundaries.add(endMs);
|
||||
}
|
||||
}
|
||||
|
||||
const orderedBoundaries = [...boundaries].sort((left, right) => left - right);
|
||||
for (let index = 0; index < orderedBoundaries.length - 1; index += 1) {
|
||||
const startMs = orderedBoundaries[index] ?? 0;
|
||||
const endMs = orderedBoundaries[index + 1] ?? 0;
|
||||
if (endMs - startMs <= 0.5) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const midpointMs = startMs + (endMs - startMs) / 2;
|
||||
const speedRegion = speedRegions.find(
|
||||
(region) => midpointMs >= region.startMs && midpointMs < region.endMs,
|
||||
);
|
||||
const speed = speedRegion?.speed ?? 1;
|
||||
if (!isSafeFiltergraphSpeed(speed)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
segments.push({
|
||||
startMs,
|
||||
endMs,
|
||||
speed,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return segments;
|
||||
}
|
||||
|
||||
export function classifyEditedTrackStrategy(input: EditedTrackStrategyInput): EditedTrackStrategy {
|
||||
if (!input.primaryAudioSourcePath) {
|
||||
return "offline-render-fallback";
|
||||
}
|
||||
|
||||
if (input.audioRegions.length > 0) {
|
||||
return "offline-render-fallback";
|
||||
}
|
||||
|
||||
if (input.speedRegions.length === 0) {
|
||||
return "offline-render-fallback";
|
||||
}
|
||||
|
||||
if (!hasSafeFiltergraphSpeedRegions(input.speedRegions)) {
|
||||
return "offline-render-fallback";
|
||||
}
|
||||
|
||||
if (input.sourceAudioFallbackPaths.length > 0) {
|
||||
return "offline-render-fallback";
|
||||
}
|
||||
|
||||
if (
|
||||
buildEditedTrackSourceSegments(
|
||||
input.sourceDurationMs,
|
||||
input.trimRegions,
|
||||
input.speedRegions,
|
||||
).length === 0
|
||||
) {
|
||||
return "offline-render-fallback";
|
||||
}
|
||||
|
||||
return "filtergraph-fast-path";
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import type {
|
||||
import { extensionHost } from "@/lib/extensions";
|
||||
import { AudioProcessor, isAacAudioEncodingSupported } from "./audioEncoder";
|
||||
import { normalizeLightningRuntimePlatform } from "./backendPolicy";
|
||||
import { buildEditedTrackSourceSegments, classifyEditedTrackStrategy } from "./editedTrackStrategy";
|
||||
import {
|
||||
type ExportBackpressureProfile,
|
||||
getExportBackpressureProfile,
|
||||
@@ -106,6 +107,14 @@ type NativeAudioPlan =
|
||||
}
|
||||
| {
|
||||
audioMode: "edited-track";
|
||||
strategy: "offline-render-fallback";
|
||||
}
|
||||
| {
|
||||
audioMode: "edited-track";
|
||||
strategy: "filtergraph-fast-path";
|
||||
audioSourcePath: string;
|
||||
audioSourceSampleRate: number;
|
||||
editedTrackSegments: Array<{ startMs: number; endMs: number; speed: number }>;
|
||||
};
|
||||
|
||||
const NATIVE_EXPORT_ENGINE_NAME = "Breeze";
|
||||
@@ -737,11 +746,62 @@ export class ModernVideoExporter {
|
||||
audioRegions.length > 0 ||
|
||||
sourceAudioFallbackPaths.length > 1
|
||||
) {
|
||||
return { audioMode: "edited-track" };
|
||||
const sourceDurationMs = Math.max(
|
||||
0,
|
||||
Math.round((videoInfo.streamDuration ?? videoInfo.duration) * 1000),
|
||||
);
|
||||
const trimRegions = this.config.trimRegions ?? [];
|
||||
const strategy =
|
||||
videoInfo.hasAudio &&
|
||||
localVideoSourcePath &&
|
||||
sourceAudioFallbackPaths.length === 0 &&
|
||||
typeof videoInfo.audioSampleRate === "number" &&
|
||||
Number.isFinite(videoInfo.audioSampleRate) &&
|
||||
videoInfo.audioSampleRate > 0
|
||||
? classifyEditedTrackStrategy({
|
||||
primaryAudioSourcePath,
|
||||
sourceDurationMs,
|
||||
trimRegions,
|
||||
speedRegions,
|
||||
audioRegions,
|
||||
sourceAudioFallbackPaths,
|
||||
})
|
||||
: "offline-render-fallback";
|
||||
|
||||
if (strategy === "filtergraph-fast-path") {
|
||||
const audioSourcePath = localVideoSourcePath;
|
||||
const audioSourceSampleRate = videoInfo.audioSampleRate;
|
||||
const editedTrackSegments = buildEditedTrackSourceSegments(
|
||||
sourceDurationMs,
|
||||
trimRegions,
|
||||
speedRegions,
|
||||
);
|
||||
if (
|
||||
audioSourcePath &&
|
||||
typeof audioSourceSampleRate === "number" &&
|
||||
editedTrackSegments.length > 0
|
||||
) {
|
||||
return {
|
||||
audioMode: "edited-track",
|
||||
strategy,
|
||||
audioSourcePath,
|
||||
audioSourceSampleRate,
|
||||
editedTrackSegments,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
audioMode: "edited-track",
|
||||
strategy: "offline-render-fallback",
|
||||
};
|
||||
}
|
||||
|
||||
if (!primaryAudioSourcePath) {
|
||||
return { audioMode: "edited-track" };
|
||||
return {
|
||||
audioMode: "edited-track",
|
||||
strategy: "offline-render-fallback",
|
||||
};
|
||||
}
|
||||
|
||||
if ((this.config.trimRegions ?? []).length > 0) {
|
||||
@@ -949,7 +1009,10 @@ export class ModernVideoExporter {
|
||||
let editedAudioBuffer: ArrayBuffer | undefined;
|
||||
let editedAudioMimeType: string | null = null;
|
||||
|
||||
if (audioPlan.audioMode === "edited-track") {
|
||||
if (
|
||||
audioPlan.audioMode === "edited-track" &&
|
||||
audioPlan.strategy === "offline-render-fallback"
|
||||
) {
|
||||
this.audioProcessor = new AudioProcessor();
|
||||
this.audioProcessor.setOnProgress((progress) => {
|
||||
this.reportFinalizingProgress(this.processedFrameCount, 99, progress);
|
||||
@@ -976,6 +1039,8 @@ export class ModernVideoExporter {
|
||||
console.log(`[VideoExporter] Finalizing ${NATIVE_EXPORT_ENGINE_NAME} export`, {
|
||||
sessionId,
|
||||
audioMode: audioPlan.audioMode,
|
||||
editedTrackStrategy:
|
||||
audioPlan.audioMode === "edited-track" ? audioPlan.strategy : undefined,
|
||||
encoderName: this.encoderName ?? "unknown",
|
||||
});
|
||||
|
||||
@@ -987,11 +1052,25 @@ export class ModernVideoExporter {
|
||||
audioMode: audioPlan.audioMode,
|
||||
audioSourcePath:
|
||||
audioPlan.audioMode === "copy-source" ||
|
||||
audioPlan.audioMode === "trim-source"
|
||||
audioPlan.audioMode === "trim-source" ||
|
||||
(audioPlan.audioMode === "edited-track" &&
|
||||
audioPlan.strategy === "filtergraph-fast-path")
|
||||
? audioPlan.audioSourcePath
|
||||
: null,
|
||||
trimSegments:
|
||||
audioPlan.audioMode === "trim-source" ? audioPlan.trimSegments : undefined,
|
||||
editedTrackStrategy:
|
||||
audioPlan.audioMode === "edited-track" ? audioPlan.strategy : undefined,
|
||||
editedTrackSegments:
|
||||
audioPlan.audioMode === "edited-track" &&
|
||||
audioPlan.strategy === "filtergraph-fast-path"
|
||||
? audioPlan.editedTrackSegments
|
||||
: undefined,
|
||||
audioSourceSampleRate:
|
||||
audioPlan.audioMode === "edited-track" &&
|
||||
audioPlan.strategy === "filtergraph-fast-path"
|
||||
? audioPlan.audioSourceSampleRate
|
||||
: undefined,
|
||||
editedAudioData: editedAudioBuffer,
|
||||
editedAudioMimeType,
|
||||
}),
|
||||
@@ -1041,7 +1120,10 @@ export class ModernVideoExporter {
|
||||
let editedAudioBuffer: ArrayBuffer | undefined;
|
||||
let editedAudioMimeType: string | null = null;
|
||||
|
||||
if (audioPlan.audioMode === "edited-track") {
|
||||
if (
|
||||
audioPlan.audioMode === "edited-track" &&
|
||||
audioPlan.strategy === "offline-render-fallback"
|
||||
) {
|
||||
this.audioProcessor = new AudioProcessor();
|
||||
this.audioProcessor.setOnProgress((progress) => {
|
||||
this.reportFinalizingProgress(this.processedFrameCount, 99, progress);
|
||||
@@ -1071,11 +1153,25 @@ export class ModernVideoExporter {
|
||||
audioMode: audioPlan.audioMode,
|
||||
audioSourcePath:
|
||||
audioPlan.audioMode === "copy-source" ||
|
||||
audioPlan.audioMode === "trim-source"
|
||||
audioPlan.audioMode === "trim-source" ||
|
||||
(audioPlan.audioMode === "edited-track" &&
|
||||
audioPlan.strategy === "filtergraph-fast-path")
|
||||
? audioPlan.audioSourcePath
|
||||
: null,
|
||||
trimSegments:
|
||||
audioPlan.audioMode === "trim-source" ? audioPlan.trimSegments : undefined,
|
||||
editedTrackStrategy:
|
||||
audioPlan.audioMode === "edited-track" ? audioPlan.strategy : undefined,
|
||||
editedTrackSegments:
|
||||
audioPlan.audioMode === "edited-track" &&
|
||||
audioPlan.strategy === "filtergraph-fast-path"
|
||||
? audioPlan.editedTrackSegments
|
||||
: undefined,
|
||||
audioSourceSampleRate:
|
||||
audioPlan.audioMode === "edited-track" &&
|
||||
audioPlan.strategy === "filtergraph-fast-path"
|
||||
? audioPlan.audioSourceSampleRate
|
||||
: undefined,
|
||||
editedAudioData: editedAudioBuffer,
|
||||
editedAudioMimeType,
|
||||
}),
|
||||
|
||||
@@ -17,6 +17,7 @@ export interface DecodedVideoInfo {
|
||||
codec: string;
|
||||
hasAudio: boolean;
|
||||
audioCodec?: string;
|
||||
audioSampleRate?: number;
|
||||
}
|
||||
|
||||
/** Caller must close the VideoFrame after use. */
|
||||
@@ -205,6 +206,10 @@ export class StreamingVideoDecoder {
|
||||
codec: videoStream?.codec_string || "unknown",
|
||||
hasAudio: !!audioStream,
|
||||
audioCodec: audioStream?.codec_string,
|
||||
audioSampleRate:
|
||||
typeof audioStream?.sample_rate === "string"
|
||||
? Number.parseInt(audioStream.sample_rate, 10)
|
||||
: undefined,
|
||||
};
|
||||
|
||||
return this.metadata;
|
||||
|
||||
@@ -13,6 +13,7 @@ import type {
|
||||
ZoomTransitionEasing,
|
||||
} from "@/components/video-editor/types";
|
||||
import { AudioProcessor, isAacAudioEncodingSupported } from "./audioEncoder";
|
||||
import { buildEditedTrackSourceSegments, classifyEditedTrackStrategy } from "./editedTrackStrategy";
|
||||
import {
|
||||
advanceFinalizationProgress,
|
||||
type FinalizationProgressWatchdog,
|
||||
@@ -94,6 +95,14 @@ type NativeAudioPlan =
|
||||
}
|
||||
| {
|
||||
audioMode: "edited-track";
|
||||
strategy: "offline-render-fallback";
|
||||
}
|
||||
| {
|
||||
audioMode: "edited-track";
|
||||
strategy: "filtergraph-fast-path";
|
||||
audioSourcePath: string;
|
||||
audioSourceSampleRate: number;
|
||||
editedTrackSegments: Array<{ startMs: number; endMs: number; speed: number }>;
|
||||
};
|
||||
|
||||
export class VideoExporter {
|
||||
@@ -504,11 +513,62 @@ export class VideoExporter {
|
||||
audioRegions.length > 0 ||
|
||||
sourceAudioFallbackPaths.length > 1
|
||||
) {
|
||||
return { audioMode: "edited-track" };
|
||||
const sourceDurationMs = Math.max(
|
||||
0,
|
||||
Math.round((videoInfo.streamDuration ?? videoInfo.duration) * 1000),
|
||||
);
|
||||
const trimRegions = this.config.trimRegions ?? [];
|
||||
const strategy =
|
||||
videoInfo.hasAudio &&
|
||||
localVideoSourcePath &&
|
||||
sourceAudioFallbackPaths.length === 0 &&
|
||||
typeof videoInfo.audioSampleRate === "number" &&
|
||||
Number.isFinite(videoInfo.audioSampleRate) &&
|
||||
videoInfo.audioSampleRate > 0
|
||||
? classifyEditedTrackStrategy({
|
||||
primaryAudioSourcePath,
|
||||
sourceDurationMs,
|
||||
trimRegions,
|
||||
speedRegions,
|
||||
audioRegions,
|
||||
sourceAudioFallbackPaths,
|
||||
})
|
||||
: "offline-render-fallback";
|
||||
|
||||
if (strategy === "filtergraph-fast-path") {
|
||||
const audioSourcePath = localVideoSourcePath;
|
||||
const audioSourceSampleRate = videoInfo.audioSampleRate;
|
||||
const editedTrackSegments = buildEditedTrackSourceSegments(
|
||||
sourceDurationMs,
|
||||
trimRegions,
|
||||
speedRegions,
|
||||
);
|
||||
if (
|
||||
audioSourcePath &&
|
||||
typeof audioSourceSampleRate === "number" &&
|
||||
editedTrackSegments.length > 0
|
||||
) {
|
||||
return {
|
||||
audioMode: "edited-track",
|
||||
strategy,
|
||||
audioSourcePath,
|
||||
audioSourceSampleRate,
|
||||
editedTrackSegments,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
audioMode: "edited-track",
|
||||
strategy: "offline-render-fallback",
|
||||
};
|
||||
}
|
||||
|
||||
if (!primaryAudioSourcePath) {
|
||||
return { audioMode: "edited-track" };
|
||||
return {
|
||||
audioMode: "edited-track",
|
||||
strategy: "offline-render-fallback",
|
||||
};
|
||||
}
|
||||
|
||||
if ((this.config.trimRegions ?? []).length > 0) {
|
||||
@@ -709,7 +769,10 @@ export class VideoExporter {
|
||||
let editedAudioBuffer: ArrayBuffer | undefined;
|
||||
let editedAudioMimeType: string | null = null;
|
||||
|
||||
if (audioPlan.audioMode === "edited-track") {
|
||||
if (
|
||||
audioPlan.audioMode === "edited-track" &&
|
||||
audioPlan.strategy === "offline-render-fallback"
|
||||
) {
|
||||
this.audioProcessor = new AudioProcessor();
|
||||
this.audioProcessor.setOnProgress((progress) => {
|
||||
this.reportFinalizingProgress(totalFrames, 99, progress);
|
||||
@@ -741,11 +804,25 @@ export class VideoExporter {
|
||||
audioMode: audioPlan.audioMode,
|
||||
audioSourcePath:
|
||||
audioPlan.audioMode === "copy-source" ||
|
||||
audioPlan.audioMode === "trim-source"
|
||||
audioPlan.audioMode === "trim-source" ||
|
||||
(audioPlan.audioMode === "edited-track" &&
|
||||
audioPlan.strategy === "filtergraph-fast-path")
|
||||
? audioPlan.audioSourcePath
|
||||
: null,
|
||||
trimSegments:
|
||||
audioPlan.audioMode === "trim-source" ? audioPlan.trimSegments : undefined,
|
||||
editedTrackStrategy:
|
||||
audioPlan.audioMode === "edited-track" ? audioPlan.strategy : undefined,
|
||||
editedTrackSegments:
|
||||
audioPlan.audioMode === "edited-track" &&
|
||||
audioPlan.strategy === "filtergraph-fast-path"
|
||||
? audioPlan.editedTrackSegments
|
||||
: undefined,
|
||||
audioSourceSampleRate:
|
||||
audioPlan.audioMode === "edited-track" &&
|
||||
audioPlan.strategy === "filtergraph-fast-path"
|
||||
? audioPlan.audioSourceSampleRate
|
||||
: undefined,
|
||||
editedAudioData: editedAudioBuffer,
|
||||
editedAudioMimeType,
|
||||
}),
|
||||
@@ -790,7 +867,10 @@ export class VideoExporter {
|
||||
let editedAudioBuffer: ArrayBuffer | undefined;
|
||||
let editedAudioMimeType: string | null = null;
|
||||
|
||||
if (audioPlan.audioMode === "edited-track") {
|
||||
if (
|
||||
audioPlan.audioMode === "edited-track" &&
|
||||
audioPlan.strategy === "offline-render-fallback"
|
||||
) {
|
||||
this.audioProcessor = new AudioProcessor();
|
||||
this.audioProcessor.setOnProgress((progress) => {
|
||||
this.reportFinalizingProgress(totalFrames, 99, progress);
|
||||
@@ -820,11 +900,25 @@ export class VideoExporter {
|
||||
audioMode: audioPlan.audioMode,
|
||||
audioSourcePath:
|
||||
audioPlan.audioMode === "copy-source" ||
|
||||
audioPlan.audioMode === "trim-source"
|
||||
audioPlan.audioMode === "trim-source" ||
|
||||
(audioPlan.audioMode === "edited-track" &&
|
||||
audioPlan.strategy === "filtergraph-fast-path")
|
||||
? audioPlan.audioSourcePath
|
||||
: null,
|
||||
trimSegments:
|
||||
audioPlan.audioMode === "trim-source" ? audioPlan.trimSegments : undefined,
|
||||
editedTrackStrategy:
|
||||
audioPlan.audioMode === "edited-track" ? audioPlan.strategy : undefined,
|
||||
editedTrackSegments:
|
||||
audioPlan.audioMode === "edited-track" &&
|
||||
audioPlan.strategy === "filtergraph-fast-path"
|
||||
? audioPlan.editedTrackSegments
|
||||
: undefined,
|
||||
audioSourceSampleRate:
|
||||
audioPlan.audioMode === "edited-track" &&
|
||||
audioPlan.strategy === "filtergraph-fast-path"
|
||||
? audioPlan.audioSourceSampleRate
|
||||
: undefined,
|
||||
editedAudioData: editedAudioBuffer,
|
||||
editedAudioMimeType,
|
||||
}),
|
||||
|
||||
Reference in New Issue
Block a user