mirror of
https://github.com/webadderallorg/Recordly.git
synced 2026-09-25 23:35:43 +00:00
Merge pull request #482 from ExtraBinoss/fix/audio-wrong-layer
Fix/audio wrong layer and audio playback
This commit is contained in:
@@ -172,4 +172,34 @@ describe("local media path policy", () => {
|
||||
expect(result.path).toBe(projectPath);
|
||||
expect(result.project).toMatchObject({ videoPath });
|
||||
});
|
||||
|
||||
it("approves editor audioRegions audioPath entries when loading a project", async () => {
|
||||
const downloadsPath = path.join(tempRoot, "Downloads");
|
||||
const videoPath = path.join(tempPath, "recording.mp4");
|
||||
const audioPath = path.join(downloadsPath, "music.ogg");
|
||||
const projectPath = path.join(tempPath, "recording.recordly");
|
||||
await fs.mkdir(downloadsPath, { recursive: true });
|
||||
await fs.writeFile(videoPath, "test-video");
|
||||
await fs.writeFile(audioPath, "test-audio");
|
||||
await fs.writeFile(
|
||||
projectPath,
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
videoPath,
|
||||
editor: {
|
||||
audioRegions: [
|
||||
{ id: "a1", startMs: 0, endMs: 1000, audioPath, volume: 1 },
|
||||
],
|
||||
},
|
||||
}),
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
const { loadProjectFromPath, resolveApprovedLocalMediaPath } = await import("./manager");
|
||||
const resolvedAudioPath = await fs.realpath(audioPath);
|
||||
|
||||
const result = await loadProjectFromPath(projectPath);
|
||||
expect(result.success).toBe(true);
|
||||
await expect(resolveApprovedLocalMediaPath(audioPath)).resolves.toBe(resolvedAudioPath);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -428,6 +428,7 @@ export async function loadProjectFromPath(projectPath: string) {
|
||||
const projectObj = project as Record<string, unknown>;
|
||||
const editorObj = projectObj?.editor as Record<string, unknown> | undefined;
|
||||
const audioTracks = editorObj?.audioTracks as { sourcePath?: unknown }[] | undefined;
|
||||
const audioRegions = editorObj?.audioRegions as { audioPath?: unknown }[] | undefined;
|
||||
const approvedProjectPaths: Array<string | null | undefined> = [
|
||||
mediaSources.videoPath,
|
||||
mediaSources.webcamPath,
|
||||
@@ -439,6 +440,13 @@ export async function loadProjectFromPath(projectPath: string) {
|
||||
}
|
||||
}
|
||||
}
|
||||
if (Array.isArray(audioRegions)) {
|
||||
for (const region of audioRegions) {
|
||||
if (typeof region?.audioPath === "string") {
|
||||
approvedProjectPaths.push(region.audioPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
await replaceApprovedSessionLocalReadPaths(approvedProjectPaths);
|
||||
await rememberRecentProject(normalizedPath);
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ export function getBrowserMicSidecarFilters(profile?: string | null) {
|
||||
return BROWSER_MIC_SIDECAR_FILTERS;
|
||||
}
|
||||
|
||||
export const RECORDING_AUDIO_SIDECAR_DEBUG_ENV = "RECORDLY_KEEP_RECORDING_AUDIO_SIDECARS";
|
||||
export const RECORDING_AUDIO_SIDECAR_DEBUG_ENV = "RECORDLY_KEEP_RECORDING_AUDIO_SIDECARS"; // not used yet, because we need to have seperate audio files for system and mic for each recording
|
||||
|
||||
export function shouldKeepRecordingAudioSidecars(env: NodeJS.ProcessEnv = process.env) {
|
||||
const value = env[RECORDING_AUDIO_SIDECAR_DEBUG_ENV]?.trim().toLowerCase();
|
||||
|
||||
@@ -499,20 +499,21 @@ export async function getCompanionAudioFallbackInfo(videoPath: string) {
|
||||
|
||||
let paths: string[];
|
||||
if (await hasEmbeddedAudioStream(videoPath)) {
|
||||
const microphoneCompanionPaths = Array.from(
|
||||
const companionPaths = Array.from(
|
||||
new Set(
|
||||
companionCandidates.flatMap((candidate) =>
|
||||
candidate.usablePaths.filter(
|
||||
(companionPath) => companionPath === candidate.micPath,
|
||||
(companionPath) =>
|
||||
companionPath === candidate.micPath || companionPath === candidate.systemPath,
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
if (microphoneCompanionPaths.length === 0) {
|
||||
if (companionPaths.length === 0) {
|
||||
return { paths: [], startDelayMsByPath: {} };
|
||||
}
|
||||
|
||||
paths = [videoPath, ...microphoneCompanionPaths];
|
||||
paths = [videoPath, ...companionPaths];
|
||||
} else {
|
||||
paths = Array.from(
|
||||
new Set(companionCandidates.flatMap((candidate) => candidate.usablePaths)),
|
||||
|
||||
@@ -277,12 +277,6 @@ export async function muxNativeMacRecordingWithAudio(
|
||||
|
||||
await moveFileWithOverwrite(mixedOutputPath, videoPath);
|
||||
console.log("[mux] Successfully muxed audio into video:", videoPath);
|
||||
|
||||
for (const audioPath of [systemAudioPath, microphonePath]) {
|
||||
if (audioPath) {
|
||||
await fs.rm(audioPath, { force: true }).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function attachNativeCaptureLifecycle(process: ChildProcessWithoutNullStreams) {
|
||||
|
||||
@@ -24,7 +24,6 @@ import {
|
||||
import type { AudioSyncAdjustment } from "../types";
|
||||
import { moveFileWithOverwrite } from "../utils";
|
||||
import {
|
||||
RECORDING_AUDIO_SIDECAR_DEBUG_ENV,
|
||||
shouldKeepRecordingAudioSidecars,
|
||||
WINDOWS_NATIVE_MIC_PRE_FILTERS,
|
||||
} from "./audioFilters";
|
||||
@@ -474,30 +473,6 @@ export async function muxNativeWindowsVideoWithAudio(
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (keepAudioSidecars) {
|
||||
console.log(
|
||||
`[mux-win] Keeping native audio sidecars because ${RECORDING_AUDIO_SIDECAR_DEBUG_ENV} is enabled`,
|
||||
);
|
||||
return {
|
||||
muxed: true,
|
||||
videoDurationSeconds: videoDuration,
|
||||
muxTimeoutMs,
|
||||
audioInputs,
|
||||
audio,
|
||||
outputPath: videoPath,
|
||||
keptAudioSidecars: true,
|
||||
};
|
||||
}
|
||||
|
||||
for (const audioPath of [systemAudioPath, micAudioPath]) {
|
||||
if (audioPath) {
|
||||
await Promise.all([
|
||||
fs.rm(audioPath, { force: true }).catch(() => undefined),
|
||||
fs.rm(`${audioPath}.json`, { force: true }).catch(() => undefined),
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
muxed: true,
|
||||
videoDurationSeconds: videoDuration,
|
||||
@@ -505,6 +480,6 @@ export async function muxNativeWindowsVideoWithAudio(
|
||||
audioInputs,
|
||||
audio,
|
||||
outputPath: videoPath,
|
||||
keptAudioSidecars: false,
|
||||
keptAudioSidecars: true,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -139,8 +139,9 @@ export function AnnotationSettingsPanel({
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex-[2] min-w-0 bg-editor-panel border border-foreground/10 rounded-2xl p-4 flex flex-col shadow-xl h-full overflow-y-auto custom-scrollbar">
|
||||
<div className="mb-6">
|
||||
<div className="flex-[2] min-w-0 bg-editor-panel border border-foreground/10 rounded-2xl flex flex-col shadow-xl h-full overflow-hidden">
|
||||
<div className="flex-1 min-h-0 p-4 overflow-y-auto custom-scrollbar">
|
||||
<div className="mb-6">
|
||||
<div className="flex items-center justify-between mb-4">
|
||||
<span className="text-sm font-medium text-foreground">
|
||||
{t("annotations.settings")}
|
||||
@@ -772,16 +773,6 @@ export function AnnotationSettingsPanel({
|
||||
</TabsContent>
|
||||
</Tabs>
|
||||
|
||||
<Button
|
||||
onClick={onDelete}
|
||||
variant="destructive"
|
||||
size="sm"
|
||||
className="w-full gap-2 bg-red-500/10 text-red-400 border border-red-500/20 hover:bg-red-500/20 hover:border-red-500/30 transition-all mt-4"
|
||||
>
|
||||
<Trash2 className="w-4 h-4" />
|
||||
{t("annotations.deleteAnnotation")}
|
||||
</Button>
|
||||
|
||||
<div className="mt-6 p-3 bg-foreground/5 rounded-lg border border-foreground/5">
|
||||
<div className="flex items-center gap-2 mb-2 text-muted-foreground">
|
||||
<Info className="w-3.5 h-3.5" />
|
||||
@@ -795,6 +786,18 @@ export function AnnotationSettingsPanel({
|
||||
<li>{t("annotations.tipCycleBackward")}</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex-shrink-0 border-t border-foreground/10 bg-editor-panel p-4 pt-3">
|
||||
<Button
|
||||
onClick={onDelete}
|
||||
variant="destructive"
|
||||
size="sm"
|
||||
className="w-full gap-2 bg-red-500/10 text-red-400 border border-red-500/20 hover:bg-red-500/20 hover:border-red-500/30 transition-all"
|
||||
>
|
||||
<Trash2 className="w-4 h-4" />
|
||||
{t("annotations.deleteAnnotation")}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -469,12 +469,21 @@ interface SettingsPanelProps {
|
||||
selectedClipId?: string | null;
|
||||
selectedClipSpeed?: number | null;
|
||||
selectedClipMuted?: boolean | null;
|
||||
selectedClipShowSourceAudio?: boolean | null;
|
||||
hasClipSourceAudio?: boolean;
|
||||
onClipSpeedChange?: (speed: number) => void;
|
||||
onClipMutedChange?: (muted: boolean) => void;
|
||||
onClipShowSourceAudioChange?: (show: boolean) => void;
|
||||
sourceAudioTrackMeta?: Array<{ id: string; label: string }>;
|
||||
sourceAudioTrackSettings?: Record<string, { volume: number; normalize: boolean }>;
|
||||
onSourceAudioTrackVolumeChange?: (id: string, volume: number) => void;
|
||||
onSourceAudioTrackNormalizeChange?: (id: string, normalize: boolean) => void;
|
||||
onClipDelete?: (id: string) => void;
|
||||
selectedAudioId?: string | null;
|
||||
selectedAudioVolume?: number | null;
|
||||
selectedAudioNormalize?: boolean | null;
|
||||
onAudioVolumeChange?: (volume: number) => void;
|
||||
onAudioNormalizeChange?: (normalize: boolean) => void;
|
||||
onAudioDelete?: (id: string) => void;
|
||||
shadowIntensity?: number;
|
||||
onShadowChange?: (intensity: number) => void;
|
||||
@@ -862,12 +871,21 @@ export function SettingsPanel({
|
||||
selectedClipId,
|
||||
selectedClipSpeed,
|
||||
selectedClipMuted,
|
||||
selectedClipShowSourceAudio = false,
|
||||
hasClipSourceAudio = false,
|
||||
onClipSpeedChange,
|
||||
onClipMutedChange,
|
||||
onClipShowSourceAudioChange,
|
||||
sourceAudioTrackMeta = [],
|
||||
sourceAudioTrackSettings = {},
|
||||
onSourceAudioTrackVolumeChange,
|
||||
onSourceAudioTrackNormalizeChange,
|
||||
onClipDelete,
|
||||
selectedAudioId,
|
||||
selectedAudioVolume,
|
||||
selectedAudioNormalize,
|
||||
onAudioVolumeChange,
|
||||
onAudioNormalizeChange,
|
||||
onAudioDelete,
|
||||
shadowIntensity = 0.67,
|
||||
onShadowChange,
|
||||
@@ -3035,6 +3053,45 @@ export function SettingsPanel({
|
||||
</section>
|
||||
);
|
||||
|
||||
const audioSectionContent = (
|
||||
<section className="flex flex-col gap-3">
|
||||
<div className="flex items-center justify-between gap-3">
|
||||
<SectionLabel>{tSettings("audio.volumeTitle", "Audio")}</SectionLabel>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
onAudioVolumeChange?.(1);
|
||||
onAudioNormalizeChange?.(false);
|
||||
}}
|
||||
className="text-[10px] text-[#2563EB] transition-opacity hover:opacity-80"
|
||||
>
|
||||
{t("common.actions.reset", "Reset")}
|
||||
</button>
|
||||
</div>
|
||||
<SliderControl
|
||||
label={tSettings("audio.volume", "Volume")}
|
||||
value={selectedAudioVolume ?? 1}
|
||||
defaultValue={1}
|
||||
min={0}
|
||||
max={1}
|
||||
step={0.01}
|
||||
onChange={(v) => onAudioVolumeChange?.(v)}
|
||||
formatValue={(v) => `${Math.round(v * 100)}%`}
|
||||
parseInput={(text) => parseFloat(text.replace(/%$/, "")) / 100}
|
||||
/>
|
||||
<div className="flex items-center justify-between rounded-lg bg-foreground/[0.03] px-2.5 py-1.5">
|
||||
<span className="text-[10px] text-muted-foreground">
|
||||
{tSettings("audio.normalize", "Normalize")}
|
||||
</span>
|
||||
<Switch
|
||||
checked={Boolean(selectedAudioNormalize)}
|
||||
onCheckedChange={(v) => onAudioNormalizeChange?.(v)}
|
||||
className="data-[state=checked]:bg-[#2563EB] scale-75"
|
||||
/>
|
||||
</div>
|
||||
</section>
|
||||
);
|
||||
|
||||
const clipSectionContent = (
|
||||
<section className="flex flex-col gap-2">
|
||||
<div className="flex items-center justify-between gap-3">
|
||||
@@ -3045,16 +3102,7 @@ export function SettingsPanel({
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex items-center justify-between rounded-lg bg-foreground/[0.03] px-2.5 py-1.5">
|
||||
<span className="text-[10px] text-muted-foreground">
|
||||
{tSettings("clip.muteAudio", "Mute Audio")}
|
||||
</span>
|
||||
<Switch
|
||||
checked={selectedClipMuted ?? false}
|
||||
onCheckedChange={(v) => onClipMutedChange?.(v)}
|
||||
className="data-[state=checked]:bg-[#06b6d4] scale-75"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-3">
|
||||
<SectionLabel>{tSettings("speed.label", "Speed")}</SectionLabel>
|
||||
</div>
|
||||
@@ -3095,19 +3143,100 @@ export function SettingsPanel({
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
{selectedClipId && (
|
||||
<Button
|
||||
onClick={() => {
|
||||
if (selectedClipId && onClipDelete) onClipDelete(selectedClipId);
|
||||
}}
|
||||
variant="destructive"
|
||||
size="sm"
|
||||
className="mt-1 h-8 w-full gap-2 border border-red-500/20 bg-red-500/10 text-xs text-red-400 transition-all hover:border-red-500/30 hover:bg-red-500/20"
|
||||
>
|
||||
<Trash2 className="h-3 w-3" />
|
||||
{tSettings("clip.delete", "Delete Clip")}
|
||||
</Button>
|
||||
)}
|
||||
|
||||
<div className="mt-2 flex flex-col gap-2 border-t border-foreground/5 pt-3">
|
||||
<SectionLabel>{tSettings("audio.title", "Audio")}</SectionLabel>
|
||||
|
||||
<div className="flex items-center justify-between rounded-lg bg-foreground/[0.03] px-2.5 py-1.5">
|
||||
<div>
|
||||
<span className="text-[10px] text-muted-foreground">
|
||||
{tSettings("clip.mute", "Mute")}
|
||||
</span>
|
||||
<p className="text-[9px] text-muted-foreground/50 mt-0.5">
|
||||
{selectedClipMuted
|
||||
? tSettings("clip.mutedState", "Audio is muted")
|
||||
: tSettings("clip.unmutedState", "Audio is playing")}
|
||||
</p>
|
||||
</div>
|
||||
<Switch
|
||||
checked={selectedClipMuted ?? false}
|
||||
onCheckedChange={(v) => onClipMutedChange?.(v)}
|
||||
className="data-[state=checked]:bg-[#06b6d4] scale-75"
|
||||
/>
|
||||
</div>
|
||||
{hasClipSourceAudio && (
|
||||
<div className="flex items-center justify-between rounded-lg bg-foreground/[0.03] px-2.5 py-1.5">
|
||||
<span className="text-[10px] text-muted-foreground">
|
||||
{tSettings("clip.separateClipFromAudio", "Separate clip from audio")}
|
||||
</span>
|
||||
<Switch
|
||||
checked={selectedClipShowSourceAudio ?? false}
|
||||
onCheckedChange={(v) => onClipShowSourceAudioChange?.(v)}
|
||||
className="data-[state=checked]:bg-[#06b6d4] scale-75"
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{selectedClipId &&
|
||||
hasClipSourceAudio &&
|
||||
sourceAudioTrackMeta.length > 0 && (
|
||||
<div className="mt-1 flex flex-col gap-3">
|
||||
{sourceAudioTrackMeta.map((track) => {
|
||||
const settings = sourceAudioTrackSettings[track.id] ?? {
|
||||
volume: 1,
|
||||
normalize: false,
|
||||
};
|
||||
return (
|
||||
<div
|
||||
key={track.id}
|
||||
className="rounded-lg border border-foreground/10 bg-foreground/[0.03] px-3 py-2"
|
||||
>
|
||||
<div className="mb-2 flex items-center justify-between">
|
||||
<span className="text-[11px] font-medium text-foreground">
|
||||
{track.label}
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
onSourceAudioTrackVolumeChange?.(track.id, 1);
|
||||
onSourceAudioTrackNormalizeChange?.(track.id, false);
|
||||
}}
|
||||
className="text-[10px] text-[#2563EB] transition-opacity hover:opacity-80"
|
||||
>
|
||||
{t("common.actions.reset", "Reset")}
|
||||
</button>
|
||||
</div>
|
||||
<div className="mb-2 flex items-center justify-between rounded-lg bg-foreground/[0.03] px-2.5 py-1.5">
|
||||
<span className="text-[10px] text-muted-foreground">
|
||||
{tSettings("audio.normalize", "Normalize")}
|
||||
</span>
|
||||
<Switch
|
||||
checked={settings.normalize}
|
||||
onCheckedChange={(v) =>
|
||||
onSourceAudioTrackNormalizeChange?.(track.id, v)
|
||||
}
|
||||
className="data-[state=checked]:bg-[#06b6d4] scale-75"
|
||||
/>
|
||||
</div>
|
||||
<SliderControl
|
||||
label={tSettings("audio.volume", "Volume")}
|
||||
value={settings.volume}
|
||||
defaultValue={1}
|
||||
min={0}
|
||||
max={2}
|
||||
step={0.01}
|
||||
onChange={(v) => onSourceAudioTrackVolumeChange?.(track.id, v)}
|
||||
formatValue={(v) => `${Math.round(v * 100)}%`}
|
||||
parseInput={(text) =>
|
||||
parseFloat(text.replace(/%$/, "")) / 100
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</section>
|
||||
);
|
||||
|
||||
@@ -3120,6 +3249,8 @@ export function SettingsPanel({
|
||||
return zoomItemSectionContent;
|
||||
case "clip":
|
||||
return clipSectionContent;
|
||||
case "audio":
|
||||
return audioSectionContent;
|
||||
case "frame":
|
||||
return sceneSectionContent;
|
||||
case "crop":
|
||||
@@ -3561,41 +3692,68 @@ export function SettingsPanel({
|
||||
|
||||
<div
|
||||
className={cn(
|
||||
"flex-shrink-0 border-t border-foreground/10 bg-editor-header p-4 pt-3",
|
||||
!selectedAudioId && "hidden",
|
||||
"flex-shrink-0 border-t border-foreground/10 bg-editor-panel p-4 pt-3",
|
||||
(() => {
|
||||
if (activeEffectSection === "clip" && selectedClipId) return false;
|
||||
if (activeEffectSection === "zoom" && selectedZoomId) return false;
|
||||
if (activeEffectSection === "audio" && selectedAudioId) return false;
|
||||
if (selectedAnnotationId) return false; // Annotation editor handles its own but let's see
|
||||
return true;
|
||||
})() && "hidden",
|
||||
)}
|
||||
>
|
||||
{selectedAudioId && (
|
||||
<div>
|
||||
<div className="mb-3 flex items-center justify-between">
|
||||
<span className="text-sm font-medium text-foreground">
|
||||
{tSettings("audio.volumeTitle", "Audio Volume")}
|
||||
</span>
|
||||
<span className="rounded-full bg-[#2563EB]/10 px-2 py-0.5 text-[10px] font-medium uppercase tracking-wider text-[#2563EB]">
|
||||
{Math.round((selectedAudioVolume ?? 1) * 100)}%
|
||||
</span>
|
||||
</div>
|
||||
<SliderControl
|
||||
label={tSettings("audio.volume", "Volume")}
|
||||
value={selectedAudioVolume ?? 1}
|
||||
defaultValue={1}
|
||||
min={0}
|
||||
max={1}
|
||||
step={0.01}
|
||||
onChange={(v) => onAudioVolumeChange?.(v)}
|
||||
formatValue={(v) => `${Math.round(v * 100)}%`}
|
||||
parseInput={(text) => parseFloat(text.replace(/%$/, "")) / 100}
|
||||
/>
|
||||
<Button
|
||||
onClick={() => selectedAudioId && onAudioDelete?.(selectedAudioId)}
|
||||
variant="destructive"
|
||||
size="sm"
|
||||
className="mt-2 h-8 w-full gap-2 border border-red-500/20 bg-red-500/10 text-xs text-red-400 transition-all hover:border-red-500/30 hover:bg-red-500/20"
|
||||
>
|
||||
<Trash2 className="h-3 w-3" />
|
||||
{tSettings("audio.deleteRegion", "Delete Audio")}
|
||||
</Button>
|
||||
</div>
|
||||
{activeEffectSection === "clip" && selectedClipId && (
|
||||
<Button
|
||||
onClick={() => {
|
||||
if (selectedClipId && onClipDelete) onClipDelete(selectedClipId);
|
||||
}}
|
||||
variant="destructive"
|
||||
size="sm"
|
||||
className="h-8 w-full gap-2 border border-red-500/20 bg-red-500/10 text-xs text-red-400 transition-all hover:border-red-500/30 hover:bg-red-500/20"
|
||||
>
|
||||
<Trash2 className="h-3 w-3" />
|
||||
{tSettings("clip.delete", "Delete Clip")}
|
||||
</Button>
|
||||
)}
|
||||
{activeEffectSection === "zoom" && selectedZoomId && (
|
||||
<Button
|
||||
onClick={() => {
|
||||
if (selectedZoomId && onZoomDelete) onZoomDelete(selectedZoomId);
|
||||
}}
|
||||
variant="destructive"
|
||||
size="sm"
|
||||
className="h-8 w-full gap-2 border border-red-500/20 bg-red-500/10 text-xs text-red-400 transition-all hover:border-red-500/30 hover:bg-red-500/20"
|
||||
>
|
||||
<Trash2 className="h-3 w-3" />
|
||||
{tSettings("zoom.deleteZoom", "Delete Zoom")}
|
||||
</Button>
|
||||
)}
|
||||
{activeEffectSection === "audio" && selectedAudioId && (
|
||||
<Button
|
||||
onClick={() => {
|
||||
if (selectedAudioId && onAudioDelete) onAudioDelete(selectedAudioId);
|
||||
}}
|
||||
variant="destructive"
|
||||
size="sm"
|
||||
className="h-8 w-full gap-2 border border-red-500/20 bg-red-500/10 text-xs text-red-400 transition-all hover:border-red-500/30 hover:bg-red-500/20"
|
||||
>
|
||||
<Trash2 className="h-3 w-3" />
|
||||
{tSettings("audio.deleteRegion", "Delete Audio")}
|
||||
</Button>
|
||||
)}
|
||||
{selectedAnnotationId && (
|
||||
<Button
|
||||
onClick={() => {
|
||||
if (selectedAnnotationId && onAnnotationDelete)
|
||||
onAnnotationDelete(selectedAnnotationId);
|
||||
}}
|
||||
variant="destructive"
|
||||
size="sm"
|
||||
className="h-8 w-full gap-2 border border-red-500/20 bg-red-500/10 text-xs text-red-400 transition-all hover:border-red-500/30 hover:bg-red-500/20"
|
||||
>
|
||||
<Trash2 className="h-3 w-3" />
|
||||
{tSettings("annotation.delete", "Delete Annotation")}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -80,14 +80,6 @@ import {
|
||||
canUseInMemoryExportSaveFallback,
|
||||
describeBlockedInMemoryExportSave,
|
||||
} from "@/lib/exporter/exportSavePolicy";
|
||||
import { resolveMediaElementSource } from "@/lib/exporter/localMediaSource";
|
||||
import { resolveSourceAudioFallbackPaths } from "@/lib/exporter/sourceAudioFallback";
|
||||
import {
|
||||
clampMediaTimeToDuration,
|
||||
enablePitchPreservingPlayback,
|
||||
estimateCompanionAudioStartDelaySeconds,
|
||||
getMediaSyncPlaybackRate,
|
||||
} from "@/lib/mediaTiming";
|
||||
import { matchesShortcut } from "@/lib/shortcuts";
|
||||
import { cn } from "@/lib/utils";
|
||||
import {
|
||||
@@ -147,6 +139,7 @@ import {
|
||||
validateProjectData,
|
||||
} from "./projectPersistence";
|
||||
import { SettingsPanel } from "./SettingsPanel";
|
||||
import { useVideoEditorAudio } from "./audio/useVideoEditorAudio";
|
||||
import {
|
||||
APP_HEADER_ICON_BUTTON_CLASS,
|
||||
DiscordLinkButton,
|
||||
@@ -156,6 +149,7 @@ import {
|
||||
} from "./TutorialHelp";
|
||||
import TimelineEditor, { type TimelineEditorHandle } from "./timeline/TimelineEditor";
|
||||
import { normalizeCursorTelemetry } from "./timeline/zoomSuggestionUtils";
|
||||
import type { SourceAudioTrackSettings } from "@/components/video-editor/audio/audioTypes";
|
||||
import {
|
||||
type AnnotationRegion,
|
||||
type AudioRegion,
|
||||
@@ -347,12 +341,6 @@ async function writeSmokeExportReport(
|
||||
|
||||
const SMOKE_EXPORT_READY_TIMEOUT_MS = 30_000;
|
||||
const DEFAULT_MP4_EXPORT_FRAME_RATE: ExportMp4FrameRate = 30;
|
||||
const SOURCE_AUDIO_FALLBACK_TOAST_ID = "source-audio-fallback-error";
|
||||
const SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS = 0.18;
|
||||
const SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS = 0.01;
|
||||
const SOURCE_AUDIO_PREVIEW_RATE_TOLERANCE_SECONDS = 0.08;
|
||||
const SOURCE_AUDIO_PREVIEW_RATE_CORRECTION_WINDOW_SECONDS = 8;
|
||||
const SOURCE_AUDIO_PREVIEW_MAX_RATE_ADJUSTMENT = 0.015;
|
||||
const PROJECT_AUTOSAVE_DELAY_MS = 1000;
|
||||
const EXPORT_ERROR_TOAST_DURATION_MS = 20000;
|
||||
|
||||
@@ -677,6 +665,13 @@ export default function VideoEditor() {
|
||||
const [selectedAnnotationId, setSelectedAnnotationId] = useState<string | null>(null);
|
||||
const [audioRegions, setAudioRegions] = useState<AudioRegion[]>([]);
|
||||
const [selectedAudioId, setSelectedAudioId] = useState<string | null>(null);
|
||||
const [sourceAudioTrackSettingsByClip, setSourceAudioTrackSettingsByClip] = useState<
|
||||
Record<string, SourceAudioTrackSettings>
|
||||
>({});
|
||||
const [defaultSourceAudioTrackSettings, setDefaultSourceAudioTrackSettings] = useState<
|
||||
SourceAudioTrackSettings
|
||||
>({});
|
||||
const [hasClipSourceAudio, setHasClipSourceAudio] = useState(false);
|
||||
const [autoCaptions, setAutoCaptions] = useState<CaptionCue[]>([]);
|
||||
const [autoCaptionSettings, setAutoCaptionSettings] = useState<AutoCaptionSettings>(
|
||||
DEFAULT_AUTO_CAPTION_SETTINGS,
|
||||
@@ -700,9 +695,6 @@ export default function VideoEditor() {
|
||||
const [exportError, setExportError] = useState<string | null>(null);
|
||||
const [showExportDropdown, setShowExportDropdown] = useState(false);
|
||||
const [previewVolume, setPreviewVolume] = useState(1);
|
||||
const [sourceAudioFallbackPaths, setSourceAudioFallbackPaths] = useState<string[]>([]);
|
||||
const [sourceAudioFallbackStartDelayMsByPath, setSourceAudioFallbackStartDelayMsByPath] =
|
||||
useState<Record<string, number>>({});
|
||||
const applySessionPresentation = useCallback(
|
||||
(
|
||||
session:
|
||||
@@ -1767,6 +1759,8 @@ export default function VideoEditor() {
|
||||
gifFrameRate: GifFrameRate;
|
||||
gifLoop: boolean;
|
||||
gifSizePreset: GifSizePreset;
|
||||
sourceAudioTrackSettingsByClip: Record<string, SourceAudioTrackSettings>;
|
||||
defaultSourceAudioTrackSettings: SourceAudioTrackSettings;
|
||||
}>,
|
||||
) => {
|
||||
return editor;
|
||||
@@ -1778,63 +1772,6 @@ export default function VideoEditor() {
|
||||
() => videoSourcePath ?? (videoPath ? fromFileUrl(videoPath) : null),
|
||||
[videoPath, videoSourcePath],
|
||||
);
|
||||
const { hasEmbeddedSourceAudio, externalAudioPaths: previewSourceAudioFallbackPaths } = useMemo(
|
||||
() => resolveSourceAudioFallbackPaths(currentSourcePath, sourceAudioFallbackPaths),
|
||||
[currentSourcePath, sourceAudioFallbackPaths],
|
||||
);
|
||||
const shouldMutePreviewVideo =
|
||||
!hasEmbeddedSourceAudio && previewSourceAudioFallbackPaths.length > 0;
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
setSourceAudioFallbackPaths([]);
|
||||
setSourceAudioFallbackStartDelayMsByPath({});
|
||||
|
||||
if (!currentSourcePath) {
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}
|
||||
|
||||
void (async () => {
|
||||
try {
|
||||
const result =
|
||||
await window.electronAPI.getVideoAudioFallbackPaths(currentSourcePath);
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
if (!result.success) {
|
||||
setSourceAudioFallbackPaths([]);
|
||||
setSourceAudioFallbackStartDelayMsByPath({});
|
||||
toast.warning(
|
||||
result.error
|
||||
? `Could not load companion audio sources: ${summarizeErrorMessage(result.error)}`
|
||||
: "Could not load companion audio sources. Playback and export may miss microphone audio.",
|
||||
{ id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 },
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
toast.dismiss(SOURCE_AUDIO_FALLBACK_TOAST_ID);
|
||||
setSourceAudioFallbackPaths(result.paths ?? []);
|
||||
setSourceAudioFallbackStartDelayMsByPath(result.startDelayMsByPath ?? {});
|
||||
} catch (error) {
|
||||
if (!cancelled) {
|
||||
setSourceAudioFallbackPaths([]);
|
||||
setSourceAudioFallbackStartDelayMsByPath({});
|
||||
toast.warning(
|
||||
`Could not load companion audio sources: ${summarizeErrorMessage(String(error))}`,
|
||||
{ id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 },
|
||||
);
|
||||
}
|
||||
}
|
||||
})();
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [currentSourcePath]);
|
||||
|
||||
const projectDisplayName = useMemo(() => {
|
||||
const fileName =
|
||||
currentProjectPath?.split(/[\\/]/).pop() ??
|
||||
@@ -1925,6 +1862,8 @@ export default function VideoEditor() {
|
||||
gifFrameRate,
|
||||
gifLoop,
|
||||
gifSizePreset,
|
||||
sourceAudioTrackSettingsByClip,
|
||||
defaultSourceAudioTrackSettings,
|
||||
}),
|
||||
[
|
||||
buildPersistedEditorState,
|
||||
@@ -1985,6 +1924,8 @@ export default function VideoEditor() {
|
||||
gifLoop,
|
||||
gifSizePreset,
|
||||
frame,
|
||||
sourceAudioTrackSettingsByClip,
|
||||
defaultSourceAudioTrackSettings,
|
||||
],
|
||||
);
|
||||
|
||||
@@ -2170,6 +2111,10 @@ export default function VideoEditor() {
|
||||
setSpeedRegions(normalizedEditor.speedRegions);
|
||||
setAnnotationRegions(normalizedEditor.annotationRegions);
|
||||
setAudioRegions(normalizedEditor.audioRegions);
|
||||
setSourceAudioTrackSettingsByClip(normalizedEditor.sourceAudioTrackSettingsByClip ?? {});
|
||||
setDefaultSourceAudioTrackSettings(
|
||||
normalizedEditor.defaultSourceAudioTrackSettings ?? {},
|
||||
);
|
||||
setAutoCaptions(normalizedEditor.autoCaptions);
|
||||
setAutoCaptionSettings(normalizedEditor.autoCaptionSettings);
|
||||
setAspectRatio(normalizedEditor.aspectRatio);
|
||||
@@ -3380,6 +3325,29 @@ export default function VideoEditor() {
|
||||
}
|
||||
return result;
|
||||
}, [clipRegions, speedRegions]);
|
||||
const audio = useVideoEditorAudio({
|
||||
currentSourcePath,
|
||||
selectedClipId,
|
||||
clipRegions,
|
||||
audioRegions,
|
||||
effectiveSpeedRegions,
|
||||
sourceAudioTrackSettingsByClip,
|
||||
setSourceAudioTrackSettingsByClip,
|
||||
defaultSourceAudioTrackSettings,
|
||||
setDefaultSourceAudioTrackSettings,
|
||||
currentTime,
|
||||
timelineTime: timelinePlayheadTime,
|
||||
duration,
|
||||
isPlaying,
|
||||
previewVolume,
|
||||
summarizeErrorMessage,
|
||||
onSourceFallbackLoadError: (error) => {
|
||||
toast.warning(
|
||||
`Could not load companion audio source: ${summarizeErrorMessage(getErrorMessage(error))}`,
|
||||
{ duration: 10000 },
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
function togglePlayPause() {
|
||||
const playback = videoPlaybackRef.current;
|
||||
@@ -3761,6 +3729,18 @@ export default function VideoEditor() {
|
||||
[selectedClipId],
|
||||
);
|
||||
|
||||
const handleClipShowSourceAudioChange = useCallback(
|
||||
(showSourceAudio: boolean) => {
|
||||
if (!selectedClipId) return;
|
||||
setClipRegions((prev) =>
|
||||
prev.map((clip) =>
|
||||
clip.id === selectedClipId ? { ...clip, showSourceAudio } : clip,
|
||||
),
|
||||
);
|
||||
},
|
||||
[selectedClipId],
|
||||
);
|
||||
|
||||
const handleClipDelete = useCallback(
|
||||
(id: string) => {
|
||||
const deletedClip = clipRegions.find((clip) => clip.id === id);
|
||||
@@ -3792,23 +3772,26 @@ export default function VideoEditor() {
|
||||
if (id) {
|
||||
setSelectedZoomId(null);
|
||||
setSelectedAnnotationId(null);
|
||||
setActiveEffectSection("audio");
|
||||
}
|
||||
}, []);
|
||||
|
||||
const handleAudioAdded = useCallback((span: Span, audioPath: string, trackIndex?: number) => {
|
||||
const id = `audio-${nextAudioIdRef.current++}`;
|
||||
const newRegion: AudioRegion = {
|
||||
id,
|
||||
startMs: Math.round(span.start),
|
||||
endMs: Math.round(span.end),
|
||||
audioPath,
|
||||
volume: 1,
|
||||
trackIndex,
|
||||
};
|
||||
const handleAudioAdded = useCallback((span: Span, audioPath: string, trackIndex?: number) => {
|
||||
const id = `audio-${nextAudioIdRef.current++}`;
|
||||
const newRegion: AudioRegion = {
|
||||
id,
|
||||
startMs: Math.round(span.start),
|
||||
endMs: Math.round(span.end),
|
||||
audioPath,
|
||||
volume: 1,
|
||||
normalize: false,
|
||||
trackIndex,
|
||||
};
|
||||
setAudioRegions((prev) => [...prev, newRegion]);
|
||||
setSelectedAudioId(id);
|
||||
setSelectedZoomId(null);
|
||||
setSelectedAnnotationId(null);
|
||||
setActiveEffectSection("audio");
|
||||
}, []);
|
||||
|
||||
const handleAudioSpanChange = useCallback((id: string, span: Span, trackIndex?: number) => {
|
||||
@@ -3853,15 +3836,29 @@ export default function VideoEditor() {
|
||||
[selectedAudioId],
|
||||
);
|
||||
|
||||
const handleAudioDelete = useCallback(
|
||||
(id: string) => {
|
||||
const handleAudioDelete = useCallback(
|
||||
(id: string) => {
|
||||
setAudioRegions((prev) => prev.filter((region) => region.id !== id));
|
||||
if (selectedAudioId === id) {
|
||||
setSelectedAudioId(null);
|
||||
}
|
||||
},
|
||||
[selectedAudioId],
|
||||
);
|
||||
[selectedAudioId],
|
||||
);
|
||||
|
||||
const handleAudioNormalizeChange = useCallback(
|
||||
(normalize: boolean) => {
|
||||
if (!selectedAudioId) {
|
||||
return;
|
||||
}
|
||||
setAudioRegions((prev) =>
|
||||
prev.map((region) =>
|
||||
region.id === selectedAudioId ? { ...region, normalize } : region,
|
||||
),
|
||||
);
|
||||
},
|
||||
[selectedAudioId],
|
||||
);
|
||||
|
||||
const handleAnnotationAdded = useCallback((span: Span, trackIndex = 0) => {
|
||||
const id = `annotation-${nextAnnotationIdRef.current++}`;
|
||||
@@ -4096,290 +4093,6 @@ export default function VideoEditor() {
|
||||
}
|
||||
}, [selectedAudioId, audioRegions]);
|
||||
|
||||
// Audio playback sync: manage Audio elements that play in sync with video
|
||||
const audioElementsRef = useRef<Map<string, HTMLAudioElement>>(new Map());
|
||||
const audioElementRevokersRef = useRef<Map<string, () => void>>(new Map());
|
||||
const audioElementResourcesRef = useRef<Map<string, string>>(new Map());
|
||||
const sourceAudioElementsRef = useRef<Map<string, HTMLAudioElement>>(new Map());
|
||||
const sourceAudioElementRevokersRef = useRef<Map<string, () => void>>(new Map());
|
||||
const sourceAudioElementResourcesRef = useRef<Map<string, string>>(new Map());
|
||||
const lastSourceAudioSyncTimeRef = useRef<number | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
const existing = audioElementsRef.current;
|
||||
const currentIds = new Set(audioRegions.map((r) => r.id));
|
||||
|
||||
// Remove old audio elements
|
||||
for (const [id, audio] of existing) {
|
||||
if (!currentIds.has(id)) {
|
||||
audio.pause();
|
||||
audio.src = "";
|
||||
audioElementRevokersRef.current.get(id)?.();
|
||||
audioElementRevokersRef.current.delete(id);
|
||||
audioElementResourcesRef.current.delete(id);
|
||||
existing.delete(id);
|
||||
}
|
||||
}
|
||||
|
||||
// Create/update audio elements
|
||||
for (const region of audioRegions) {
|
||||
let audio = existing.get(region.id);
|
||||
if (!audio) {
|
||||
audio = new Audio();
|
||||
audio.preload = "auto";
|
||||
existing.set(region.id, audio);
|
||||
}
|
||||
|
||||
if (audioElementResourcesRef.current.get(region.id) !== region.audioPath) {
|
||||
audio.pause();
|
||||
audio.src = "";
|
||||
audioElementRevokersRef.current.get(region.id)?.();
|
||||
audioElementRevokersRef.current.delete(region.id);
|
||||
audioElementResourcesRef.current.set(region.id, region.audioPath);
|
||||
|
||||
void (async () => {
|
||||
const resolved = await resolveMediaElementSource(region.audioPath);
|
||||
const latestAudio = existing.get(region.id);
|
||||
|
||||
if (
|
||||
cancelled ||
|
||||
latestAudio !== audio ||
|
||||
audioElementResourcesRef.current.get(region.id) !== region.audioPath
|
||||
) {
|
||||
resolved.revoke();
|
||||
return;
|
||||
}
|
||||
|
||||
audioElementRevokersRef.current.set(region.id, resolved.revoke);
|
||||
latestAudio.src = resolved.src;
|
||||
})();
|
||||
}
|
||||
|
||||
audio.volume = Math.max(0, Math.min(1, region.volume * previewVolume));
|
||||
}
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [audioRegions, previewVolume]);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
const existing = sourceAudioElementsRef.current;
|
||||
const currentIds = new Set(previewSourceAudioFallbackPaths);
|
||||
|
||||
for (const [id, audio] of existing) {
|
||||
if (!currentIds.has(id)) {
|
||||
audio.pause();
|
||||
audio.src = "";
|
||||
sourceAudioElementRevokersRef.current.get(id)?.();
|
||||
sourceAudioElementRevokersRef.current.delete(id);
|
||||
sourceAudioElementResourcesRef.current.delete(id);
|
||||
existing.delete(id);
|
||||
}
|
||||
}
|
||||
|
||||
for (const audioPath of previewSourceAudioFallbackPaths) {
|
||||
let audio = existing.get(audioPath);
|
||||
if (!audio) {
|
||||
audio = new Audio();
|
||||
audio.preload = "auto";
|
||||
existing.set(audioPath, audio);
|
||||
}
|
||||
audio.dataset.sourceAudioPath = audioPath;
|
||||
|
||||
if (sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath) {
|
||||
audio.pause();
|
||||
audio.src = "";
|
||||
sourceAudioElementRevokersRef.current.get(audioPath)?.();
|
||||
sourceAudioElementRevokersRef.current.delete(audioPath);
|
||||
sourceAudioElementResourcesRef.current.set(audioPath, audioPath);
|
||||
|
||||
void (async () => {
|
||||
try {
|
||||
const resolved = await resolveMediaElementSource(audioPath);
|
||||
const latestAudio = existing.get(audioPath);
|
||||
|
||||
if (
|
||||
cancelled ||
|
||||
latestAudio !== audio ||
|
||||
sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath
|
||||
) {
|
||||
resolved.revoke();
|
||||
return;
|
||||
}
|
||||
|
||||
sourceAudioElementRevokersRef.current.set(audioPath, resolved.revoke);
|
||||
latestAudio.src = resolved.src;
|
||||
} catch (error) {
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
sourceAudioElementRevokersRef.current.get(audioPath)?.();
|
||||
sourceAudioElementRevokersRef.current.delete(audioPath);
|
||||
sourceAudioElementResourcesRef.current.delete(audioPath);
|
||||
const latestAudio = existing.get(audioPath);
|
||||
if (latestAudio === audio) {
|
||||
latestAudio.pause();
|
||||
latestAudio.src = "";
|
||||
}
|
||||
toast.warning(
|
||||
`Could not load companion audio source: ${summarizeErrorMessage(getErrorMessage(error))}`,
|
||||
{ id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 },
|
||||
);
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
audio.volume = Math.max(0, Math.min(1, previewVolume));
|
||||
}
|
||||
|
||||
if (previewSourceAudioFallbackPaths.length === 0) {
|
||||
lastSourceAudioSyncTimeRef.current = null;
|
||||
}
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [previewSourceAudioFallbackPaths, previewVolume]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
for (const audio of audioElementsRef.current.values()) {
|
||||
audio.pause();
|
||||
audio.src = "";
|
||||
}
|
||||
for (const revoke of audioElementRevokersRef.current.values()) {
|
||||
revoke();
|
||||
}
|
||||
audioElementsRef.current.clear();
|
||||
audioElementRevokersRef.current.clear();
|
||||
audioElementResourcesRef.current.clear();
|
||||
for (const audio of sourceAudioElementsRef.current.values()) {
|
||||
audio.pause();
|
||||
audio.src = "";
|
||||
}
|
||||
for (const revoke of sourceAudioElementRevokersRef.current.values()) {
|
||||
revoke();
|
||||
}
|
||||
sourceAudioElementsRef.current.clear();
|
||||
sourceAudioElementRevokersRef.current.clear();
|
||||
sourceAudioElementResourcesRef.current.clear();
|
||||
lastSourceAudioSyncTimeRef.current = null;
|
||||
};
|
||||
}, []);
|
||||
|
||||
// Sync audio playback with video currentTime and isPlaying state
|
||||
useEffect(() => {
|
||||
const currentTimeMs = currentTime * 1000;
|
||||
const activeSpeedRegion = effectiveSpeedRegions.find(
|
||||
(region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs,
|
||||
);
|
||||
const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1;
|
||||
|
||||
for (const region of audioRegions) {
|
||||
const audio = audioElementsRef.current.get(region.id);
|
||||
if (!audio) continue;
|
||||
|
||||
const isInRegion = currentTimeMs >= region.startMs && currentTimeMs < region.endMs;
|
||||
|
||||
if (isPlaying && isInRegion) {
|
||||
enablePitchPreservingPlayback(audio);
|
||||
const audioOffset = (currentTimeMs - region.startMs) / 1000;
|
||||
// Only seek if significantly out of sync (> 200ms)
|
||||
if (Math.abs(audio.currentTime - audioOffset) > 0.2) {
|
||||
audio.currentTime = audioOffset;
|
||||
}
|
||||
const syncedPlaybackRate = getMediaSyncPlaybackRate({
|
||||
basePlaybackRate: targetPlaybackRate,
|
||||
currentTime: audio.currentTime,
|
||||
targetTime: audioOffset,
|
||||
});
|
||||
if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) {
|
||||
audio.playbackRate = syncedPlaybackRate;
|
||||
}
|
||||
if (audio.paused) {
|
||||
audio.play().catch(() => undefined);
|
||||
}
|
||||
} else {
|
||||
if (!audio.paused) {
|
||||
audio.pause();
|
||||
}
|
||||
}
|
||||
}
|
||||
}, [isPlaying, currentTime, audioRegions, effectiveSpeedRegions]);
|
||||
|
||||
useEffect(() => {
|
||||
if (previewSourceAudioFallbackPaths.length === 0) {
|
||||
lastSourceAudioSyncTimeRef.current = null;
|
||||
return;
|
||||
}
|
||||
|
||||
const activeSpeedRegion = effectiveSpeedRegions.find(
|
||||
(region) => currentTime * 1000 >= region.startMs && currentTime * 1000 < region.endMs,
|
||||
);
|
||||
const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1;
|
||||
const previousTimelineTime = lastSourceAudioSyncTimeRef.current;
|
||||
const timelineJumped =
|
||||
previousTimelineTime === null || Math.abs(currentTime - previousTimelineTime) > 0.25;
|
||||
const driftThreshold = isPlaying
|
||||
? SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS
|
||||
: SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS;
|
||||
|
||||
for (const audio of sourceAudioElementsRef.current.values()) {
|
||||
enablePitchPreservingPlayback(audio);
|
||||
const audioDuration = Number.isFinite(audio.duration) ? audio.duration : null;
|
||||
const startDelaySeconds = estimateCompanionAudioStartDelaySeconds(
|
||||
duration,
|
||||
audioDuration,
|
||||
sourceAudioFallbackStartDelayMsByPath[audio.dataset.sourceAudioPath ?? ""],
|
||||
);
|
||||
const beforeAudioStart = currentTime + 0.001 < startDelaySeconds;
|
||||
const targetTime = clampMediaTimeToDuration(
|
||||
currentTime - startDelaySeconds,
|
||||
audioDuration,
|
||||
);
|
||||
|
||||
if (timelineJumped || Math.abs(audio.currentTime - targetTime) > driftThreshold) {
|
||||
try {
|
||||
audio.currentTime = targetTime;
|
||||
} catch {
|
||||
// no-op
|
||||
}
|
||||
}
|
||||
|
||||
const syncedPlaybackRate = getMediaSyncPlaybackRate({
|
||||
basePlaybackRate: targetPlaybackRate,
|
||||
currentTime: audio.currentTime,
|
||||
targetTime,
|
||||
toleranceSeconds: SOURCE_AUDIO_PREVIEW_RATE_TOLERANCE_SECONDS,
|
||||
correctionWindowSeconds: SOURCE_AUDIO_PREVIEW_RATE_CORRECTION_WINDOW_SECONDS,
|
||||
maxAdjustment: SOURCE_AUDIO_PREVIEW_MAX_RATE_ADJUSTMENT,
|
||||
});
|
||||
if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) {
|
||||
audio.playbackRate = syncedPlaybackRate;
|
||||
}
|
||||
|
||||
const atEnd = audioDuration !== null && targetTime >= audioDuration;
|
||||
if (isPlaying && !beforeAudioStart && !atEnd) {
|
||||
audio.play().catch(() => undefined);
|
||||
} else if (!audio.paused) {
|
||||
audio.pause();
|
||||
}
|
||||
}
|
||||
|
||||
lastSourceAudioSyncTimeRef.current = currentTime;
|
||||
}, [
|
||||
currentTime,
|
||||
duration,
|
||||
isPlaying,
|
||||
previewSourceAudioFallbackPaths,
|
||||
sourceAudioFallbackStartDelayMsByPath,
|
||||
effectiveSpeedRegions,
|
||||
]);
|
||||
|
||||
const showExportSuccessToast = useCallback((filePath: string) => {
|
||||
toast.success(`Exported successfully to ${filePath}`, {
|
||||
action: {
|
||||
@@ -4639,6 +4352,10 @@ export default function VideoEditor() {
|
||||
encodingMode,
|
||||
useModernNativeStaticLayout: useExperimentalNativeExport,
|
||||
});
|
||||
const sourceAudioTrackSettingsForExport =
|
||||
selectedClipId !== null
|
||||
? audio.selectedClipSourceAudioTrackSettings
|
||||
: audio.activeSourceAudioTrackSettings;
|
||||
|
||||
const exporterConfig = {
|
||||
videoUrl: videoPath,
|
||||
@@ -4704,8 +4421,11 @@ export default function VideoEditor() {
|
||||
cursorSway,
|
||||
frame,
|
||||
audioRegions,
|
||||
sourceAudioFallbackPaths,
|
||||
sourceAudioFallbackStartDelayMsByPath,
|
||||
clipRegions,
|
||||
sourceAudioFallbackPaths: audio.sourceAudioFallbackPaths,
|
||||
sourceAudioFallbackStartDelayMsByPath:
|
||||
audio.sourceAudioFallbackStartDelayMsByPath,
|
||||
sourceAudioTrackSettings: sourceAudioTrackSettingsForExport,
|
||||
previewWidth,
|
||||
previewHeight,
|
||||
onProgress: (progress: ExportProgress) => {
|
||||
@@ -4952,8 +4672,10 @@ export default function VideoEditor() {
|
||||
cursorClickBounceDuration,
|
||||
cursorSway,
|
||||
audioRegions,
|
||||
sourceAudioFallbackPaths,
|
||||
sourceAudioFallbackStartDelayMsByPath,
|
||||
audio.sourceAudioFallbackPaths,
|
||||
audio.sourceAudioFallbackStartDelayMsByPath,
|
||||
audio.activeSourceAudioTrackSettings,
|
||||
audio.selectedClipSourceAudioTrackSettings,
|
||||
exportEncodingMode,
|
||||
exportBackendPreference,
|
||||
exportPipelineModel,
|
||||
@@ -4985,6 +4707,7 @@ export default function VideoEditor() {
|
||||
smokeExportConfig.shadowIntensity,
|
||||
effectiveSpeedRegions,
|
||||
frame,
|
||||
selectedClipId,
|
||||
smokeExportConfig.encodingMode,
|
||||
smokeExportConfig.fps,
|
||||
smokeExportConfig.quality,
|
||||
@@ -6032,32 +5755,50 @@ export default function VideoEditor() {
|
||||
selectedClipId={selectedClipId}
|
||||
selectedClipSpeed={
|
||||
selectedClipId
|
||||
? (clipRegions.find((c) => c.id === selectedClipId)
|
||||
?.speed ?? 1)
|
||||
? clipRegions.find((c) => c.id === selectedClipId)?.speed ?? 1
|
||||
: null
|
||||
}
|
||||
selectedClipMuted={
|
||||
selectedClipId
|
||||
? (clipRegions.find((c) => c.id === selectedClipId)
|
||||
?.muted ?? false)
|
||||
? clipRegions.find((c) => c.id === selectedClipId)?.muted ??
|
||||
false
|
||||
: null
|
||||
}
|
||||
onClipSpeedChange={(speed) =>
|
||||
selectedClipId && handleClipSpeedChange(speed)
|
||||
}
|
||||
onClipMutedChange={(muted) =>
|
||||
selectedClipId && handleClipMutedChange(muted)
|
||||
selectedClipShowSourceAudio={
|
||||
selectedClipId
|
||||
? clipRegions.find((c) => c.id === selectedClipId)
|
||||
?.showSourceAudio ?? false
|
||||
: null
|
||||
}
|
||||
onClipSpeedChange={handleClipSpeedChange}
|
||||
onClipMutedChange={handleClipMutedChange}
|
||||
onClipShowSourceAudioChange={handleClipShowSourceAudioChange}
|
||||
onClipDelete={handleClipDelete}
|
||||
selectedAudioId={selectedAudioId}
|
||||
selectedAudioVolume={
|
||||
selectedAudioId
|
||||
? (audioRegions.find((r) => r.id === selectedAudioId)
|
||||
?.volume ?? null)
|
||||
: null
|
||||
hasClipSourceAudio={hasClipSourceAudio}
|
||||
sourceAudioTrackMeta={audio.sourceAudioTrackMeta}
|
||||
sourceAudioTrackSettings={audio.selectedClipSourceAudioTrackSettings}
|
||||
onSourceAudioTrackVolumeChange={
|
||||
audio.onSelectedClipSourceAudioTrackVolumeChange
|
||||
}
|
||||
onAudioVolumeChange={handleAudioVolumeChange}
|
||||
onAudioDelete={handleAudioDelete}
|
||||
onSourceAudioTrackNormalizeChange={
|
||||
audio.onSelectedClipSourceAudioTrackNormalizeChange
|
||||
}
|
||||
selectedAudioId={selectedAudioId}
|
||||
selectedAudioVolume={
|
||||
selectedAudioId
|
||||
? (audioRegions.find((r) => r.id === selectedAudioId)
|
||||
?.volume ?? null)
|
||||
: null
|
||||
}
|
||||
selectedAudioNormalize={
|
||||
selectedAudioId
|
||||
? (audioRegions.find((r) => r.id === selectedAudioId)
|
||||
?.normalize ?? false)
|
||||
: null
|
||||
}
|
||||
onAudioVolumeChange={handleAudioVolumeChange}
|
||||
onAudioNormalizeChange={handleAudioNormalizeChange}
|
||||
onAudioDelete={handleAudioDelete}
|
||||
shadowIntensity={shadowIntensity}
|
||||
onShadowChange={setShadowIntensity}
|
||||
backgroundBlur={backgroundBlur}
|
||||
@@ -6353,7 +6094,17 @@ export default function VideoEditor() {
|
||||
cursorClickBounceDuration
|
||||
}
|
||||
cursorSway={cursorSway}
|
||||
volume={shouldMutePreviewVideo ? 0 : previewVolume}
|
||||
volume={
|
||||
audio.shouldMutePreviewVideo || audio.isCurrentClipMuted
|
||||
? 0
|
||||
: Math.max(
|
||||
0,
|
||||
Math.min(
|
||||
1,
|
||||
previewVolume * audio.embeddedSourcePreviewGain,
|
||||
),
|
||||
)
|
||||
}
|
||||
suspendRendering={shouldSuspendPreviewRendering}
|
||||
/>
|
||||
</div>
|
||||
@@ -6628,6 +6379,17 @@ export default function VideoEditor() {
|
||||
selectedAnnotationId={selectedAnnotationId}
|
||||
onSelectAnnotation={handleSelectAnnotation}
|
||||
aspectRatio={aspectRatio}
|
||||
showSourceAudioTrack={clipRegions.some((c) => c.showSourceAudio)}
|
||||
sourceAudioTrackSettings={audio.activeSourceAudioTrackSettings}
|
||||
getSourceAudioTrackSettingsForClip={
|
||||
audio.getSourceAudioTrackSettingsForClip
|
||||
}
|
||||
onSourceAudioAvailabilityChange={(available) => {
|
||||
setHasClipSourceAudio(available);
|
||||
}}
|
||||
onSourceAudioTracksMetaChange={(tracks) => {
|
||||
audio.onSourceAudioTracksMetaChange(tracks);
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
import type { AudioPeaksData } from "../timeline/core/timelineTypes";
|
||||
|
||||
export type SourceAudioTrackId = "mixed" | "system" | "mic" | (string & {});
|
||||
|
||||
export interface SourceAudioTrackSetting {
|
||||
volume: number;
|
||||
normalize: boolean;
|
||||
}
|
||||
|
||||
export type SourceAudioTrackSettings = Record<string, SourceAudioTrackSetting>;
|
||||
|
||||
export interface SourceAudioTrackMetaItem {
|
||||
id: SourceAudioTrackId;
|
||||
label: string;
|
||||
}
|
||||
|
||||
export type SourceAudioTrackMeta = SourceAudioTrackMetaItem[];
|
||||
|
||||
export interface SourceAudioTrackWithPeaks extends SourceAudioTrackMetaItem {
|
||||
peaks: AudioPeaksData;
|
||||
}
|
||||
|
||||
export const SOURCE_AUDIO_FALLBACK_TOAST_ID = "source-audio-fallback-error";
|
||||
export const SOURCE_AUDIO_NORMALIZE_GAIN = 1.35;
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import { getClipSourceEndMs, sortClipRegions } from "../types";
|
||||
import type { ClipRegion } from "../types";
|
||||
|
||||
export function getActiveClipIdAtSourceTime(
|
||||
sourceTimeSeconds: number,
|
||||
clipRegions: ClipRegion[],
|
||||
): string | null {
|
||||
const sourceMs = Math.round(sourceTimeSeconds * 1000);
|
||||
const activeClip = sortClipRegions(clipRegions).find(
|
||||
(clip) => sourceMs >= clip.startMs && sourceMs < getClipSourceEndMs(clip),
|
||||
);
|
||||
return activeClip?.id ?? null;
|
||||
}
|
||||
|
||||
export function isClipMutedById(clipId: string | null, clipRegions: ClipRegion[]): boolean {
|
||||
if (!clipId) return false;
|
||||
return clipRegions.find((clip) => clip.id === clipId)?.muted ?? false;
|
||||
}
|
||||
@@ -0,0 +1,459 @@
|
||||
import { useEffect, useMemo, useRef } from "react";
|
||||
import { buildResolvedAudioPlan } from "@/lib/exporter/audioRoutingEngine";
|
||||
import { resolveMediaElementSource } from "@/lib/exporter/localMediaSource";
|
||||
import {
|
||||
clampMediaTimeToDuration,
|
||||
enablePitchPreservingPlayback,
|
||||
estimateCompanionAudioStartDelaySeconds,
|
||||
getMediaSyncPlaybackRate,
|
||||
} from "@/lib/mediaTiming";
|
||||
import type { AudioRegion, SpeedRegion } from "../types";
|
||||
|
||||
const SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS = 0.18;
|
||||
const SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS = 0.01;
|
||||
|
||||
interface UseAudioPreviewSyncParams {
|
||||
audioRegions: AudioRegion[];
|
||||
previewVolume: number;
|
||||
isPlaying: boolean;
|
||||
currentTime: number;
|
||||
timelineTime: number;
|
||||
duration: number;
|
||||
effectiveSpeedRegions: SpeedRegion[];
|
||||
previewSourceAudioFallbackPaths: string[];
|
||||
sourceAudioFallbackStartDelayMsByPath: Record<string, number>;
|
||||
isCurrentClipMuted: boolean;
|
||||
getSourceTrackPreviewGain: (audioPath: string) => number;
|
||||
onSourceFallbackLoadError: (error: unknown) => void;
|
||||
}
|
||||
|
||||
export function useAudioPreviewSync({
|
||||
audioRegions,
|
||||
previewVolume,
|
||||
isPlaying,
|
||||
currentTime,
|
||||
timelineTime,
|
||||
duration,
|
||||
effectiveSpeedRegions,
|
||||
previewSourceAudioFallbackPaths,
|
||||
sourceAudioFallbackStartDelayMsByPath,
|
||||
isCurrentClipMuted,
|
||||
getSourceTrackPreviewGain,
|
||||
onSourceFallbackLoadError,
|
||||
}: UseAudioPreviewSyncParams) {
|
||||
const resolvedPlan = useMemo(
|
||||
() =>
|
||||
buildResolvedAudioPlan({
|
||||
videoResource: null,
|
||||
sourceAudioFallbackPaths: previewSourceAudioFallbackPaths,
|
||||
audioRegions,
|
||||
}),
|
||||
[audioRegions, previewSourceAudioFallbackPaths],
|
||||
);
|
||||
const resolvedUserTracks = useMemo(
|
||||
() => resolvedPlan.tracks.filter((track) => track.kind === "user"),
|
||||
[resolvedPlan],
|
||||
);
|
||||
const resolvedSourceTracks = useMemo(
|
||||
() => resolvedPlan.tracks.filter((track) => track.kind !== "user"),
|
||||
[resolvedPlan],
|
||||
);
|
||||
|
||||
const audioElementsRef = useRef<Map<string, HTMLAudioElement>>(new Map());
|
||||
const audioElementRevokersRef = useRef<Map<string, () => void>>(new Map());
|
||||
const audioElementResourcesRef = useRef<Map<string, string>>(new Map());
|
||||
const sourceAudioElementsRef = useRef<Map<string, HTMLAudioElement>>(new Map());
|
||||
const sourceAudioMediaNodesRef = useRef<Map<string, MediaElementAudioSourceNode>>(new Map());
|
||||
const sourceAudioGainNodesRef = useRef<Map<string, GainNode>>(new Map());
|
||||
const sourceAudioElementRevokersRef = useRef<Map<string, () => void>>(new Map());
|
||||
const sourceAudioElementResourcesRef = useRef<Map<string, string>>(new Map());
|
||||
const sourceAudioContextRef = useRef<AudioContext | null>(null);
|
||||
const sourceAudioMasterGainRef = useRef<GainNode | null>(null);
|
||||
const sourceAudioResumePromiseRef = useRef<Promise<void> | null>(null);
|
||||
const lastSourceAudioSyncTimeRef = useRef<number | null>(null);
|
||||
|
||||
const ensureSourceAudioContext = () => {
|
||||
if (!sourceAudioContextRef.current) {
|
||||
const context = new AudioContext({ latencyHint: "interactive" });
|
||||
const masterGain = context.createGain();
|
||||
masterGain.gain.value = 1;
|
||||
masterGain.connect(context.destination);
|
||||
sourceAudioContextRef.current = context;
|
||||
sourceAudioMasterGainRef.current = masterGain;
|
||||
}
|
||||
return sourceAudioContextRef.current;
|
||||
};
|
||||
|
||||
const ensureSourceAudioRunning = () => {
|
||||
const context = ensureSourceAudioContext();
|
||||
if (context.state === "running") {
|
||||
return Promise.resolve();
|
||||
}
|
||||
if (!sourceAudioResumePromiseRef.current) {
|
||||
sourceAudioResumePromiseRef.current = context
|
||||
.resume()
|
||||
.catch(() => undefined)
|
||||
.finally(() => {
|
||||
sourceAudioResumePromiseRef.current = null;
|
||||
});
|
||||
}
|
||||
return sourceAudioResumePromiseRef.current;
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
const existing = audioElementsRef.current;
|
||||
const currentIds = new Set(resolvedUserTracks.map((track) => track.id));
|
||||
|
||||
for (const [id, audio] of existing) {
|
||||
if (!currentIds.has(id)) {
|
||||
audio.pause();
|
||||
audio.src = "";
|
||||
audioElementRevokersRef.current.get(id)?.();
|
||||
audioElementRevokersRef.current.delete(id);
|
||||
audioElementResourcesRef.current.delete(id);
|
||||
existing.delete(id);
|
||||
}
|
||||
}
|
||||
|
||||
for (const track of resolvedUserTracks) {
|
||||
let audio = existing.get(track.id);
|
||||
if (!audio) {
|
||||
audio = new Audio();
|
||||
audio.preload = "auto";
|
||||
existing.set(track.id, audio);
|
||||
}
|
||||
|
||||
if (audioElementResourcesRef.current.get(track.id) !== track.sourceRef.path) {
|
||||
audio.pause();
|
||||
audio.src = "";
|
||||
audioElementRevokersRef.current.get(track.id)?.();
|
||||
audioElementRevokersRef.current.delete(track.id);
|
||||
audioElementResourcesRef.current.set(track.id, track.sourceRef.path);
|
||||
|
||||
void (async () => {
|
||||
const resolved = await resolveMediaElementSource(track.sourceRef.path);
|
||||
const latestAudio = existing.get(track.id);
|
||||
|
||||
if (
|
||||
cancelled ||
|
||||
latestAudio !== audio ||
|
||||
audioElementResourcesRef.current.get(track.id) !== track.sourceRef.path
|
||||
) {
|
||||
resolved.revoke();
|
||||
return;
|
||||
}
|
||||
|
||||
audioElementRevokersRef.current.set(track.id, resolved.revoke);
|
||||
latestAudio.src = resolved.src;
|
||||
})();
|
||||
}
|
||||
|
||||
audio.volume = Math.max(0, Math.min(1, track.gain * previewVolume));
|
||||
}
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [previewVolume, resolvedUserTracks]);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
const existing = sourceAudioElementsRef.current;
|
||||
const currentIds = new Set(resolvedSourceTracks.map((track) => track.sourceRef.path));
|
||||
|
||||
for (const [id, audio] of existing) {
|
||||
if (!currentIds.has(id)) {
|
||||
audio.pause();
|
||||
audio.src = "";
|
||||
sourceAudioMediaNodesRef.current.get(id)?.disconnect();
|
||||
sourceAudioMediaNodesRef.current.delete(id);
|
||||
sourceAudioGainNodesRef.current.get(id)?.disconnect();
|
||||
sourceAudioGainNodesRef.current.delete(id);
|
||||
sourceAudioElementRevokersRef.current.get(id)?.();
|
||||
sourceAudioElementRevokersRef.current.delete(id);
|
||||
sourceAudioElementResourcesRef.current.delete(id);
|
||||
existing.delete(id);
|
||||
}
|
||||
}
|
||||
|
||||
for (const track of resolvedSourceTracks) {
|
||||
const audioPath = track.sourceRef.path;
|
||||
let audio = existing.get(audioPath);
|
||||
if (!audio) {
|
||||
audio = new Audio();
|
||||
audio.preload = "auto";
|
||||
audio.crossOrigin = "anonymous";
|
||||
existing.set(audioPath, audio);
|
||||
}
|
||||
audio.volume = 1;
|
||||
audio.dataset.sourceAudioPath = audioPath;
|
||||
|
||||
const context = ensureSourceAudioContext();
|
||||
const masterGain = sourceAudioMasterGainRef.current;
|
||||
if (context && masterGain && !sourceAudioMediaNodesRef.current.has(audioPath)) {
|
||||
try {
|
||||
const mediaNode = context.createMediaElementSource(audio);
|
||||
const trackGainNode = context.createGain();
|
||||
mediaNode.connect(trackGainNode);
|
||||
trackGainNode.connect(masterGain);
|
||||
sourceAudioMediaNodesRef.current.set(audioPath, mediaNode);
|
||||
sourceAudioGainNodesRef.current.set(audioPath, trackGainNode);
|
||||
} catch (error) {
|
||||
onSourceFallbackLoadError(error);
|
||||
}
|
||||
}
|
||||
|
||||
if (sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath) {
|
||||
audio.pause();
|
||||
audio.src = "";
|
||||
sourceAudioElementRevokersRef.current.get(audioPath)?.();
|
||||
sourceAudioElementRevokersRef.current.delete(audioPath);
|
||||
sourceAudioElementResourcesRef.current.set(audioPath, audioPath);
|
||||
|
||||
void (async () => {
|
||||
try {
|
||||
const resolved = await resolveMediaElementSource(audioPath);
|
||||
const latestAudio = existing.get(audioPath);
|
||||
|
||||
if (
|
||||
cancelled ||
|
||||
latestAudio !== audio ||
|
||||
sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath
|
||||
) {
|
||||
resolved.revoke();
|
||||
return;
|
||||
}
|
||||
|
||||
sourceAudioElementRevokersRef.current.set(audioPath, resolved.revoke);
|
||||
latestAudio.src = resolved.src;
|
||||
} catch (error) {
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
sourceAudioElementRevokersRef.current.get(audioPath)?.();
|
||||
sourceAudioElementRevokersRef.current.delete(audioPath);
|
||||
sourceAudioElementResourcesRef.current.delete(audioPath);
|
||||
const latestAudio = existing.get(audioPath);
|
||||
if (latestAudio === audio) {
|
||||
latestAudio.pause();
|
||||
latestAudio.src = "";
|
||||
}
|
||||
onSourceFallbackLoadError(error);
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
const trackGainNode = sourceAudioGainNodesRef.current.get(audioPath);
|
||||
if (trackGainNode) {
|
||||
trackGainNode.gain.value = Math.max(0, Math.min(2, getSourceTrackPreviewGain(audioPath)));
|
||||
}
|
||||
}
|
||||
|
||||
if (sourceAudioMasterGainRef.current) {
|
||||
sourceAudioMasterGainRef.current.gain.value = isCurrentClipMuted
|
||||
? 0
|
||||
: Math.max(0, Math.min(1, previewVolume));
|
||||
}
|
||||
|
||||
if (resolvedSourceTracks.length === 0) {
|
||||
lastSourceAudioSyncTimeRef.current = null;
|
||||
}
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [
|
||||
getSourceTrackPreviewGain,
|
||||
isCurrentClipMuted,
|
||||
onSourceFallbackLoadError,
|
||||
resolvedSourceTracks,
|
||||
previewVolume,
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
for (const audio of audioElementsRef.current.values()) {
|
||||
audio.pause();
|
||||
audio.src = "";
|
||||
}
|
||||
for (const revoke of audioElementRevokersRef.current.values()) {
|
||||
revoke();
|
||||
}
|
||||
audioElementsRef.current.clear();
|
||||
audioElementRevokersRef.current.clear();
|
||||
audioElementResourcesRef.current.clear();
|
||||
for (const audio of sourceAudioElementsRef.current.values()) {
|
||||
audio.pause();
|
||||
audio.src = "";
|
||||
}
|
||||
for (const node of sourceAudioMediaNodesRef.current.values()) {
|
||||
node.disconnect();
|
||||
}
|
||||
for (const node of sourceAudioGainNodesRef.current.values()) {
|
||||
node.disconnect();
|
||||
}
|
||||
for (const revoke of sourceAudioElementRevokersRef.current.values()) {
|
||||
revoke();
|
||||
}
|
||||
sourceAudioElementsRef.current.clear();
|
||||
sourceAudioMediaNodesRef.current.clear();
|
||||
sourceAudioGainNodesRef.current.clear();
|
||||
sourceAudioElementRevokersRef.current.clear();
|
||||
sourceAudioElementResourcesRef.current.clear();
|
||||
if (sourceAudioMasterGainRef.current) {
|
||||
sourceAudioMasterGainRef.current.disconnect();
|
||||
sourceAudioMasterGainRef.current = null;
|
||||
}
|
||||
const context = sourceAudioContextRef.current;
|
||||
sourceAudioContextRef.current = null;
|
||||
sourceAudioResumePromiseRef.current = null;
|
||||
if (context) {
|
||||
void context.close();
|
||||
}
|
||||
lastSourceAudioSyncTimeRef.current = null;
|
||||
};
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
const currentTimeMs = timelineTime * 1000;
|
||||
const activeSpeedRegion = effectiveSpeedRegions.find(
|
||||
(region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs,
|
||||
);
|
||||
const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1;
|
||||
|
||||
for (const track of resolvedUserTracks) {
|
||||
const audio = audioElementsRef.current.get(track.id);
|
||||
if (!audio) continue;
|
||||
|
||||
const startMs = track.timelineBinding.startMs;
|
||||
const endMs = track.timelineBinding.endMs;
|
||||
const isInRegion = currentTimeMs >= startMs && currentTimeMs < endMs;
|
||||
|
||||
if (isPlaying && isInRegion) {
|
||||
enablePitchPreservingPlayback(audio);
|
||||
const audioOffset = (currentTimeMs - startMs) / 1000;
|
||||
if (Math.abs(audio.currentTime - audioOffset) > 0.2) {
|
||||
audio.currentTime = audioOffset;
|
||||
}
|
||||
const syncedPlaybackRate = getMediaSyncPlaybackRate({
|
||||
basePlaybackRate: targetPlaybackRate,
|
||||
currentTime: audio.currentTime,
|
||||
targetTime: audioOffset,
|
||||
});
|
||||
if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) {
|
||||
audio.playbackRate = syncedPlaybackRate;
|
||||
}
|
||||
if (audio.paused) {
|
||||
audio.play().catch(() => undefined);
|
||||
}
|
||||
} else if (!audio.paused) {
|
||||
audio.pause();
|
||||
}
|
||||
}
|
||||
}, [effectiveSpeedRegions, isPlaying, resolvedUserTracks, timelineTime]);
|
||||
|
||||
useEffect(() => {
|
||||
if (resolvedSourceTracks.length === 0) {
|
||||
lastSourceAudioSyncTimeRef.current = null;
|
||||
return;
|
||||
}
|
||||
|
||||
const activeSpeedRegion = effectiveSpeedRegions.find(
|
||||
(region) => currentTime * 1000 >= region.startMs && currentTime * 1000 < region.endMs,
|
||||
);
|
||||
const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1;
|
||||
const previousTimelineTime = lastSourceAudioSyncTimeRef.current;
|
||||
const timelineJumped =
|
||||
previousTimelineTime === null || Math.abs(currentTime - previousTimelineTime) > 0.25;
|
||||
const driftThreshold = isPlaying
|
||||
? SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS
|
||||
: SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS;
|
||||
if (sourceAudioMasterGainRef.current) {
|
||||
sourceAudioMasterGainRef.current.gain.value = isCurrentClipMuted
|
||||
? 0
|
||||
: Math.max(0, Math.min(1, previewVolume));
|
||||
}
|
||||
|
||||
for (const audio of sourceAudioElementsRef.current.values()) {
|
||||
const sourceAudioPath = audio.dataset.sourceAudioPath ?? "";
|
||||
const trackGainNode = sourceAudioGainNodesRef.current.get(sourceAudioPath);
|
||||
if (trackGainNode) {
|
||||
trackGainNode.gain.value = Math.max(0, Math.min(2, getSourceTrackPreviewGain(sourceAudioPath)));
|
||||
}
|
||||
|
||||
enablePitchPreservingPlayback(audio);
|
||||
const audioDuration = Number.isFinite(audio.duration) ? audio.duration : null;
|
||||
const isMicCompanionTrack = /\.mic\./i.test(sourceAudioPath);
|
||||
const rawStartDelaySeconds = estimateCompanionAudioStartDelaySeconds(
|
||||
duration,
|
||||
audioDuration,
|
||||
sourceAudioFallbackStartDelayMsByPath[sourceAudioPath],
|
||||
);
|
||||
const maxPreviewStartDelaySeconds = isMicCompanionTrack ? 2 : 5;
|
||||
const startDelaySeconds = isMicCompanionTrack
|
||||
? 0
|
||||
: Number.isFinite(duration) &&
|
||||
(rawStartDelaySeconds >= Math.max(0, duration - 0.01) ||
|
||||
rawStartDelaySeconds > Math.max(maxPreviewStartDelaySeconds, duration * 0.9))
|
||||
? 0
|
||||
: rawStartDelaySeconds;
|
||||
const beforeAudioStart = currentTime + 0.001 < startDelaySeconds;
|
||||
const targetTime = clampMediaTimeToDuration(currentTime - startDelaySeconds, audioDuration);
|
||||
|
||||
const shouldSeek =
|
||||
timelineJumped ||
|
||||
(!isPlaying && Math.abs(audio.currentTime - targetTime) > driftThreshold) ||
|
||||
(isPlaying && Math.abs(audio.currentTime - targetTime) > 0.9);
|
||||
if (shouldSeek) {
|
||||
try {
|
||||
audio.currentTime = targetTime;
|
||||
} catch {
|
||||
// no-op
|
||||
}
|
||||
}
|
||||
|
||||
// KISS for companion source tracks: fixed playback rate avoids audible flutter/stutter
|
||||
// from continuous micro-corrections on system audio.
|
||||
const syncedPlaybackRate = targetPlaybackRate;
|
||||
if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) {
|
||||
audio.playbackRate = syncedPlaybackRate;
|
||||
}
|
||||
|
||||
const atEnd = audioDuration !== null && targetTime >= audioDuration;
|
||||
if (isPlaying && !beforeAudioStart && !atEnd) {
|
||||
void ensureSourceAudioRunning().then(() => {
|
||||
audio.play().catch(() => undefined);
|
||||
});
|
||||
} else if (!audio.paused) {
|
||||
audio.pause();
|
||||
}
|
||||
}
|
||||
|
||||
lastSourceAudioSyncTimeRef.current = currentTime;
|
||||
}, [
|
||||
currentTime,
|
||||
duration,
|
||||
effectiveSpeedRegions,
|
||||
getSourceTrackPreviewGain,
|
||||
isCurrentClipMuted,
|
||||
isPlaying,
|
||||
previewVolume,
|
||||
resolvedSourceTracks,
|
||||
sourceAudioFallbackStartDelayMsByPath,
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isPlaying || resolvedSourceTracks.length === 0) {
|
||||
return;
|
||||
}
|
||||
void ensureSourceAudioRunning().then(() => {
|
||||
for (const audio of sourceAudioElementsRef.current.values()) {
|
||||
if (audio.paused) {
|
||||
audio.play().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
});
|
||||
}, [isPlaying, resolvedSourceTracks.length]);
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import React, { useCallback, useMemo } from "react";
|
||||
import {
|
||||
SOURCE_AUDIO_NORMALIZE_GAIN,
|
||||
type SourceAudioTrackSettings,
|
||||
} from "@/components/video-editor/audio/audioTypes";
|
||||
import { useSourceAudioTrackSettings } from "./useSourceAudioTrackSettings";
|
||||
import { getSourceTrackIdFromPath } from "@/lib/exporter/audioRoutingEngine";
|
||||
|
||||
interface UseClipAudioSettingsControllerParams {
|
||||
selectedClipId: string | null;
|
||||
activeClipId: string | null;
|
||||
sourceAudioTrackSettingsByClip: Record<string, SourceAudioTrackSettings>;
|
||||
setSourceAudioTrackSettingsByClip: React.Dispatch<
|
||||
React.SetStateAction<Record<string, SourceAudioTrackSettings>>
|
||||
>;
|
||||
defaultSourceAudioTrackSettings: SourceAudioTrackSettings;
|
||||
setDefaultSourceAudioTrackSettings: React.Dispatch<
|
||||
React.SetStateAction<SourceAudioTrackSettings>
|
||||
>;
|
||||
}
|
||||
|
||||
export function useClipAudioSettingsController({
|
||||
selectedClipId,
|
||||
activeClipId,
|
||||
sourceAudioTrackSettingsByClip,
|
||||
setSourceAudioTrackSettingsByClip,
|
||||
defaultSourceAudioTrackSettings,
|
||||
setDefaultSourceAudioTrackSettings,
|
||||
}: UseClipAudioSettingsControllerParams) {
|
||||
const {
|
||||
sourceAudioTrackMeta,
|
||||
activeSourceAudioTrackSettings,
|
||||
selectedClipSourceAudioTrackSettings,
|
||||
getSourceAudioTrackSettingsForClip,
|
||||
onSourceAudioTracksMetaChange,
|
||||
onSelectedClipSourceAudioTrackVolumeChange,
|
||||
onSelectedClipSourceAudioTrackNormalizeChange,
|
||||
} = useSourceAudioTrackSettings({
|
||||
selectedClipId,
|
||||
activeClipId,
|
||||
sourceAudioTrackSettingsByClip,
|
||||
setSourceAudioTrackSettingsByClip,
|
||||
defaultSourceAudioTrackSettings,
|
||||
setDefaultSourceAudioTrackSettings,
|
||||
});
|
||||
|
||||
const previewSourceAudioTrackSettings = useMemo(
|
||||
() =>
|
||||
activeClipId ? activeSourceAudioTrackSettings : selectedClipSourceAudioTrackSettings,
|
||||
[activeClipId, activeSourceAudioTrackSettings, selectedClipSourceAudioTrackSettings],
|
||||
);
|
||||
|
||||
const embeddedTrackId = useMemo<"mixed" | "system">(() => {
|
||||
const hasMixedTrack = sourceAudioTrackMeta.some((track) => track.id === "mixed");
|
||||
if (hasMixedTrack) return "mixed";
|
||||
const hasSystemTrack = sourceAudioTrackMeta.some((track) => track.id === "system");
|
||||
return hasSystemTrack ? "system" : "mixed";
|
||||
}, [sourceAudioTrackMeta]);
|
||||
|
||||
const embeddedSourcePreviewGain = useMemo(() => {
|
||||
const settings = previewSourceAudioTrackSettings[embeddedTrackId] ?? {
|
||||
volume: 1,
|
||||
normalize: false,
|
||||
};
|
||||
const normalizeGain = settings.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1;
|
||||
return Math.max(0, Math.min(2, settings.volume * normalizeGain));
|
||||
}, [embeddedTrackId, previewSourceAudioTrackSettings]);
|
||||
|
||||
const getSourceTrackPreviewGain = useCallback(
|
||||
(audioPath: string) => {
|
||||
const trackId = getSourceTrackIdFromPath(audioPath);
|
||||
const settings = previewSourceAudioTrackSettings[trackId] ?? {
|
||||
volume: 1,
|
||||
normalize: false,
|
||||
};
|
||||
const normalizeGain = settings.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1;
|
||||
return Math.max(0, Math.min(2, settings.volume * normalizeGain));
|
||||
},
|
||||
[previewSourceAudioTrackSettings],
|
||||
);
|
||||
|
||||
return {
|
||||
sourceAudioTrackMeta,
|
||||
activeSourceAudioTrackSettings,
|
||||
selectedClipSourceAudioTrackSettings,
|
||||
getSourceAudioTrackSettingsForClip,
|
||||
onSourceAudioTracksMetaChange,
|
||||
onSelectedClipSourceAudioTrackVolumeChange,
|
||||
onSelectedClipSourceAudioTrackNormalizeChange,
|
||||
embeddedSourcePreviewGain,
|
||||
getSourceTrackPreviewGain,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
import { SOURCE_AUDIO_FALLBACK_TOAST_ID } from "@/components/video-editor/audio/audioTypes";
|
||||
|
||||
interface UseSourceAudioFallbackParams {
|
||||
currentSourcePath: string | null;
|
||||
summarizeErrorMessage: (message: string) => string;
|
||||
}
|
||||
|
||||
export function useSourceAudioFallback({
|
||||
currentSourcePath,
|
||||
summarizeErrorMessage,
|
||||
}: UseSourceAudioFallbackParams) {
|
||||
const [sourceAudioFallbackPaths, setSourceAudioFallbackPaths] = useState<string[]>([]);
|
||||
const [sourceAudioFallbackStartDelayMsByPath, setSourceAudioFallbackStartDelayMsByPath] =
|
||||
useState<Record<string, number>>({});
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
setSourceAudioFallbackPaths([]);
|
||||
setSourceAudioFallbackStartDelayMsByPath({});
|
||||
|
||||
if (!currentSourcePath) {
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}
|
||||
|
||||
void (async () => {
|
||||
try {
|
||||
const result = await window.electronAPI.getVideoAudioFallbackPaths(currentSourcePath);
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
if (!result.success) {
|
||||
setSourceAudioFallbackPaths([]);
|
||||
setSourceAudioFallbackStartDelayMsByPath({});
|
||||
toast.warning(
|
||||
result.error
|
||||
? `Could not load companion audio sources: ${summarizeErrorMessage(result.error)}`
|
||||
: "Could not load companion audio sources. Playback and export may miss microphone audio.",
|
||||
{ id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 },
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
toast.dismiss(SOURCE_AUDIO_FALLBACK_TOAST_ID);
|
||||
setSourceAudioFallbackPaths(result.paths ?? []);
|
||||
setSourceAudioFallbackStartDelayMsByPath(result.startDelayMsByPath ?? {});
|
||||
} catch (error) {
|
||||
if (!cancelled) {
|
||||
setSourceAudioFallbackPaths([]);
|
||||
setSourceAudioFallbackStartDelayMsByPath({});
|
||||
toast.warning(
|
||||
`Could not load companion audio sources: ${summarizeErrorMessage(String(error))}`,
|
||||
{ id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 },
|
||||
);
|
||||
}
|
||||
}
|
||||
})();
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [currentSourcePath, summarizeErrorMessage]);
|
||||
|
||||
return { sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath };
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
import React, { useCallback, useMemo, useState } from "react";
|
||||
import type {
|
||||
SourceAudioTrackMeta,
|
||||
SourceAudioTrackSettings,
|
||||
} from "@/components/video-editor/audio/audioTypes";
|
||||
|
||||
interface UseSourceAudioTrackSettingsParams {
|
||||
selectedClipId: string | null;
|
||||
activeClipId: string | null;
|
||||
sourceAudioTrackSettingsByClip: Record<string, SourceAudioTrackSettings>;
|
||||
setSourceAudioTrackSettingsByClip: React.Dispatch<
|
||||
React.SetStateAction<Record<string, SourceAudioTrackSettings>>
|
||||
>;
|
||||
defaultSourceAudioTrackSettings: SourceAudioTrackSettings;
|
||||
setDefaultSourceAudioTrackSettings: React.Dispatch<React.SetStateAction<SourceAudioTrackSettings>>;
|
||||
}
|
||||
|
||||
export interface UseSourceAudioTrackSettingsResult {
|
||||
sourceAudioTrackMeta: SourceAudioTrackMeta;
|
||||
activeSourceAudioTrackSettings: SourceAudioTrackSettings;
|
||||
selectedClipSourceAudioTrackSettings: SourceAudioTrackSettings;
|
||||
getSourceAudioTrackSettingsForClip: (clipId: string | null) => SourceAudioTrackSettings;
|
||||
onSourceAudioTracksMetaChange: (tracks: SourceAudioTrackMeta) => void;
|
||||
onSelectedClipSourceAudioTrackVolumeChange: (id: string, volume: number) => void;
|
||||
onSelectedClipSourceAudioTrackNormalizeChange: (id: string, normalize: boolean) => void;
|
||||
}
|
||||
|
||||
function isSameTrackMeta(left: SourceAudioTrackMeta, right: SourceAudioTrackMeta): boolean {
|
||||
if (left.length !== right.length) return false;
|
||||
for (let index = 0; index < left.length; index += 1) {
|
||||
const leftTrack = left[index];
|
||||
const rightTrack = right[index];
|
||||
if (!leftTrack || !rightTrack) return false;
|
||||
if (leftTrack.id !== rightTrack.id || leftTrack.label !== rightTrack.label) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
export function useSourceAudioTrackSettings({
|
||||
selectedClipId,
|
||||
activeClipId,
|
||||
sourceAudioTrackSettingsByClip,
|
||||
setSourceAudioTrackSettingsByClip,
|
||||
defaultSourceAudioTrackSettings,
|
||||
setDefaultSourceAudioTrackSettings,
|
||||
}: UseSourceAudioTrackSettingsParams): UseSourceAudioTrackSettingsResult {
|
||||
const [sourceAudioTrackMeta, setSourceAudioTrackMeta] = useState<SourceAudioTrackMeta>([]);
|
||||
|
||||
const activeSourceAudioTrackSettings = useMemo(() => {
|
||||
if (!activeClipId) {
|
||||
return defaultSourceAudioTrackSettings;
|
||||
}
|
||||
return {
|
||||
...defaultSourceAudioTrackSettings,
|
||||
...(sourceAudioTrackSettingsByClip[activeClipId] ?? {}),
|
||||
};
|
||||
}, [activeClipId, defaultSourceAudioTrackSettings, sourceAudioTrackSettingsByClip]);
|
||||
|
||||
const selectedClipSourceAudioTrackSettings = useMemo(() => {
|
||||
if (!selectedClipId) {
|
||||
return defaultSourceAudioTrackSettings;
|
||||
}
|
||||
return {
|
||||
...defaultSourceAudioTrackSettings,
|
||||
...(sourceAudioTrackSettingsByClip[selectedClipId] ?? {}),
|
||||
};
|
||||
}, [defaultSourceAudioTrackSettings, selectedClipId, sourceAudioTrackSettingsByClip]);
|
||||
|
||||
const onSourceAudioTracksMetaChange = useCallback((tracks: SourceAudioTrackMeta) => {
|
||||
setSourceAudioTrackMeta((prev) => (isSameTrackMeta(prev, tracks) ? prev : tracks));
|
||||
setDefaultSourceAudioTrackSettings((prev) => {
|
||||
const next: SourceAudioTrackSettings = {};
|
||||
for (const track of tracks) {
|
||||
next[track.id] = prev[track.id] ?? { volume: 1, normalize: false };
|
||||
}
|
||||
const prevKeys = Object.keys(prev);
|
||||
const nextKeys = Object.keys(next);
|
||||
if (prevKeys.length !== nextKeys.length) {
|
||||
return next;
|
||||
}
|
||||
for (const key of nextKeys) {
|
||||
const prevSetting = prev[key];
|
||||
const nextSetting = next[key];
|
||||
if (!prevSetting || !nextSetting) {
|
||||
return next;
|
||||
}
|
||||
if (
|
||||
prevSetting.volume !== nextSetting.volume ||
|
||||
prevSetting.normalize !== nextSetting.normalize
|
||||
) {
|
||||
return next;
|
||||
}
|
||||
}
|
||||
return prev;
|
||||
});
|
||||
}, []);
|
||||
|
||||
const getSourceAudioTrackSettingsForClip = useCallback(
|
||||
(clipId: string | null): SourceAudioTrackSettings => {
|
||||
if (!clipId) {
|
||||
return defaultSourceAudioTrackSettings;
|
||||
}
|
||||
return {
|
||||
...defaultSourceAudioTrackSettings,
|
||||
...(sourceAudioTrackSettingsByClip[clipId] ?? {}),
|
||||
};
|
||||
},
|
||||
[defaultSourceAudioTrackSettings, sourceAudioTrackSettingsByClip],
|
||||
);
|
||||
|
||||
const onSelectedClipSourceAudioTrackVolumeChange = useCallback(
|
||||
(id: string, volume: number) => {
|
||||
if (!selectedClipId) return;
|
||||
setSourceAudioTrackSettingsByClip((prev) => {
|
||||
const prevClip = prev[selectedClipId] ?? defaultSourceAudioTrackSettings;
|
||||
const nextVolume = Number.isFinite(volume)
|
||||
? Math.max(0, Math.min(2, volume))
|
||||
: (prevClip[id]?.volume ?? 1);
|
||||
const prevNormalize = prevClip[id]?.normalize ?? false;
|
||||
if (
|
||||
prevClip[id]?.volume === nextVolume &&
|
||||
prevClip[id]?.normalize === prevNormalize
|
||||
) {
|
||||
return prev;
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[selectedClipId]: {
|
||||
...prevClip,
|
||||
[id]: {
|
||||
volume: nextVolume,
|
||||
normalize: prevNormalize,
|
||||
},
|
||||
},
|
||||
};
|
||||
});
|
||||
},
|
||||
[defaultSourceAudioTrackSettings, selectedClipId],
|
||||
);
|
||||
|
||||
const onSelectedClipSourceAudioTrackNormalizeChange = useCallback(
|
||||
(id: string, normalize: boolean) => {
|
||||
if (!selectedClipId) return;
|
||||
setSourceAudioTrackSettingsByClip((prev) => {
|
||||
const prevClip = prev[selectedClipId] ?? defaultSourceAudioTrackSettings;
|
||||
const prevVolume = prevClip[id]?.volume ?? 1;
|
||||
if (prevClip[id]?.normalize === normalize) {
|
||||
return prev;
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[selectedClipId]: {
|
||||
...prevClip,
|
||||
[id]: {
|
||||
volume: prevVolume,
|
||||
normalize,
|
||||
},
|
||||
},
|
||||
};
|
||||
});
|
||||
},
|
||||
[defaultSourceAudioTrackSettings, selectedClipId],
|
||||
);
|
||||
|
||||
return {
|
||||
sourceAudioTrackMeta,
|
||||
activeSourceAudioTrackSettings,
|
||||
selectedClipSourceAudioTrackSettings,
|
||||
getSourceAudioTrackSettingsForClip,
|
||||
onSourceAudioTracksMetaChange,
|
||||
onSelectedClipSourceAudioTrackVolumeChange,
|
||||
onSelectedClipSourceAudioTrackNormalizeChange,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
import React, { useMemo } from "react";
|
||||
import { resolveSourceTrackRoutingPolicy } from "@/lib/exporter/sourceTrackRoutingPolicy";
|
||||
import type {
|
||||
AudioRegion,
|
||||
ClipRegion,
|
||||
SpeedRegion,
|
||||
} from "../types";
|
||||
import type { SourceAudioTrackSettings } from "@/components/video-editor/audio/audioTypes";
|
||||
import { getActiveClipIdAtSourceTime, isClipMutedById } from "./clipAudio";
|
||||
import { useAudioPreviewSync } from "./useAudioPreviewSync";
|
||||
import { useClipAudioSettingsController } from "./useClipAudioSettingsController";
|
||||
import { useSourceAudioFallback } from "./useSourceAudioFallback";
|
||||
|
||||
function extractLocalPathFromMediaServerUrl(input: string | null | undefined): string | null {
|
||||
if (!input) return null;
|
||||
try {
|
||||
const url = new URL(input);
|
||||
const isLocalMediaServer =
|
||||
(url.protocol === "http:" || url.protocol === "https:") &&
|
||||
(url.hostname === "127.0.0.1" || url.hostname === "localhost") &&
|
||||
url.pathname === "/video";
|
||||
if (!isLocalMediaServer) return null;
|
||||
return url.searchParams.get("path");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
interface UseVideoEditorAudioParams {
|
||||
currentSourcePath: string | null;
|
||||
selectedClipId: string | null;
|
||||
clipRegions: ClipRegion[];
|
||||
audioRegions: AudioRegion[];
|
||||
effectiveSpeedRegions: SpeedRegion[];
|
||||
sourceAudioTrackSettingsByClip: Record<string, SourceAudioTrackSettings>;
|
||||
setSourceAudioTrackSettingsByClip: React.Dispatch<
|
||||
React.SetStateAction<Record<string, SourceAudioTrackSettings>>
|
||||
>;
|
||||
defaultSourceAudioTrackSettings: SourceAudioTrackSettings;
|
||||
setDefaultSourceAudioTrackSettings: React.Dispatch<
|
||||
React.SetStateAction<SourceAudioTrackSettings>
|
||||
>;
|
||||
currentTime: number;
|
||||
timelineTime: number;
|
||||
duration: number;
|
||||
isPlaying: boolean;
|
||||
previewVolume: number;
|
||||
summarizeErrorMessage: (message: string) => string;
|
||||
onSourceFallbackLoadError: (error: unknown) => void;
|
||||
}
|
||||
|
||||
export function useVideoEditorAudio({
|
||||
currentSourcePath,
|
||||
selectedClipId,
|
||||
clipRegions,
|
||||
audioRegions,
|
||||
effectiveSpeedRegions,
|
||||
sourceAudioTrackSettingsByClip,
|
||||
setSourceAudioTrackSettingsByClip,
|
||||
defaultSourceAudioTrackSettings,
|
||||
setDefaultSourceAudioTrackSettings,
|
||||
currentTime,
|
||||
timelineTime,
|
||||
duration,
|
||||
isPlaying,
|
||||
previewVolume,
|
||||
summarizeErrorMessage,
|
||||
onSourceFallbackLoadError,
|
||||
}: UseVideoEditorAudioParams) {
|
||||
const fallbackLookupSourcePath = useMemo(
|
||||
() => extractLocalPathFromMediaServerUrl(currentSourcePath) ?? currentSourcePath,
|
||||
[currentSourcePath],
|
||||
);
|
||||
|
||||
const { sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath } =
|
||||
useSourceAudioFallback({
|
||||
currentSourcePath: fallbackLookupSourcePath,
|
||||
summarizeErrorMessage,
|
||||
});
|
||||
|
||||
const sourceTrackRoutingPolicy = useMemo(
|
||||
() => resolveSourceTrackRoutingPolicy(currentSourcePath, sourceAudioFallbackPaths),
|
||||
[currentSourcePath, sourceAudioFallbackPaths],
|
||||
);
|
||||
const previewSourceAudioFallbackPaths = sourceTrackRoutingPolicy.playbackPaths;
|
||||
const shouldMutePreviewVideo = sourceTrackRoutingPolicy.muteEmbeddedPreview;
|
||||
|
||||
const activeClipIdAtCurrentTime = useMemo(
|
||||
() => getActiveClipIdAtSourceTime(currentTime, clipRegions),
|
||||
[clipRegions, currentTime],
|
||||
);
|
||||
const isCurrentClipMuted = useMemo(
|
||||
() => isClipMutedById(activeClipIdAtCurrentTime, clipRegions),
|
||||
[activeClipIdAtCurrentTime, clipRegions],
|
||||
);
|
||||
|
||||
const {
|
||||
sourceAudioTrackMeta,
|
||||
activeSourceAudioTrackSettings,
|
||||
selectedClipSourceAudioTrackSettings,
|
||||
getSourceAudioTrackSettingsForClip,
|
||||
onSourceAudioTracksMetaChange,
|
||||
onSelectedClipSourceAudioTrackVolumeChange,
|
||||
onSelectedClipSourceAudioTrackNormalizeChange,
|
||||
embeddedSourcePreviewGain,
|
||||
getSourceTrackPreviewGain,
|
||||
} = useClipAudioSettingsController({
|
||||
selectedClipId,
|
||||
activeClipId: activeClipIdAtCurrentTime,
|
||||
sourceAudioTrackSettingsByClip,
|
||||
setSourceAudioTrackSettingsByClip,
|
||||
defaultSourceAudioTrackSettings,
|
||||
setDefaultSourceAudioTrackSettings,
|
||||
});
|
||||
|
||||
useAudioPreviewSync({
|
||||
audioRegions,
|
||||
previewVolume,
|
||||
isPlaying,
|
||||
currentTime,
|
||||
timelineTime,
|
||||
duration,
|
||||
effectiveSpeedRegions,
|
||||
previewSourceAudioFallbackPaths,
|
||||
sourceAudioFallbackStartDelayMsByPath,
|
||||
isCurrentClipMuted,
|
||||
getSourceTrackPreviewGain,
|
||||
onSourceFallbackLoadError,
|
||||
});
|
||||
|
||||
return {
|
||||
sourceAudioFallbackPaths,
|
||||
sourceAudioFallbackStartDelayMsByPath,
|
||||
previewSourceAudioFallbackPaths,
|
||||
shouldMutePreviewVideo,
|
||||
activeClipIdAtCurrentTime,
|
||||
isCurrentClipMuted,
|
||||
sourceAudioTrackMeta,
|
||||
activeSourceAudioTrackSettings,
|
||||
selectedClipSourceAudioTrackSettings,
|
||||
getSourceAudioTrackSettingsForClip,
|
||||
onSourceAudioTracksMetaChange,
|
||||
onSelectedClipSourceAudioTrackVolumeChange,
|
||||
onSelectedClipSourceAudioTrackNormalizeChange,
|
||||
embeddedSourcePreviewGain,
|
||||
getSourceTrackPreviewGain,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
import WorkerConstructor from "./waveform.worker?worker";
|
||||
import type { AudioPeaksData } from "../../timeline/core/timelineTypes";
|
||||
import { WAVEFORM_DEFAULT_PEAK_COUNT } from "../../timeline/core/constants";
|
||||
|
||||
export class WaveformGenerator {
|
||||
private audioContext: AudioContext;
|
||||
private worker: Worker;
|
||||
private peaksCache = new Map<string, AudioPeaksData>();
|
||||
private pending = new Map<string, Promise<AudioPeaksData>>();
|
||||
private workerRequestSeq = 0;
|
||||
private workerResolvers = new Map<number, { resolve: (peaks: Float32Array) => void; reject: (err: Error) => void }>();
|
||||
|
||||
constructor() {
|
||||
this.audioContext = new (window.AudioContext || (window as typeof window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext)();
|
||||
this.worker = new WorkerConstructor();
|
||||
|
||||
this.worker.addEventListener(
|
||||
"message",
|
||||
(event: MessageEvent<{ requestId: number; peaks?: Float32Array; error?: string }>) => {
|
||||
const { requestId, peaks, error } = event.data;
|
||||
const resolver = this.workerResolvers.get(requestId);
|
||||
if (!resolver) return;
|
||||
|
||||
this.workerResolvers.delete(requestId);
|
||||
if (error) {
|
||||
resolver.reject(new Error(error));
|
||||
} else if (peaks) {
|
||||
resolver.resolve(peaks);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
this.worker.addEventListener("error", (error: ErrorEvent) => {
|
||||
console.error("[WaveformGenerator] Worker fatal error:", error);
|
||||
const fatalError = error.error ?? new Error(error.message || "Worker crashed");
|
||||
|
||||
// Reject all pending requests if the worker itself crashes
|
||||
for (const resolver of this.workerResolvers.values()) {
|
||||
resolver.reject(fatalError);
|
||||
}
|
||||
this.workerResolvers.clear();
|
||||
});
|
||||
}
|
||||
|
||||
private computePeaksWithWorker(channels: Float32Array[], samples: number): Promise<Float32Array> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const requestId = ++this.workerRequestSeq;
|
||||
this.workerResolvers.set(requestId, { resolve, reject });
|
||||
|
||||
this.worker.postMessage(
|
||||
{
|
||||
requestId,
|
||||
channels,
|
||||
samples,
|
||||
},
|
||||
channels.map(c => c.buffer),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
public async generate(url: string, peakCount = WAVEFORM_DEFAULT_PEAK_COUNT): Promise<AudioPeaksData> {
|
||||
const cacheKey = `${url}::${peakCount}`;
|
||||
const cached = this.peaksCache.get(cacheKey);
|
||||
if (cached) return cached;
|
||||
|
||||
const inflight = this.pending.get(cacheKey);
|
||||
if (inflight) return inflight;
|
||||
|
||||
const request = (async () => {
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to load media: ${response.status}`);
|
||||
}
|
||||
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
const decoded = await this.audioContext.decodeAudioData(arrayBuffer);
|
||||
|
||||
const channels: Float32Array[] = [];
|
||||
for (let i = 0; i < decoded.numberOfChannels; i++) {
|
||||
// We slice to transfer the underlying buffer to the worker
|
||||
channels.push(decoded.getChannelData(i).slice());
|
||||
}
|
||||
|
||||
const peaks = await this.computePeaksWithWorker(channels, peakCount);
|
||||
|
||||
let max = 0;
|
||||
for (let i = 0; i < peaks.length; i++) {
|
||||
if (peaks[i] > max) max = peaks[i];
|
||||
}
|
||||
if (max > 0) {
|
||||
for (let i = 0; i < peaks.length; i++) {
|
||||
peaks[i] /= max;
|
||||
}
|
||||
}
|
||||
|
||||
const result: AudioPeaksData = {
|
||||
peaks,
|
||||
durationMs: decoded.duration * 1000,
|
||||
};
|
||||
this.peaksCache.set(cacheKey, result);
|
||||
this.pending.delete(cacheKey);
|
||||
return result;
|
||||
})().catch((error) => {
|
||||
this.pending.delete(cacheKey);
|
||||
throw error;
|
||||
});
|
||||
|
||||
this.pending.set(cacheKey, request);
|
||||
return request;
|
||||
}
|
||||
}
|
||||
|
||||
export const waveformGenerator = new WaveformGenerator();
|
||||
@@ -0,0 +1,48 @@
|
||||
type WaveformWorkerRequest = {
|
||||
requestId: number;
|
||||
channels: Float32Array[];
|
||||
samples: number;
|
||||
};
|
||||
|
||||
interface WorkerContext {
|
||||
onmessage: (e: MessageEvent<WaveformWorkerRequest>) => void;
|
||||
postMessage: (message: any, transfer?: Transferable[]) => void;
|
||||
}
|
||||
|
||||
const workerScope = self as unknown as WorkerContext;
|
||||
|
||||
workerScope.onmessage = (e: MessageEvent<WaveformWorkerRequest>) => {
|
||||
const { requestId, channels, samples } = e.data;
|
||||
|
||||
if (!channels || channels.length === 0 || samples <= 0) {
|
||||
const empty = new Float32Array(0);
|
||||
workerScope.postMessage({ requestId, peaks: empty }, [empty.buffer]);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const firstChannel = channels[0];
|
||||
const result = new Float32Array(samples);
|
||||
const total = firstChannel.length;
|
||||
|
||||
for (let i = 0; i < samples; i++) {
|
||||
const start = Math.floor((i * total) / samples);
|
||||
const end = Math.floor(((i + 1) * total) / samples);
|
||||
let max = 0;
|
||||
for (let j = start; j < end; j++) {
|
||||
for (let c = 0; c < channels.length; c++) {
|
||||
const val = Math.abs(channels[c][j]);
|
||||
if (val > max) max = val;
|
||||
}
|
||||
}
|
||||
result[i] = max;
|
||||
}
|
||||
|
||||
workerScope.postMessage({ requestId, peaks: result }, [result.buffer]);
|
||||
} catch (err) {
|
||||
workerScope.postMessage({
|
||||
requestId,
|
||||
error: err instanceof Error ? err.message : "Unknown worker error",
|
||||
});
|
||||
}
|
||||
};
|
||||
@@ -20,6 +20,7 @@ import {
|
||||
import { DEFAULT_WALLPAPER_PATH } from "@/lib/wallpapers";
|
||||
import { ASPECT_RATIOS, type AspectRatio, isCustomAspectRatio } from "@/utils/aspectRatioUtils";
|
||||
import { CURSOR_MOTION_PRESETS, resolveCursorMotionPresetId } from "./cursorMotionPresets";
|
||||
import type { SourceAudioTrackSettings } from "@/components/video-editor/audio/audioTypes";
|
||||
import {
|
||||
type AnnotationRegion,
|
||||
type AudioRegion,
|
||||
@@ -127,6 +128,8 @@ export interface ProjectEditorState {
|
||||
autoCaptionSettings: AutoCaptionSettings;
|
||||
webcam: WebcamOverlaySettings;
|
||||
aspectRatio: AspectRatio;
|
||||
sourceAudioTrackSettingsByClip?: Record<string, SourceAudioTrackSettings>;
|
||||
defaultSourceAudioTrackSettings?: SourceAudioTrackSettings;
|
||||
exportEncodingMode: ExportEncodingMode;
|
||||
exportBackendPreference: ExportBackendPreference;
|
||||
exportPipelineModel: ExportPipelineModel;
|
||||
@@ -496,6 +499,10 @@ export function normalizeProjectEditor(editor: Partial<ProjectEditorState>): Pro
|
||||
endMs,
|
||||
speed: isFiniteNumber(region.speed) ? region.speed : 1,
|
||||
muted: typeof region.muted === "boolean" ? region.muted : false,
|
||||
showSourceAudio:
|
||||
typeof region.showSourceAudio === "boolean"
|
||||
? region.showSourceAudio
|
||||
: false,
|
||||
};
|
||||
})
|
||||
: [];
|
||||
@@ -647,16 +654,17 @@ export function normalizeProjectEditor(editor: Partial<ProjectEditorState>): Pro
|
||||
const startMs = Math.max(0, Math.min(rawStart, rawEnd));
|
||||
const endMs = Math.max(startMs + 1, rawEnd);
|
||||
|
||||
return {
|
||||
id: region.id,
|
||||
startMs,
|
||||
endMs,
|
||||
audioPath: typeof region.audioPath === "string" ? region.audioPath : "",
|
||||
volume: isFiniteNumber(region.volume) ? clamp(region.volume, 0, 1) : 1,
|
||||
trackIndex: isFiniteNumber(region.trackIndex)
|
||||
? Math.max(0, Math.floor(region.trackIndex))
|
||||
: 0,
|
||||
};
|
||||
return {
|
||||
id: region.id,
|
||||
startMs,
|
||||
endMs,
|
||||
audioPath: typeof region.audioPath === "string" ? region.audioPath : "",
|
||||
volume: isFiniteNumber(region.volume) ? clamp(region.volume, 0, 1) : 1,
|
||||
normalize: Boolean(region.normalize),
|
||||
trackIndex: isFiniteNumber(region.trackIndex)
|
||||
? Math.max(0, Math.floor(region.trackIndex))
|
||||
: 0,
|
||||
};
|
||||
})
|
||||
: [];
|
||||
|
||||
@@ -983,6 +991,16 @@ export function normalizeProjectEditor(editor: Partial<ProjectEditorState>): Pro
|
||||
? clamp(webcam.margin, 0, 96)
|
||||
: DEFAULT_WEBCAM_MARGIN,
|
||||
},
|
||||
sourceAudioTrackSettingsByClip:
|
||||
editor.sourceAudioTrackSettingsByClip &&
|
||||
typeof editor.sourceAudioTrackSettingsByClip === "object"
|
||||
? editor.sourceAudioTrackSettingsByClip
|
||||
: {},
|
||||
defaultSourceAudioTrackSettings:
|
||||
editor.defaultSourceAudioTrackSettings &&
|
||||
typeof editor.defaultSourceAudioTrackSettings === "object"
|
||||
? editor.defaultSourceAudioTrackSettings
|
||||
: {},
|
||||
aspectRatio:
|
||||
typeof editor.aspectRatio === "string" &&
|
||||
(validAspectRatios.has(editor.aspectRatio as AspectRatio) ||
|
||||
|
||||
@@ -5,18 +5,22 @@ import {
|
||||
MusicNotes as Music,
|
||||
MouseLeftClickIcon as PhMouseLeftClick,
|
||||
Scissors,
|
||||
SpeakerX,
|
||||
MagnifyingGlassPlus as ZoomIn,
|
||||
} from "@phosphor-icons/react";
|
||||
import type { Span } from "dnd-timeline";
|
||||
import { useItem } from "dnd-timeline";
|
||||
import { useMemo } from "react";
|
||||
import { cn } from "@/lib/utils";
|
||||
import AudioWaveform from "./components/waveform/AudioWaveform";
|
||||
import type { AudioPeaksData } from "./core/timelineTypes";
|
||||
import glassStyles from "./ItemGlass.module.css";
|
||||
|
||||
interface ItemProps {
|
||||
id: string;
|
||||
span: Span;
|
||||
rowId: string;
|
||||
disabled?: boolean;
|
||||
children: React.ReactNode;
|
||||
isSelected?: boolean;
|
||||
onSelect?: () => void;
|
||||
@@ -24,6 +28,11 @@ interface ItemProps {
|
||||
zoomDepth?: number;
|
||||
zoomMode?: "auto" | "manual";
|
||||
speedValue?: number;
|
||||
waveformPeaks?: AudioPeaksData | null;
|
||||
waveformSegmentSpan?: Span;
|
||||
waveformGain?: number;
|
||||
waveformNormalize?: boolean;
|
||||
muted?: boolean;
|
||||
variant?: "zoom" | "trim" | "clip" | "annotation" | "speed" | "audio";
|
||||
}
|
||||
|
||||
@@ -51,18 +60,25 @@ export default function Item({
|
||||
id,
|
||||
span,
|
||||
rowId,
|
||||
disabled = false,
|
||||
isSelected = false,
|
||||
onSelect,
|
||||
onSelectId,
|
||||
zoomDepth = 1,
|
||||
zoomMode = "auto",
|
||||
speedValue,
|
||||
waveformPeaks = null,
|
||||
waveformSegmentSpan,
|
||||
waveformGain = 1,
|
||||
waveformNormalize = false,
|
||||
muted = false,
|
||||
variant = "zoom",
|
||||
children,
|
||||
}: ItemProps) {
|
||||
const { setNodeRef, attributes, listeners, itemStyle, itemContentStyle } = useItem({
|
||||
id,
|
||||
span,
|
||||
disabled,
|
||||
data: { rowId },
|
||||
});
|
||||
|
||||
@@ -71,6 +87,7 @@ export default function Item({
|
||||
const isClip = variant === "clip";
|
||||
const isSpeed = variant === "speed";
|
||||
const isAudio = variant === "audio";
|
||||
const showAudioWaveform = isAudio && Boolean(waveformPeaks);
|
||||
|
||||
const glassClass = isZoom
|
||||
? glassStyles.glassPurple
|
||||
@@ -146,6 +163,22 @@ export default function Item({
|
||||
style={{ cursor: "col-resize", pointerEvents: "auto" }}
|
||||
title="Resize right"
|
||||
/>
|
||||
{showAudioWaveform && waveformPeaks && (
|
||||
<AudioWaveform
|
||||
peaks={waveformPeaks}
|
||||
segmentStartMs={waveformSegmentSpan?.start ?? span.start}
|
||||
segmentEndMs={waveformSegmentSpan?.end ?? span.end}
|
||||
gain={waveformGain}
|
||||
normalize={waveformNormalize}
|
||||
className="absolute inset-0 w-full h-full pointer-events-none opacity-45"
|
||||
/>
|
||||
)}
|
||||
{/* Muted overlay for source audio track items */}
|
||||
{isAudio && muted && (
|
||||
<div className="absolute inset-0 z-20 flex items-center justify-center gap-1 bg-red-900/40 pointer-events-none">
|
||||
<SpeakerX className="w-3 h-3 text-red-300/90 shrink-0" />
|
||||
</div>
|
||||
)}
|
||||
{/* Content */}
|
||||
<div className="relative z-10 flex flex-col items-center justify-center text-black/70 dark:text-white/90 opacity-80 group-hover:opacity-100 transition-opacity select-none overflow-hidden">
|
||||
<div className="flex items-center gap-1.5">
|
||||
|
||||
@@ -17,6 +17,12 @@ import {
|
||||
} from "@/utils/aspectRatioUtils";
|
||||
import { formatShortcut } from "@/utils/platformUtils";
|
||||
import { loadEditorPreferences, saveEditorPreferences } from "../editorPreferences";
|
||||
import { fromFileUrl } from "../projectPersistence";
|
||||
import type {
|
||||
SourceAudioTrackMeta,
|
||||
SourceAudioTrackSettings,
|
||||
SourceAudioTrackWithPeaks,
|
||||
} from "@/components/video-editor/audio/audioTypes";
|
||||
import type {
|
||||
AnnotationRegion,
|
||||
AudioRegion,
|
||||
@@ -80,6 +86,38 @@ export interface TimelineEditorProps {
|
||||
isCropped?: boolean;
|
||||
videoPath?: string | null;
|
||||
hideToolbar?: boolean;
|
||||
showSourceAudioTrack?: boolean;
|
||||
onSourceAudioAvailabilityChange?: (available: boolean) => void;
|
||||
sourceAudioTrackSettings?: SourceAudioTrackSettings;
|
||||
getSourceAudioTrackSettingsForClip?: (
|
||||
clipId: string | null,
|
||||
) => SourceAudioTrackSettings;
|
||||
onSourceAudioTracksMetaChange?: (tracks: SourceAudioTrackMeta) => void;
|
||||
}
|
||||
|
||||
function extractLocalPathFromMediaServerUrl(input: string | null | undefined): string | null {
|
||||
if (!input) return null;
|
||||
try {
|
||||
const url = new URL(input);
|
||||
const isLocalMediaServer =
|
||||
(url.protocol === "http:" || url.protocol === "https:") &&
|
||||
(url.hostname === "127.0.0.1" || url.hostname === "localhost") &&
|
||||
url.pathname === "/video";
|
||||
if (!isLocalMediaServer) return null;
|
||||
return url.searchParams.get("path");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function buildSourceSidecarPath(source: string, suffix: "mic" | "system"): string {
|
||||
const normalized = source.replace(/\\/g, "/");
|
||||
const lastSlash = normalized.lastIndexOf("/");
|
||||
const dir = lastSlash >= 0 ? normalized.slice(0, lastSlash + 1) : "";
|
||||
const fileName = lastSlash >= 0 ? normalized.slice(lastSlash + 1) : normalized;
|
||||
const dotIndex = fileName.lastIndexOf(".");
|
||||
const baseName = dotIndex > 0 ? fileName.slice(0, dotIndex) : fileName;
|
||||
return `${dir}${baseName}.${suffix}.wav`;
|
||||
}
|
||||
|
||||
export interface TimelineEditorHandle {
|
||||
@@ -138,6 +176,11 @@ const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
|
||||
isCropped = false,
|
||||
videoPath,
|
||||
hideToolbar = false,
|
||||
showSourceAudioTrack = false,
|
||||
onSourceAudioAvailabilityChange,
|
||||
sourceAudioTrackSettings = {},
|
||||
getSourceAudioTrackSettingsForClip,
|
||||
onSourceAudioTracksMetaChange,
|
||||
},
|
||||
ref,
|
||||
) {
|
||||
@@ -178,8 +221,111 @@ const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
|
||||
pan: "Shift + Ctrl + Scroll",
|
||||
zoom: "Ctrl + Scroll",
|
||||
});
|
||||
const [liveSpanPreviewById, setLiveSpanPreviewById] = useState<Record<string, Span>>({});
|
||||
const liveZoomPreview = useMemo(() => {
|
||||
const previewSpans: Record<string, Span> = { ...liveSpanPreviewById };
|
||||
const hiddenZoomIds = new Set<string>();
|
||||
|
||||
for (const [previewId, previewSpan] of Object.entries(liveSpanPreviewById)) {
|
||||
const oldClip = clipRegions.find((clip) => clip.id === previewId);
|
||||
if (!oldClip) continue;
|
||||
|
||||
const newStart = Math.round(previewSpan.start);
|
||||
const newEnd = Math.round(previewSpan.end);
|
||||
const removedSegments = [
|
||||
...(newStart > oldClip.startMs
|
||||
? [{ startMs: oldClip.startMs, endMs: newStart }]
|
||||
: []),
|
||||
...(newEnd < oldClip.endMs
|
||||
? [{ startMs: newEnd, endMs: oldClip.endMs }]
|
||||
: []),
|
||||
];
|
||||
|
||||
const startDelta = newStart - oldClip.startMs;
|
||||
const endDelta = newEnd - oldClip.endMs;
|
||||
const isMove = Math.abs(startDelta - endDelta) < 1 && Math.abs(startDelta) > 0;
|
||||
|
||||
if (isMove) {
|
||||
const delta = startDelta;
|
||||
for (const zoom of zoomRegions) {
|
||||
const overlaps =
|
||||
zoom.startMs < oldClip.endMs && zoom.endMs > oldClip.startMs;
|
||||
if (!overlaps) continue;
|
||||
previewSpans[zoom.id] = {
|
||||
start: zoom.startMs + delta,
|
||||
end: zoom.endMs + delta,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (removedSegments.length > 0) {
|
||||
for (const zoom of zoomRegions) {
|
||||
const removed = removedSegments.some(
|
||||
(segment) =>
|
||||
zoom.startMs < segment.endMs && zoom.endMs > segment.startMs,
|
||||
);
|
||||
if (removed) hiddenZoomIds.add(zoom.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { previewSpans, hiddenZoomIds };
|
||||
}, [clipRegions, liveSpanPreviewById, zoomRegions]);
|
||||
const { shortcuts: keyShortcuts, isMac } = useShortcuts();
|
||||
const audioPeaks = useTimelineAudioPeaks(videoPath);
|
||||
const sourceAudioPeaks = useTimelineAudioPeaks(videoPath, {
|
||||
enableSourceSidecarFallback: true,
|
||||
});
|
||||
const localSourcePath = useMemo(() => {
|
||||
if (!videoPath) return null;
|
||||
return (
|
||||
extractLocalPathFromMediaServerUrl(videoPath) ||
|
||||
(/^file:\/\//i.test(videoPath) ? fromFileUrl(videoPath) : videoPath)
|
||||
);
|
||||
}, [videoPath]);
|
||||
const micSidecarPath = useMemo(
|
||||
() => (localSourcePath ? buildSourceSidecarPath(localSourcePath, "mic") : null),
|
||||
[localSourcePath],
|
||||
);
|
||||
const systemSidecarPath = useMemo(
|
||||
() => (localSourcePath ? buildSourceSidecarPath(localSourcePath, "system") : null),
|
||||
[localSourcePath],
|
||||
);
|
||||
const micSidecarPeaks = useTimelineAudioPeaks(micSidecarPath);
|
||||
const systemSidecarPeaks = useTimelineAudioPeaks(systemSidecarPath);
|
||||
const sourceAudioTracks = useMemo<SourceAudioTrackWithPeaks[]>(() => {
|
||||
if (systemSidecarPeaks || micSidecarPeaks) {
|
||||
const tracks: SourceAudioTrackWithPeaks[] = [];
|
||||
if (systemSidecarPeaks)
|
||||
tracks.push({
|
||||
id: "system",
|
||||
label: t("audio.systemLabel", "Source System"),
|
||||
peaks: systemSidecarPeaks,
|
||||
});
|
||||
if (micSidecarPeaks)
|
||||
tracks.push({
|
||||
id: "mic",
|
||||
label: t("audio.micLabel", "Source Mic"),
|
||||
peaks: micSidecarPeaks,
|
||||
});
|
||||
return tracks;
|
||||
}
|
||||
return sourceAudioPeaks
|
||||
? [
|
||||
{
|
||||
id: "mixed",
|
||||
label: t("audio.mixedLabel", "Source"),
|
||||
peaks: sourceAudioPeaks,
|
||||
},
|
||||
]
|
||||
: [];
|
||||
}, [micSidecarPeaks, sourceAudioPeaks, systemSidecarPeaks, t]);
|
||||
useEffect(() => {
|
||||
onSourceAudioTracksMetaChange?.(sourceAudioTracks.map((t) => ({ id: t.id, label: t.label })));
|
||||
}, [onSourceAudioTracksMetaChange, sourceAudioTracks]);
|
||||
void sourceAudioTrackSettings;
|
||||
useEffect(() => {
|
||||
onSourceAudioAvailabilityChange?.(sourceAudioTracks.length > 0);
|
||||
}, [onSourceAudioAvailabilityChange, sourceAudioTracks.length]);
|
||||
|
||||
useEffect(() => {
|
||||
if (aspectRatio === "native") {
|
||||
@@ -376,6 +522,25 @@ const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
|
||||
onItemSpanChange={handleItemSpanChange}
|
||||
resolveTargetRowId={getResolvedDropRowId}
|
||||
allRegionSpans={allRegionSpans}
|
||||
onLiveSpanPreviewChange={(id, span) => {
|
||||
setLiveSpanPreviewById((prev) => {
|
||||
if (!span) {
|
||||
if (!(id in prev)) return prev;
|
||||
const next = { ...prev };
|
||||
delete next[id];
|
||||
return next;
|
||||
}
|
||||
const current = prev[id];
|
||||
if (
|
||||
current &&
|
||||
current.start === span.start &&
|
||||
current.end === span.end
|
||||
) {
|
||||
return prev;
|
||||
}
|
||||
return { ...prev, [id]: span };
|
||||
});
|
||||
}}
|
||||
>
|
||||
<KeyframeMarkers
|
||||
keyframes={keyframes}
|
||||
@@ -403,7 +568,11 @@ const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
|
||||
selectAllBlocksActive={selectAllBlocksActive}
|
||||
onClearBlockSelection={clearSelectedBlocks}
|
||||
keyframes={keyframes}
|
||||
audioPeaks={audioPeaks}
|
||||
sourceAudioTracks={sourceAudioTracks}
|
||||
getSourceAudioTrackSettingsForClip={getSourceAudioTrackSettingsForClip}
|
||||
showSourceAudioTrack={showSourceAudioTrack}
|
||||
liveSpanPreviewById={liveZoomPreview.previewSpans}
|
||||
liveHiddenItemIds={Array.from(liveZoomPreview.hiddenZoomIds)}
|
||||
/>
|
||||
</TimelineWrapper>
|
||||
</div>
|
||||
|
||||
@@ -11,18 +11,21 @@ import {
|
||||
type MouseEventHandler,
|
||||
} from "react";
|
||||
import { cn } from "@/lib/utils";
|
||||
import type {
|
||||
SourceAudioTrackSettings,
|
||||
SourceAudioTrackWithPeaks,
|
||||
} from "@/components/video-editor/audio/audioTypes";
|
||||
import {
|
||||
getTimelineContentMinHeightPx,
|
||||
getTimelineRowsMinHeightPx,
|
||||
getTimelineViewportStretchFactor,
|
||||
TIMELINE_AXIS_HEIGHT_PX,
|
||||
} from "../../timelineLayout";
|
||||
import AudioWaveform from "../waveform/AudioWaveform";
|
||||
import glassStyles from "../../ItemGlass.module.css";
|
||||
import Item from "../../Item";
|
||||
import Row from "../../Row";
|
||||
import { CLIP_ROW_ID, ZOOM_ROW_ID } from "../../core/constants";
|
||||
import type { AudioPeaksData, TimelineRenderItem } from "../../core/timelineTypes";
|
||||
import { CLIP_ROW_ID, SOURCE_AUDIO_ROW_ID, ZOOM_ROW_ID } from "../../core/constants";
|
||||
import type { TimelineRenderItem } from "../../core/timelineTypes";
|
||||
import {
|
||||
getAnnotationTrackIndex,
|
||||
getAnnotationTrackRowId,
|
||||
@@ -34,6 +37,7 @@ import {
|
||||
import TimelineAxis from "../axis/TimelineAxis";
|
||||
import ClipMarkerOverlay from "../overlays/ClipMarkerOverlay";
|
||||
import PlaybackCursor from "../playhead/PlaybackCursor";
|
||||
import { useTimelineAudioPeaks } from "../../hooks/useTimelineAudioPeaks";
|
||||
|
||||
const HINT_CLIP = "Press C to split clip";
|
||||
const HINT_ANNOTATION = "Press A to add annotation";
|
||||
@@ -57,7 +61,13 @@ interface TimelineCanvasProps {
|
||||
selectAllBlocksActive?: boolean;
|
||||
onClearBlockSelection?: () => void;
|
||||
keyframes?: { id: string; time: number }[];
|
||||
audioPeaks?: AudioPeaksData | null;
|
||||
sourceAudioTracks?: SourceAudioTrackWithPeaks[];
|
||||
getSourceAudioTrackSettingsForClip?: (
|
||||
clipId: string | null,
|
||||
) => SourceAudioTrackSettings;
|
||||
showSourceAudioTrack?: boolean;
|
||||
liveSpanPreviewById?: Record<string, { start: number; end: number }>;
|
||||
liveHiddenItemIds?: string[];
|
||||
}
|
||||
|
||||
interface TimelineHoverParams {
|
||||
@@ -218,7 +228,13 @@ interface TimelineCanvasRowsProps {
|
||||
onSelectClip?: (id: string | null) => void;
|
||||
onSelectAnnotation?: (id: string | null) => void;
|
||||
onSelectAudio?: (id: string | null) => void;
|
||||
audioPeaks?: AudioPeaksData | null;
|
||||
sourceAudioTracks?: SourceAudioTrackWithPeaks[];
|
||||
getSourceAudioTrackSettingsForClip?: (
|
||||
clipId: string | null,
|
||||
) => SourceAudioTrackSettings;
|
||||
showSourceAudioTrack?: boolean;
|
||||
liveSpanPreviewById?: Record<string, { start: number; end: number }>;
|
||||
liveHiddenItemIds?: string[];
|
||||
direction: string;
|
||||
canShowGhostZoom: boolean;
|
||||
ghostStartMs: number | null;
|
||||
@@ -230,6 +246,44 @@ interface TimelineCanvasRowsProps {
|
||||
onZoomRowClick: MouseEventHandler<HTMLDivElement>;
|
||||
}
|
||||
|
||||
interface AudioItemWithWaveformProps {
|
||||
item: TimelineRenderItem;
|
||||
span: { start: number; end: number };
|
||||
waveformSpan: { start: number; end: number };
|
||||
isSelected: boolean;
|
||||
onSelectAudio?: (id: string | null) => void;
|
||||
}
|
||||
|
||||
function AudioItemWithWaveform({
|
||||
item,
|
||||
span,
|
||||
waveformSpan,
|
||||
isSelected,
|
||||
onSelectAudio,
|
||||
}: AudioItemWithWaveformProps) {
|
||||
const peaks = useTimelineAudioPeaks(item.audioPath ?? null);
|
||||
const normalizedWaveformSpan = useMemo(() => {
|
||||
const duration = Math.max(0, waveformSpan.end - waveformSpan.start);
|
||||
return { start: 0, end: duration };
|
||||
}, [waveformSpan.end, waveformSpan.start]);
|
||||
return (
|
||||
<Item
|
||||
id={item.id}
|
||||
rowId={item.rowId}
|
||||
span={span}
|
||||
isSelected={isSelected}
|
||||
onSelectId={onSelectAudio}
|
||||
variant="audio"
|
||||
waveformPeaks={peaks}
|
||||
waveformSegmentSpan={normalizedWaveformSpan}
|
||||
waveformGain={Math.max(0, Math.min(2, item.audioGain ?? 1))}
|
||||
waveformNormalize={Boolean(item.audioNormalize)}
|
||||
>
|
||||
{item.label}
|
||||
</Item>
|
||||
);
|
||||
}
|
||||
|
||||
const TimelineCanvasRows = memo(function TimelineCanvasRows({
|
||||
items,
|
||||
videoDurationMs,
|
||||
@@ -242,7 +296,11 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({
|
||||
onSelectClip,
|
||||
onSelectAnnotation,
|
||||
onSelectAudio,
|
||||
audioPeaks,
|
||||
sourceAudioTracks = [],
|
||||
getSourceAudioTrackSettingsForClip,
|
||||
showSourceAudioTrack = false,
|
||||
liveSpanPreviewById,
|
||||
liveHiddenItemIds,
|
||||
direction,
|
||||
canShowGhostZoom,
|
||||
ghostStartMs,
|
||||
@@ -253,6 +311,7 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({
|
||||
onZoomRowMouseLeave,
|
||||
onZoomRowClick,
|
||||
}: TimelineCanvasRowsProps) {
|
||||
const hiddenIds = useMemo(() => new Set(liveHiddenItemIds ?? []), [liveHiddenItemIds]);
|
||||
const { clipItems, zoomItems, annotationRows, audioRows } = useMemo(() => {
|
||||
const nextClipItems: TimelineRenderItem[] = [];
|
||||
const nextZoomItems: TimelineRenderItem[] = [];
|
||||
@@ -307,7 +366,6 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({
|
||||
return (
|
||||
<>
|
||||
<Row id={CLIP_ROW_ID} isEmpty={clipItems.length === 0} hint={HINT_CLIP}>
|
||||
{audioPeaks && <AudioWaveform peaks={audioPeaks} />}
|
||||
<ClipMarkerOverlay videoDurationMs={videoDurationMs} />
|
||||
{clipItems.map((item) => (
|
||||
<Item
|
||||
@@ -323,6 +381,35 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({
|
||||
</Item>
|
||||
))}
|
||||
</Row>
|
||||
{showSourceAudioTrack &&
|
||||
sourceAudioTracks.map((track) => (
|
||||
<Row key={track.id} id={`${SOURCE_AUDIO_ROW_ID}-${track.id}`}>
|
||||
{clipItems.filter(item => item.showSourceAudio).map((item) => {
|
||||
const settings = getSourceAudioTrackSettingsForClip?.(item.id)?.[
|
||||
track.id
|
||||
] ?? { volume: 1, normalize: false };
|
||||
return (
|
||||
<Item
|
||||
key={`source-audio-${track.id}-${item.id}`}
|
||||
id={`source-audio-${track.id}-${item.id}`}
|
||||
rowId={`${SOURCE_AUDIO_ROW_ID}-${track.id}`}
|
||||
span={liveSpanPreviewById?.[item.id] ?? item.span}
|
||||
disabled
|
||||
isSelected={selectAllBlocksActive || item.id === selectedClipId}
|
||||
onSelect={() => onSelectClip?.(item.id)}
|
||||
variant="audio"
|
||||
waveformPeaks={track.peaks}
|
||||
waveformSegmentSpan={item.sourceSpan ?? item.span}
|
||||
waveformGain={Math.max(0, Math.min(2, settings.volume))}
|
||||
waveformNormalize={Boolean(settings.normalize)}
|
||||
muted={item.muted}
|
||||
>
|
||||
{track.label}
|
||||
</Item>
|
||||
);
|
||||
})}
|
||||
</Row>
|
||||
))}
|
||||
|
||||
<Row
|
||||
id={ZOOM_ROW_ID}
|
||||
@@ -357,7 +444,9 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{zoomItems.map((item) => (
|
||||
{zoomItems
|
||||
.filter((item) => !hiddenIds.has(item.id))
|
||||
.map((item) => (
|
||||
<Item
|
||||
id={item.id}
|
||||
key={item.id}
|
||||
@@ -395,17 +484,14 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({
|
||||
{audioRows.map(({ rowId, items: rowItems }, index) => (
|
||||
<Row key={rowId} id={rowId} isEmpty={rowItems.length === 0} hint={index === 0 ? HINT_AUDIO : undefined}>
|
||||
{rowItems.map((item) => (
|
||||
<Item
|
||||
id={item.id}
|
||||
<AudioItemWithWaveform
|
||||
key={item.id}
|
||||
rowId={item.rowId}
|
||||
item={item}
|
||||
span={item.span}
|
||||
waveformSpan={liveSpanPreviewById?.[item.id] ?? item.span}
|
||||
isSelected={selectAllBlocksActive || item.id === selectedAudioId}
|
||||
onSelectId={onSelectAudio}
|
||||
variant="audio"
|
||||
>
|
||||
{item.label}
|
||||
</Item>
|
||||
onSelectAudio={onSelectAudio}
|
||||
/>
|
||||
))}
|
||||
</Row>
|
||||
))}
|
||||
@@ -431,7 +517,11 @@ export default function TimelineCanvas({
|
||||
selectAllBlocksActive = false,
|
||||
onClearBlockSelection,
|
||||
keyframes = [],
|
||||
audioPeaks,
|
||||
sourceAudioTracks = [],
|
||||
getSourceAudioTrackSettingsForClip,
|
||||
showSourceAudioTrack = false,
|
||||
liveSpanPreviewById,
|
||||
liveHiddenItemIds,
|
||||
}: TimelineCanvasProps) {
|
||||
const { setTimelineRef, style, sidebarWidth, direction, range, valueToPixels, pixelsToValue } =
|
||||
useTimelineContext();
|
||||
@@ -583,8 +673,9 @@ export default function TimelineCanvas({
|
||||
if (isAnnotationTrackRowId(item.rowId)) annotationRowIds.add(item.rowId);
|
||||
if (isAudioTrackRowId(item.rowId)) audioRowIds.add(item.rowId);
|
||||
}
|
||||
return 2 + annotationRowIds.size + audioRowIds.size;
|
||||
}, [items]);
|
||||
const sourceAudioRows = showSourceAudioTrack ? sourceAudioTracks.length : 0;
|
||||
return 2 + sourceAudioRows + annotationRowIds.size + audioRowIds.size;
|
||||
}, [items, showSourceAudioTrack, sourceAudioTracks.length]);
|
||||
const timelineRowsMinHeightPx = getTimelineRowsMinHeightPx(timelineRowCount);
|
||||
const timelineContentMinHeightPx = getTimelineContentMinHeightPx(timelineRowCount);
|
||||
const timelineViewportStretchFactor = getTimelineViewportStretchFactor(timelineRowCount);
|
||||
@@ -660,7 +751,11 @@ export default function TimelineCanvas({
|
||||
onSelectClip={onSelectClip}
|
||||
onSelectAnnotation={onSelectAnnotation}
|
||||
onSelectAudio={onSelectAudio}
|
||||
audioPeaks={audioPeaks}
|
||||
sourceAudioTracks={sourceAudioTracks}
|
||||
getSourceAudioTrackSettingsForClip={getSourceAudioTrackSettingsForClip}
|
||||
showSourceAudioTrack={showSourceAudioTrack}
|
||||
liveSpanPreviewById={liveSpanPreviewById}
|
||||
liveHiddenItemIds={liveHiddenItemIds}
|
||||
direction={direction}
|
||||
canShowGhostZoom={canShowGhostZoom}
|
||||
ghostStartMs={ghostStartMs}
|
||||
|
||||
@@ -4,6 +4,11 @@ import type { AudioPeaksData } from "../../core/timelineTypes";
|
||||
|
||||
interface AudioWaveformProps {
|
||||
peaks: AudioPeaksData;
|
||||
segmentStartMs?: number;
|
||||
segmentEndMs?: number;
|
||||
gain?: number;
|
||||
normalize?: boolean;
|
||||
className?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -11,10 +16,18 @@ interface AudioWaveformProps {
|
||||
* Automatically syncs with the timeline's visible range so the waveform
|
||||
* scrolls and zooms together with the clip items above it.
|
||||
*/
|
||||
function AudioWaveformComponent({ peaks }: AudioWaveformProps) {
|
||||
function AudioWaveformComponent({
|
||||
peaks,
|
||||
segmentStartMs,
|
||||
segmentEndMs,
|
||||
gain = 1,
|
||||
normalize = false,
|
||||
className,
|
||||
}: AudioWaveformProps) {
|
||||
const canvasRef = useRef<HTMLCanvasElement>(null);
|
||||
const { range } = useTimelineContext();
|
||||
const [resizeKey, setResizeKey] = useState(0);
|
||||
const lastDrawAtRef = useRef(0);
|
||||
|
||||
// Bump resizeKey when the canvas element changes size.
|
||||
const observerRef = useRef<ResizeObserver | null>(null);
|
||||
@@ -34,55 +47,77 @@ function AudioWaveformComponent({ peaks }: AudioWaveformProps) {
|
||||
useEffect(() => {
|
||||
const canvas = canvasRef.current;
|
||||
if (!canvas) return;
|
||||
let rafId = 0;
|
||||
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) return;
|
||||
const draw = () => {
|
||||
const now = performance.now();
|
||||
if (now - lastDrawAtRef.current < 33) {
|
||||
rafId = requestAnimationFrame(draw);
|
||||
return;
|
||||
}
|
||||
lastDrawAtRef.current = now;
|
||||
|
||||
const rect = canvas.getBoundingClientRect();
|
||||
const dpr = window.devicePixelRatio || 1;
|
||||
const width = Math.round(rect.width * dpr);
|
||||
const height = Math.round(rect.height * dpr);
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) return;
|
||||
|
||||
if (width === 0 || height === 0) return;
|
||||
const rect = canvas.getBoundingClientRect();
|
||||
const dpr = window.devicePixelRatio || 1;
|
||||
const width = Math.round(rect.width * dpr);
|
||||
const height = Math.round(rect.height * dpr);
|
||||
|
||||
canvas.width = width;
|
||||
canvas.height = height;
|
||||
if (width === 0 || height === 0) return;
|
||||
|
||||
ctx.clearRect(0, 0, width, height);
|
||||
canvas.width = width;
|
||||
canvas.height = height;
|
||||
|
||||
const { peaks: peakData, durationMs } = peaks;
|
||||
if (durationMs <= 0 || peakData.length === 0) return;
|
||||
ctx.clearRect(0, 0, width, height);
|
||||
|
||||
const visibleStartMs = range.start;
|
||||
const visibleEndMs = range.end;
|
||||
const visibleDurationMs = visibleEndMs - visibleStartMs;
|
||||
if (visibleDurationMs <= 0) return;
|
||||
const { peaks: peakData, durationMs } = peaks;
|
||||
if (durationMs <= 0 || peakData.length === 0) return;
|
||||
|
||||
const midY = height / 2;
|
||||
const rawVisibleStartMs = segmentStartMs ?? range.start;
|
||||
const rawVisibleEndMs = segmentEndMs ?? range.end;
|
||||
const msPerBin = durationMs / peakData.length;
|
||||
const visibleStartMs =
|
||||
msPerBin > 0 ? Math.round(rawVisibleStartMs / msPerBin) * msPerBin : rawVisibleStartMs;
|
||||
const visibleEndMs =
|
||||
msPerBin > 0 ? Math.round(rawVisibleEndMs / msPerBin) * msPerBin : rawVisibleEndMs;
|
||||
const visibleDurationMs = visibleEndMs - visibleStartMs;
|
||||
if (visibleDurationMs <= 0) return;
|
||||
|
||||
ctx.beginPath();
|
||||
for (let px = 0; px < width; px++) {
|
||||
const t = visibleStartMs + (px / width) * visibleDurationMs;
|
||||
const binIndex = Math.min(
|
||||
peakData.length - 1,
|
||||
Math.max(0, Math.floor((t / durationMs) * peakData.length)),
|
||||
);
|
||||
const amplitude = peakData[binIndex];
|
||||
const barHeight = amplitude * midY * 0.85;
|
||||
const midY = height / 2;
|
||||
|
||||
ctx.moveTo(px, midY - barHeight);
|
||||
ctx.lineTo(px, midY + barHeight);
|
||||
}
|
||||
ctx.beginPath();
|
||||
for (let px = 0; px < width; px++) {
|
||||
const t = visibleStartMs + (px / width) * visibleDurationMs;
|
||||
const exactIndex = Math.max(
|
||||
0,
|
||||
Math.min(peakData.length - 1, (t / durationMs) * (peakData.length - 1)),
|
||||
);
|
||||
const leftIndex = Math.floor(exactIndex);
|
||||
const rightIndex = Math.min(peakData.length - 1, leftIndex + 1);
|
||||
const mix = exactIndex - leftIndex;
|
||||
let amplitude = peakData[leftIndex] * (1 - mix) + peakData[rightIndex] * mix;
|
||||
if (normalize) amplitude = Math.sqrt(Math.max(0, amplitude));
|
||||
amplitude = Math.max(0, Math.min(1, amplitude * gain));
|
||||
const barHeight = amplitude * midY * 0.85;
|
||||
|
||||
ctx.strokeStyle = "rgba(255, 255, 255, 0.55)";
|
||||
ctx.lineWidth = dpr;
|
||||
ctx.stroke();
|
||||
}, [peaks, range.start, range.end, resizeKey]);
|
||||
ctx.moveTo(px, midY - barHeight);
|
||||
ctx.lineTo(px, midY + barHeight);
|
||||
}
|
||||
|
||||
ctx.strokeStyle = "rgba(255, 255, 255, 0.55)";
|
||||
ctx.lineWidth = dpr;
|
||||
ctx.stroke();
|
||||
};
|
||||
rafId = requestAnimationFrame(draw);
|
||||
return () => cancelAnimationFrame(rafId);
|
||||
}, [gain, normalize, peaks, range.start, range.end, resizeKey, segmentStartMs, segmentEndMs]);
|
||||
|
||||
return (
|
||||
<canvas
|
||||
ref={setCanvasRef}
|
||||
className="absolute inset-0 w-full h-full pointer-events-none"
|
||||
className={className ?? "absolute inset-0 w-full h-full pointer-events-none"}
|
||||
style={{ display: "block" }}
|
||||
/>
|
||||
);
|
||||
|
||||
@@ -29,6 +29,7 @@ interface TimelineWrapperProps {
|
||||
onItemSpanChange: (id: string, span: Span, rowId?: string) => void;
|
||||
resolveTargetRowId?: (id: string, proposedRowId: string) => string;
|
||||
allRegionSpans?: TimelineRegionSpan[];
|
||||
onLiveSpanPreviewChange?: (id: string, span: Span | null) => void;
|
||||
}
|
||||
|
||||
export default function TimelineWrapper({
|
||||
@@ -43,6 +44,7 @@ export default function TimelineWrapper({
|
||||
onItemSpanChange,
|
||||
resolveTargetRowId,
|
||||
allRegionSpans = [],
|
||||
onLiveSpanPreviewChange,
|
||||
}: TimelineWrapperProps) {
|
||||
const totalMs = Math.max(0, Math.round(videoDuration * 1000));
|
||||
|
||||
@@ -144,8 +146,12 @@ export default function TimelineWrapper({
|
||||
? (event.activatorEvent as PointerEvent).clientX + (event.delta?.x ?? 0)
|
||||
: undefined;
|
||||
if (span) showTooltip(span, screenX);
|
||||
const moved = Math.hypot(event.delta?.x ?? 0, event.delta?.y ?? 0) > 0.01;
|
||||
if (moved) {
|
||||
onLiveSpanPreviewChange?.(event.active.id as string, span ?? null);
|
||||
}
|
||||
},
|
||||
[showTooltip],
|
||||
[onLiveSpanPreviewChange, showTooltip],
|
||||
);
|
||||
|
||||
const onResizeMove = useCallback(
|
||||
@@ -156,8 +162,9 @@ export default function TimelineWrapper({
|
||||
? (event.activatorEvent as PointerEvent).clientX + (event.delta?.x ?? 0)
|
||||
: undefined;
|
||||
if (span) showTooltip(span, screenX);
|
||||
onLiveSpanPreviewChange?.(event.active.id as string, span ?? null);
|
||||
},
|
||||
[showTooltip],
|
||||
[onLiveSpanPreviewChange, showTooltip],
|
||||
);
|
||||
|
||||
const hideTooltip = useCallback(() => showTooltip(null), [showTooltip]);
|
||||
@@ -166,16 +173,18 @@ export default function TimelineWrapper({
|
||||
(event: ResizeEndEvent) => {
|
||||
hideTooltip();
|
||||
onResizeEnd(event);
|
||||
onLiveSpanPreviewChange?.(event.active.id as string, null);
|
||||
},
|
||||
[hideTooltip, onResizeEnd],
|
||||
[hideTooltip, onLiveSpanPreviewChange, onResizeEnd],
|
||||
);
|
||||
|
||||
const onDragEndWithTooltip = useCallback(
|
||||
(event: DragEndEvent) => {
|
||||
hideTooltip();
|
||||
onDragEnd(event);
|
||||
onLiveSpanPreviewChange?.(event.active.id as string, null);
|
||||
},
|
||||
[hideTooltip, onDragEnd],
|
||||
[hideTooltip, onDragEnd, onLiveSpanPreviewChange],
|
||||
);
|
||||
|
||||
const handleRangeChange = useCallback(
|
||||
|
||||
@@ -2,8 +2,10 @@ export const ZOOM_ROW_ID = "row-zoom";
|
||||
export const CLIP_ROW_ID = "row-clip";
|
||||
export const ANNOTATION_ROW_ID = "row-annotation";
|
||||
export const AUDIO_ROW_ID = "row-audio";
|
||||
export const SOURCE_AUDIO_ROW_ID = "row-source-audio";
|
||||
export const ANNOTATION_ROW_PREFIX = `${ANNOTATION_ROW_ID}-`;
|
||||
export const AUDIO_ROW_PREFIX = `${AUDIO_ROW_ID}-`;
|
||||
|
||||
export const FALLBACK_RANGE_MS = 1000;
|
||||
export const TARGET_MARKER_COUNT = 12;
|
||||
export const WAVEFORM_DEFAULT_PEAK_COUNT = 2048;
|
||||
|
||||
@@ -31,10 +31,16 @@ export interface TimelineRenderItem {
|
||||
id: string;
|
||||
rowId: string;
|
||||
span: Span;
|
||||
sourceSpan?: Span;
|
||||
label: string;
|
||||
audioPath?: string;
|
||||
audioGain?: number;
|
||||
audioNormalize?: boolean;
|
||||
zoomDepth?: number;
|
||||
zoomMode?: ZoomMode;
|
||||
speedValue?: number;
|
||||
showSourceAudio?: boolean;
|
||||
muted?: boolean;
|
||||
variant: "zoom" | "trim" | "clip" | "annotation" | "speed" | "audio";
|
||||
}
|
||||
|
||||
|
||||
@@ -1,81 +1,100 @@
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { resolveMediaResourceUrl } from "@/lib/exporter/localMediaSource";
|
||||
import { fromFileUrl } from "../../projectPersistence";
|
||||
import { waveformGenerator } from "../../audio/waveform/WaveformGenerator";
|
||||
import { WAVEFORM_DEFAULT_PEAK_COUNT } from "../core/constants";
|
||||
import type { AudioPeaksData } from "../core/timelineTypes";
|
||||
|
||||
/** Number of peak bins to produce — enough for smooth display at any zoom. */
|
||||
const TARGET_PEAK_COUNT = 2048;
|
||||
function buildSidecarAudioCandidates(sourcePath: string): string[] {
|
||||
const normalized = sourcePath.replace(/\\/g, "/");
|
||||
const lastSlash = normalized.lastIndexOf("/");
|
||||
const dir = lastSlash >= 0 ? normalized.slice(0, lastSlash + 1) : "";
|
||||
const fileName = lastSlash >= 0 ? normalized.slice(lastSlash + 1) : normalized;
|
||||
const dotIndex = fileName.lastIndexOf(".");
|
||||
const baseName = dotIndex > 0 ? fileName.slice(0, dotIndex) : fileName;
|
||||
|
||||
/**
|
||||
* Decode audio from a media file URL and produce a fixed-length array of peak
|
||||
* amplitudes suitable for waveform visualisation.
|
||||
*
|
||||
* Returns `null` while loading or if the file has no decodeable audio.
|
||||
*/
|
||||
export function useTimelineAudioPeaks(fileUrl: string | null | undefined): AudioPeaksData | null {
|
||||
return [
|
||||
`${dir}${baseName}.system.wav`,
|
||||
`${dir}${baseName}.mic.wav`,
|
||||
`${dir}${baseName}.system.m4a`,
|
||||
`${dir}${baseName}.mic.m4a`,
|
||||
];
|
||||
}
|
||||
|
||||
function extractLocalPathFromMediaServerUrl(input: string): string | null {
|
||||
try {
|
||||
const url = new URL(input);
|
||||
const isLocalMediaServer =
|
||||
(url.protocol === "http:" || url.protocol === "https:") &&
|
||||
(url.hostname === "127.0.0.1" || url.hostname === "localhost") &&
|
||||
url.pathname === "/video";
|
||||
if (!isLocalMediaServer) return null;
|
||||
return url.searchParams.get("path");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
interface TimelineAudioPeaksOptions {
|
||||
enableSourceSidecarFallback?: boolean;
|
||||
peakCount?: number;
|
||||
}
|
||||
|
||||
export function useTimelineAudioPeaks(
|
||||
mediaResource: string | null | undefined,
|
||||
options: TimelineAudioPeaksOptions = {},
|
||||
): AudioPeaksData | null {
|
||||
const [data, setData] = useState<AudioPeaksData | null>(null);
|
||||
const urlRef = useRef(fileUrl);
|
||||
const sourceRef = useRef(mediaResource);
|
||||
const enableSourceSidecarFallback = options.enableSourceSidecarFallback ?? false;
|
||||
const peakCount = options.peakCount ?? WAVEFORM_DEFAULT_PEAK_COUNT;
|
||||
|
||||
useEffect(() => {
|
||||
urlRef.current = fileUrl;
|
||||
sourceRef.current = mediaResource;
|
||||
setData(null);
|
||||
|
||||
if (!fileUrl) {
|
||||
return;
|
||||
}
|
||||
if (!mediaResource) return;
|
||||
|
||||
let cancelled = false;
|
||||
|
||||
(async () => {
|
||||
const run = async () => {
|
||||
const tryGenerate = async (resource: string): Promise<AudioPeaksData> => {
|
||||
const resolvedUrl = await resolveMediaResourceUrl(resource);
|
||||
return waveformGenerator.generate(resolvedUrl, peakCount);
|
||||
};
|
||||
|
||||
try {
|
||||
const response = await fetch(fileUrl);
|
||||
if (cancelled) return;
|
||||
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
if (cancelled) return;
|
||||
|
||||
const audioCtx = new OfflineAudioContext(1, 1, 44100);
|
||||
const decoded = await audioCtx.decodeAudioData(arrayBuffer);
|
||||
if (cancelled) return;
|
||||
|
||||
const channelData = decoded.getChannelData(0);
|
||||
const durationMs = decoded.duration * 1000;
|
||||
const binSize = Math.max(1, Math.floor(channelData.length / TARGET_PEAK_COUNT));
|
||||
const peakCount = Math.ceil(channelData.length / binSize);
|
||||
const peaks = new Float32Array(peakCount);
|
||||
|
||||
for (let i = 0; i < peakCount; i++) {
|
||||
const start = i * binSize;
|
||||
const end = Math.min(start + binSize, channelData.length);
|
||||
let max = 0;
|
||||
for (let j = start; j < end; j++) {
|
||||
const abs = Math.abs(channelData[j]);
|
||||
if (abs > max) max = abs;
|
||||
}
|
||||
peaks[i] = max;
|
||||
}
|
||||
|
||||
// Normalise to 0–1 range.
|
||||
let globalMax = 0;
|
||||
for (let i = 0; i < peaks.length; i++) {
|
||||
if (peaks[i] > globalMax) globalMax = peaks[i];
|
||||
}
|
||||
if (globalMax > 0) {
|
||||
for (let i = 0; i < peaks.length; i++) {
|
||||
peaks[i] /= globalMax;
|
||||
}
|
||||
}
|
||||
|
||||
if (!cancelled && urlRef.current === fileUrl) {
|
||||
setData({ peaks, durationMs });
|
||||
}
|
||||
const result = await tryGenerate(mediaResource);
|
||||
if (!cancelled && sourceRef.current === mediaResource) setData(result);
|
||||
return;
|
||||
} catch {
|
||||
// File has no audio or decoding failed — leave as null.
|
||||
// fallthrough
|
||||
}
|
||||
})();
|
||||
|
||||
if (!enableSourceSidecarFallback) return;
|
||||
|
||||
const localPathFromServer = extractLocalPathFromMediaServerUrl(mediaResource);
|
||||
const localSourcePath =
|
||||
localPathFromServer ||
|
||||
(/^file:\/\//i.test(mediaResource) ? fromFileUrl(mediaResource) : mediaResource);
|
||||
if (!localSourcePath) return;
|
||||
|
||||
for (const candidate of buildSidecarAudioCandidates(localSourcePath)) {
|
||||
try {
|
||||
const result = await tryGenerate(candidate);
|
||||
if (!cancelled && sourceRef.current === mediaResource) setData(result);
|
||||
return;
|
||||
} catch {
|
||||
// try next
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
void run();
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [fileUrl]);
|
||||
}, [mediaResource, enableSourceSidecarFallback, peakCount]);
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
@@ -47,13 +47,22 @@ export function buildTimelineItems(params: {
|
||||
variant: "zoom",
|
||||
}));
|
||||
|
||||
const clips: TimelineRenderItem[] = clipRegions.map((region, index) => ({
|
||||
id: region.id,
|
||||
rowId: CLIP_ROW_ID,
|
||||
span: { start: region.startMs, end: region.endMs },
|
||||
label: `Clip ${index + 1}`,
|
||||
variant: "clip",
|
||||
}));
|
||||
const clips: TimelineRenderItem[] = clipRegions.map((region, index) => {
|
||||
const displayDurationMs = Math.max(0, region.endMs - region.startMs);
|
||||
const speed = Number.isFinite(region.speed) && region.speed > 0 ? region.speed : 1;
|
||||
const sourceEndMs = region.startMs + displayDurationMs * speed;
|
||||
|
||||
return {
|
||||
id: region.id,
|
||||
rowId: CLIP_ROW_ID,
|
||||
span: { start: region.startMs, end: region.endMs },
|
||||
sourceSpan: { start: region.startMs, end: sourceEndMs },
|
||||
label: `Clip ${index + 1}`,
|
||||
showSourceAudio: region.showSourceAudio,
|
||||
muted: Boolean(region.muted),
|
||||
variant: "clip",
|
||||
};
|
||||
});
|
||||
|
||||
const annotations: TimelineRenderItem[] = annotationRegions.map((region) => ({
|
||||
id: region.id,
|
||||
@@ -68,6 +77,9 @@ export function buildTimelineItems(params: {
|
||||
rowId: getAudioTrackRowId(region.trackIndex ?? 0),
|
||||
span: { start: region.startMs, end: region.endMs },
|
||||
label: getAudioLabel(region),
|
||||
audioPath: region.audioPath,
|
||||
audioGain: region.volume,
|
||||
audioNormalize: Boolean(region.normalize),
|
||||
variant: "audio",
|
||||
}));
|
||||
|
||||
|
||||
@@ -64,6 +64,7 @@ export type EditorEffectSection =
|
||||
| "crop"
|
||||
| "extensions"
|
||||
| "clip"
|
||||
| "audio"
|
||||
| `ext:${string}`;
|
||||
|
||||
export type ZoomTransitionEasing = "recordly" | "glide" | "smooth" | "snappy" | "linear";
|
||||
@@ -169,6 +170,7 @@ export interface ClipRegion {
|
||||
endMs: number;
|
||||
speed: number;
|
||||
muted?: boolean;
|
||||
showSourceAudio?: boolean;
|
||||
}
|
||||
|
||||
export function getClipSourceEndMs(clip: ClipRegion): number {
|
||||
@@ -464,6 +466,7 @@ export const DEFAULT_PADDING: Padding = {
|
||||
right: 20,
|
||||
linked: true,
|
||||
};
|
||||
export type { SourceAudioTrackSetting, SourceAudioTrackSettings } from "@/components/video-editor/audio/audioTypes";
|
||||
|
||||
export interface AudioRegion {
|
||||
id: string;
|
||||
@@ -471,6 +474,7 @@ export interface AudioRegion {
|
||||
endMs: number;
|
||||
audioPath: string;
|
||||
volume: number;
|
||||
normalize?: boolean;
|
||||
trackIndex?: number;
|
||||
}
|
||||
|
||||
|
||||
@@ -19,7 +19,10 @@
|
||||
},
|
||||
"clip": {
|
||||
"title": "Clip",
|
||||
"muteAudio": "Mute Audio",
|
||||
"mute": "Mute",
|
||||
"mutedState": "Audio is muted",
|
||||
"unmutedState": "Audio is playing",
|
||||
"separateClipFromAudio": "Separate clip from audio",
|
||||
"delete": "Delete Clip"
|
||||
},
|
||||
"effects": {
|
||||
@@ -201,5 +204,16 @@
|
||||
"exportVideo": "Export {{format}}",
|
||||
"reportBug": "Report Bug",
|
||||
"starOnGithub": "Star on GitHub"
|
||||
},
|
||||
"audio": {
|
||||
"title": "Audio",
|
||||
"volumeTitle": "Audio",
|
||||
"volume": "Volume",
|
||||
"normalize": "Normalize",
|
||||
"sourceTracksTitle": "Clip Source Audio",
|
||||
"systemLabel": "Source System",
|
||||
"micLabel": "Source Mic",
|
||||
"mixedLabel": "Source",
|
||||
"deleteRegion": "Delete Audio"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,10 @@
|
||||
},
|
||||
"clip": {
|
||||
"title": "Clip",
|
||||
"muteAudio": "Silenciar audio",
|
||||
"mute": "Silenciar",
|
||||
"mutedState": "El audio está silenciado",
|
||||
"unmutedState": "El audio se está reproduciendo",
|
||||
"separateClipFromAudio": "Separar clip del audio",
|
||||
"delete": "Eliminar clip"
|
||||
},
|
||||
"effects": {
|
||||
@@ -181,5 +184,16 @@
|
||||
"exportVideo": "Exportar {{format}}",
|
||||
"reportBug": "Reportar error",
|
||||
"starOnGithub": "Estrella en GitHub"
|
||||
},
|
||||
"audio": {
|
||||
"title": "Audio",
|
||||
"volumeTitle": "Audio",
|
||||
"volume": "Volumen",
|
||||
"normalize": "Normalizar",
|
||||
"sourceTracksTitle": "Audio fuente del clip",
|
||||
"systemLabel": "Sonido del sistema",
|
||||
"micLabel": "Micrófono",
|
||||
"mixedLabel": "Fuente",
|
||||
"deleteRegion": "Eliminar audio"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,10 @@
|
||||
},
|
||||
"clip": {
|
||||
"title": "Clip",
|
||||
"muteAudio": "Couper le son",
|
||||
"mute": "Sourdine",
|
||||
"mutedState": "Le son est coupé",
|
||||
"unmutedState": "Le son est activé",
|
||||
"separateClipFromAudio": "Séparer le clip de l'audio",
|
||||
"delete": "Supprimer le clip"
|
||||
},
|
||||
"effects": {
|
||||
@@ -181,5 +184,16 @@
|
||||
"exportVideo": "Exporter en {{format}}",
|
||||
"reportBug": "Signaler un bug",
|
||||
"starOnGithub": "Mettre une étoile sur GitHub"
|
||||
},
|
||||
"audio": {
|
||||
"title": "Audio",
|
||||
"volumeTitle": "Audio",
|
||||
"volume": "Volume",
|
||||
"normalize": "Normaliser",
|
||||
"sourceTracksTitle": "Source audio du clip",
|
||||
"systemLabel": "Son système",
|
||||
"micLabel": "Microphone",
|
||||
"mixedLabel": "Source",
|
||||
"deleteRegion": "Supprimer la zone audio"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,10 @@
|
||||
},
|
||||
"clip": {
|
||||
"title": "클립",
|
||||
"muteAudio": "오디오 음소거",
|
||||
"mute": "음소거",
|
||||
"mutedState": "오디오가 음소거됨",
|
||||
"unmutedState": "오디오가 재생 중",
|
||||
"separateClipFromAudio": "클립에서 오디오 분리",
|
||||
"delete": "클립 삭제"
|
||||
},
|
||||
"effects": {
|
||||
@@ -181,5 +184,16 @@
|
||||
"exportVideo": "{{format}} 내보내기",
|
||||
"reportBug": "버그 신고",
|
||||
"starOnGithub": "GitHub에서 별표 주기"
|
||||
},
|
||||
"audio": {
|
||||
"title": "오디오",
|
||||
"volumeTitle": "오디오",
|
||||
"volume": "볼륨",
|
||||
"normalize": "정규화",
|
||||
"sourceTracksTitle": "클립 소스 오디오",
|
||||
"systemLabel": "시스템 소스",
|
||||
"micLabel": "마이크 소스",
|
||||
"mixedLabel": "소스",
|
||||
"deleteRegion": "오디오 삭제"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,10 @@
|
||||
},
|
||||
"clip": {
|
||||
"title": "Clip",
|
||||
"muteAudio": "Audio dempen",
|
||||
"mute": "Dempen",
|
||||
"mutedState": "Audio is gedempt",
|
||||
"unmutedState": "Audio wordt afgespeeld",
|
||||
"separateClipFromAudio": "Clip van audio scheiden",
|
||||
"delete": "Clip verwijderen"
|
||||
},
|
||||
"effects": {
|
||||
@@ -181,5 +184,16 @@
|
||||
"exportVideo": "{{format}} exporteren",
|
||||
"reportBug": "Bug melden",
|
||||
"starOnGithub": "Ster op GitHub"
|
||||
},
|
||||
"audio": {
|
||||
"title": "Audio",
|
||||
"volumeTitle": "Audio",
|
||||
"volume": "Volume",
|
||||
"normalize": "Normaliseren",
|
||||
"sourceTracksTitle": "Bronaudio clip",
|
||||
"systemLabel": "Systeemaudio",
|
||||
"micLabel": "Microfoon",
|
||||
"mixedLabel": "Bron",
|
||||
"deleteRegion": "Audio verwijderen"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,10 @@
|
||||
},
|
||||
"clip": {
|
||||
"title": "Clipe",
|
||||
"muteAudio": "Silenciar áudio",
|
||||
"mute": "Mudo",
|
||||
"mutedState": "O áudio está mudo",
|
||||
"unmutedState": "O áudio está tocando",
|
||||
"separateClipFromAudio": "Separar áudio do clipe",
|
||||
"delete": "Excluir clipe"
|
||||
},
|
||||
"effects": {
|
||||
@@ -181,5 +184,16 @@
|
||||
"exportVideo": "Exportar {{format}}",
|
||||
"reportBug": "Reportar bug",
|
||||
"starOnGithub": "Dar estrela no GitHub"
|
||||
},
|
||||
"audio": {
|
||||
"title": "Áudio",
|
||||
"volumeTitle": "Áudio",
|
||||
"volume": "Volume",
|
||||
"normalize": "Normalizar",
|
||||
"sourceTracksTitle": "Áudio original do clipe",
|
||||
"systemLabel": "Sistema",
|
||||
"micLabel": "Microfone",
|
||||
"mixedLabel": "Fonte",
|
||||
"deleteRegion": "Excluir áudio"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,8 +19,11 @@
|
||||
},
|
||||
"clip": {
|
||||
"title": "Клип",
|
||||
"muteAudio": "Выключить звук",
|
||||
"delete": "Удалить"
|
||||
"mute": "Без звука",
|
||||
"mutedState": "Звук выключен",
|
||||
"unmutedState": "Звук включен",
|
||||
"separateClipFromAudio": "Отделить аудио от клипа",
|
||||
"delete": "Удалить клип"
|
||||
},
|
||||
"effects": {
|
||||
"title": "Эффекты",
|
||||
@@ -201,5 +204,16 @@
|
||||
"exportVideo": "Экспортировать {{format}}",
|
||||
"reportBug": "Сообщить об ошибке",
|
||||
"starOnGithub": "Оценить на GitHub"
|
||||
},
|
||||
"audio": {
|
||||
"title": "Аудио",
|
||||
"volumeTitle": "Аудио",
|
||||
"volume": "Громкость",
|
||||
"normalize": "Нормализовать",
|
||||
"sourceTracksTitle": "Исходное аудио клипа",
|
||||
"systemLabel": "Системный звук",
|
||||
"micLabel": "Микрофон",
|
||||
"mixedLabel": "Источник",
|
||||
"deleteRegion": "Удалить аудио"
|
||||
}
|
||||
}
|
||||
@@ -19,7 +19,10 @@
|
||||
},
|
||||
"clip": {
|
||||
"title": "片段",
|
||||
"muteAudio": "静音音频",
|
||||
"mute": "静音",
|
||||
"mutedState": "音频已静音",
|
||||
"unmutedState": "音频正在播放",
|
||||
"separateClipFromAudio": "将剪辑与音频分离",
|
||||
"delete": "删除片段"
|
||||
},
|
||||
"effects": {
|
||||
@@ -195,6 +198,17 @@
|
||||
"saveProject": "保存项目",
|
||||
"exportVideo": "导出{{format}}",
|
||||
"reportBug": "报告问题",
|
||||
"starOnGithub": "在 GitHub 上加星"
|
||||
"starOnGithub": "在 GitHub 上点赞"
|
||||
},
|
||||
"audio": {
|
||||
"title": "音频",
|
||||
"volumeTitle": "音频",
|
||||
"volume": "音量",
|
||||
"normalize": "标准化",
|
||||
"sourceTracksTitle": "剪辑源音频",
|
||||
"systemLabel": "系统声音",
|
||||
"micLabel": "麦克风",
|
||||
"mixedLabel": "来源",
|
||||
"deleteRegion": "删除音频"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,10 @@
|
||||
},
|
||||
"clip": {
|
||||
"title": "片段",
|
||||
"muteAudio": "靜音",
|
||||
"mute": "靜音",
|
||||
"mutedState": "音訊已靜音",
|
||||
"unmutedState": "音訊正在播放",
|
||||
"separateClipFromAudio": "將片段與音訊分離",
|
||||
"delete": "刪除片段"
|
||||
},
|
||||
"effects": {
|
||||
@@ -181,5 +184,16 @@
|
||||
"exportVideo": "匯出 {{format}}",
|
||||
"reportBug": "回報錯誤",
|
||||
"starOnGithub": "在 GitHub 按讚"
|
||||
},
|
||||
"audio": {
|
||||
"title": "音訊",
|
||||
"volumeTitle": "音訊",
|
||||
"volume": "音量",
|
||||
"normalize": "正規化",
|
||||
"sourceTracksTitle": "剪輯源音訊",
|
||||
"systemLabel": "系統源",
|
||||
"micLabel": "麥克風源",
|
||||
"mixedLabel": "源",
|
||||
"deleteRegion": "刪除音訊"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,12 +3,18 @@ import type {
|
||||
AudioRegion,
|
||||
ClipRegion,
|
||||
SpeedRegion,
|
||||
SourceAudioTrackSettings,
|
||||
TrimRegion,
|
||||
} from "@/components/video-editor/types";
|
||||
import {
|
||||
buildResolvedAudioPlan,
|
||||
SourceTrackId,
|
||||
} from "@/lib/exporter/audioRoutingEngine";
|
||||
import { estimateCompanionAudioStartDelaySeconds } from "@/lib/mediaTiming";
|
||||
import { resolveMediaElementSource } from "./localMediaSource";
|
||||
import type { VideoMuxer } from "./muxer";
|
||||
import { resolveSourceAudioFallbackPaths } from "./sourceAudioFallback";
|
||||
import { resolveSourceTrackRoutingPolicy } from "./sourceTrackRoutingPolicy";
|
||||
import { SOURCE_AUDIO_NORMALIZE_GAIN } from "@/components/video-editor/audio/audioTypes";
|
||||
|
||||
const AUDIO_BITRATE = 128_000;
|
||||
const DECODE_BACKPRESSURE_LIMIT = 20;
|
||||
@@ -19,6 +25,60 @@ const OFFLINE_AUDIO_SAMPLE_RATE = 48_000;
|
||||
const OFFLINE_ENCODE_CHUNK_FRAMES = 1024;
|
||||
const OFFLINE_CHUNK_DURATION_SEC = 30;
|
||||
|
||||
function resolveSourceTrackGain(
|
||||
sourceAudioTrackSettings: SourceAudioTrackSettings | undefined,
|
||||
trackId: "mic" | "system" | "mixed",
|
||||
) {
|
||||
const settings = sourceAudioTrackSettings?.[trackId];
|
||||
if (!settings) {
|
||||
return 1;
|
||||
}
|
||||
const normalizeGain = settings.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1;
|
||||
return Math.max(0, Math.min(2, settings.volume * normalizeGain));
|
||||
}
|
||||
|
||||
export function getSourceTrackIdFromPath(audioPath: string): SourceTrackId {
|
||||
const normalized = audioPath.toLowerCase();
|
||||
// Check for common patterns like .mic., -mic., mic.mp4, etc.
|
||||
if (
|
||||
normalized.includes(".mic.") ||
|
||||
normalized.includes("-mic.") ||
|
||||
normalized.includes("_mic_") ||
|
||||
normalized.includes("/mic.") ||
|
||||
normalized.includes("\\mic.") ||
|
||||
normalized.endsWith("mic.mp4") ||
|
||||
normalized.endsWith("mic.m4a") ||
|
||||
normalized.endsWith("mic.wav")
|
||||
) {
|
||||
return "mic";
|
||||
}
|
||||
if (
|
||||
normalized.includes(".system.") ||
|
||||
normalized.includes("-system.") ||
|
||||
normalized.includes("_system_") ||
|
||||
normalized.includes("/system.") ||
|
||||
normalized.includes("\\system.") ||
|
||||
normalized.endsWith("system.mp4") ||
|
||||
normalized.endsWith("system.m4a") ||
|
||||
normalized.endsWith("system.wav")
|
||||
) {
|
||||
return "system";
|
||||
}
|
||||
return "mixed";
|
||||
}
|
||||
|
||||
export function hasNonDefaultSourceTrackSettings(
|
||||
sourceAudioTrackSettings?: SourceAudioTrackSettings,
|
||||
) {
|
||||
if (!sourceAudioTrackSettings) {
|
||||
return false;
|
||||
}
|
||||
return Object.values(sourceAudioTrackSettings).some(
|
||||
(settings) =>
|
||||
Math.abs((settings?.volume ?? 1) - 1) > 0.0005 || Boolean(settings?.normalize),
|
||||
);
|
||||
}
|
||||
|
||||
interface TimelineSlice {
|
||||
sourceStartMs: number;
|
||||
sourceEndMs: number;
|
||||
@@ -26,9 +86,10 @@ interface TimelineSlice {
|
||||
}
|
||||
|
||||
interface PreparedOfflineRender {
|
||||
mainBuffer: AudioBuffer | null;
|
||||
companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number }>;
|
||||
mainBufferEntry: { buffer: AudioBuffer; gain: number } | null;
|
||||
companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number; gain: number }>;
|
||||
regionEntries: Array<{ buffer: AudioBuffer; region: AudioRegion }>;
|
||||
mutedSourceOutputRangesSec: Array<{ startSec: number; endSec: number }>;
|
||||
slices: TimelineSlice[];
|
||||
outputDurationMs: number;
|
||||
numChannels: number;
|
||||
@@ -144,6 +205,8 @@ export class AudioProcessor {
|
||||
audioRegions?: AudioRegion[],
|
||||
sourceAudioFallbackPaths?: string[],
|
||||
sourceAudioFallbackStartDelayMsByPath?: Record<string, number>,
|
||||
sourceAudioTrackSettings?: SourceAudioTrackSettings,
|
||||
clipRegions?: ClipRegion[],
|
||||
): Promise<void> {
|
||||
const sortedTrims = trimRegions
|
||||
? [...trimRegions].sort((a, b) => a.startMs - b.startMs)
|
||||
@@ -161,23 +224,25 @@ export class AudioProcessor {
|
||||
(audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0,
|
||||
)
|
||||
: [];
|
||||
const { hasEmbeddedSourceAudio, externalAudioPaths } = resolveSourceAudioFallbackPaths(
|
||||
const routingPolicy = resolveSourceTrackRoutingPolicy(
|
||||
videoUrl,
|
||||
sortedSourceAudioFallbackPaths,
|
||||
);
|
||||
const hasTimedCompanionAudio = externalAudioPaths.some(
|
||||
const hasTimedCompanionAudio = routingPolicy.playbackPaths.some(
|
||||
(audioPath) => (sourceAudioFallbackStartDelayMsByPath?.[audioPath] ?? 0) > 0,
|
||||
);
|
||||
const needsSourceAudioMixing =
|
||||
externalAudioPaths.length > 1 ||
|
||||
(hasEmbeddedSourceAudio && externalAudioPaths.length > 0) ||
|
||||
routingPolicy.playbackPaths.length > 1 ||
|
||||
(routingPolicy.hasEmbeddedSourceAudio && routingPolicy.playbackPaths.length > 0) ||
|
||||
hasTimedCompanionAudio;
|
||||
|
||||
// When speed edits, audio regions, or multiple audio sources need mixing, use offline AudioContext pipeline.
|
||||
if (
|
||||
sortedSpeedRegions.length > 0 ||
|
||||
sortedAudioRegions.length > 0 ||
|
||||
needsSourceAudioMixing
|
||||
needsSourceAudioMixing ||
|
||||
hasNonDefaultSourceTrackSettings(sourceAudioTrackSettings) ||
|
||||
(clipRegions ?? []).some((clip) => Boolean(clip.muted))
|
||||
) {
|
||||
await this.renderAndMuxOfflineAudio(
|
||||
videoUrl,
|
||||
@@ -186,14 +251,16 @@ export class AudioProcessor {
|
||||
sortedAudioRegions,
|
||||
sortedSourceAudioFallbackPaths,
|
||||
sourceAudioFallbackStartDelayMsByPath,
|
||||
sourceAudioTrackSettings,
|
||||
clipRegions,
|
||||
muxer,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Single sidecar audio with no speed/audio edits: demux directly (skips slow real-time rendering).
|
||||
if (!hasEmbeddedSourceAudio && externalAudioPaths.length === 1) {
|
||||
const sidecarDemuxer = await this.loadAudioFileDemuxer(externalAudioPaths[0]);
|
||||
if (!routingPolicy.hasEmbeddedSourceAudio && routingPolicy.playbackPaths.length === 1) {
|
||||
const sidecarDemuxer = await this.loadAudioFileDemuxer(routingPolicy.playbackPaths[0]);
|
||||
if (sidecarDemuxer) {
|
||||
try {
|
||||
await this.processTrimOnlyAudio(sidecarDemuxer, muxer, sortedTrims);
|
||||
@@ -215,8 +282,10 @@ export class AudioProcessor {
|
||||
sortedTrims,
|
||||
[],
|
||||
[],
|
||||
externalAudioPaths,
|
||||
routingPolicy.playbackPaths,
|
||||
sourceAudioFallbackStartDelayMsByPath,
|
||||
sourceAudioTrackSettings,
|
||||
clipRegions,
|
||||
muxer,
|
||||
);
|
||||
return;
|
||||
@@ -263,6 +332,8 @@ export class AudioProcessor {
|
||||
audioRegions?: AudioRegion[],
|
||||
sourceAudioFallbackPaths?: string[],
|
||||
sourceAudioFallbackStartDelayMsByPath?: Record<string, number>,
|
||||
sourceAudioTrackSettings?: SourceAudioTrackSettings,
|
||||
clipRegions?: ClipRegion[],
|
||||
): Promise<Blob> {
|
||||
const sortedTrims = trimRegions
|
||||
? [...trimRegions].sort((a, b) => a.startMs - b.startMs)
|
||||
@@ -288,6 +359,8 @@ export class AudioProcessor {
|
||||
sortedAudioRegions,
|
||||
sortedSourceAudioFallbackPaths,
|
||||
sourceAudioFallbackStartDelayMsByPath,
|
||||
sourceAudioTrackSettings,
|
||||
clipRegions,
|
||||
);
|
||||
return this.renderToWavBlobChunked(prepared);
|
||||
}
|
||||
@@ -563,6 +636,8 @@ export class AudioProcessor {
|
||||
audioRegions: AudioRegion[],
|
||||
sourceAudioFallbackPaths: string[],
|
||||
sourceAudioFallbackStartDelayMsByPath: Record<string, number> | undefined,
|
||||
sourceAudioTrackSettings: SourceAudioTrackSettings | undefined,
|
||||
clipRegions: ClipRegion[] | undefined,
|
||||
muxer: VideoMuxer,
|
||||
): Promise<void> {
|
||||
const prepared = await this.prepareOfflineRender(
|
||||
@@ -572,6 +647,8 @@ export class AudioProcessor {
|
||||
audioRegions,
|
||||
sourceAudioFallbackPaths,
|
||||
sourceAudioFallbackStartDelayMsByPath,
|
||||
sourceAudioTrackSettings,
|
||||
clipRegions,
|
||||
);
|
||||
if (this.cancelled) return;
|
||||
await this.renderAndEncodeChunked(prepared, muxer);
|
||||
@@ -584,31 +661,59 @@ export class AudioProcessor {
|
||||
audioRegions: AudioRegion[],
|
||||
sourceAudioFallbackPaths: string[],
|
||||
sourceAudioFallbackStartDelayMsByPath?: Record<string, number>,
|
||||
sourceAudioTrackSettings?: SourceAudioTrackSettings,
|
||||
clipRegions?: ClipRegion[],
|
||||
): Promise<PreparedOfflineRender> {
|
||||
if (this.cancelled) throw new Error("Export cancelled");
|
||||
this.onProgress?.(0);
|
||||
|
||||
const { externalAudioPaths } = resolveSourceAudioFallbackPaths(
|
||||
videoUrl,
|
||||
const resolvedPlan = buildResolvedAudioPlan({
|
||||
videoResource: videoUrl,
|
||||
sourceAudioFallbackPaths,
|
||||
);
|
||||
audioRegions,
|
||||
sourceTrackGainById: {
|
||||
mic: resolveSourceTrackGain(sourceAudioTrackSettings, "mic"),
|
||||
system: resolveSourceTrackGain(sourceAudioTrackSettings, "system"),
|
||||
mixed: resolveSourceTrackGain(sourceAudioTrackSettings, "mixed"),
|
||||
},
|
||||
embeddedGain: Math.max(
|
||||
0,
|
||||
Math.min(
|
||||
2,
|
||||
sourceAudioTrackSettings?.mixed
|
||||
? resolveSourceTrackGain(sourceAudioTrackSettings, "mixed")
|
||||
: sourceAudioTrackSettings?.system
|
||||
? resolveSourceTrackGain(sourceAudioTrackSettings, "system")
|
||||
: 1,
|
||||
),
|
||||
),
|
||||
});
|
||||
|
||||
// Decode embedded source audio separately from companion sidecars.
|
||||
const mainBuffer = await this.decodeAudioFromUrl(videoUrl);
|
||||
const mainBuffer = resolvedPlan.includeEmbeddedInExport
|
||||
? await this.decodeAudioFromUrl(videoUrl)
|
||||
: null;
|
||||
const mainBufferGain = resolveSourceTrackGain(sourceAudioTrackSettings, "mixed");
|
||||
const mainBufferEntry = mainBuffer ? { buffer: mainBuffer, gain: mainBufferGain } : null;
|
||||
if (this.cancelled) throw new Error("Export cancelled");
|
||||
|
||||
// Decode companion / sidecar audio files
|
||||
const companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number }> = [];
|
||||
const companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number; gain: number }> =
|
||||
[];
|
||||
const refDuration =
|
||||
mainBuffer?.duration ??
|
||||
(externalAudioPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0);
|
||||
for (const audioPath of externalAudioPaths) {
|
||||
(resolvedPlan.playbackPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0);
|
||||
for (const audioPath of resolvedPlan.playbackPaths) {
|
||||
if (this.cancelled) throw new Error("Export cancelled");
|
||||
const buffer = await this.decodeAudioFromUrl(audioPath);
|
||||
if (!buffer) continue;
|
||||
|
||||
companionEntries.push({
|
||||
buffer,
|
||||
gain: resolveSourceTrackGain(
|
||||
sourceAudioTrackSettings,
|
||||
getSourceTrackIdFromPath(audioPath),
|
||||
),
|
||||
startDelaySec: estimateCompanionAudioStartDelaySeconds(
|
||||
refDuration,
|
||||
buffer.duration,
|
||||
@@ -629,15 +734,15 @@ export class AudioProcessor {
|
||||
this.onProgress?.(0.2);
|
||||
|
||||
// Determine source duration for timeline calculation
|
||||
const primaryBuffer = mainBuffer ?? companionEntries[0]?.buffer ?? null;
|
||||
const primaryBuffer = mainBufferEntry?.buffer ?? companionEntries[0]?.buffer ?? null;
|
||||
if (!primaryBuffer && regionEntries.length === 0) {
|
||||
throw new Error("No decodable audio sources found");
|
||||
}
|
||||
|
||||
let sourceDurationSec: number;
|
||||
if (mainBuffer) {
|
||||
sourceDurationSec = mainBuffer.duration;
|
||||
} else if (externalAudioPaths.length > 0 || regionEntries.length > 0) {
|
||||
if (mainBufferEntry?.buffer) {
|
||||
sourceDurationSec = mainBufferEntry.buffer.duration;
|
||||
} else if (resolvedPlan.playbackPaths.length > 0 || regionEntries.length > 0) {
|
||||
sourceDurationSec = await this.getMediaDurationSec(videoUrl);
|
||||
} else {
|
||||
sourceDurationSec = primaryBuffer?.duration ?? 0;
|
||||
@@ -659,11 +764,24 @@ export class AudioProcessor {
|
||||
}
|
||||
|
||||
const numChannels = Math.min(primaryBuffer?.numberOfChannels ?? 2, 2);
|
||||
const mutedSourceOutputRangesSec = (clipRegions ?? [])
|
||||
.filter(
|
||||
(clip) =>
|
||||
Boolean(clip.muted) &&
|
||||
Number.isFinite(clip.startMs) &&
|
||||
Number.isFinite(clip.endMs) &&
|
||||
clip.endMs > clip.startMs,
|
||||
)
|
||||
.map((clip) => ({
|
||||
startSec: Math.max(0, clip.startMs / 1000),
|
||||
endSec: Math.max(0, clip.endMs / 1000),
|
||||
}));
|
||||
|
||||
return {
|
||||
mainBuffer,
|
||||
mainBufferEntry,
|
||||
companionEntries,
|
||||
regionEntries,
|
||||
mutedSourceOutputRangesSec,
|
||||
slices,
|
||||
outputDurationMs,
|
||||
numChannels,
|
||||
@@ -788,14 +906,16 @@ export class AudioProcessor {
|
||||
);
|
||||
|
||||
// Schedule main audio
|
||||
if (prepared.mainBuffer) {
|
||||
if (prepared.mainBufferEntry) {
|
||||
this.scheduleBufferThroughTimeline(
|
||||
offlineCtx,
|
||||
prepared.mainBuffer,
|
||||
prepared.mainBufferEntry.buffer,
|
||||
slices,
|
||||
0,
|
||||
prepared.mainBufferEntry.gain,
|
||||
outputOffsetSec,
|
||||
chunkSec,
|
||||
prepared.mutedSourceOutputRangesSec,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -806,8 +926,10 @@ export class AudioProcessor {
|
||||
entry.buffer,
|
||||
slices,
|
||||
entry.startDelaySec,
|
||||
entry.gain,
|
||||
outputOffsetSec,
|
||||
chunkSec,
|
||||
prepared.mutedSourceOutputRangesSec,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -865,7 +987,8 @@ export class AudioProcessor {
|
||||
if (duration <= 0.001) return;
|
||||
|
||||
const gainNode = ctx.createGain();
|
||||
gainNode.gain.value = Math.max(0, Math.min(1, region.volume));
|
||||
const normalizeGain = region.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1;
|
||||
gainNode.gain.value = Math.max(0, Math.min(1, region.volume * normalizeGain));
|
||||
gainNode.connect(ctx.destination);
|
||||
|
||||
const source = ctx.createBufferSource();
|
||||
@@ -1265,8 +1388,10 @@ export class AudioProcessor {
|
||||
buffer: AudioBuffer,
|
||||
slices: TimelineSlice[],
|
||||
sourceStartDelaySec: number,
|
||||
gain = 1,
|
||||
chunkOutputStartSec = 0,
|
||||
chunkDurationSec = Number.POSITIVE_INFINITY,
|
||||
mutedOutputRangesSec: Array<{ startSec: number; endSec: number }> = [],
|
||||
): void {
|
||||
let outputOffsetSec = 0;
|
||||
|
||||
@@ -1330,12 +1455,51 @@ export class AudioProcessor {
|
||||
continue;
|
||||
}
|
||||
|
||||
const source = ctx.createBufferSource();
|
||||
source.buffer = buffer;
|
||||
source.playbackRate.value = slice.speed;
|
||||
source.connect(ctx.destination);
|
||||
const audibleRanges: Array<{ startSec: number; endSec: number }> = [
|
||||
{
|
||||
startSec: localOutputStartSec + chunkOutputStartSec,
|
||||
endSec:
|
||||
localOutputStartSec + chunkOutputStartSec + effectiveSourceDurationSec / slice.speed,
|
||||
},
|
||||
];
|
||||
for (const mutedRange of mutedOutputRangesSec) {
|
||||
for (let rangeIndex = audibleRanges.length - 1; rangeIndex >= 0; rangeIndex -= 1) {
|
||||
const current = audibleRanges[rangeIndex];
|
||||
const overlapStart = Math.max(current.startSec, mutedRange.startSec);
|
||||
const overlapEnd = Math.min(current.endSec, mutedRange.endSec);
|
||||
if (overlapEnd <= overlapStart) {
|
||||
continue;
|
||||
}
|
||||
audibleRanges.splice(rangeIndex, 1);
|
||||
if (current.startSec < overlapStart) {
|
||||
audibleRanges.push({ startSec: current.startSec, endSec: overlapStart });
|
||||
}
|
||||
if (overlapEnd < current.endSec) {
|
||||
audibleRanges.push({ startSec: overlapEnd, endSec: current.endSec });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
source.start(localOutputStartSec, effectiveBufferStartSec, effectiveSourceDurationSec);
|
||||
for (const audibleRange of audibleRanges) {
|
||||
const audibleDurationSec = audibleRange.endSec - audibleRange.startSec;
|
||||
if (audibleDurationSec <= 0.001) {
|
||||
continue;
|
||||
}
|
||||
const source = ctx.createBufferSource();
|
||||
const gainNode = ctx.createGain();
|
||||
gainNode.gain.value = Math.max(0, Math.min(2, gain));
|
||||
source.buffer = buffer;
|
||||
source.playbackRate.value = slice.speed;
|
||||
source.connect(gainNode);
|
||||
gainNode.connect(ctx.destination);
|
||||
|
||||
const sourceOffsetSec =
|
||||
effectiveBufferStartSec +
|
||||
(audibleRange.startSec - (localOutputStartSec + chunkOutputStartSec)) * slice.speed;
|
||||
const localStartSec = audibleRange.startSec - chunkOutputStartSec;
|
||||
const sourceDurationSec = audibleDurationSec * slice.speed;
|
||||
source.start(localStartSec, sourceOffsetSec, sourceDurationSec);
|
||||
}
|
||||
|
||||
outputOffsetSec += sliceOutputDurationSec;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
import type { AudioRegion } from "@/components/video-editor/types";
|
||||
import { SOURCE_AUDIO_NORMALIZE_GAIN } from "@/components/video-editor/audio/audioTypes";
|
||||
import { resolveSourceAudioFallbackPaths } from "./sourceAudioFallback";
|
||||
|
||||
export type SourceTrackId = "mic" | "system" | "mixed";
|
||||
export type ResolvedAudioTrackKind = "user" | "system" | "mic" | "mixed" | "embedded";
|
||||
|
||||
export interface ResolvedAudioTrack {
|
||||
id: string;
|
||||
kind: ResolvedAudioTrackKind;
|
||||
sourceRef: {
|
||||
path: string;
|
||||
startDelayMs: number;
|
||||
};
|
||||
gain: number;
|
||||
timelineBinding: {
|
||||
startMs: number;
|
||||
endMs: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface ResolvedAudioPlan {
|
||||
hasEmbeddedSourceAudio: boolean;
|
||||
pathsByTrack: Partial<Record<SourceTrackId, string>>;
|
||||
playbackPaths: string[];
|
||||
muteEmbeddedPreview: boolean;
|
||||
includeEmbeddedInExport: boolean;
|
||||
tracks: ResolvedAudioTrack[];
|
||||
masterGain: number;
|
||||
}
|
||||
|
||||
export function getSourceTrackIdFromPath(audioPath: string): SourceTrackId {
|
||||
const normalized = audioPath.toLowerCase();
|
||||
if (normalized.includes(".mic.")) return "mic";
|
||||
if (normalized.includes(".system.")) return "system";
|
||||
return "mixed";
|
||||
}
|
||||
|
||||
function clampGain(value: number, max: number) {
|
||||
if (!Number.isFinite(value)) return 1;
|
||||
return Math.max(0, Math.min(max, value));
|
||||
}
|
||||
|
||||
export function buildResolvedAudioPlan(input: {
|
||||
videoResource: string | null | undefined;
|
||||
sourceAudioFallbackPaths: string[] | null | undefined;
|
||||
audioRegions?: AudioRegion[];
|
||||
sourceTrackGainById?: Partial<Record<SourceTrackId, number>>;
|
||||
embeddedGain?: number;
|
||||
masterGain?: number;
|
||||
}): ResolvedAudioPlan {
|
||||
const { hasEmbeddedSourceAudio, externalAudioPaths } = resolveSourceAudioFallbackPaths(
|
||||
input.videoResource,
|
||||
input.sourceAudioFallbackPaths,
|
||||
);
|
||||
|
||||
const pathsByTrack: Partial<Record<SourceTrackId, string>> = {};
|
||||
for (const path of externalAudioPaths) {
|
||||
const trackId = getSourceTrackIdFromPath(path);
|
||||
if (!pathsByTrack[trackId]) {
|
||||
pathsByTrack[trackId] = path;
|
||||
}
|
||||
}
|
||||
|
||||
const hasDedicatedTracks = Boolean(pathsByTrack.system || pathsByTrack.mic);
|
||||
const playbackPaths: string[] = [];
|
||||
if (pathsByTrack.system) playbackPaths.push(pathsByTrack.system);
|
||||
if (pathsByTrack.mic) playbackPaths.push(pathsByTrack.mic);
|
||||
if (!hasDedicatedTracks && pathsByTrack.mixed) playbackPaths.push(pathsByTrack.mixed);
|
||||
|
||||
const includeEmbeddedInExport = !pathsByTrack.system && !pathsByTrack.mixed;
|
||||
const resolvedRegions = (input.audioRegions ?? []).slice().sort((a, b) => a.startMs - b.startMs);
|
||||
const tracks: ResolvedAudioTrack[] = resolvedRegions.map((region) => ({
|
||||
id: `user:${region.id}`,
|
||||
kind: "user",
|
||||
sourceRef: {
|
||||
path: region.audioPath,
|
||||
startDelayMs: 0,
|
||||
},
|
||||
gain: clampGain(region.volume * (region.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1), 1),
|
||||
timelineBinding: {
|
||||
startMs: Math.max(0, region.startMs),
|
||||
endMs: Math.max(0, region.endMs),
|
||||
},
|
||||
}));
|
||||
|
||||
for (const audioPath of playbackPaths) {
|
||||
const trackId = getSourceTrackIdFromPath(audioPath);
|
||||
tracks.push({
|
||||
id: `${trackId}:${audioPath}`,
|
||||
kind: trackId,
|
||||
sourceRef: {
|
||||
path: audioPath,
|
||||
startDelayMs: 0,
|
||||
},
|
||||
gain: clampGain(input.sourceTrackGainById?.[trackId] ?? 1, 2),
|
||||
timelineBinding: {
|
||||
startMs: 0,
|
||||
endMs: Number.POSITIVE_INFINITY,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (hasEmbeddedSourceAudio && input.videoResource) {
|
||||
tracks.push({
|
||||
id: `embedded:${input.videoResource}`,
|
||||
kind: "embedded",
|
||||
sourceRef: {
|
||||
path: input.videoResource,
|
||||
startDelayMs: 0,
|
||||
},
|
||||
gain: clampGain(input.embeddedGain ?? input.sourceTrackGainById?.mixed ?? 1, 2),
|
||||
timelineBinding: {
|
||||
startMs: 0,
|
||||
endMs: Number.POSITIVE_INFINITY,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
hasEmbeddedSourceAudio,
|
||||
pathsByTrack,
|
||||
playbackPaths,
|
||||
muteEmbeddedPreview: hasDedicatedTracks && !includeEmbeddedInExport,
|
||||
includeEmbeddedInExport,
|
||||
tracks,
|
||||
masterGain: clampGain(input.masterGain ?? 1, 1),
|
||||
};
|
||||
}
|
||||
@@ -3,11 +3,13 @@ import type {
|
||||
AudioRegion,
|
||||
AutoCaptionSettings,
|
||||
CaptionCue,
|
||||
ClipRegion,
|
||||
CropRegion,
|
||||
CursorStyle,
|
||||
CursorTelemetryPoint,
|
||||
Padding,
|
||||
SpeedRegion,
|
||||
SourceAudioTrackSettings,
|
||||
TrimRegion,
|
||||
WebcamOverlaySettings,
|
||||
ZoomMotionBlurTuning,
|
||||
@@ -135,8 +137,10 @@ interface VideoExporterConfig extends ExportConfig {
|
||||
zoomClassicMode?: boolean;
|
||||
frame?: string | null;
|
||||
audioRegions?: AudioRegion[];
|
||||
clipRegions?: ClipRegion[];
|
||||
sourceAudioFallbackPaths?: string[];
|
||||
sourceAudioFallbackStartDelayMsByPath?: Record<string, number>;
|
||||
sourceAudioTrackSettings?: SourceAudioTrackSettings;
|
||||
previewWidth?: number;
|
||||
previewHeight?: number;
|
||||
onProgress?: (progress: ExportProgress) => void;
|
||||
@@ -167,6 +171,16 @@ type NativeAudioPlan =
|
||||
};
|
||||
|
||||
const FILTERGRAPH_FALLBACK_AUDIO_SAMPLE_RATE = 48_000;
|
||||
|
||||
function hasNonDefaultSourceTrackSettings(sourceAudioTrackSettings?: SourceAudioTrackSettings) {
|
||||
if (!sourceAudioTrackSettings) {
|
||||
return false;
|
||||
}
|
||||
return Object.values(sourceAudioTrackSettings).some(
|
||||
(settings) =>
|
||||
Math.abs((settings?.volume ?? 1) - 1) > 0.0005 || Boolean(settings?.normalize),
|
||||
);
|
||||
}
|
||||
const MIN_NATIVE_STATIC_LAYOUT_SPEED = 0.25;
|
||||
const MAX_NATIVE_STATIC_LAYOUT_SPEED = 30;
|
||||
|
||||
@@ -752,6 +766,8 @@ export class ModernVideoExporter {
|
||||
this.config.audioRegions,
|
||||
this.config.sourceAudioFallbackPaths,
|
||||
this.config.sourceAudioFallbackStartDelayMsByPath,
|
||||
this.config.sourceAudioTrackSettings,
|
||||
this.config.clipRegions,
|
||||
),
|
||||
"audio processing",
|
||||
"audio",
|
||||
@@ -1181,7 +1197,9 @@ export class ModernVideoExporter {
|
||||
speedRegions.length > 0 ||
|
||||
audioRegions.length > 0 ||
|
||||
sourceAudioFallbackPaths.length > 1 ||
|
||||
hasTimedSourceAudioFallback
|
||||
hasTimedSourceAudioFallback ||
|
||||
hasNonDefaultSourceTrackSettings(this.config.sourceAudioTrackSettings) ||
|
||||
(this.config.clipRegions ?? []).some((clip) => Boolean(clip.muted))
|
||||
) {
|
||||
const sourceDurationMs = Math.max(
|
||||
0,
|
||||
@@ -1201,16 +1219,20 @@ export class ModernVideoExporter {
|
||||
typeof primaryAudioSourceSampleRate === "number" &&
|
||||
Number.isFinite(primaryAudioSourceSampleRate) &&
|
||||
primaryAudioSourceSampleRate > 0;
|
||||
const strategy = canUsePrimaryAudioFiltergraph
|
||||
? classifyEditedTrackStrategy({
|
||||
primaryAudioSourcePath,
|
||||
sourceDurationMs,
|
||||
trimRegions,
|
||||
speedRegions,
|
||||
audioRegions,
|
||||
sourceAudioFallbackPaths,
|
||||
})
|
||||
: "offline-render-fallback";
|
||||
const requiresRenderedEditedTrack =
|
||||
hasNonDefaultSourceTrackSettings(this.config.sourceAudioTrackSettings) ||
|
||||
(this.config.clipRegions ?? []).some((clip) => Boolean(clip.muted));
|
||||
const strategy =
|
||||
canUsePrimaryAudioFiltergraph && !requiresRenderedEditedTrack
|
||||
? classifyEditedTrackStrategy({
|
||||
primaryAudioSourcePath,
|
||||
sourceDurationMs,
|
||||
trimRegions,
|
||||
speedRegions,
|
||||
audioRegions,
|
||||
sourceAudioFallbackPaths,
|
||||
})
|
||||
: "offline-render-fallback";
|
||||
|
||||
if (strategy === "filtergraph-fast-path") {
|
||||
const audioSourcePath = primaryAudioSourcePath;
|
||||
@@ -1805,6 +1827,8 @@ export class ModernVideoExporter {
|
||||
this.config.audioRegions,
|
||||
this.config.sourceAudioFallbackPaths,
|
||||
this.config.sourceAudioFallbackStartDelayMsByPath,
|
||||
this.config.sourceAudioTrackSettings,
|
||||
this.config.clipRegions,
|
||||
),
|
||||
description,
|
||||
"audio",
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { resolveSourceTrackRoutingPolicy } from "./sourceTrackRoutingPolicy";
|
||||
|
||||
describe("resolveSourceTrackRoutingPolicy", () => {
|
||||
it("prioritizes system+mic sidecars and mutes embedded preview", () => {
|
||||
const policy = resolveSourceTrackRoutingPolicy("/tmp/recording.mp4", [
|
||||
"/tmp/recording.mp4",
|
||||
"/tmp/recording.system.wav",
|
||||
"/tmp/recording.mic.wav",
|
||||
"/tmp/recording.mixed.wav",
|
||||
]);
|
||||
|
||||
expect(policy.playbackPaths).toEqual([
|
||||
"/tmp/recording.system.wav",
|
||||
"/tmp/recording.mic.wav",
|
||||
]);
|
||||
expect(policy.muteEmbeddedPreview).toBe(true);
|
||||
expect(policy.includeEmbeddedInExport).toBe(false);
|
||||
});
|
||||
|
||||
it("falls back to mixed when dedicated tracks are absent", () => {
|
||||
const policy = resolveSourceTrackRoutingPolicy("/tmp/recording.mp4", [
|
||||
"/tmp/recording.mixed.wav",
|
||||
]);
|
||||
|
||||
expect(policy.playbackPaths).toEqual(["/tmp/recording.mixed.wav"]);
|
||||
expect(policy.muteEmbeddedPreview).toBe(false);
|
||||
expect(policy.includeEmbeddedInExport).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps embedded audio when only mic sidecar is present", () => {
|
||||
const policy = resolveSourceTrackRoutingPolicy("/tmp/recording.mp4", [
|
||||
"/tmp/recording.mp4",
|
||||
"/tmp/recording.mic.wav",
|
||||
]);
|
||||
|
||||
expect(policy.playbackPaths).toEqual(["/tmp/recording.mic.wav"]);
|
||||
expect(policy.muteEmbeddedPreview).toBe(false);
|
||||
expect(policy.includeEmbeddedInExport).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,30 @@
|
||||
import {
|
||||
buildResolvedAudioPlan,
|
||||
type SourceTrackId,
|
||||
} from "./audioRoutingEngine";
|
||||
|
||||
export interface SourceTrackRoutingPolicy {
|
||||
hasEmbeddedSourceAudio: boolean;
|
||||
pathsByTrack: Partial<Record<SourceTrackId, string>>;
|
||||
playbackPaths: string[];
|
||||
muteEmbeddedPreview: boolean;
|
||||
includeEmbeddedInExport: boolean;
|
||||
}
|
||||
|
||||
export function resolveSourceTrackRoutingPolicy(
|
||||
videoResource: string | null | undefined,
|
||||
sourceAudioFallbackPaths: string[] | null | undefined,
|
||||
): SourceTrackRoutingPolicy {
|
||||
const plan = buildResolvedAudioPlan({
|
||||
videoResource,
|
||||
sourceAudioFallbackPaths,
|
||||
});
|
||||
|
||||
return {
|
||||
hasEmbeddedSourceAudio: plan.hasEmbeddedSourceAudio,
|
||||
pathsByTrack: plan.pathsByTrack,
|
||||
playbackPaths: plan.playbackPaths,
|
||||
muteEmbeddedPreview: plan.muteEmbeddedPreview,
|
||||
includeEmbeddedInExport: plan.includeEmbeddedInExport,
|
||||
};
|
||||
}
|
||||
@@ -3,11 +3,13 @@ import type {
|
||||
AudioRegion,
|
||||
AutoCaptionSettings,
|
||||
CaptionCue,
|
||||
ClipRegion,
|
||||
CropRegion,
|
||||
CursorStyle,
|
||||
CursorTelemetryPoint,
|
||||
Padding,
|
||||
SpeedRegion,
|
||||
SourceAudioTrackSettings,
|
||||
TrimRegion,
|
||||
WebcamOverlaySettings,
|
||||
ZoomMotionBlurTuning,
|
||||
@@ -90,8 +92,10 @@ interface VideoExporterConfig extends ExportConfig {
|
||||
zoomSmoothness?: number;
|
||||
frame?: string | null;
|
||||
audioRegions?: AudioRegion[];
|
||||
clipRegions?: ClipRegion[];
|
||||
sourceAudioFallbackPaths?: string[];
|
||||
sourceAudioFallbackStartDelayMsByPath?: Record<string, number>;
|
||||
sourceAudioTrackSettings?: SourceAudioTrackSettings;
|
||||
previewWidth?: number;
|
||||
previewHeight?: number;
|
||||
onProgress?: (progress: ExportProgress) => void;
|
||||
@@ -121,6 +125,16 @@ type NativeAudioPlan =
|
||||
|
||||
const FILTERGRAPH_FALLBACK_AUDIO_SAMPLE_RATE = 48_000;
|
||||
|
||||
function hasNonDefaultSourceTrackSettings(sourceAudioTrackSettings?: SourceAudioTrackSettings) {
|
||||
if (!sourceAudioTrackSettings) {
|
||||
return false;
|
||||
}
|
||||
return Object.values(sourceAudioTrackSettings).some(
|
||||
(settings) =>
|
||||
Math.abs((settings?.volume ?? 1) - 1) > 0.0005 || Boolean(settings?.normalize),
|
||||
);
|
||||
}
|
||||
|
||||
export class VideoExporter {
|
||||
private config: VideoExporterConfig;
|
||||
private streamingDecoder: StreamingVideoDecoder | null = null;
|
||||
@@ -398,6 +412,7 @@ export class VideoExporter {
|
||||
this.config.audioRegions,
|
||||
this.config.sourceAudioFallbackPaths,
|
||||
this.config.sourceAudioFallbackStartDelayMsByPath,
|
||||
this.config.sourceAudioTrackSettings,
|
||||
),
|
||||
"audio processing",
|
||||
"audio",
|
||||
@@ -560,7 +575,9 @@ export class VideoExporter {
|
||||
speedRegions.length > 0 ||
|
||||
audioRegions.length > 0 ||
|
||||
sourceAudioFallbackPaths.length > 1 ||
|
||||
hasTimedSourceAudioFallback
|
||||
hasTimedSourceAudioFallback ||
|
||||
hasNonDefaultSourceTrackSettings(this.config.sourceAudioTrackSettings) ||
|
||||
(this.config.clipRegions ?? []).some((clip) => Boolean(clip.muted))
|
||||
) {
|
||||
const sourceDurationMs = Math.max(
|
||||
0,
|
||||
@@ -847,6 +864,8 @@ export class VideoExporter {
|
||||
this.config.audioRegions,
|
||||
this.config.sourceAudioFallbackPaths,
|
||||
this.config.sourceAudioFallbackStartDelayMsByPath,
|
||||
this.config.sourceAudioTrackSettings,
|
||||
this.config.clipRegions,
|
||||
),
|
||||
"native edited audio rendering",
|
||||
"audio",
|
||||
@@ -943,6 +962,8 @@ export class VideoExporter {
|
||||
this.config.audioRegions,
|
||||
this.config.sourceAudioFallbackPaths,
|
||||
this.config.sourceAudioFallbackStartDelayMsByPath,
|
||||
this.config.sourceAudioTrackSettings,
|
||||
this.config.clipRegions,
|
||||
),
|
||||
"ffmpeg edited audio rendering",
|
||||
"audio",
|
||||
|
||||
Reference in New Issue
Block a user