lee/Recordly
1
0
Fork 0
mirror of https://github.com/webadderallorg/Recordly.git synced 2026-09-27 00:05:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix embedded audio fallback handling and audio diagnostics (#309)

Browse Source 
* Fix embedded audio fallback handling and audio diagnostics

* Add recording fallback diagnostics toasts

* Fix Windows audio fallback cleanup and path matching

* Format rebased recording fallback changes

* Fix embedded audio preview and export fallback handling
This commit is contained in:
webadderall
2026-04-24 11:30:10 +10:00
committed by GitHub
parent c76e3dc84c
commit 05a6d3f1b6
13 changed files with 2091 additions and 1333 deletions
Download Patch File Download Diff File Expand all files Collapse all files
electron/ipc/handlers.ts
+24 -18
View File
@@ -1,25 +1,26 @@
import { BrowserWindow } from "electron";
import {
windowsCaptureProcess,
setWindowsCaptureProcess,
setWindowsCaptureTargetPath,
setWindowsNativeCaptureActive,
setNativeScreenRecordingActive,
setWindowsCaptureStopRequested,
setWindowsCapturePaused,
setWindowsSystemAudioPath,
setWindowsMicAudioPath,
setWindowsPendingVideoPath,
selectedSource,
} from "./state";
import { registerSourceHandlers } from "./register/sources";
import { registerRecordingHandlers } from "./register/recording";
import { registerPermissionHandlers } from "./register/permissions";
import { registerAssetHandlers } from "./register/assets";
import { registerExportHandlers } from "./register/export";
import { registerCaptionHandlers } from "./register/captions";
import { registerExportHandlers } from "./register/export";
import { registerPermissionHandlers } from "./register/permissions";
import { registerProjectHandlers } from "./register/project";
import { registerRecordingHandlers } from "./register/recording";
import { registerSettingsHandlers } from "./register/settings";
import { registerSourceHandlers } from "./register/sources";
import {
selectedSource,
setNativeScreenRecordingActive,
setWindowsCapturePaused,
setWindowsCaptureProcess,
setWindowsCaptureStopRequested,
setWindowsCaptureTargetPath,
setWindowsMicAudioPath,
setWindowsNativeCaptureActive,
setWindowsOrphanedMicAudioPath,
setWindowsPendingVideoPath,
setWindowsSystemAudioPath,
windowsCaptureProcess,
} from "./state";
export { cleanupNativeVideoExportSessions } from "./export/native-video";
@@ -43,6 +44,7 @@ export function killWindowsCaptureProcess() {
setWindowsCapturePaused(false);
setWindowsSystemAudioPath(null);
setWindowsMicAudioPath(null);
setWindowsOrphanedMicAudioPath(null);
setWindowsPendingVideoPath(null);
}
}
@@ -54,7 +56,11 @@ export function registerIpcHandlers(
getSourceSelectorWindow: () => BrowserWindow | null,
onRecordingStateChange?: (recording: boolean, sourceName: string) => void,
) {
registerSourceHandlers({ createEditorWindow, createSourceSelectorWindow, getSourceSelectorWindow });
registerSourceHandlers({
createEditorWindow,
createSourceSelectorWindow,
getSourceSelectorWindow,
});
registerRecordingHandlers(onRecordingStateChange);
registerPermissionHandlers();
registerAssetHandlers();
electron/ipc/recording/windowsFallbacks.test.ts
+23
View File
@@ -0,0 +1,23 @@
import { describe, expect, it } from "vitest";
import { shouldUseWindowsBrowserMicrophoneFallback } from "./windowsFallbacks";
describe("shouldUseWindowsBrowserMicrophoneFallback", () => {
it("returns true when native Windows mic initialization fails", () => {
expect(
shouldUseWindowsBrowserMicrophoneFallback(
"WARNING: Failed to initialize WASAPI mic capture\nRecording started",
{ capturesMicrophone: true },
),
).toBe(true);
});
it("returns false when microphone capture was not requested", () => {
expect(
shouldUseWindowsBrowserMicrophoneFallback(
"WARNING: Failed to initialize WASAPI mic capture\nRecording started",
{ capturesMicrophone: false },
),
).toBe(false);
});
});
electron/ipc/recording/windowsFallbacks.ts
+11
View File
@@ -0,0 +1,11 @@
const WINDOWS_MIC_CAPTURE_INIT_WARNING = "WARNING: Failed to initialize WASAPI mic capture";
export function shouldUseWindowsBrowserMicrophoneFallback(
captureOutput: string,
options?: { capturesMicrophone?: boolean },
) {
return (
Boolean(options?.capturesMicrophone) &&
captureOutput.includes(WINDOWS_MIC_CAPTURE_INIT_WARNING)
);
}
electron/ipc/register/recording.ts
+1293 -1104
View File
File diff suppressed because it is too large Load Diff
electron/ipc/state.ts
+169 -53
View File
@@ -44,6 +44,7 @@ export let windowsCaptureStopRequested = false;
export let windowsCapturePaused = false;
export let windowsSystemAudioPath: string | null = null;
export let windowsMicAudioPath: string | null = null;
export let windowsOrphanedMicAudioPath: string | null = null;
export let windowsPendingVideoPath: string | null = null;
// ── Diagnostics ───────────────────────────────────────────────────────────────
@@ -90,7 +91,11 @@ export let cachedNativeMacWindowSources: import("./types").NativeMacWindowSource
export let cachedNativeMacWindowSourcesAtMs = 0;
// ── Native video export ───────────────────────────────────────────────────────
export let cachedNativeVideoEncoder: { ffmpegPath: string; encodingMode: string; encoderName: string } | null = null;
export let cachedNativeVideoEncoder: {
ffmpegPath: string;
encodingMode: string;
encoderName: string;
} | null = null;
// ── Native helper migration ───────────────────────────────────────────────────
export let nativeHelperMigrationPromise: Promise<void> | null = null;
@@ -102,68 +107,179 @@ export type { CursorInteractionType, CursorTelemetryPoint };
// TypeScript exported `let` can be reassigned by the owning module but importers
// cannot assign to them directly. Provide simple setters for cross-module writes.
export function setSelectedSource(v: SelectedSource | null) { selectedSource = v; }
export function setCurrentProjectPath(v: string | null) { currentProjectPath = v; }
export function setCurrentVideoPath(v: string | null) { currentVideoPath = v; }
export function setCurrentRecordingSession(v: RecordingSessionData | null) { currentRecordingSession = v; }
export function setSelectedSource(v: SelectedSource | null) {
selectedSource = v;
}
export function setCurrentProjectPath(v: string | null) {
currentProjectPath = v;
}
export function setCurrentVideoPath(v: string | null) {
currentVideoPath = v;
}
export function setCurrentRecordingSession(v: RecordingSessionData | null) {
currentRecordingSession = v;
}
export function setNativeScreenRecordingActive(v: boolean) { nativeScreenRecordingActive = v; }
export function setNativeCaptureProcess(v: ChildProcessWithoutNullStreams | null) { nativeCaptureProcess = v; }
export function setNativeCaptureOutputBuffer(v: string) { nativeCaptureOutputBuffer = v; }
export function setNativeCaptureTargetPath(v: string | null) { nativeCaptureTargetPath = v; }
export function setNativeCaptureStopRequested(v: boolean) { nativeCaptureStopRequested = v; }
export function setNativeCaptureSystemAudioPath(v: string | null) { nativeCaptureSystemAudioPath = v; }
export function setNativeCaptureMicrophonePath(v: string | null) { nativeCaptureMicrophonePath = v; }
export function setNativeCapturePaused(v: boolean) { nativeCapturePaused = v; }
export function setNativeScreenRecordingActive(v: boolean) {
nativeScreenRecordingActive = v;
}
export function setNativeCaptureProcess(v: ChildProcessWithoutNullStreams | null) {
nativeCaptureProcess = v;
}
export function setNativeCaptureOutputBuffer(v: string) {
nativeCaptureOutputBuffer = v;
}
export function setNativeCaptureTargetPath(v: string | null) {
nativeCaptureTargetPath = v;
}
export function setNativeCaptureStopRequested(v: boolean) {
nativeCaptureStopRequested = v;
}
export function setNativeCaptureSystemAudioPath(v: string | null) {
nativeCaptureSystemAudioPath = v;
}
export function setNativeCaptureMicrophonePath(v: string | null) {
nativeCaptureMicrophonePath = v;
}
export function setNativeCapturePaused(v: boolean) {
nativeCapturePaused = v;
}
export function setNativeCursorMonitorProcess(v: ChildProcessWithoutNullStreams | null) { nativeCursorMonitorProcess = v; }
export function setNativeCursorMonitorOutputBuffer(v: string) { nativeCursorMonitorOutputBuffer = v; }
export function setNativeCursorMonitorProcess(v: ChildProcessWithoutNullStreams | null) {
nativeCursorMonitorProcess = v;
}
export function setNativeCursorMonitorOutputBuffer(v: string) {
nativeCursorMonitorOutputBuffer = v;
}
export function setWindowsCaptureProcess(v: ChildProcessWithoutNullStreams | null) { windowsCaptureProcess = v; }
export function setWindowsCaptureOutputBuffer(v: string) { windowsCaptureOutputBuffer = v; }
export function setWindowsCaptureTargetPath(v: string | null) { windowsCaptureTargetPath = v; }
export function setWindowsNativeCaptureActive(v: boolean) { windowsNativeCaptureActive = v; }
export function setWindowsCaptureStopRequested(v: boolean) { windowsCaptureStopRequested = v; }
export function setWindowsCapturePaused(v: boolean) { windowsCapturePaused = v; }
export function setWindowsSystemAudioPath(v: string | null) { windowsSystemAudioPath = v; }
export function setWindowsMicAudioPath(v: string | null) { windowsMicAudioPath = v; }
export function setWindowsPendingVideoPath(v: string | null) { windowsPendingVideoPath = v; }
export function setWindowsCaptureProcess(v: ChildProcessWithoutNullStreams | null) {
windowsCaptureProcess = v;
}
export function setWindowsCaptureOutputBuffer(v: string) {
windowsCaptureOutputBuffer = v;
}
export function setWindowsCaptureTargetPath(v: string | null) {
windowsCaptureTargetPath = v;
}
export function setWindowsNativeCaptureActive(v: boolean) {
windowsNativeCaptureActive = v;
}
export function setWindowsCaptureStopRequested(v: boolean) {
windowsCaptureStopRequested = v;
}
export function setWindowsCapturePaused(v: boolean) {
windowsCapturePaused = v;
}
export function setWindowsSystemAudioPath(v: string | null) {
windowsSystemAudioPath = v;
}
export function setWindowsMicAudioPath(v: string | null) {
windowsMicAudioPath = v;
}
export function setWindowsOrphanedMicAudioPath(v: string | null) {
windowsOrphanedMicAudioPath = v;
}
export function setWindowsPendingVideoPath(v: string | null) {
windowsPendingVideoPath = v;
}
export function setLastNativeCaptureDiagnostics(v: NativeCaptureDiagnostics | null) { lastNativeCaptureDiagnostics = v; }
export function setLastNativeCaptureDiagnostics(v: NativeCaptureDiagnostics | null) {
lastNativeCaptureDiagnostics = v;
}
export function setFfmpegScreenRecordingActive(v: boolean) { ffmpegScreenRecordingActive = v; }
export function setFfmpegCaptureProcess(v: ChildProcessWithoutNullStreams | null) { ffmpegCaptureProcess = v; }
export function setFfmpegCaptureOutputBuffer(v: string) { ffmpegCaptureOutputBuffer = v; }
export function setFfmpegCaptureTargetPath(v: string | null) { ffmpegCaptureTargetPath = v; }
export function setFfmpegScreenRecordingActive(v: boolean) {
ffmpegScreenRecordingActive = v;
}
export function setFfmpegCaptureProcess(v: ChildProcessWithoutNullStreams | null) {
ffmpegCaptureProcess = v;
}
export function setFfmpegCaptureOutputBuffer(v: string) {
ffmpegCaptureOutputBuffer = v;
}
export function setFfmpegCaptureTargetPath(v: string | null) {
ffmpegCaptureTargetPath = v;
}
export function setCustomRecordingsDir(v: string | null) { customRecordingsDir = v; }
export function setRecordingsDirLoaded(v: boolean) { recordingsDirLoaded = v; }
export function setCustomRecordingsDir(v: string | null) {
customRecordingsDir = v;
}
export function setRecordingsDirLoaded(v: boolean) {
recordingsDirLoaded = v;
}
export function setCachedSystemCursorAssets(v: Record<string, SystemCursorAsset> | null) { cachedSystemCursorAssets = v; }
export function setCachedSystemCursorAssetsSourceMtimeMs(v: number | null) { cachedSystemCursorAssetsSourceMtimeMs = v; }
export function setCachedSystemCursorAssets(v: Record<string, SystemCursorAsset> | null) {
cachedSystemCursorAssets = v;
}
export function setCachedSystemCursorAssetsSourceMtimeMs(v: number | null) {
cachedSystemCursorAssetsSourceMtimeMs = v;
}
export function setCountdownTimer(v: ReturnType<typeof setInterval> | null) { countdownTimer = v; }
export function setCountdownCancelled(v: boolean) { countdownCancelled = v; }
export function setCountdownInProgress(v: boolean) { countdownInProgress = v; }
export function setCountdownRemaining(v: number | null) { countdownRemaining = v; }
export function setCountdownTimer(v: ReturnType<typeof setInterval> | null) {
countdownTimer = v;
}
export function setCountdownCancelled(v: boolean) {
countdownCancelled = v;
}
export function setCountdownInProgress(v: boolean) {
countdownInProgress = v;
}
export function setCountdownRemaining(v: number | null) {
countdownRemaining = v;
}
export function setCurrentCursorVisualType(v: CursorVisualType | undefined) { currentCursorVisualType = v; }
export function setCurrentCursorVisualType(v: CursorVisualType | undefined) {
currentCursorVisualType = v;
}
export function setCursorCaptureInterval(v: NodeJS.Timeout | null) { cursorCaptureInterval = v; }
export function setCursorCaptureStartTimeMs(v: number) { cursorCaptureStartTimeMs = v; }
export function setActiveCursorSamples(v: CursorTelemetryPoint[]) { activeCursorSamples = v; }
export function setPendingCursorSamples(v: CursorTelemetryPoint[]) { pendingCursorSamples = v; }
export function setIsCursorCaptureActive(v: boolean) { isCursorCaptureActive = v; }
export function setInteractionCaptureCleanup(v: (() => void) | null) { interactionCaptureCleanup = v; }
export function setHasLoggedInteractionHookFailure(v: boolean) { hasLoggedInteractionHookFailure = v; }
export function setLastLeftClick(v: { timeMs: number; cx: number; cy: number } | null) { lastLeftClick = v; }
export function setLinuxCursorScreenPoint(v: { x: number; y: number; updatedAt: number } | null) { linuxCursorScreenPoint = v; }
export function setSelectedWindowBounds(v: WindowBounds | null) { selectedWindowBounds = v; }
export function setWindowBoundsCaptureInterval(v: NodeJS.Timeout | null) { windowBoundsCaptureInterval = v; }
export function setCursorCaptureInterval(v: NodeJS.Timeout | null) {
cursorCaptureInterval = v;
}
export function setCursorCaptureStartTimeMs(v: number) {
cursorCaptureStartTimeMs = v;
}
export function setActiveCursorSamples(v: CursorTelemetryPoint[]) {
activeCursorSamples = v;
}
export function setPendingCursorSamples(v: CursorTelemetryPoint[]) {
pendingCursorSamples = v;
}
export function setIsCursorCaptureActive(v: boolean) {
isCursorCaptureActive = v;
}
export function setInteractionCaptureCleanup(v: (() => void) | null) {
interactionCaptureCleanup = v;
}
export function setHasLoggedInteractionHookFailure(v: boolean) {
hasLoggedInteractionHookFailure = v;
}
export function setLastLeftClick(v: { timeMs: number; cx: number; cy: number } | null) {
lastLeftClick = v;
}
export function setLinuxCursorScreenPoint(v: { x: number; y: number; updatedAt: number } | null) {
linuxCursorScreenPoint = v;
}
export function setSelectedWindowBounds(v: WindowBounds | null) {
selectedWindowBounds = v;
}
export function setWindowBoundsCaptureInterval(v: NodeJS.Timeout | null) {
windowBoundsCaptureInterval = v;
}
export function setCachedNativeMacWindowSources(v: import("./types").NativeMacWindowSource[] | null) { cachedNativeMacWindowSources = v; }
export function setCachedNativeMacWindowSourcesAtMs(v: number) { cachedNativeMacWindowSourcesAtMs = v; }
export function setCachedNativeMacWindowSources(
v: import("./types").NativeMacWindowSource[] | null,
) {
cachedNativeMacWindowSources = v;
}
export function setCachedNativeMacWindowSourcesAtMs(v: number) {
cachedNativeMacWindowSourcesAtMs = v;
}
export function setCachedNativeVideoEncoder(v: { ffmpegPath: string; encodingMode: string; encoderName: string } | null) { cachedNativeVideoEncoder = v; }
export function setCachedNativeVideoEncoder(
v: { ffmpegPath: string; encodingMode: string; encoderName: string } | null,
) {
cachedNativeVideoEncoder = v;
}
export function setNativeHelperMigrationPromise(v: Promise<void> | null) { nativeHelperMigrationPromise = v; }
export function setNativeHelperMigrationPromise(v: Promise<void> | null) {
nativeHelperMigrationPromise = v;
}
src/components/video-editor/VideoEditor.tsx
+86 -31
View File
@@ -63,6 +63,7 @@ import {
VideoExporter,
} from "@/lib/exporter";
import { resolveMediaElementSource } from "@/lib/exporter/localMediaSource";
import { resolveSourceAudioFallbackPaths } from "@/lib/exporter/sourceAudioFallback";
import {
clampMediaTimeToDuration,
estimateCompanionAudioStartDelaySeconds,
@@ -157,8 +158,8 @@ import {
extendAutoFullTrackClip,
type FigureData,
getClipSourceEndMs,
type PlaybackSpeed,
type Padding,
type PlaybackSpeed,
type SpeedRegion,
type TrimRegion,
type WebcamOverlaySettings,
@@ -239,6 +240,7 @@ async function writeSmokeExportReport(
}
const DEFAULT_MP4_EXPORT_FRAME_RATE: ExportMp4FrameRate = 30;
const SOURCE_AUDIO_FALLBACK_TOAST_ID = "source-audio-fallback-error";
function getEncodingModeBitrateMultiplier(encodingMode: ExportEncodingMode): number {
switch (encodingMode) {
@@ -750,7 +752,9 @@ export default function VideoEditor() {
}
context.imageSmoothingEnabled = true;
context.imageSmoothingQuality = "high";
const editorBgHsl = getComputedStyle(document.documentElement).getPropertyValue("--editor-bg").trim();
const editorBgHsl = getComputedStyle(document.documentElement)
.getPropertyValue("--editor-bg")
.trim();
context.fillStyle = editorBgHsl ? `hsl(${editorBgHsl})` : "#111113";
context.fillRect(0, 0, targetWidth, targetHeight);
@@ -786,7 +790,9 @@ export default function VideoEditor() {
padding,
cropRegion,
webcam,
webcamUrl: resolvedWebcamVideoUrl ?? (webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null),
webcamUrl:
resolvedWebcamVideoUrl ??
(webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null),
videoWidth: previewVideo.videoWidth,
videoHeight: previewVideo.videoHeight,
annotationRegions,
@@ -1203,7 +1209,12 @@ export default function VideoEditor() {
() => videoSourcePath ?? (videoPath ? fromFileUrl(videoPath) : null),
[videoPath, videoSourcePath],
);
const hasSourceAudioFallback = sourceAudioFallbackPaths.length > 0;
const { hasEmbeddedSourceAudio, externalAudioPaths: previewSourceAudioFallbackPaths } = useMemo(
() => resolveSourceAudioFallbackPaths(currentSourcePath, sourceAudioFallbackPaths),
[currentSourcePath, sourceAudioFallbackPaths],
);
const shouldMutePreviewVideo =
!hasEmbeddedSourceAudio && previewSourceAudioFallbackPaths.length > 0;
useEffect(() => {
let cancelled = false;
@@ -1222,10 +1233,26 @@ export default function VideoEditor() {
if (cancelled) {
return;
}
setSourceAudioFallbackPaths(result.success ? (result.paths ?? []) : []);
} catch {
if (!result.success) {
setSourceAudioFallbackPaths([]);
toast.warning(
result.error
? `Could not load companion audio sources: ${summarizeErrorMessage(result.error)}`
: "Could not load companion audio sources. Playback and export may miss microphone audio.",
{ id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 },
);
return;
}
toast.dismiss(SOURCE_AUDIO_FALLBACK_TOAST_ID);
setSourceAudioFallbackPaths(result.paths ?? []);
} catch (error) {
if (!cancelled) {
setSourceAudioFallbackPaths([]);
toast.warning(
`Could not load companion audio sources: ${summarizeErrorMessage(String(error))}`,
{ id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 },
);
}
}
})();
@@ -2134,7 +2161,7 @@ export default function VideoEditor() {
currentSourcePath,
currentPersistedEditorState,
lastSavedSnapshot?.projectId ?? null,
);
);
const fileNameBase =
currentSourcePath
@@ -2249,7 +2276,7 @@ export default function VideoEditor() {
currentSourcePath,
currentPersistedEditorState,
lastSavedSnapshot?.projectId ?? null,
);
);
const thumbnailDataUrl = await captureProjectThumbnail();
const result = await window.electronAPI.saveProjectFileNamed(
projectData,
@@ -2949,7 +2976,9 @@ export default function VideoEditor() {
regions.filter(
(region) =>
!removedSegments.some(
(segment) => region.startMs < segment.endMs && region.endMs > segment.startMs,
(segment) =>
region.startMs < segment.endMs &&
region.endMs > segment.startMs,
),
);
setZoomRegions((prev) => removeTrimmedRegions(prev));
@@ -3455,7 +3484,7 @@ export default function VideoEditor() {
useEffect(() => {
let cancelled = false;
const existing = sourceAudioElementsRef.current;
const currentIds = new Set(sourceAudioFallbackPaths);
const currentIds = new Set(previewSourceAudioFallbackPaths);
for (const [id, audio] of existing) {
if (!currentIds.has(id)) {
@@ -3468,7 +3497,7 @@ export default function VideoEditor() {
}
}
for (const audioPath of sourceAudioFallbackPaths) {
for (const audioPath of previewSourceAudioFallbackPaths) {
let audio = existing.get(audioPath);
if (!audio) {
audio = new Audio();
@@ -3484,34 +3513,53 @@ export default function VideoEditor() {
sourceAudioElementResourcesRef.current.set(audioPath, audioPath);
void (async () => {
const resolved = await resolveMediaElementSource(audioPath);
const latestAudio = existing.get(audioPath);
try {
const resolved = await resolveMediaElementSource(audioPath);
const latestAudio = existing.get(audioPath);
if (
cancelled ||
latestAudio !== audio ||
sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath
) {
resolved.revoke();
return;
if (
cancelled ||
latestAudio !== audio ||
sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath
) {
resolved.revoke();
return;
}
sourceAudioElementRevokersRef.current.set(audioPath, resolved.revoke);
latestAudio.src = resolved.src;
} catch (error) {
if (cancelled) {
return;
}
sourceAudioElementRevokersRef.current.get(audioPath)?.();
sourceAudioElementRevokersRef.current.delete(audioPath);
sourceAudioElementResourcesRef.current.delete(audioPath);
const latestAudio = existing.get(audioPath);
if (latestAudio === audio) {
latestAudio.pause();
latestAudio.src = "";
}
toast.warning(
`Could not load companion audio source: ${summarizeErrorMessage(getErrorMessage(error))}`,
{ id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 },
);
}
sourceAudioElementRevokersRef.current.set(audioPath, resolved.revoke);
latestAudio.src = resolved.src;
})();
}
audio.volume = Math.max(0, Math.min(1, previewVolume));
}
if (sourceAudioFallbackPaths.length === 0) {
if (previewSourceAudioFallbackPaths.length === 0) {
lastSourceAudioSyncTimeRef.current = null;
}
return () => {
cancelled = true;
};
}, [previewVolume, sourceAudioFallbackPaths]);
}, [previewSourceAudioFallbackPaths, previewVolume]);
useEffect(() => {
return () => {
@@ -3579,7 +3627,7 @@ export default function VideoEditor() {
}, [isPlaying, currentTime, audioRegions, speedRegions]);
useEffect(() => {
if (sourceAudioFallbackPaths.length === 0) {
if (previewSourceAudioFallbackPaths.length === 0) {
lastSourceAudioSyncTimeRef.current = null;
return;
}
@@ -3631,7 +3679,7 @@ export default function VideoEditor() {
}
lastSourceAudioSyncTimeRef.current = currentTime;
}, [currentTime, duration, isPlaying, sourceAudioFallbackPaths, speedRegions]);
}, [currentTime, duration, isPlaying, previewSourceAudioFallbackPaths, speedRegions]);
const showExportSuccessToast = useCallback((filePath: string) => {
toast.success(`Exported successfully to ${filePath}`, {
@@ -3759,7 +3807,9 @@ export default function VideoEditor() {
videoPadding: padding,
cropRegion,
webcam,
webcamUrl: resolvedWebcamVideoUrl ?? (webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null),
webcamUrl:
resolvedWebcamVideoUrl ??
(webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null),
annotationRegions,
autoCaptions,
autoCaptionSettings,
@@ -3928,7 +3978,9 @@ export default function VideoEditor() {
padding,
cropRegion,
webcam,
webcamUrl: resolvedWebcamVideoUrl ?? (webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null),
webcamUrl:
resolvedWebcamVideoUrl ??
(webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null),
annotationRegions,
autoCaptions,
autoCaptionSettings,
@@ -4682,7 +4734,10 @@ export default function VideoEditor() {
</p>
{isRenderingAudio ? (
<p className="mt-1 text-[11px] text-muted-foreground/70">
{t("editor.export.processingAudioEdits", "Processing audio with speed/overlay edits")}
{t(
"editor.export.processingAudioEdits",
"Processing audio with speed/overlay edits",
)}
</p>
) : exportRenderSpeedLabel ? (
<p className="mt-1 text-[11px] text-muted-foreground/70">
@@ -5180,7 +5235,7 @@ export default function VideoEditor() {
cursorClickBounceDuration
}
cursorSway={cursorSway}
volume={hasSourceAudioFallback ? 0 : previewVolume}
volume={shouldMutePreviewVideo ? 0 : previewVolume}
/>
</div>
</div>
src/hooks/useScreenRecorder.ts
+82 -6
View File
@@ -33,6 +33,10 @@ const WEBCAM_WIDTH = 1280;
const WEBCAM_HEIGHT = 720;
const WEBCAM_FRAME_RATE = 30;
const WEBCAM_SUFFIX = "-webcam";
const SOURCE_AUDIO_MUX_TOAST_ID = "recording-audio-mux-warning";
const MICROPHONE_FALLBACK_TOAST_ID = "recording-microphone-fallback";
const MICROPHONE_FALLBACK_ERROR_TOAST_ID = "recording-microphone-fallback-error";
const MICROPHONE_SIDECAR_ERROR_TOAST_ID = "recording-microphone-sidecar-error";
const LINUX_PORTAL_SOURCE: ProcessedDesktopSource = {
id: "screen:linux-portal",
name: "Linux Portal",
@@ -76,6 +80,36 @@ type UseScreenRecorderReturn = {
setCountdownDelay: (delay: number) => void;
};
function getErrorMessage(error: unknown) {
if (error instanceof Error && error.message) {
return error.message;
}
if (typeof error === "string" && error.trim().length > 0) {
return error;
}
if (typeof error === "object" && error !== null) {
try {
const serialized = JSON.stringify(error);
if (serialized && serialized !== "{}") {
return serialized;
}
} catch {
// Ignore stringify failures and fall through to a generic message.
}
if (typeof (error as { toString?: () => string }).toString === "function") {
const stringified = (error as { toString: () => string }).toString();
if (stringified && stringified !== "[object Object]") {
return stringified;
}
}
}
return "An unexpected error occurred";
}
export function useScreenRecorder(): UseScreenRecorderReturn {
const [recording, setRecording] = useState(false);
const [paused, setPaused] = useState(false);
@@ -446,9 +480,25 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
try {
const arrayBuffer = await micFallbackBlob.arrayBuffer();
await window.electronAPI.storeMicrophoneSidecar(arrayBuffer, finalPath);
const result = await window.electronAPI.storeMicrophoneSidecar(
arrayBuffer,
finalPath,
);
if (!result.success) {
const errorMessage =
result.error || "Failed to save the fallback microphone audio track";
console.warn("Failed to store microphone sidecar:", errorMessage);
toast.error(
`${errorMessage}. Recording was saved without the fallback microphone track.`,
{ id: MICROPHONE_SIDECAR_ERROR_TOAST_ID, duration: 10000 },
);
}
} catch (error) {
console.warn("Failed to store microphone sidecar:", error);
toast.error(
`${getErrorMessage(error)}. Recording was saved without the fallback microphone track.`,
{ id: MICROPHONE_SIDECAR_ERROR_TOAST_ID, duration: 10000 },
);
}
},
[],
@@ -678,13 +728,24 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
await window.electronAPI.muxNativeWindowsRecording(pauseSegments);
if (!muxResult?.success || !muxResult.path) {
void logNativeCaptureDiagnostics("mux-native-windows-recording");
const failureMessage = await buildNativeCaptureFailureMessage(
"mux-native-windows-recording",
if (!muxResult?.path) {
const failureMessage = await buildNativeCaptureFailureMessage(
"mux-native-windows-recording",
muxResult?.message ||
"Failed to finalize the Windows recording, so the editor was not opened.",
);
await notifyRecordingFinalizationFailure(failureMessage);
return;
}
const warningMessage =
muxResult?.error ||
muxResult?.message ||
"Failed to finalize the Windows recording, so the editor was not opened.",
"Failed to finish the native Windows audio mux";
toast.warning(
`${warningMessage}. Recording was saved, but audio playback or export may be incomplete.`,
{ id: SOURCE_AUDIO_MUX_TOAST_ID, duration: 10000 },
);
await notifyRecordingFinalizationFailure(failureMessage);
return;
}
finalPath = muxResult.path;
}
@@ -976,6 +1037,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
// When native mic capture is unavailable (macOS < 14), record mic
// via browser getUserMedia so it can be saved as a sidecar file.
if (nativeResult.microphoneFallbackRequired && microphoneEnabled) {
void logNativeCaptureDiagnostics("start-browser-microphone-fallback");
toast.warning(
"Native microphone capture is unavailable. Using browser microphone fallback for this recording.",
{ id: MICROPHONE_FALLBACK_TOAST_ID, duration: 8000 },
);
try {
const micStream = await navigator.mediaDevices.getUserMedia({
audio: microphoneDeviceId
@@ -1005,6 +1071,16 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
micFallbackRecorder.current = recorder;
} catch (micError) {
console.warn("Browser microphone fallback failed:", micError);
const permissionDenied =
micError instanceof DOMException &&
(micError.name === "NotAllowedError" ||
micError.name === "SecurityError");
toast.error(
permissionDenied
? "Microphone permission denied. Recording will continue without microphone audio."
: `${getErrorMessage(micError)}. Recording will continue without microphone audio.`,
{ id: MICROPHONE_FALLBACK_ERROR_TOAST_ID, duration: 10000 },
);
}
}
src/lib/exporter/audioEncoder.test.ts
+85
View File
@@ -0,0 +1,85 @@
import { describe, expect, it, vi } from "vitest";
import { AudioProcessor } from "./audioEncoder";
type OfflineRenderTestHarness = AudioProcessor & {
decodeAudioFromUrl(url: string): Promise<AudioBuffer | null>;
getMediaDurationSec(url: string): Promise<number>;
loadAudioFileDemuxer(audioPath: string): Promise<unknown>;
prepareOfflineRender(
videoUrl: string,
trimRegions: never[],
speedRegions: never[],
audioRegions: never[],
sourceAudioFallbackPaths: string[],
): Promise<{
mainBuffer: AudioBuffer | null;
companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number }>;
}>;
renderAndMuxOfflineAudio(
videoUrl: string,
trimRegions: never[],
speedRegions: never[],
audioRegions: never[],
sourceAudioFallbackPaths: string[],
muxer: unknown,
): Promise<void>;
};
describe("AudioProcessor offline render preparation", () => {
it("keeps embedded source audio separate from external companion sidecars", async () => {
const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness;
const mainBuffer = { duration: 10, numberOfChannels: 2 } as AudioBuffer;
const micBuffer = { duration: 9.5, numberOfChannels: 1 } as AudioBuffer;
const decodeAudioFromUrl = vi
.spyOn(processor, "decodeAudioFromUrl")
.mockImplementation(async (url: string) => {
if (url === "file:///tmp/recording.mp4") {
return mainBuffer;
}
if (url === "/tmp/recording.mic.wav") {
return micBuffer;
}
return null;
});
vi.spyOn(processor, "getMediaDurationSec").mockResolvedValue(10);
const prepared = await processor.prepareOfflineRender(
"file:///tmp/recording.mp4",
[],
[],
[],
["/tmp/recording.mp4", "/tmp/recording.mic.wav"],
);
expect(prepared.mainBuffer).toBe(mainBuffer);
expect(prepared.companionEntries).toHaveLength(1);
expect(prepared.companionEntries[0]?.buffer).toBe(micBuffer);
expect(decodeAudioFromUrl).toHaveBeenCalledWith("file:///tmp/recording.mp4");
expect(decodeAudioFromUrl).toHaveBeenCalledWith("/tmp/recording.mic.wav");
expect(decodeAudioFromUrl).not.toHaveBeenCalledWith("/tmp/recording.mp4");
});
it("does not treat a single embedded fallback path as an external sidecar", async () => {
const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness;
const loadAudioFileDemuxer = vi.spyOn(processor, "loadAudioFileDemuxer");
const renderAndMuxOfflineAudio = vi
.spyOn(processor, "renderAndMuxOfflineAudio")
.mockResolvedValue();
await processor.process(
null,
{} as never,
"file:///tmp/recording.mp4",
[],
[],
undefined,
[],
["/tmp/recording.mp4"],
);
expect(loadAudioFileDemuxer).not.toHaveBeenCalled();
expect(renderAndMuxOfflineAudio).not.toHaveBeenCalled();
});
});
src/lib/exporter/audioEncoder.ts
+61 -121
View File
@@ -5,11 +5,10 @@ import type {
SpeedRegion,
TrimRegion,
} from "@/components/video-editor/types";
import {
estimateCompanionAudioStartDelaySeconds,
} from "@/lib/mediaTiming";
import { estimateCompanionAudioStartDelaySeconds } from "@/lib/mediaTiming";
import { resolveMediaElementSource } from "./localMediaSource";
import type { VideoMuxer } from "./muxer";
import { resolveSourceAudioFallbackPaths } from "./sourceAudioFallback";
const AUDIO_BITRATE = 128_000;
const DECODE_BACKPRESSURE_LIMIT = 20;
@@ -36,20 +35,20 @@ interface PreparedOfflineRender {
}
export async function isAacAudioEncodingSupported(
sampleRate = 48_000,
numberOfChannels = 2,
sampleRate = 48_000,
numberOfChannels = 2,
): Promise<boolean> {
try {
const support = await AudioEncoder.isConfigSupported({
codec: MP4_AUDIO_CODEC,
sampleRate,
numberOfChannels,
bitrate: AUDIO_BITRATE,
});
return support.supported === true;
} catch {
return false;
}
try {
const support = await AudioEncoder.isConfigSupported({
codec: MP4_AUDIO_CODEC,
sampleRate,
numberOfChannels,
bitrate: AUDIO_BITRATE,
});
return support.supported === true;
} catch {
return false;
}
}
type TrimLikeRegion = TrimRegion | ClipRegion;
@@ -161,12 +160,19 @@ export class AudioProcessor {
(audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0,
)
: [];
const { hasEmbeddedSourceAudio, externalAudioPaths } = resolveSourceAudioFallbackPaths(
videoUrl,
sortedSourceAudioFallbackPaths,
);
const needsSourceAudioMixing =
externalAudioPaths.length > 1 ||
(hasEmbeddedSourceAudio && externalAudioPaths.length > 0);
// When speed edits, audio regions, or multiple audio sources need mixing, use offline AudioContext pipeline.
if (
sortedSpeedRegions.length > 0 ||
sortedAudioRegions.length > 0 ||
sortedSourceAudioFallbackPaths.length > 1
needsSourceAudioMixing
) {
await this.renderAndMuxOfflineAudio(
videoUrl,
@@ -180,10 +186,8 @@ export class AudioProcessor {
}
// Single sidecar audio with no speed/audio edits: demux directly (skips slow real-time rendering).
if (sortedSourceAudioFallbackPaths.length === 1) {
const sidecarDemuxer = await this.loadAudioFileDemuxer(
sortedSourceAudioFallbackPaths[0],
);
if (!hasEmbeddedSourceAudio && externalAudioPaths.length === 1) {
const sidecarDemuxer = await this.loadAudioFileDemuxer(externalAudioPaths[0]);
if (sidecarDemuxer) {
try {
await this.processTrimOnlyAudio(sidecarDemuxer, muxer, sortedTrims);
@@ -205,7 +209,7 @@ export class AudioProcessor {
sortedTrims,
[],
[],
sortedSourceAudioFallbackPaths,
externalAudioPaths,
muxer,
);
return;
@@ -547,23 +551,23 @@ export class AudioProcessor {
if (this.cancelled) throw new Error("Export cancelled");
this.onProgress?.(0);
const hasExternalSources = sourceAudioFallbackPaths.length > 0;
const { externalAudioPaths } = resolveSourceAudioFallbackPaths(
videoUrl,
sourceAudioFallbackPaths,
);
// Decode primary audio source (streaming decode with bulk fallback)
const mainBuffer = !hasExternalSources
? await this.decodeAudioFromUrl(videoUrl)
: null;
// Decode embedded source audio separately from companion sidecars.
const mainBuffer = await this.decodeAudioFromUrl(videoUrl);
if (this.cancelled) throw new Error("Export cancelled");
// Decode companion / sidecar audio files
const companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number }> = [];
for (const audioPath of sourceAudioFallbackPaths) {
for (const audioPath of externalAudioPaths) {
if (this.cancelled) throw new Error("Export cancelled");
const buffer = await this.decodeAudioFromUrl(audioPath);
if (!buffer) continue;
const refDuration =
mainBuffer?.duration ?? (await this.getMediaDurationSec(videoUrl));
const refDuration = mainBuffer?.duration ?? (await this.getMediaDurationSec(videoUrl));
companionEntries.push({
buffer,
startDelaySec: estimateCompanionAudioStartDelaySeconds(
@@ -593,7 +597,7 @@ export class AudioProcessor {
let sourceDurationSec: number;
if (mainBuffer) {
sourceDurationSec = mainBuffer.duration;
} else if (hasExternalSources || regionEntries.length > 0) {
} else if (externalAudioPaths.length > 0 || regionEntries.length > 0) {
sourceDurationSec = await this.getMediaDurationSec(videoUrl);
} else {
sourceDurationSec = primaryBuffer?.duration ?? 0;
@@ -659,10 +663,7 @@ export class AudioProcessor {
pendingMuxing = pendingMuxing
.then(async () => {
if (this.cancelled) return;
await muxer.addAudioChunk(
chunk,
!wroteFirstChunk ? meta : undefined,
);
await muxer.addAudioChunk(chunk, !wroteFirstChunk ? meta : undefined);
wroteFirstChunk = true;
})
.catch((error) => {
@@ -706,27 +707,17 @@ export class AudioProcessor {
// Render timeline to a WAV blob for the native/FFmpeg export path.
// Processes in chunks to avoid holding the entire output in memory.
private async renderToWavBlobChunked(
prepared: PreparedOfflineRender,
): Promise<Blob> {
private async renderToWavBlobChunked(prepared: PreparedOfflineRender): Promise<Blob> {
const totalOutputSec = Math.max(prepared.outputDurationMs / 1000, 0.01);
const totalFrames = Math.ceil(totalOutputSec * OFFLINE_AUDIO_SAMPLE_RATE);
const numChannels = prepared.numChannels;
const header = this.createWavHeader(
OFFLINE_AUDIO_SAMPLE_RATE,
numChannels,
totalFrames,
);
const header = this.createWavHeader(OFFLINE_AUDIO_SAMPLE_RATE, numChannels, totalFrames);
const pcmParts: ArrayBuffer[] = [header];
await this.renderChunked(
prepared,
totalOutputSec,
async (rendered) => {
pcmParts.push(...this.audioBufferToPcmParts(rendered));
},
);
await this.renderChunked(prepared, totalOutputSec, async (rendered) => {
pcmParts.push(...this.audioBufferToPcmParts(rendered));
});
return new Blob(pcmParts, { type: "audio/wav" });
}
@@ -747,10 +738,7 @@ export class AudioProcessor {
const chunkCount = Math.ceil(totalOutputSec / OFFLINE_CHUNK_DURATION_SEC);
for (let i = 0; i < chunkCount && !this.cancelled; i++) {
const chunkSec = Math.min(
OFFLINE_CHUNK_DURATION_SEC,
totalOutputSec - outputOffsetSec,
);
const chunkSec = Math.min(OFFLINE_CHUNK_DURATION_SEC, totalOutputSec - outputOffsetSec);
const chunkFrames = Math.ceil(chunkSec * OFFLINE_AUDIO_SAMPLE_RATE);
const offlineCtx = new OfflineAudioContext(
@@ -833,10 +821,7 @@ export class AudioProcessor {
localEndSec = chunkDurationSec;
}
const duration = Math.min(
localEndSec - localStartSec,
buffer.duration - bufferOffsetSec,
);
const duration = Math.min(localEndSec - localStartSec, buffer.duration - bufferOffsetSec);
if (duration <= 0.001) return;
const gainNode = ctx.createGain();
@@ -869,10 +854,7 @@ export class AudioProcessor {
const planarData = new Float32Array(frameCount * numChannels);
for (let ch = 0; ch < numChannels; ch++) {
const channelData = buffer.getChannelData(ch);
planarData.set(
channelData.subarray(offset, offset + frameCount),
ch * frameCount,
);
planarData.set(channelData.subarray(offset, offset + frameCount), ch * frameCount);
}
const audioData = new AudioData({
@@ -880,19 +862,14 @@ export class AudioProcessor {
sampleRate,
numberOfFrames: frameCount,
numberOfChannels: numChannels,
timestamp: Math.round(
(offset / sampleRate + timestampOffsetSec) * 1_000_000,
),
timestamp: Math.round((offset / sampleRate + timestampOffsetSec) * 1_000_000),
data: planarData,
});
encoder.encode(audioData);
audioData.close();
while (
encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT &&
!this.cancelled
) {
while (encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT && !this.cancelled) {
await new Promise((r) => setTimeout(r, 1));
}
}
@@ -929,18 +906,13 @@ export class AudioProcessor {
type: blob.type || "video/mp4",
});
const wasmUrl = new URL(
"./wasm/web-demuxer.wasm",
window.location.href,
).href;
const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href;
demuxer = new WebDemuxer({ wasmFilePath: wasmUrl });
await demuxer.load(file);
let audioConfig: AudioDecoderConfig;
try {
audioConfig = (await demuxer.getDecoderConfig(
"audio",
)) as AudioDecoderConfig;
audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig;
} catch {
return null; // No audio track
}
@@ -949,10 +921,7 @@ export class AudioProcessor {
const numChannels = Math.min(audioConfig.numberOfChannels || 2, 2);
// Accumulate decoded PCM per channel
const channelChunks: Float32Array[][] = Array.from(
{ length: numChannels },
() => [],
);
const channelChunks: Float32Array[][] = Array.from({ length: numChannels }, () => []);
let totalFrames = 0;
let decodeError: Error | null = null;
@@ -960,10 +929,7 @@ export class AudioProcessor {
output: (data: AudioData) => {
try {
const frames = data.numberOfFrames;
const dataChannels = Math.min(
data.numberOfChannels,
numChannels,
);
const dataChannels = Math.min(data.numberOfChannels, numChannels);
const format = data.format;
if (format?.includes("planar")) {
@@ -973,9 +939,7 @@ export class AudioProcessor {
});
const bytes = new ArrayBuffer(size);
data.copyTo(bytes, { planeIndex: ch });
channelChunks[ch].push(
this.rawToFloat32(bytes, format, frames),
);
channelChunks[ch].push(this.rawToFloat32(bytes, format, frames));
}
} else if (format) {
// Interleaved format — deinterleave into per-channel arrays.
@@ -993,8 +957,7 @@ export class AudioProcessor {
for (let ch = 0; ch < dataChannels; ch++) {
const chData = new Float32Array(frames);
for (let i = 0; i < frames; i++) {
chData[i] =
interleaved[i * srcChannels + ch];
chData[i] = interleaved[i * srcChannels + ch];
}
channelChunks[ch].push(chData);
}
@@ -1011,18 +974,14 @@ export class AudioProcessor {
}
},
error: (err: DOMException) => {
decodeError = new Error(
`Streaming audio decode error: ${err.message}`,
);
decodeError = new Error(`Streaming audio decode error: ${err.message}`);
},
});
decoder.configure(audioConfig);
const audioStream = demuxer.read("audio");
const reader = (
audioStream as ReadableStream<EncodedAudioChunk>
).getReader();
const reader = (audioStream as ReadableStream<EncodedAudioChunk>).getReader();
try {
while (!this.cancelled) {
@@ -1032,10 +991,7 @@ export class AudioProcessor {
decoder.decode(chunk);
while (
decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT &&
!this.cancelled
) {
while (decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT && !this.cancelled) {
if (decodeError) throw decodeError;
await new Promise((r) => setTimeout(r, 1));
}
@@ -1085,11 +1041,7 @@ export class AudioProcessor {
}
// Convert raw bytes from AudioData to Float32Array based on the sample format.
private rawToFloat32(
bytes: ArrayBuffer,
format: string,
sampleCount: number,
): Float32Array {
private rawToFloat32(bytes: ArrayBuffer, format: string, sampleCount: number): Float32Array {
if (format.startsWith("f32")) {
return new Float32Array(bytes);
}
@@ -1122,10 +1074,7 @@ export class AudioProcessor {
}
// Bulk decode fallback: loads entire file into memory and uses decodeAudioData.
private async bulkDecodeFromUrl(
url: string,
sampleRate: number,
): Promise<AudioBuffer | null> {
private async bulkDecodeFromUrl(url: string, sampleRate: number): Promise<AudioBuffer | null> {
try {
const source = await resolveMediaElementSource(url);
try {
@@ -1197,8 +1146,7 @@ export class AudioProcessor {
boundaries.add(sourceDurationMs);
for (const trim of trimRegions) {
if (trim.startMs >= 0 && trim.startMs <= sourceDurationMs)
boundaries.add(trim.startMs);
if (trim.startMs >= 0 && trim.startMs <= sourceDurationMs) boundaries.add(trim.startMs);
if (trim.endMs >= 0 && trim.endMs <= sourceDurationMs) boundaries.add(trim.endMs);
}
for (const speed of speedRegions) {
@@ -1234,10 +1182,7 @@ export class AudioProcessor {
}
// Map a source-timeline timestamp to the corresponding output-timeline timestamp.
private sourceTimeToOutputTime(
sourceMs: number,
slices: TimelineSlice[],
): number {
private sourceTimeToOutputTime(sourceMs: number, slices: TimelineSlice[]): number {
let outputMs = 0;
for (const slice of slices) {
@@ -1303,8 +1248,7 @@ export class AudioProcessor {
// Calculate output position (global then chunk-local)
let localOutputStartSec =
outputOffsetSec + trimmedFromStartSec / slice.speed - chunkOutputStartSec;
let localOutputEndSec =
localOutputStartSec + effectiveSourceDurationSec / slice.speed;
let localOutputEndSec = localOutputStartSec + effectiveSourceDurationSec / slice.speed;
// Skip if entirely outside chunk window
if (localOutputEndSec <= 0 || localOutputStartSec >= chunkDurationSec) {
@@ -1401,11 +1345,7 @@ export class AudioProcessor {
for (let i = 0; i < chunkFrames; i++) {
for (let ch = 0; ch < numChannels; ch++) {
const sample = Math.max(-1, Math.min(1, channels[ch][frameOffset + i]));
view.setInt16(
byteOffset,
sample < 0 ? sample * 0x8000 : sample * 0x7fff,
true,
);
view.setInt16(byteOffset, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true);
byteOffset += 2;
}
}
src/lib/exporter/mediaResource.test.ts
+34
View File
@@ -0,0 +1,34 @@
import { describe, expect, it } from "vitest";
import { getLocalFilePathFromResource, getResourceFileName } from "./mediaResource";
describe("getLocalFilePathFromResource", () => {
it("extracts the path from file URLs", () => {
expect(getLocalFilePathFromResource("file:///tmp/example%20video.mp4")).toBe(
"/tmp/example video.mp4",
);
});
it("extracts the approved file path from loopback media server URLs", () => {
expect(
getLocalFilePathFromResource(
"http://127.0.0.1:4321/video?path=%2Ftmp%2Fexample%20video.mp4",
),
).toBe("/tmp/example video.mp4");
});
it("does not treat arbitrary remote URLs as local files", () => {
expect(getLocalFilePathFromResource("https://example.com/video.mp4")).toBeNull();
});
});
describe("getResourceFileName", () => {
it("uses the source file name for loopback media server URLs", () => {
expect(
getResourceFileName(
"http://127.0.0.1:4321/video?path=%2Ftmp%2Fexample%20video.mp4",
"fallback.mp4",
),
).toBe("example video.mp4");
});
});
src/lib/exporter/mediaResource.ts
+111
View File
@@ -0,0 +1,111 @@
const LOOPBACK_MEDIA_SERVER_HOSTS = new Set(["127.0.0.1", "localhost", "::1", "[::1]"]);
export function isAbsoluteLocalPath(resource: string): boolean {
return (
resource.startsWith("/") ||
/^[A-Za-z]:[\\/]/.test(resource) ||
/^\\\\[^\\]+\\[^\\]+/.test(resource)
);
}
function fromFileUrl(resource: string): string {
try {
const url = new URL(resource);
const pathname = decodeURIComponent(url.pathname);
if (url.host && url.host !== "localhost") {
const uncPath = `//${url.host}${pathname.startsWith("/") ? pathname : `/${pathname}`}`;
return uncPath.replace(/\//g, "\\");
}
if (/^\/[A-Za-z]:/.test(pathname)) {
return pathname.slice(1);
}
return pathname;
} catch {
const rawFallbackPath = resource.replace(/^file:\/\//i, "");
let fallbackPath = rawFallbackPath;
try {
fallbackPath = decodeURIComponent(rawFallbackPath);
} catch {
// Keep raw best-effort path if percent decoding fails.
}
return fallbackPath.replace(/^\/([A-Za-z]:)/, "$1");
}
}
function getLoopbackMediaServerPath(resource: string): string | null {
try {
const url = new URL(resource);
if (url.pathname !== "/video") {
return null;
}
if (!/^https?:$/i.test(url.protocol)) {
return null;
}
if (!LOOPBACK_MEDIA_SERVER_HOSTS.has(url.hostname.toLowerCase())) {
return null;
}
const pathParam = url.searchParams.get("path");
if (!pathParam) {
return null;
}
if (/^file:\/\//i.test(pathParam)) {
return fromFileUrl(pathParam);
}
return isAbsoluteLocalPath(pathParam) ? pathParam : null;
} catch {
return null;
}
}
function getPathBaseName(filePath: string): string {
const normalized = filePath.replace(/\\/g, "/");
const segments = normalized.split("/").filter(Boolean);
return segments[segments.length - 1] ?? "";
}
export function getLocalFilePathFromResource(resource: string): string | null {
if (!resource) {
return null;
}
if (/^file:\/\//i.test(resource)) {
return fromFileUrl(resource);
}
const mediaServerPath = getLoopbackMediaServerPath(resource);
if (mediaServerPath) {
return mediaServerPath;
}
return isAbsoluteLocalPath(resource) ? resource : null;
}
export function getResourceFileName(resource: string, fallback: string): string {
const localFilePath = getLocalFilePathFromResource(resource);
if (localFilePath) {
const fileName = getPathBaseName(localFilePath);
if (fileName) {
return fileName;
}
}
try {
const url = new URL(resource);
const fileName = getPathBaseName(decodeURIComponent(url.pathname));
if (fileName) {
return fileName;
}
} catch {
// Ignore parse errors and fall back to the provided default.
}
return fallback;
}
src/lib/exporter/sourceAudioFallback.test.ts
+64
View File
@@ -0,0 +1,64 @@
import { describe, expect, it } from "vitest";
import { resolveSourceAudioFallbackPaths } from "./sourceAudioFallback";
describe("resolveSourceAudioFallbackPaths", () => {
it("treats the video file path as embedded source audio when present in the fallback list", () => {
const videoPath = "/tmp/recording.mp4";
expect(
resolveSourceAudioFallbackPaths(videoPath, [videoPath, "/tmp/recording.mic.wav"]),
).toEqual({
hasEmbeddedSourceAudio: true,
externalAudioPaths: ["/tmp/recording.mic.wav"],
});
});
it("keeps all fallback paths external when the video has no embedded source audio", () => {
expect(
resolveSourceAudioFallbackPaths("/tmp/recording.mp4", [
"/tmp/recording.system.wav",
"/tmp/recording.mic.wav",
]),
).toEqual({
hasEmbeddedSourceAudio: false,
externalAudioPaths: ["/tmp/recording.system.wav", "/tmp/recording.mic.wav"],
});
});
it("matches embedded source audio when the video resource is a file URL", () => {
expect(
resolveSourceAudioFallbackPaths("file:///tmp/recording.mp4", [
"/tmp/recording.mp4",
"/tmp/recording.mic.wav",
]),
).toEqual({
hasEmbeddedSourceAudio: true,
externalAudioPaths: ["/tmp/recording.mic.wav"],
});
});
it("normalizes Windows file URLs and local paths when checking embedded audio", () => {
expect(
resolveSourceAudioFallbackPaths("file:///C:/Users/Egg/Videos/recording.mp4", [
"C:\\Users\\Egg\\Videos\\recording.mp4",
"C:\\Users\\Egg\\Videos\\recording.mic.wav",
]),
).toEqual({
hasEmbeddedSourceAudio: true,
externalAudioPaths: ["C:\\Users\\Egg\\Videos\\recording.mic.wav"],
});
});
it("matches Windows paths case-insensitively for embedded audio detection", () => {
expect(
resolveSourceAudioFallbackPaths("file:///C:/Users/Egg/Videos/recording.mp4", [
"c:\\users\\egg\\videos\\recording.mp4",
"c:\\users\\egg\\videos\\recording.mic.wav",
]),
).toEqual({
hasEmbeddedSourceAudio: true,
externalAudioPaths: ["c:\\users\\egg\\videos\\recording.mic.wav"],
});
});
});
src/lib/exporter/sourceAudioFallback.ts
+48
View File
@@ -0,0 +1,48 @@
import { getLocalFilePathFromResource } from "./mediaResource";
function normalizeSourceAudioFallbackPath(resourceOrPath: string): string | null {
if (typeof resourceOrPath !== "string") {
return null;
}
const resolvedPath = getLocalFilePathFromResource(resourceOrPath) ?? resourceOrPath;
const trimmedPath = resolvedPath.trim();
if (!trimmedPath) {
return null;
}
const isWindowsPath =
/^[A-Za-z]:[\\/]/.test(trimmedPath) || /^\\\\[^\\]+\\[^\\]+/.test(trimmedPath);
if (isWindowsPath) {
return trimmedPath.replace(/\//g, "\\").toLowerCase();
}
return trimmedPath.replace(/\\/g, "/");
}
export function resolveSourceAudioFallbackPaths(
videoResource: string | null | undefined,
sourceAudioFallbackPaths: string[] | null | undefined,
) {
const normalizedPaths = (sourceAudioFallbackPaths ?? [])
.filter((audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0)
.map((audioPath) => ({
audioPath,
normalizedPath: normalizeSourceAudioFallbackPath(audioPath),
}));
const localVideoSourcePath = videoResource
? normalizeSourceAudioFallbackPath(videoResource)
: null;
const hasEmbeddedSourceAudio =
Boolean(localVideoSourcePath) &&
normalizedPaths.some(({ normalizedPath }) => normalizedPath === localVideoSourcePath);
return {
hasEmbeddedSourceAudio,
externalAudioPaths: hasEmbeddedSourceAudio
? normalizedPaths
.filter(({ normalizedPath }) => normalizedPath !== localVideoSourcePath)
.map(({ audioPath }) => audioPath)
: normalizedPaths.map(({ audioPath }) => audioPath),
};
}