From 05a6d3f1b6030e63d2d83f906a411e138b636f97 Mon Sep 17 00:00:00 2001 From: webadderall <131426131+webadderall@users.noreply.github.com> Date: Fri, 24 Apr 2026 11:30:10 +1000 Subject: [PATCH] Fix embedded audio fallback handling and audio diagnostics (#309) * Fix embedded audio fallback handling and audio diagnostics * Add recording fallback diagnostics toasts * Fix Windows audio fallback cleanup and path matching * Format rebased recording fallback changes * Fix embedded audio preview and export fallback handling --- electron/ipc/handlers.ts | 42 +- .../ipc/recording/windowsFallbacks.test.ts | 23 + electron/ipc/recording/windowsFallbacks.ts | 11 + electron/ipc/register/recording.ts | 2397 +++++++++-------- electron/ipc/state.ts | 222 +- src/components/video-editor/VideoEditor.tsx | 117 +- src/hooks/useScreenRecorder.ts | 88 +- src/lib/exporter/audioEncoder.test.ts | 85 + src/lib/exporter/audioEncoder.ts | 182 +- src/lib/exporter/mediaResource.test.ts | 34 + src/lib/exporter/mediaResource.ts | 111 + src/lib/exporter/sourceAudioFallback.test.ts | 64 + src/lib/exporter/sourceAudioFallback.ts | 48 + 13 files changed, 2091 insertions(+), 1333 deletions(-) create mode 100644 electron/ipc/recording/windowsFallbacks.test.ts create mode 100644 electron/ipc/recording/windowsFallbacks.ts create mode 100644 src/lib/exporter/audioEncoder.test.ts create mode 100644 src/lib/exporter/mediaResource.test.ts create mode 100644 src/lib/exporter/mediaResource.ts create mode 100644 src/lib/exporter/sourceAudioFallback.test.ts create mode 100644 src/lib/exporter/sourceAudioFallback.ts diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 760bbe23..4f179417 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -1,25 +1,26 @@ import { BrowserWindow } from "electron"; -import { - windowsCaptureProcess, - setWindowsCaptureProcess, - setWindowsCaptureTargetPath, - setWindowsNativeCaptureActive, - setNativeScreenRecordingActive, - setWindowsCaptureStopRequested, - setWindowsCapturePaused, - setWindowsSystemAudioPath, - setWindowsMicAudioPath, - setWindowsPendingVideoPath, - selectedSource, -} from "./state"; -import { registerSourceHandlers } from "./register/sources"; -import { registerRecordingHandlers } from "./register/recording"; -import { registerPermissionHandlers } from "./register/permissions"; import { registerAssetHandlers } from "./register/assets"; -import { registerExportHandlers } from "./register/export"; import { registerCaptionHandlers } from "./register/captions"; +import { registerExportHandlers } from "./register/export"; +import { registerPermissionHandlers } from "./register/permissions"; import { registerProjectHandlers } from "./register/project"; +import { registerRecordingHandlers } from "./register/recording"; import { registerSettingsHandlers } from "./register/settings"; +import { registerSourceHandlers } from "./register/sources"; +import { + selectedSource, + setNativeScreenRecordingActive, + setWindowsCapturePaused, + setWindowsCaptureProcess, + setWindowsCaptureStopRequested, + setWindowsCaptureTargetPath, + setWindowsMicAudioPath, + setWindowsNativeCaptureActive, + setWindowsOrphanedMicAudioPath, + setWindowsPendingVideoPath, + setWindowsSystemAudioPath, + windowsCaptureProcess, +} from "./state"; export { cleanupNativeVideoExportSessions } from "./export/native-video"; @@ -43,6 +44,7 @@ export function killWindowsCaptureProcess() { setWindowsCapturePaused(false); setWindowsSystemAudioPath(null); setWindowsMicAudioPath(null); + setWindowsOrphanedMicAudioPath(null); setWindowsPendingVideoPath(null); } } @@ -54,7 +56,11 @@ export function registerIpcHandlers( getSourceSelectorWindow: () => BrowserWindow | null, onRecordingStateChange?: (recording: boolean, sourceName: string) => void, ) { - registerSourceHandlers({ createEditorWindow, createSourceSelectorWindow, getSourceSelectorWindow }); + registerSourceHandlers({ + createEditorWindow, + createSourceSelectorWindow, + getSourceSelectorWindow, + }); registerRecordingHandlers(onRecordingStateChange); registerPermissionHandlers(); registerAssetHandlers(); diff --git a/electron/ipc/recording/windowsFallbacks.test.ts b/electron/ipc/recording/windowsFallbacks.test.ts new file mode 100644 index 00000000..9945210e --- /dev/null +++ b/electron/ipc/recording/windowsFallbacks.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from "vitest"; + +import { shouldUseWindowsBrowserMicrophoneFallback } from "./windowsFallbacks"; + +describe("shouldUseWindowsBrowserMicrophoneFallback", () => { + it("returns true when native Windows mic initialization fails", () => { + expect( + shouldUseWindowsBrowserMicrophoneFallback( + "WARNING: Failed to initialize WASAPI mic capture\nRecording started", + { capturesMicrophone: true }, + ), + ).toBe(true); + }); + + it("returns false when microphone capture was not requested", () => { + expect( + shouldUseWindowsBrowserMicrophoneFallback( + "WARNING: Failed to initialize WASAPI mic capture\nRecording started", + { capturesMicrophone: false }, + ), + ).toBe(false); + }); +}); \ No newline at end of file diff --git a/electron/ipc/recording/windowsFallbacks.ts b/electron/ipc/recording/windowsFallbacks.ts new file mode 100644 index 00000000..b8e6e709 --- /dev/null +++ b/electron/ipc/recording/windowsFallbacks.ts @@ -0,0 +1,11 @@ +const WINDOWS_MIC_CAPTURE_INIT_WARNING = "WARNING: Failed to initialize WASAPI mic capture"; + +export function shouldUseWindowsBrowserMicrophoneFallback( + captureOutput: string, + options?: { capturesMicrophone?: boolean }, +) { + return ( + Boolean(options?.capturesMicrophone) && + captureOutput.includes(WINDOWS_MIC_CAPTURE_INIT_WARNING) + ); +} \ No newline at end of file diff --git a/electron/ipc/register/recording.ts b/electron/ipc/register/recording.ts index 4fd5f16a..1f19cb51 100644 --- a/electron/ipc/register/recording.ts +++ b/electron/ipc/register/recording.ts @@ -3,124 +3,137 @@ import { execFile, spawn } from "node:child_process"; import fs from "node:fs/promises"; import path from "node:path"; import { promisify } from "node:util"; -import { app, BrowserWindow, desktopCapturer, dialog, ipcMain, shell, systemPreferences } from "electron"; +import { + app, + BrowserWindow, + desktopCapturer, + dialog, + ipcMain, + shell, + systemPreferences, +} from "electron"; import { showCursor } from "../../cursorHider"; import { ALLOW_RECORDLY_WINDOW_CAPTURE } from "../constants"; -import type { SelectedSource, NativeMacRecordingOptions, PauseSegment, CursorTelemetryPoint } from "../types"; +import { startWindowBoundsCapture, stopWindowBoundsCapture } from "../cursor/bounds"; +import { startInteractionCapture, stopInteractionCapture } from "../cursor/interaction"; +import { startNativeCursorMonitor, stopNativeCursorMonitor } from "../cursor/monitor"; import { - selectedSource, - nativeScreenRecordingActive, - setNativeScreenRecordingActive, - currentVideoPath, - nativeCaptureProcess, - setNativeCaptureProcess, - nativeCaptureOutputBuffer, - setNativeCaptureOutputBuffer, - nativeCaptureTargetPath, - setNativeCaptureTargetPath, - setNativeCaptureStopRequested, - nativeCaptureSystemAudioPath, - setNativeCaptureSystemAudioPath, - nativeCaptureMicrophonePath, - setNativeCaptureMicrophonePath, - nativeCapturePaused, - setNativeCapturePaused, - windowsCaptureProcess, - setWindowsCaptureProcess, - windowsCaptureTargetPath, - setWindowsCaptureTargetPath, - windowsNativeCaptureActive, - setWindowsNativeCaptureActive, - setWindowsCaptureStopRequested, - windowsCapturePaused, - setWindowsCapturePaused, - windowsSystemAudioPath, - setWindowsSystemAudioPath, - windowsMicAudioPath, - setWindowsMicAudioPath, - windowsPendingVideoPath, - setWindowsPendingVideoPath, - lastNativeCaptureDiagnostics, - ffmpegScreenRecordingActive, - setFfmpegScreenRecordingActive, - ffmpegCaptureProcess, - setFfmpegCaptureProcess, - ffmpegCaptureOutputBuffer, - setFfmpegCaptureOutputBuffer, - ffmpegCaptureTargetPath, - setFfmpegCaptureTargetPath, - cachedSystemCursorAssets, - setCachedSystemCursorAssets, - cachedSystemCursorAssetsSourceMtimeMs, - setCachedSystemCursorAssetsSourceMtimeMs, - setCursorCaptureStartTimeMs, - setActiveCursorSamples, - setPendingCursorSamples, - setIsCursorCaptureActive, - setLastLeftClick, - setLinuxCursorScreenPoint, - windowsCaptureOutputBuffer, - setWindowsCaptureOutputBuffer, -} from "../state"; -import { - getRecordingsDir, - getScreen, - getMacPrivacySettingsUrl, - moveFileWithOverwrite, - parseWindowId, - normalizeVideoSourcePath, - getTelemetryPathForVideo, -} from "../utils"; -import { - ensureSwiftHelperBinary, - getSystemCursorHelperSourcePath, - getSystemCursorHelperBinaryPath, - getNativeCaptureHelperBinaryPath, - ensureNativeCaptureHelperBinary, - getWindowsCaptureExePath, -} from "../paths/binaries"; + clamp, + sampleCursorPoint, + snapshotCursorTelemetryForPersistence, + startCursorSampling, + stopCursorCapture, +} from "../cursor/telemetry"; import { getFfmpegBinaryPath } from "../ffmpeg/binary"; import { - recordNativeCaptureDiagnostics, - getFileSizeIfPresent, - getCompanionAudioFallbackPaths, - validateRecordedVideo, -} from "../recording/diagnostics"; + ensureNativeCaptureHelperBinary, + ensureSwiftHelperBinary, + getNativeCaptureHelperBinaryPath, + getSystemCursorHelperBinaryPath, + getSystemCursorHelperSourcePath, + getWindowsCaptureExePath, +} from "../paths/binaries"; import { rememberApprovedLocalReadPath } from "../project/manager"; import { - isNativeWindowsCaptureAvailable, - waitForWindowsCaptureStart, - waitForWindowsCaptureStop, - attachWindowsCaptureLifecycle, - muxNativeWindowsVideoWithAudio, -} from "../recording/windows"; -import { - waitForNativeCaptureStart, - waitForNativeCaptureStop, - muxNativeMacRecordingWithAudio, - attachNativeCaptureLifecycle, - finalizeStoredVideo, - recoverNativeMacCaptureOutput, -} from "../recording/mac"; + getCompanionAudioFallbackPaths, + getFileSizeIfPresent, + recordNativeCaptureDiagnostics, + validateRecordedVideo, +} from "../recording/diagnostics"; import { buildFfmpegCaptureArgs, waitForFfmpegCaptureStart, waitForFfmpegCaptureStop, } from "../recording/ffmpeg"; +import { + attachNativeCaptureLifecycle, + finalizeStoredVideo, + muxNativeMacRecordingWithAudio, + recoverNativeMacCaptureOutput, + waitForNativeCaptureStart, + waitForNativeCaptureStop, +} from "../recording/mac"; +import { + attachWindowsCaptureLifecycle, + isNativeWindowsCaptureAvailable, + muxNativeWindowsVideoWithAudio, + waitForWindowsCaptureStart, + waitForWindowsCaptureStop, +} from "../recording/windows"; +import { shouldUseWindowsBrowserMicrophoneFallback } from "../recording/windowsFallbacks"; +import { + cachedSystemCursorAssets, + cachedSystemCursorAssetsSourceMtimeMs, + currentVideoPath, + ffmpegCaptureOutputBuffer, + ffmpegCaptureProcess, + ffmpegCaptureTargetPath, + ffmpegScreenRecordingActive, + lastNativeCaptureDiagnostics, + nativeCaptureMicrophonePath, + nativeCaptureOutputBuffer, + nativeCapturePaused, + nativeCaptureProcess, + nativeCaptureSystemAudioPath, + nativeCaptureTargetPath, + nativeScreenRecordingActive, + selectedSource, + setActiveCursorSamples, + setCachedSystemCursorAssets, + setCachedSystemCursorAssetsSourceMtimeMs, + setCursorCaptureStartTimeMs, + setFfmpegCaptureOutputBuffer, + setFfmpegCaptureProcess, + setFfmpegCaptureTargetPath, + setFfmpegScreenRecordingActive, + setIsCursorCaptureActive, + setLastLeftClick, + setLinuxCursorScreenPoint, + setNativeCaptureMicrophonePath, + setNativeCaptureOutputBuffer, + setNativeCapturePaused, + setNativeCaptureProcess, + setNativeCaptureStopRequested, + setNativeCaptureSystemAudioPath, + setNativeCaptureTargetPath, + setNativeScreenRecordingActive, + setPendingCursorSamples, + setWindowsCaptureOutputBuffer, + setWindowsCapturePaused, + setWindowsCaptureProcess, + setWindowsCaptureStopRequested, + setWindowsCaptureTargetPath, + setWindowsMicAudioPath, + setWindowsNativeCaptureActive, + setWindowsOrphanedMicAudioPath, + setWindowsPendingVideoPath, + setWindowsSystemAudioPath, + windowsCaptureOutputBuffer, + windowsCapturePaused, + windowsCaptureProcess, + windowsCaptureTargetPath, + windowsMicAudioPath, + windowsNativeCaptureActive, + windowsOrphanedMicAudioPath, + windowsPendingVideoPath, + windowsSystemAudioPath, +} from "../state"; +import type { + CursorTelemetryPoint, + NativeMacRecordingOptions, + PauseSegment, + SelectedSource, +} from "../types"; +import { + getMacPrivacySettingsUrl, + getRecordingsDir, + getScreen, + getTelemetryPathForVideo, + moveFileWithOverwrite, + normalizeVideoSourcePath, + parseWindowId, +} from "../utils"; import { resolveWindowsCaptureDisplay } from "../windowsCaptureSelection"; -import { - clamp, - stopCursorCapture, - sampleCursorPoint, - startCursorSampling, - snapshotCursorTelemetryForPersistence, -} from "../cursor/telemetry"; -import { - startWindowBoundsCapture, - stopWindowBoundsCapture, -} from "../cursor/bounds"; -import { startInteractionCapture, stopInteractionCapture } from "../cursor/interaction"; -import { stopNativeCursorMonitor, startNativeCursorMonitor } from "../cursor/monitor"; const execFileAsync = promisify(execFile); @@ -141,15 +154,22 @@ async function getSystemCursorAssets() { "system cursor helper", "recordly-system-cursors", ); - const { stdout } = await execFileAsync(binaryPath, [], { timeout: 15000, maxBuffer: 20 * 1024 * 1024 }); - const parsed = JSON.parse(stdout) as Record>; + const { stdout } = await execFileAsync(binaryPath, [], { + timeout: 15000, + maxBuffer: 20 * 1024 * 1024, + }); + const parsed = JSON.parse(stdout) as Record< + string, + Partial + >; const result = Object.fromEntries( - Object.entries(parsed).filter(([, asset]) => - typeof asset?.dataUrl === "string" && - typeof asset?.hotspotX === "number" && - typeof asset?.hotspotY === "number" && - typeof asset?.width === "number" && - typeof asset?.height === "number" + Object.entries(parsed).filter( + ([, asset]) => + typeof asset?.dataUrl === "string" && + typeof asset?.hotspotX === "number" && + typeof asset?.hotspotY === "number" && + typeof asset?.width === "number" && + typeof asset?.height === "number", ), ) as Record; setCachedSystemCursorAssets(result); @@ -161,1015 +181,1184 @@ function normalizeDesktopSourceName(value: string) { return value.trim().replace(/\s+/g, " ").toLowerCase(); } +async function cleanupWindowsOrphanedMicAudioPath(filePath: string | null) { + if (!filePath) { + return; + } + + await fs.rm(filePath, { force: true }).catch(() => undefined); +} + export function registerRecordingHandlers( onRecordingStateChange?: (recording: boolean, sourceName: string) => void, ) { - ipcMain.handle('start-native-screen-recording', async (_, source: SelectedSource, options?: NativeMacRecordingOptions) => { - // Windows native capture path - if (process.platform === 'win32') { - const windowsCaptureAvailable = await isNativeWindowsCaptureAvailable() - if (!windowsCaptureAvailable) { - return { success: false, message: 'Native Windows capture is not available on this system.' } - } - - if (windowsCaptureProcess && !windowsNativeCaptureActive) { - try { windowsCaptureProcess.kill() } catch { /* ignore */ } - setWindowsCaptureProcess(null) - setWindowsCaptureTargetPath(null) - setWindowsCaptureStopRequested(false) - } - - if (windowsCaptureProcess) { - return { success: false, message: 'A native Windows screen recording is already active.' } - } - - let wcProc: ChildProcessWithoutNullStreams | null = null - try { - const exePath = getWindowsCaptureExePath() - const recordingsDir = await getRecordingsDir() - const timestamp = Date.now() - const outputPath = path.join(recordingsDir, `recording-${timestamp}.mp4`) - const resolvedDisplay = resolveWindowsCaptureDisplay( - source, - getScreen().getAllDisplays(), - getScreen().getPrimaryDisplay(), - ) - const displayBounds = resolvedDisplay.bounds - - const config: Record = { - outputPath, - fps: 60, - displayId: resolvedDisplay.displayId, - displayX: Math.round(resolvedDisplay.bounds.x), - displayY: Math.round(resolvedDisplay.bounds.y), - displayW: Math.round(resolvedDisplay.bounds.width), - displayH: Math.round(resolvedDisplay.bounds.height), - } - - if (options?.capturesSystemAudio) { - const audioPath = path.join(recordingsDir, `recording-${timestamp}.system.wav`) - config.captureSystemAudio = true - config.audioOutputPath = audioPath - setWindowsSystemAudioPath(audioPath) - } - - if (options?.capturesMicrophone) { - const micPath = path.join(recordingsDir, `recording-${timestamp}.mic.wav`) - config.captureMic = true - config.micOutputPath = micPath - if (options.microphoneLabel) { - config.micDeviceName = options.microphoneLabel - } - setWindowsMicAudioPath(micPath) - } - - recordNativeCaptureDiagnostics({ - backend: 'windows-wgc', - phase: 'start', - sourceId: source?.id ?? null, - sourceType: source?.sourceType ?? 'unknown', - displayId: typeof config.displayId === 'number' ? config.displayId : null, - displayBounds, - windowHandle: typeof config.windowHandle === 'number' ? config.windowHandle : null, - helperPath: exePath, - outputPath, - systemAudioPath: windowsSystemAudioPath, - microphonePath: windowsMicAudioPath, - }) - - setWindowsCaptureOutputBuffer('') - setWindowsCaptureTargetPath(outputPath) - setWindowsCaptureStopRequested(false) - setWindowsCapturePaused(false) - wcProc = spawn(exePath, [JSON.stringify(config)], { - cwd: recordingsDir, - stdio: ['pipe', 'pipe', 'pipe'], - }) - setWindowsCaptureProcess(wcProc) - attachWindowsCaptureLifecycle(wcProc) - - wcProc.stdout.on('data', (chunk: Buffer) => { - setWindowsCaptureOutputBuffer(windowsCaptureOutputBuffer + chunk.toString()) - }) - wcProc.stderr.on('data', (chunk: Buffer) => { - setWindowsCaptureOutputBuffer(windowsCaptureOutputBuffer + chunk.toString()) - }) - - await waitForWindowsCaptureStart(wcProc) - setWindowsNativeCaptureActive(true) - setNativeScreenRecordingActive(true) - recordNativeCaptureDiagnostics({ - backend: 'windows-wgc', - phase: 'start', - sourceId: source?.id ?? null, - sourceType: source?.sourceType ?? 'unknown', - displayId: typeof config.displayId === 'number' ? config.displayId : null, - displayBounds, - windowHandle: typeof config.windowHandle === 'number' ? config.windowHandle : null, - helperPath: exePath, - outputPath, - systemAudioPath: windowsSystemAudioPath, - microphonePath: windowsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - }) - return { success: true } - } catch (error) { - recordNativeCaptureDiagnostics({ - backend: 'windows-wgc', - phase: 'start', - sourceId: source?.id ?? null, - sourceType: source?.sourceType ?? 'unknown', - helperPath: windowsCaptureTargetPath ? getWindowsCaptureExePath() : null, - outputPath: windowsCaptureTargetPath, - systemAudioPath: windowsSystemAudioPath, - microphonePath: windowsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - error: String(error), - }) - console.error('Failed to start native Windows capture:', error) - try { if (wcProc) wcProc.kill() } catch { /* ignore */ } - setWindowsNativeCaptureActive(false) - setNativeScreenRecordingActive(false) - setWindowsCaptureProcess(null) - setWindowsCaptureTargetPath(null) - setWindowsCaptureStopRequested(false) - setWindowsCapturePaused(false) - return { - success: false, - message: 'Failed to start native Windows capture', - error: String(error), - } - } - } - - if (process.platform !== 'darwin') { - return { success: false, message: 'Native screen recording is only available on macOS.' } - } - - if (nativeCaptureProcess && !nativeScreenRecordingActive) { - try { - nativeCaptureProcess.kill() - } catch { - // ignore stale helper cleanup failures - } - setNativeCaptureProcess(null) - setNativeCaptureTargetPath(null) - setNativeCaptureStopRequested(false) - } - - if (nativeCaptureProcess) { - return { success: false, message: 'A native screen recording is already active.' } - } - - let captProc: ChildProcessWithoutNullStreams | null = null - try { - const recordingsDir = await getRecordingsDir() - - // Warm up TCC: trigger an Electron-level screen capture API call so macOS - // activates the screen-recording grant for this process tree before the - // native helper binary spawns and calls SCStream.startCapture(). - try { - await desktopCapturer.getSources({ types: ['screen'], thumbnailSize: { width: 1, height: 1 } }) - } catch { - // non-fatal – the helper will report its own TCC status - } - - // Ensure microphone TCC is granted for this process tree when mic capture - // is requested, so the child helper inherits the grant. - if (options?.capturesMicrophone) { - const micStatus = systemPreferences.getMediaAccessStatus('microphone') - if (micStatus !== 'granted') { - await systemPreferences.askForMediaAccess('microphone') - } - } - - const appName = normalizeDesktopSourceName(String(source?.appName ?? '')) - const ownAppName = normalizeDesktopSourceName(app.getName()) - if ( - !ALLOW_RECORDLY_WINDOW_CAPTURE - && - source?.id?.startsWith('window:') - && appName - && (appName === ownAppName || appName === 'recordly') - ) { - return { success: false, message: 'Cannot record Recordly windows. Please select another app window.' } - } - - const helperPath = await ensureNativeCaptureHelperBinary() - const timestamp = Date.now() - const outputPath = path.join(recordingsDir, `recording-${timestamp}.mp4`) - const capturesSystemAudio = Boolean(options?.capturesSystemAudio) - const capturesMicrophone = Boolean(options?.capturesMicrophone) - const systemAudioOutputPath = capturesSystemAudio - ? path.join(recordingsDir, `recording-${timestamp}.system.m4a`) - : null - const microphoneOutputPath = capturesMicrophone - ? path.join(recordingsDir, `recording-${timestamp}.mic.m4a`) - : null - const config: Record = { - fps: 60, - outputPath, - capturesSystemAudio, - capturesMicrophone, - } - - if (options?.microphoneDeviceId) { - config.microphoneDeviceId = options.microphoneDeviceId - } - - if (options?.microphoneLabel) { - config.microphoneLabel = options.microphoneLabel - } - - if (systemAudioOutputPath) { - config.systemAudioOutputPath = systemAudioOutputPath - } - - if (microphoneOutputPath) { - config.microphoneOutputPath = microphoneOutputPath - } - - const windowId = parseWindowId(source?.id) - const screenId = Number(source?.display_id) - - if (Number.isFinite(windowId) && windowId && source?.id?.startsWith('window:')) { - config.windowId = windowId - } else if (Number.isFinite(screenId) && screenId > 0) { - config.displayId = screenId - } else { - config.displayId = Number(getScreen().getPrimaryDisplay().id) - } - - setNativeCaptureOutputBuffer('') - setNativeCaptureTargetPath(outputPath) - setNativeCaptureSystemAudioPath(systemAudioOutputPath) - setNativeCaptureMicrophonePath(microphoneOutputPath) - setNativeCaptureStopRequested(false) - setNativeCapturePaused(false) - captProc = spawn(helperPath, [JSON.stringify(config)], { - cwd: recordingsDir, - stdio: ['pipe', 'pipe', 'pipe'], - }) - setNativeCaptureProcess(captProc) - attachNativeCaptureLifecycle(captProc) - - captProc.stdout.on('data', (chunk: Buffer) => { - setNativeCaptureOutputBuffer(nativeCaptureOutputBuffer + chunk.toString()) - }) - captProc.stderr.on('data', (chunk: Buffer) => { - setNativeCaptureOutputBuffer(nativeCaptureOutputBuffer + chunk.toString()) - }) - - await waitForNativeCaptureStart(captProc) - setNativeScreenRecordingActive(true) - - // If the native helper reported MICROPHONE_CAPTURE_UNAVAILABLE, it started - // capture without microphone. Clear the mic path so the renderer can fall - // back to a browser-side sidecar recording for the microphone track. - const micUnavailableNatively = nativeCaptureOutputBuffer.includes('MICROPHONE_CAPTURE_UNAVAILABLE') - if (micUnavailableNatively) { - setNativeCaptureMicrophonePath(null) - } - - recordNativeCaptureDiagnostics({ - backend: 'mac-screencapturekit', - phase: 'start', - sourceId: source?.id ?? null, - sourceType: source?.sourceType ?? 'unknown', - displayId: typeof config.displayId === 'number' ? config.displayId : null, - helperPath, - outputPath, - systemAudioPath: systemAudioOutputPath, - microphonePath: nativeCaptureMicrophonePath, - processOutput: nativeCaptureOutputBuffer.trim() || undefined, - }) - return { success: true, microphoneFallbackRequired: micUnavailableNatively } - } catch (error) { - console.error('Failed to start native ScreenCaptureKit recording:', error) - const errorStr = String(error) - - // Detect TCC (screen recording permission) errors and show a helpful dialog - if (errorStr.includes('declined TCC') || errorStr.includes('declined TCCs') || errorStr.includes('SCREEN_RECORDING_PERMISSION_DENIED')) { - const { response } = await dialog.showMessageBox({ - type: 'warning', - title: 'Screen Recording Permission Required', - message: 'Recordly needs screen recording permission to capture your screen.', - detail: 'Please open System Settings > Privacy & Security > Screen Recording, make sure Recordly is toggled ON, then try recording again.', - buttons: ['Open System Settings', 'Cancel'], - defaultId: 0, - cancelId: 1, - }) - if (response === 0) { - await shell.openExternal(getMacPrivacySettingsUrl('screen')) - } - try { if (captProc) captProc.kill() } catch { /* ignore */ } - setNativeScreenRecordingActive(false) - setNativeCaptureProcess(null) - setNativeCaptureTargetPath(null) - setNativeCaptureSystemAudioPath(null) - setNativeCaptureMicrophonePath(null) - setNativeCaptureStopRequested(false) - setNativeCapturePaused(false) - return { - success: false, - message: 'Screen recording permission not granted. Please allow access in System Settings and restart the app.', - userNotified: true, - } - } - - if (errorStr.includes('MICROPHONE_PERMISSION_DENIED')) { - const { response } = await dialog.showMessageBox({ - type: 'warning', - title: 'Microphone Permission Required', - message: 'Recordly needs microphone permission to record audio.', - detail: 'Please open System Settings > Privacy & Security > Microphone, make sure Recordly is toggled ON, then try recording again.', - buttons: ['Open System Settings', 'Cancel'], - defaultId: 0, - cancelId: 1, - }) - if (response === 0) { - await shell.openExternal(getMacPrivacySettingsUrl('microphone')) - } - try { if (captProc) captProc.kill() } catch { /* ignore */ } - setNativeScreenRecordingActive(false) - setNativeCaptureProcess(null) - setNativeCaptureTargetPath(null) - setNativeCaptureSystemAudioPath(null) - setNativeCaptureMicrophonePath(null) - setNativeCaptureStopRequested(false) - setNativeCapturePaused(false) - return { - success: false, - message: 'Microphone permission not granted. Please allow access in System Settings.', - userNotified: true, - } - } - - recordNativeCaptureDiagnostics({ - backend: 'mac-screencapturekit', - phase: 'start', - sourceId: source?.id ?? null, - sourceType: source?.sourceType ?? 'unknown', - helperPath: getNativeCaptureHelperBinaryPath(), - outputPath: nativeCaptureTargetPath, - systemAudioPath: nativeCaptureSystemAudioPath, - microphonePath: nativeCaptureMicrophonePath, - processOutput: nativeCaptureOutputBuffer.trim() || undefined, - fileSizeBytes: await getFileSizeIfPresent(nativeCaptureTargetPath), - error: String(error), - }) - try { - if (captProc) captProc.kill() - } catch { - // ignore cleanup failures - } - setNativeScreenRecordingActive(false) - setNativeCaptureProcess(null) - setNativeCaptureTargetPath(null) - setNativeCaptureSystemAudioPath(null) - setNativeCaptureMicrophonePath(null) - setNativeCaptureStopRequested(false) - setNativeCapturePaused(false) - return { - success: false, - message: 'Failed to start native ScreenCaptureKit recording', - error: String(error), - } - } - }) - - ipcMain.handle('stop-native-screen-recording', async () => { - // Windows native capture stop path - if (process.platform === 'win32' && windowsNativeCaptureActive) { - try { - if (!windowsCaptureProcess) { - throw new Error('Native Windows capture process is not running') - } - - const proc = windowsCaptureProcess - const preferredVideoPath = windowsCaptureTargetPath - setWindowsCaptureStopRequested(true) - proc.stdin.write('stop\n') - const tempVideoPath = await waitForWindowsCaptureStop(proc) - - const finalVideoPath = preferredVideoPath ?? tempVideoPath - if (tempVideoPath !== finalVideoPath) { - await moveFileWithOverwrite(tempVideoPath, finalVideoPath) - } - const validation = await validateRecordedVideo(finalVideoPath) - - setWindowsCaptureProcess(null) - setWindowsNativeCaptureActive(false) - setNativeScreenRecordingActive(false) - setWindowsCaptureTargetPath(null) - setWindowsCaptureStopRequested(false) - setWindowsCapturePaused(false) - setWindowsPendingVideoPath(finalVideoPath) - recordNativeCaptureDiagnostics({ - backend: 'windows-wgc', - phase: 'stop', - outputPath: finalVideoPath, - systemAudioPath: windowsSystemAudioPath, - microphonePath: windowsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - fileSizeBytes: validation.fileSizeBytes, - }) - return { success: true, path: finalVideoPath } - } catch (error) { - console.error('Failed to stop native Windows capture:', error) - const fallbackPath = windowsCaptureTargetPath - setWindowsNativeCaptureActive(false) - setNativeScreenRecordingActive(false) - setWindowsCaptureProcess(null) - setWindowsCaptureTargetPath(null) - setWindowsCaptureStopRequested(false) - setWindowsCapturePaused(false) - setWindowsSystemAudioPath(null) - setWindowsMicAudioPath(null) - setWindowsPendingVideoPath(null) - - if (fallbackPath) { - try { - await fs.access(fallbackPath) - const validation = await validateRecordedVideo(fallbackPath) - setWindowsPendingVideoPath(fallbackPath) - recordNativeCaptureDiagnostics({ - backend: 'windows-wgc', - phase: 'stop', - outputPath: fallbackPath, - systemAudioPath: windowsSystemAudioPath, - microphonePath: windowsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - fileSizeBytes: validation.fileSizeBytes, - error: String(error), - }) - return { success: true, path: fallbackPath } - } catch { - // File is absent or failed validation. - } - } - - recordNativeCaptureDiagnostics({ - backend: 'windows-wgc', - phase: 'stop', - outputPath: fallbackPath, - systemAudioPath: windowsSystemAudioPath, - microphonePath: windowsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - fileSizeBytes: await getFileSizeIfPresent(fallbackPath), - error: String(error), - }) - - return { - success: false, - message: 'Failed to stop native Windows capture', - error: String(error), - } - } - } - - if (process.platform !== 'darwin') { - return { success: false, message: 'Native screen recording is only available on macOS.' } - } - - if (!nativeScreenRecordingActive) { - const recovered = await recoverNativeMacCaptureOutput() - if (recovered) { - return recovered - } - - return { success: false, message: 'No native screen recording is active.' } - } - - try { - if (!nativeCaptureProcess) { - throw new Error('Native capture helper process is not running') - } - - const process = nativeCaptureProcess - const preferredVideoPath = nativeCaptureTargetPath - const preferredSystemAudioPath = nativeCaptureSystemAudioPath - const preferredMicrophonePath = nativeCaptureMicrophonePath - console.log('[stop-native] Audio paths — system:', preferredSystemAudioPath, 'mic:', preferredMicrophonePath) - setNativeCaptureStopRequested(true) - process.stdin.write('stop\n') - const tempVideoPath = await waitForNativeCaptureStop(process) - console.log('[stop-native] Helper stopped, tempVideoPath:', tempVideoPath) - setNativeCaptureProcess(null) - setNativeScreenRecordingActive(false) - setNativeCaptureTargetPath(null) - setNativeCaptureSystemAudioPath(null) - setNativeCaptureMicrophonePath(null) - setNativeCaptureStopRequested(false) - setNativeCapturePaused(false) - - const finalVideoPath = preferredVideoPath ?? tempVideoPath - if (tempVideoPath !== finalVideoPath) { - await moveFileWithOverwrite(tempVideoPath, finalVideoPath) - } - - if (preferredSystemAudioPath || preferredMicrophonePath) { - console.log('[stop-native] Attempting audio mux (merging separate tracks) into:', finalVideoPath) - try { - await muxNativeMacRecordingWithAudio(finalVideoPath, preferredSystemAudioPath, preferredMicrophonePath) - console.log('[stop-native] Audio mux completed successfully') - } catch (error) { - console.warn('[stop-native] Audio mux failed (video still has inline audio):', error) - } - } else { - console.log('[stop-native] No separate audio tracks to mux') - } - - return await finalizeStoredVideo(finalVideoPath) - } catch (error) { - console.error('Failed to stop native ScreenCaptureKit recording:', error) - const fallbackPath = nativeCaptureTargetPath - const fallbackSystemAudioPath = nativeCaptureSystemAudioPath - const fallbackMicrophonePath = nativeCaptureMicrophonePath - const fallbackFileSizeBytes = await getFileSizeIfPresent(fallbackPath) - setNativeScreenRecordingActive(false) - setNativeCaptureProcess(null) - setNativeCaptureTargetPath(null) - setNativeCaptureSystemAudioPath(null) - setNativeCaptureMicrophonePath(null) - setNativeCaptureStopRequested(false) - setNativeCapturePaused(false) - - recordNativeCaptureDiagnostics({ - backend: 'mac-screencapturekit', - phase: 'stop', - sourceId: lastNativeCaptureDiagnostics?.sourceId ?? null, - sourceType: lastNativeCaptureDiagnostics?.sourceType ?? 'unknown', - displayId: lastNativeCaptureDiagnostics?.displayId ?? null, - displayBounds: lastNativeCaptureDiagnostics?.displayBounds ?? null, - windowHandle: lastNativeCaptureDiagnostics?.windowHandle ?? null, - helperPath: lastNativeCaptureDiagnostics?.helperPath ?? null, - outputPath: fallbackPath, - systemAudioPath: fallbackSystemAudioPath, - microphonePath: fallbackMicrophonePath, - osRelease: lastNativeCaptureDiagnostics?.osRelease, - supported: lastNativeCaptureDiagnostics?.supported, - helperExists: lastNativeCaptureDiagnostics?.helperExists, - processOutput: nativeCaptureOutputBuffer.trim() || undefined, - fileSizeBytes: fallbackFileSizeBytes, - error: String(error), - }) - - // Try to recover: if the target file exists on disk, finalize with it - if (fallbackPath) { - try { - await fs.access(fallbackPath) - console.log('[stop-native-screen-recording] Recovering with fallback path:', fallbackPath) - if (fallbackSystemAudioPath || fallbackMicrophonePath) { - try { - await muxNativeMacRecordingWithAudio( - fallbackPath, - fallbackSystemAudioPath, - fallbackMicrophonePath, - ) - } catch (muxError) { - console.warn('Failed to mux recovered native macOS audio into capture:', muxError) - } - } - return await finalizeStoredVideo(fallbackPath) - } catch { - // File doesn't exist or isn't accessible - } - } - - const recovered = await recoverNativeMacCaptureOutput() - if (recovered) { - return recovered - } - - return { - success: false, - message: 'Failed to stop native ScreenCaptureKit recording', - error: String(error), - } - } - }) - - ipcMain.handle('recover-native-screen-recording', async () => { - if (process.platform !== 'darwin') { - return { success: false, message: 'Native screen recording recovery is only available on macOS.' } - } - - const recovered = await recoverNativeMacCaptureOutput() - if (recovered) { - return recovered - } - - return { - success: false, - message: 'No recoverable native macOS recording output was found.', - } - }) - - ipcMain.handle('pause-native-screen-recording', async () => { - if (process.platform === 'win32') { - if (!windowsNativeCaptureActive || !windowsCaptureProcess) { - return { success: false, message: 'No native Windows screen recording is active.' } - } - - if (windowsCapturePaused) { - return { success: true } - } - - try { - windowsCaptureProcess.stdin.write('pause\n') - setWindowsCapturePaused(true) - return { success: true } - } catch (error) { - return { success: false, message: 'Failed to pause native Windows capture', error: String(error) } - } - } - - if (process.platform !== 'darwin') { - return { success: false, message: 'Native screen recording is only available on macOS.' } - } - - if (!nativeScreenRecordingActive || !nativeCaptureProcess) { - return { success: false, message: 'No native screen recording is active.' } - } - - if (nativeCapturePaused) { - return { success: true } - } - - try { - nativeCaptureProcess.stdin.write('pause\n') - setNativeCapturePaused(true) - return { success: true } - } catch (error) { - return { success: false, message: 'Failed to pause native screen recording', error: String(error) } - } - }) - - ipcMain.handle('resume-native-screen-recording', async () => { - if (process.platform === 'win32') { - if (!windowsNativeCaptureActive || !windowsCaptureProcess) { - return { success: false, message: 'No native Windows screen recording is active.' } - } - - if (!windowsCapturePaused) { - return { success: true } - } - - try { - windowsCaptureProcess.stdin.write('resume\n') - setWindowsCapturePaused(false) - return { success: true } - } catch (error) { - return { success: false, message: 'Failed to resume native Windows capture', error: String(error) } - } - } - - if (process.platform !== 'darwin') { - return { success: false, message: 'Native screen recording is only available on macOS.' } - } - - if (!nativeScreenRecordingActive || !nativeCaptureProcess) { - return { success: false, message: 'No native screen recording is active.' } - } - - if (!nativeCapturePaused) { - return { success: true } - } - - try { - nativeCaptureProcess.stdin.write('resume\n') - setNativeCapturePaused(false) - return { success: true } - } catch (error) { - return { success: false, message: 'Failed to resume native screen recording', error: String(error) } - } - }) - - ipcMain.handle('get-system-cursor-assets', async () => { - try { - return { success: true, cursors: await getSystemCursorAssets() } - } catch (error) { - console.error('Failed to load system cursor assets:', error) - return { success: false, cursors: {}, error: String(error) } - } - }) - - ipcMain.handle('is-native-windows-capture-available', async () => { - return { available: await isNativeWindowsCaptureAvailable() } - }) - - ipcMain.handle('get-last-native-capture-diagnostics', async () => { - return { success: true, diagnostics: lastNativeCaptureDiagnostics } - }) - - ipcMain.handle('get-video-audio-fallback-paths', async (_event, videoPath: string) => { - if (!videoPath) { - return { success: true, paths: [] } - } - - try { - const paths = await getCompanionAudioFallbackPaths(videoPath) - await Promise.all([ - rememberApprovedLocalReadPath(videoPath), - ...paths.map((fallbackPath) => rememberApprovedLocalReadPath(fallbackPath)), - ]) - return { success: true, paths } - } catch (error) { - console.error('Failed to resolve companion audio fallback paths:', error) - return { success: false, paths: [], error: String(error) } - } - }) - - ipcMain.handle('mux-native-windows-recording', async (_event, pauseSegments?: PauseSegment[]) => { - const videoPath = windowsPendingVideoPath - setWindowsPendingVideoPath(null) - - if (!videoPath) { - return { success: false, message: 'No native Windows video pending for mux' } - } - - try { - if (windowsSystemAudioPath || windowsMicAudioPath) { - await muxNativeWindowsVideoWithAudio(videoPath, windowsSystemAudioPath, windowsMicAudioPath, pauseSegments ?? []) - setWindowsSystemAudioPath(null) - setWindowsMicAudioPath(null) - } - - recordNativeCaptureDiagnostics({ - backend: 'windows-wgc', - phase: 'mux', - outputPath: videoPath, - fileSizeBytes: await getFileSizeIfPresent(videoPath), - }) - return await finalizeStoredVideo(videoPath) - } catch (error) { - console.error('Failed to mux native Windows recording:', error) - recordNativeCaptureDiagnostics({ - backend: 'windows-wgc', - phase: 'mux', - outputPath: videoPath, - systemAudioPath: windowsSystemAudioPath, - microphonePath: windowsMicAudioPath, - fileSizeBytes: await getFileSizeIfPresent(videoPath), - error: String(error), - }) - setWindowsSystemAudioPath(null) - setWindowsMicAudioPath(null) - return { - success: false, - message: 'Failed to finalize native Windows recording', - error: String(error), - } - } - }) - - ipcMain.handle('start-ffmpeg-recording', async (_, source: SelectedSource) => { - if (ffmpegCaptureProcess) { - return { success: false, message: 'An FFmpeg recording is already active.' } - } - - try { - const recordingsDir = await getRecordingsDir() - const ffmpegPath = getFfmpegBinaryPath() - const outputPath = path.join(recordingsDir, `recording-${Date.now()}.mp4`) - const args = await buildFfmpegCaptureArgs(source, outputPath) - - setFfmpegCaptureOutputBuffer('') - setFfmpegCaptureTargetPath(outputPath) - const ffProc = spawn(ffmpegPath, args, { - cwd: recordingsDir, - stdio: ['pipe', 'pipe', 'pipe'], - }) - setFfmpegCaptureProcess(ffProc) - - ffProc.stdout.on('data', (chunk: Buffer) => { - setFfmpegCaptureOutputBuffer(ffmpegCaptureOutputBuffer + chunk.toString()) - }) - ffProc.stderr.on('data', (chunk: Buffer) => { - setFfmpegCaptureOutputBuffer(ffmpegCaptureOutputBuffer + chunk.toString()) - }) - - await waitForFfmpegCaptureStart(ffProc) - setFfmpegScreenRecordingActive(true) - return { success: true } - } catch (error) { - console.error('Failed to start FFmpeg recording:', error) - setFfmpegScreenRecordingActive(false) - setFfmpegCaptureProcess(null) - setFfmpegCaptureTargetPath(null) - return { - success: false, - message: 'Failed to start FFmpeg recording', - error: String(error), - } - } - }) - - ipcMain.handle('stop-ffmpeg-recording', async () => { - if (!ffmpegScreenRecordingActive) { - return { success: false, message: 'No FFmpeg recording is active.' } - } - - try { - if (!ffmpegCaptureProcess || !ffmpegCaptureTargetPath) { - throw new Error('FFmpeg process is not running') - } - - const process = ffmpegCaptureProcess - const outputPath = ffmpegCaptureTargetPath - process.stdin.write('q\n') - const finalVideoPath = await waitForFfmpegCaptureStop(process, outputPath) - - setFfmpegCaptureProcess(null) - setFfmpegCaptureTargetPath(null) - setFfmpegScreenRecordingActive(false) - - return await finalizeStoredVideo(finalVideoPath) - } catch (error) { - console.error('Failed to stop FFmpeg recording:', error) + ipcMain.handle( + "start-native-screen-recording", + async (_, source: SelectedSource, options?: NativeMacRecordingOptions) => { + // Windows native capture path + if (process.platform === "win32") { + const windowsCaptureAvailable = await isNativeWindowsCaptureAvailable(); + if (!windowsCaptureAvailable) { + return { + success: false, + message: "Native Windows capture is not available on this system.", + }; + } + + if (windowsCaptureProcess && !windowsNativeCaptureActive) { + try { + windowsCaptureProcess.kill(); + } catch { + /* ignore */ + } + setWindowsCaptureProcess(null); + setWindowsCaptureTargetPath(null); + setWindowsCaptureStopRequested(false); + } + + if (windowsCaptureProcess) { + return { + success: false, + message: "A native Windows screen recording is already active.", + }; + } + + let wcProc: ChildProcessWithoutNullStreams | null = null; + try { + const exePath = getWindowsCaptureExePath(); + const recordingsDir = await getRecordingsDir(); + const timestamp = Date.now(); + const outputPath = path.join(recordingsDir, `recording-${timestamp}.mp4`); + let captureOutput = ""; + let systemAudioPath: string | null = null; + let microphonePath: string | null = null; + let orphanedMicAudioPath: string | null = null; + const resolvedDisplay = resolveWindowsCaptureDisplay( + source, + getScreen().getAllDisplays(), + getScreen().getPrimaryDisplay(), + ); + const displayBounds = resolvedDisplay.bounds; + setWindowsOrphanedMicAudioPath(null); + + const config: Record = { + outputPath, + fps: 60, + displayId: resolvedDisplay.displayId, + displayX: Math.round(resolvedDisplay.bounds.x), + displayY: Math.round(resolvedDisplay.bounds.y), + displayW: Math.round(resolvedDisplay.bounds.width), + displayH: Math.round(resolvedDisplay.bounds.height), + }; + + if (options?.capturesSystemAudio) { + systemAudioPath = path.join( + recordingsDir, + `recording-${timestamp}.system.wav`, + ); + config.captureSystemAudio = true; + config.audioOutputPath = systemAudioPath; + setWindowsSystemAudioPath(systemAudioPath); + } + + if (options?.capturesMicrophone) { + microphonePath = path.join(recordingsDir, `recording-${timestamp}.mic.wav`); + config.captureMic = true; + config.micOutputPath = microphonePath; + if (options.microphoneLabel) { + config.micDeviceName = options.microphoneLabel; + } + setWindowsMicAudioPath(microphonePath); + } + + recordNativeCaptureDiagnostics({ + backend: "windows-wgc", + phase: "start", + sourceId: source?.id ?? null, + sourceType: source?.sourceType ?? "unknown", + displayId: typeof config.displayId === "number" ? config.displayId : null, + displayBounds, + windowHandle: + typeof config.windowHandle === "number" ? config.windowHandle : null, + helperPath: exePath, + outputPath, + systemAudioPath, + microphonePath, + }); + + setWindowsCaptureOutputBuffer(""); + setWindowsCaptureTargetPath(outputPath); + setWindowsCaptureStopRequested(false); + setWindowsCapturePaused(false); + wcProc = spawn(exePath, [JSON.stringify(config)], { + cwd: recordingsDir, + stdio: ["pipe", "pipe", "pipe"], + }); + setWindowsCaptureProcess(wcProc); + attachWindowsCaptureLifecycle(wcProc); + + wcProc.stdout.on("data", (chunk: Buffer) => { + captureOutput += chunk.toString(); + setWindowsCaptureOutputBuffer(captureOutput); + }); + wcProc.stderr.on("data", (chunk: Buffer) => { + captureOutput += chunk.toString(); + setWindowsCaptureOutputBuffer(captureOutput); + }); + + await waitForWindowsCaptureStart(wcProc); + const microphoneFallbackRequired = shouldUseWindowsBrowserMicrophoneFallback( + captureOutput, + options, + ); + if (microphoneFallbackRequired) { + orphanedMicAudioPath = microphonePath; + setWindowsOrphanedMicAudioPath(orphanedMicAudioPath); + microphonePath = null; + setWindowsMicAudioPath(null); + } + setWindowsNativeCaptureActive(true); + setNativeScreenRecordingActive(true); + recordNativeCaptureDiagnostics({ + backend: "windows-wgc", + phase: "start", + sourceId: source?.id ?? null, + sourceType: source?.sourceType ?? "unknown", + displayId: typeof config.displayId === "number" ? config.displayId : null, + displayBounds, + windowHandle: + typeof config.windowHandle === "number" ? config.windowHandle : null, + helperPath: exePath, + outputPath, + systemAudioPath, + microphonePath, + processOutput: captureOutput.trim() || undefined, + }); + return { success: true, microphoneFallbackRequired }; + } catch (error) { + recordNativeCaptureDiagnostics({ + backend: "windows-wgc", + phase: "start", + sourceId: source?.id ?? null, + sourceType: source?.sourceType ?? "unknown", + helperPath: windowsCaptureTargetPath ? getWindowsCaptureExePath() : null, + outputPath: windowsCaptureTargetPath, + systemAudioPath: windowsSystemAudioPath, + microphonePath: windowsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, + error: String(error), + }); + console.error("Failed to start native Windows capture:", error); + try { + if (wcProc) wcProc.kill(); + } catch { + /* ignore */ + } + setWindowsNativeCaptureActive(false); + setNativeScreenRecordingActive(false); + setWindowsCaptureProcess(null); + setWindowsCaptureTargetPath(null); + setWindowsCaptureStopRequested(false); + setWindowsCapturePaused(false); + return { + success: false, + message: "Failed to start native Windows capture", + error: String(error), + }; + } + } + + if (process.platform !== "darwin") { + return { + success: false, + message: "Native screen recording is only available on macOS.", + }; + } + + if (nativeCaptureProcess && !nativeScreenRecordingActive) { + try { + nativeCaptureProcess.kill(); + } catch { + // ignore stale helper cleanup failures + } + setNativeCaptureProcess(null); + setNativeCaptureTargetPath(null); + setNativeCaptureStopRequested(false); + } + + if (nativeCaptureProcess) { + return { success: false, message: "A native screen recording is already active." }; + } + + let captProc: ChildProcessWithoutNullStreams | null = null; try { - ffmpegCaptureProcess?.kill() + const recordingsDir = await getRecordingsDir(); + + // Warm up TCC: trigger an Electron-level screen capture API call so macOS + // activates the screen-recording grant for this process tree before the + // native helper binary spawns and calls SCStream.startCapture(). + try { + await desktopCapturer.getSources({ + types: ["screen"], + thumbnailSize: { width: 1, height: 1 }, + }); + } catch { + // non-fatal – the helper will report its own TCC status + } + + // Ensure microphone TCC is granted for this process tree when mic capture + // is requested, so the child helper inherits the grant. + if (options?.capturesMicrophone) { + const micStatus = systemPreferences.getMediaAccessStatus("microphone"); + if (micStatus !== "granted") { + await systemPreferences.askForMediaAccess("microphone"); + } + } + + const appName = normalizeDesktopSourceName(String(source?.appName ?? "")); + const ownAppName = normalizeDesktopSourceName(app.getName()); + if ( + !ALLOW_RECORDLY_WINDOW_CAPTURE && + source?.id?.startsWith("window:") && + appName && + (appName === ownAppName || appName === "recordly") + ) { + return { + success: false, + message: + "Cannot record Recordly windows. Please select another app window.", + }; + } + + const helperPath = await ensureNativeCaptureHelperBinary(); + const timestamp = Date.now(); + const outputPath = path.join(recordingsDir, `recording-${timestamp}.mp4`); + const capturesSystemAudio = Boolean(options?.capturesSystemAudio); + const capturesMicrophone = Boolean(options?.capturesMicrophone); + const systemAudioOutputPath = capturesSystemAudio + ? path.join(recordingsDir, `recording-${timestamp}.system.m4a`) + : null; + const microphoneOutputPath = capturesMicrophone + ? path.join(recordingsDir, `recording-${timestamp}.mic.m4a`) + : null; + const config: Record = { + fps: 60, + outputPath, + capturesSystemAudio, + capturesMicrophone, + }; + + if (options?.microphoneDeviceId) { + config.microphoneDeviceId = options.microphoneDeviceId; + } + + if (options?.microphoneLabel) { + config.microphoneLabel = options.microphoneLabel; + } + + if (systemAudioOutputPath) { + config.systemAudioOutputPath = systemAudioOutputPath; + } + + if (microphoneOutputPath) { + config.microphoneOutputPath = microphoneOutputPath; + } + + const windowId = parseWindowId(source?.id); + const screenId = Number(source?.display_id); + + if (Number.isFinite(windowId) && windowId && source?.id?.startsWith("window:")) { + config.windowId = windowId; + } else if (Number.isFinite(screenId) && screenId > 0) { + config.displayId = screenId; + } else { + config.displayId = Number(getScreen().getPrimaryDisplay().id); + } + + setNativeCaptureOutputBuffer(""); + setNativeCaptureTargetPath(outputPath); + setNativeCaptureSystemAudioPath(systemAudioOutputPath); + setNativeCaptureMicrophonePath(microphoneOutputPath); + setNativeCaptureStopRequested(false); + setNativeCapturePaused(false); + captProc = spawn(helperPath, [JSON.stringify(config)], { + cwd: recordingsDir, + stdio: ["pipe", "pipe", "pipe"], + }); + setNativeCaptureProcess(captProc); + attachNativeCaptureLifecycle(captProc); + + captProc.stdout.on("data", (chunk: Buffer) => { + setNativeCaptureOutputBuffer(nativeCaptureOutputBuffer + chunk.toString()); + }); + captProc.stderr.on("data", (chunk: Buffer) => { + setNativeCaptureOutputBuffer(nativeCaptureOutputBuffer + chunk.toString()); + }); + + await waitForNativeCaptureStart(captProc); + setNativeScreenRecordingActive(true); + + // If the native helper reported MICROPHONE_CAPTURE_UNAVAILABLE, it started + // capture without microphone. Clear the mic path so the renderer can fall + // back to a browser-side sidecar recording for the microphone track. + const micUnavailableNatively = nativeCaptureOutputBuffer.includes( + "MICROPHONE_CAPTURE_UNAVAILABLE", + ); + if (micUnavailableNatively) { + setNativeCaptureMicrophonePath(null); + } + + recordNativeCaptureDiagnostics({ + backend: "mac-screencapturekit", + phase: "start", + sourceId: source?.id ?? null, + sourceType: source?.sourceType ?? "unknown", + displayId: typeof config.displayId === "number" ? config.displayId : null, + helperPath, + outputPath, + systemAudioPath: systemAudioOutputPath, + microphonePath: nativeCaptureMicrophonePath, + processOutput: nativeCaptureOutputBuffer.trim() || undefined, + }); + return { success: true, microphoneFallbackRequired: micUnavailableNatively }; + } catch (error) { + console.error("Failed to start native ScreenCaptureKit recording:", error); + const errorStr = String(error); + + // Detect TCC (screen recording permission) errors and show a helpful dialog + if ( + errorStr.includes("declined TCC") || + errorStr.includes("declined TCCs") || + errorStr.includes("SCREEN_RECORDING_PERMISSION_DENIED") + ) { + const { response } = await dialog.showMessageBox({ + type: "warning", + title: "Screen Recording Permission Required", + message: + "Recordly needs screen recording permission to capture your screen.", + detail: "Please open System Settings > Privacy & Security > Screen Recording, make sure Recordly is toggled ON, then try recording again.", + buttons: ["Open System Settings", "Cancel"], + defaultId: 0, + cancelId: 1, + }); + if (response === 0) { + await shell.openExternal(getMacPrivacySettingsUrl("screen")); + } + try { + if (captProc) captProc.kill(); + } catch { + /* ignore */ + } + setNativeScreenRecordingActive(false); + setNativeCaptureProcess(null); + setNativeCaptureTargetPath(null); + setNativeCaptureSystemAudioPath(null); + setNativeCaptureMicrophonePath(null); + setNativeCaptureStopRequested(false); + setNativeCapturePaused(false); + return { + success: false, + message: + "Screen recording permission not granted. Please allow access in System Settings and restart the app.", + userNotified: true, + }; + } + + if (errorStr.includes("MICROPHONE_PERMISSION_DENIED")) { + const { response } = await dialog.showMessageBox({ + type: "warning", + title: "Microphone Permission Required", + message: "Recordly needs microphone permission to record audio.", + detail: "Please open System Settings > Privacy & Security > Microphone, make sure Recordly is toggled ON, then try recording again.", + buttons: ["Open System Settings", "Cancel"], + defaultId: 0, + cancelId: 1, + }); + if (response === 0) { + await shell.openExternal(getMacPrivacySettingsUrl("microphone")); + } + try { + if (captProc) captProc.kill(); + } catch { + /* ignore */ + } + setNativeScreenRecordingActive(false); + setNativeCaptureProcess(null); + setNativeCaptureTargetPath(null); + setNativeCaptureSystemAudioPath(null); + setNativeCaptureMicrophonePath(null); + setNativeCaptureStopRequested(false); + setNativeCapturePaused(false); + return { + success: false, + message: + "Microphone permission not granted. Please allow access in System Settings.", + userNotified: true, + }; + } + + recordNativeCaptureDiagnostics({ + backend: "mac-screencapturekit", + phase: "start", + sourceId: source?.id ?? null, + sourceType: source?.sourceType ?? "unknown", + helperPath: getNativeCaptureHelperBinaryPath(), + outputPath: nativeCaptureTargetPath, + systemAudioPath: nativeCaptureSystemAudioPath, + microphonePath: nativeCaptureMicrophonePath, + processOutput: nativeCaptureOutputBuffer.trim() || undefined, + fileSizeBytes: await getFileSizeIfPresent(nativeCaptureTargetPath), + error: String(error), + }); + try { + if (captProc) captProc.kill(); + } catch { + // ignore cleanup failures + } + setNativeScreenRecordingActive(false); + setNativeCaptureProcess(null); + setNativeCaptureTargetPath(null); + setNativeCaptureSystemAudioPath(null); + setNativeCaptureMicrophonePath(null); + setNativeCaptureStopRequested(false); + setNativeCapturePaused(false); + return { + success: false, + message: "Failed to start native ScreenCaptureKit recording", + error: String(error), + }; + } + }, + ); + + ipcMain.handle("stop-native-screen-recording", async () => { + // Windows native capture stop path + if (process.platform === "win32" && windowsNativeCaptureActive) { + try { + if (!windowsCaptureProcess) { + throw new Error("Native Windows capture process is not running"); + } + + const proc = windowsCaptureProcess; + const preferredVideoPath = windowsCaptureTargetPath; + const preferredOrphanedMicAudioPath = windowsOrphanedMicAudioPath; + setWindowsCaptureStopRequested(true); + proc.stdin.write("stop\n"); + const tempVideoPath = await waitForWindowsCaptureStop(proc); + + const finalVideoPath = preferredVideoPath ?? tempVideoPath; + if (tempVideoPath !== finalVideoPath) { + await moveFileWithOverwrite(tempVideoPath, finalVideoPath); + } + const validation = await validateRecordedVideo(finalVideoPath); + + setWindowsCaptureProcess(null); + setWindowsNativeCaptureActive(false); + setNativeScreenRecordingActive(false); + setWindowsCaptureTargetPath(null); + setWindowsCaptureStopRequested(false); + setWindowsCapturePaused(false); + setWindowsOrphanedMicAudioPath(null); + await cleanupWindowsOrphanedMicAudioPath(preferredOrphanedMicAudioPath); + setWindowsPendingVideoPath(finalVideoPath); + recordNativeCaptureDiagnostics({ + backend: "windows-wgc", + phase: "stop", + outputPath: finalVideoPath, + systemAudioPath: windowsSystemAudioPath, + microphonePath: windowsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, + fileSizeBytes: validation.fileSizeBytes, + }); + return { success: true, path: finalVideoPath }; + } catch (error) { + console.error("Failed to stop native Windows capture:", error); + const fallbackPath = windowsCaptureTargetPath; + const fallbackOrphanedMicAudioPath = windowsOrphanedMicAudioPath; + setWindowsNativeCaptureActive(false); + setNativeScreenRecordingActive(false); + setWindowsCaptureProcess(null); + setWindowsCaptureTargetPath(null); + setWindowsCaptureStopRequested(false); + setWindowsCapturePaused(false); + setWindowsSystemAudioPath(null); + setWindowsMicAudioPath(null); + setWindowsOrphanedMicAudioPath(null); + setWindowsPendingVideoPath(null); + await cleanupWindowsOrphanedMicAudioPath(fallbackOrphanedMicAudioPath); + + if (fallbackPath) { + try { + await fs.access(fallbackPath); + const validation = await validateRecordedVideo(fallbackPath); + setWindowsPendingVideoPath(fallbackPath); + recordNativeCaptureDiagnostics({ + backend: "windows-wgc", + phase: "stop", + outputPath: fallbackPath, + systemAudioPath: windowsSystemAudioPath, + microphonePath: windowsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, + fileSizeBytes: validation.fileSizeBytes, + error: String(error), + }); + return { success: true, path: fallbackPath }; + } catch { + // File is absent or failed validation. + } + } + + recordNativeCaptureDiagnostics({ + backend: "windows-wgc", + phase: "stop", + outputPath: fallbackPath, + systemAudioPath: windowsSystemAudioPath, + microphonePath: windowsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, + fileSizeBytes: await getFileSizeIfPresent(fallbackPath), + error: String(error), + }); + + return { + success: false, + message: "Failed to stop native Windows capture", + error: String(error), + }; + } + } + + if (process.platform !== "darwin") { + return { + success: false, + message: "Native screen recording is only available on macOS.", + }; + } + + if (!nativeScreenRecordingActive) { + const recovered = await recoverNativeMacCaptureOutput(); + if (recovered) { + return recovered; + } + + return { success: false, message: "No native screen recording is active." }; + } + + try { + if (!nativeCaptureProcess) { + throw new Error("Native capture helper process is not running"); + } + + const process = nativeCaptureProcess; + const preferredVideoPath = nativeCaptureTargetPath; + const preferredSystemAudioPath = nativeCaptureSystemAudioPath; + const preferredMicrophonePath = nativeCaptureMicrophonePath; + console.log( + "[stop-native] Audio paths — system:", + preferredSystemAudioPath, + "mic:", + preferredMicrophonePath, + ); + setNativeCaptureStopRequested(true); + process.stdin.write("stop\n"); + const tempVideoPath = await waitForNativeCaptureStop(process); + console.log("[stop-native] Helper stopped, tempVideoPath:", tempVideoPath); + setNativeCaptureProcess(null); + setNativeScreenRecordingActive(false); + setNativeCaptureTargetPath(null); + setNativeCaptureSystemAudioPath(null); + setNativeCaptureMicrophonePath(null); + setNativeCaptureStopRequested(false); + setNativeCapturePaused(false); + + const finalVideoPath = preferredVideoPath ?? tempVideoPath; + if (tempVideoPath !== finalVideoPath) { + await moveFileWithOverwrite(tempVideoPath, finalVideoPath); + } + + if (preferredSystemAudioPath || preferredMicrophonePath) { + console.log( + "[stop-native] Attempting audio mux (merging separate tracks) into:", + finalVideoPath, + ); + try { + await muxNativeMacRecordingWithAudio( + finalVideoPath, + preferredSystemAudioPath, + preferredMicrophonePath, + ); + console.log("[stop-native] Audio mux completed successfully"); + } catch (error) { + console.warn( + "[stop-native] Audio mux failed (video still has inline audio):", + error, + ); + } + } else { + console.log("[stop-native] No separate audio tracks to mux"); + } + + return await finalizeStoredVideo(finalVideoPath); + } catch (error) { + console.error("Failed to stop native ScreenCaptureKit recording:", error); + const fallbackPath = nativeCaptureTargetPath; + const fallbackSystemAudioPath = nativeCaptureSystemAudioPath; + const fallbackMicrophonePath = nativeCaptureMicrophonePath; + const fallbackFileSizeBytes = await getFileSizeIfPresent(fallbackPath); + setNativeScreenRecordingActive(false); + setNativeCaptureProcess(null); + setNativeCaptureTargetPath(null); + setNativeCaptureSystemAudioPath(null); + setNativeCaptureMicrophonePath(null); + setNativeCaptureStopRequested(false); + setNativeCapturePaused(false); + + recordNativeCaptureDiagnostics({ + backend: "mac-screencapturekit", + phase: "stop", + sourceId: lastNativeCaptureDiagnostics?.sourceId ?? null, + sourceType: lastNativeCaptureDiagnostics?.sourceType ?? "unknown", + displayId: lastNativeCaptureDiagnostics?.displayId ?? null, + displayBounds: lastNativeCaptureDiagnostics?.displayBounds ?? null, + windowHandle: lastNativeCaptureDiagnostics?.windowHandle ?? null, + helperPath: lastNativeCaptureDiagnostics?.helperPath ?? null, + outputPath: fallbackPath, + systemAudioPath: fallbackSystemAudioPath, + microphonePath: fallbackMicrophonePath, + osRelease: lastNativeCaptureDiagnostics?.osRelease, + supported: lastNativeCaptureDiagnostics?.supported, + helperExists: lastNativeCaptureDiagnostics?.helperExists, + processOutput: nativeCaptureOutputBuffer.trim() || undefined, + fileSizeBytes: fallbackFileSizeBytes, + error: String(error), + }); + + // Try to recover: if the target file exists on disk, finalize with it + if (fallbackPath) { + try { + await fs.access(fallbackPath); + console.log( + "[stop-native-screen-recording] Recovering with fallback path:", + fallbackPath, + ); + if (fallbackSystemAudioPath || fallbackMicrophonePath) { + try { + await muxNativeMacRecordingWithAudio( + fallbackPath, + fallbackSystemAudioPath, + fallbackMicrophonePath, + ); + } catch (muxError) { + console.warn( + "Failed to mux recovered native macOS audio into capture:", + muxError, + ); + } + } + return await finalizeStoredVideo(fallbackPath); + } catch { + // File doesn't exist or isn't accessible + } + } + + const recovered = await recoverNativeMacCaptureOutput(); + if (recovered) { + return recovered; + } + + return { + success: false, + message: "Failed to stop native ScreenCaptureKit recording", + error: String(error), + }; + } + }); + + ipcMain.handle("recover-native-screen-recording", async () => { + if (process.platform !== "darwin") { + return { + success: false, + message: "Native screen recording recovery is only available on macOS.", + }; + } + + const recovered = await recoverNativeMacCaptureOutput(); + if (recovered) { + return recovered; + } + + return { + success: false, + message: "No recoverable native macOS recording output was found.", + }; + }); + + ipcMain.handle("pause-native-screen-recording", async () => { + if (process.platform === "win32") { + if (!windowsNativeCaptureActive || !windowsCaptureProcess) { + return { success: false, message: "No native Windows screen recording is active." }; + } + + if (windowsCapturePaused) { + return { success: true }; + } + + try { + windowsCaptureProcess.stdin.write("pause\n"); + setWindowsCapturePaused(true); + return { success: true }; + } catch (error) { + return { + success: false, + message: "Failed to pause native Windows capture", + error: String(error), + }; + } + } + + if (process.platform !== "darwin") { + return { + success: false, + message: "Native screen recording is only available on macOS.", + }; + } + + if (!nativeScreenRecordingActive || !nativeCaptureProcess) { + return { success: false, message: "No native screen recording is active." }; + } + + if (nativeCapturePaused) { + return { success: true }; + } + + try { + nativeCaptureProcess.stdin.write("pause\n"); + setNativeCapturePaused(true); + return { success: true }; + } catch (error) { + return { + success: false, + message: "Failed to pause native screen recording", + error: String(error), + }; + } + }); + + ipcMain.handle("resume-native-screen-recording", async () => { + if (process.platform === "win32") { + if (!windowsNativeCaptureActive || !windowsCaptureProcess) { + return { success: false, message: "No native Windows screen recording is active." }; + } + + if (!windowsCapturePaused) { + return { success: true }; + } + + try { + windowsCaptureProcess.stdin.write("resume\n"); + setWindowsCapturePaused(false); + return { success: true }; + } catch (error) { + return { + success: false, + message: "Failed to resume native Windows capture", + error: String(error), + }; + } + } + + if (process.platform !== "darwin") { + return { + success: false, + message: "Native screen recording is only available on macOS.", + }; + } + + if (!nativeScreenRecordingActive || !nativeCaptureProcess) { + return { success: false, message: "No native screen recording is active." }; + } + + if (!nativeCapturePaused) { + return { success: true }; + } + + try { + nativeCaptureProcess.stdin.write("resume\n"); + setNativeCapturePaused(false); + return { success: true }; + } catch (error) { + return { + success: false, + message: "Failed to resume native screen recording", + error: String(error), + }; + } + }); + + ipcMain.handle("get-system-cursor-assets", async () => { + try { + return { success: true, cursors: await getSystemCursorAssets() }; + } catch (error) { + console.error("Failed to load system cursor assets:", error); + return { success: false, cursors: {}, error: String(error) }; + } + }); + + ipcMain.handle("is-native-windows-capture-available", async () => { + return { available: await isNativeWindowsCaptureAvailable() }; + }); + + ipcMain.handle("get-last-native-capture-diagnostics", async () => { + return { success: true, diagnostics: lastNativeCaptureDiagnostics }; + }); + + ipcMain.handle("get-video-audio-fallback-paths", async (_event, videoPath: string) => { + if (!videoPath) { + return { success: true, paths: [] }; + } + + try { + const paths = await getCompanionAudioFallbackPaths(videoPath); + await Promise.all([ + rememberApprovedLocalReadPath(videoPath), + ...paths.map((fallbackPath) => rememberApprovedLocalReadPath(fallbackPath)), + ]); + return { success: true, paths }; + } catch (error) { + console.error("Failed to resolve companion audio fallback paths:", error); + return { success: false, paths: [], error: String(error) }; + } + }); + + ipcMain.handle( + "mux-native-windows-recording", + async (_event, pauseSegments?: PauseSegment[]) => { + const videoPath = windowsPendingVideoPath; + const orphanedMicAudioPath = windowsOrphanedMicAudioPath; + setWindowsPendingVideoPath(null); + setWindowsOrphanedMicAudioPath(null); + + if (!videoPath) { + return { success: false, message: "No native Windows video pending for mux" }; + } + + try { + if (windowsSystemAudioPath || windowsMicAudioPath) { + await muxNativeWindowsVideoWithAudio( + videoPath, + windowsSystemAudioPath, + windowsMicAudioPath, + pauseSegments ?? [], + ); + setWindowsSystemAudioPath(null); + setWindowsMicAudioPath(null); + } + + recordNativeCaptureDiagnostics({ + backend: "windows-wgc", + phase: "mux", + outputPath: videoPath, + fileSizeBytes: await getFileSizeIfPresent(videoPath), + }); + await cleanupWindowsOrphanedMicAudioPath(orphanedMicAudioPath); + return await finalizeStoredVideo(videoPath); + } catch (error) { + console.error("Failed to mux native Windows recording:", error); + recordNativeCaptureDiagnostics({ + backend: "windows-wgc", + phase: "mux", + outputPath: videoPath, + systemAudioPath: windowsSystemAudioPath, + microphonePath: windowsMicAudioPath, + fileSizeBytes: await getFileSizeIfPresent(videoPath), + error: String(error), + }); + setWindowsSystemAudioPath(null); + setWindowsMicAudioPath(null); + await cleanupWindowsOrphanedMicAudioPath(orphanedMicAudioPath); + try { + return await finalizeStoredVideo(videoPath); + } catch { + return { + success: false, + message: "Failed to mux native Windows recording", + error: String(error), + }; + } + } + }, + ); + + ipcMain.handle("start-ffmpeg-recording", async (_, source: SelectedSource) => { + if (ffmpegCaptureProcess) { + return { success: false, message: "An FFmpeg recording is already active." }; + } + + try { + const recordingsDir = await getRecordingsDir(); + const ffmpegPath = getFfmpegBinaryPath(); + const outputPath = path.join(recordingsDir, `recording-${Date.now()}.mp4`); + const args = await buildFfmpegCaptureArgs(source, outputPath); + + setFfmpegCaptureOutputBuffer(""); + setFfmpegCaptureTargetPath(outputPath); + const ffProc = spawn(ffmpegPath, args, { + cwd: recordingsDir, + stdio: ["pipe", "pipe", "pipe"], + }); + setFfmpegCaptureProcess(ffProc); + + ffProc.stdout.on("data", (chunk: Buffer) => { + setFfmpegCaptureOutputBuffer(ffmpegCaptureOutputBuffer + chunk.toString()); + }); + ffProc.stderr.on("data", (chunk: Buffer) => { + setFfmpegCaptureOutputBuffer(ffmpegCaptureOutputBuffer + chunk.toString()); + }); + + await waitForFfmpegCaptureStart(ffProc); + setFfmpegScreenRecordingActive(true); + return { success: true }; + } catch (error) { + console.error("Failed to start FFmpeg recording:", error); + setFfmpegScreenRecordingActive(false); + setFfmpegCaptureProcess(null); + setFfmpegCaptureTargetPath(null); + return { + success: false, + message: "Failed to start FFmpeg recording", + error: String(error), + }; + } + }); + + ipcMain.handle("stop-ffmpeg-recording", async () => { + if (!ffmpegScreenRecordingActive) { + return { success: false, message: "No FFmpeg recording is active." }; + } + + try { + if (!ffmpegCaptureProcess || !ffmpegCaptureTargetPath) { + throw new Error("FFmpeg process is not running"); + } + + const process = ffmpegCaptureProcess; + const outputPath = ffmpegCaptureTargetPath; + process.stdin.write("q\n"); + const finalVideoPath = await waitForFfmpegCaptureStop(process, outputPath); + + setFfmpegCaptureProcess(null); + setFfmpegCaptureTargetPath(null); + setFfmpegScreenRecordingActive(false); + + return await finalizeStoredVideo(finalVideoPath); + } catch (error) { + console.error("Failed to stop FFmpeg recording:", error); + try { + ffmpegCaptureProcess?.kill(); } catch { // ignore cleanup failures } - setFfmpegCaptureProcess(null) - setFfmpegCaptureTargetPath(null) - setFfmpegScreenRecordingActive(false) - return { - success: false, - message: 'Failed to stop FFmpeg recording', - error: String(error), - } - } - }) + setFfmpegCaptureProcess(null); + setFfmpegCaptureTargetPath(null); + setFfmpegScreenRecordingActive(false); + return { + success: false, + message: "Failed to stop FFmpeg recording", + error: String(error), + }; + } + }); + ipcMain.handle( + "store-microphone-sidecar", + async (_, audioData: ArrayBuffer, videoPath: string) => { + try { + const baseName = videoPath.replace(/\.[^.]+$/, ""); + const sidecarPath = `${baseName}.mic.webm`; + await fs.writeFile(sidecarPath, Buffer.from(audioData)); + return { success: true, path: sidecarPath }; + } catch (error) { + console.error("Failed to store microphone sidecar:", error); + return { success: false, error: String(error) }; + } + }, + ); + ipcMain.handle("store-recorded-video", async (_, videoData: ArrayBuffer, fileName: string) => { + try { + const recordingsDir = await getRecordingsDir(); + const videoPath = path.join(recordingsDir, fileName); + await fs.writeFile(videoPath, Buffer.from(videoData)); + return await finalizeStoredVideo(videoPath); + } catch (error) { + console.error("Failed to store video:", error); + return { + success: false, + message: "Failed to store video", + error: String(error), + }; + } + }); - ipcMain.handle('store-microphone-sidecar', async (_, audioData: ArrayBuffer, videoPath: string) => { - try { - const baseName = videoPath.replace(/\.[^.]+$/, '') - const sidecarPath = `${baseName}.mic.webm` - await fs.writeFile(sidecarPath, Buffer.from(audioData)) - return { success: true, path: sidecarPath } - } catch (error) { - console.error('Failed to store microphone sidecar:', error) - return { success: false, error: String(error) } - } - }) + ipcMain.handle("get-recorded-video-path", async () => { + try { + const recordingsDir = await getRecordingsDir(); + const entries = await fs.readdir(recordingsDir, { withFileTypes: true }); + const candidates = await Promise.all( + entries + .filter( + (entry) => + entry.isFile() && /^recording-\d+\.(webm|mov|mp4)$/i.test(entry.name), + ) + .map(async (entry) => { + const fullPath = path.join(recordingsDir, entry.name); + const stat = await fs.stat(fullPath).catch(() => null); + return stat ? { path: fullPath, mtimeMs: stat.mtimeMs } : null; + }), + ); + const sortedCandidates = candidates + .filter( + (candidate): candidate is { path: string; mtimeMs: number } => + candidate !== null, + ) + .sort((left, right) => right.mtimeMs - left.mtimeMs); - ipcMain.handle('store-recorded-video', async (_, videoData: ArrayBuffer, fileName: string) => { - try { - const recordingsDir = await getRecordingsDir() - const videoPath = path.join(recordingsDir, fileName) - await fs.writeFile(videoPath, Buffer.from(videoData)) - return await finalizeStoredVideo(videoPath) - } catch (error) { - console.error('Failed to store video:', error) - return { - success: false, - message: 'Failed to store video', - error: String(error) - } - } - }) + for (const candidate of sortedCandidates) { + try { + await validateRecordedVideo(candidate.path); + return { success: true, path: candidate.path }; + } catch (error) { + console.warn( + "Skipping unusable recovered recording candidate:", + candidate.path, + error, + ); + } + } + if (sortedCandidates.length === 0) { + return { success: false, message: "No recorded video found" }; + } + return { success: false, message: "No usable recorded video found" }; + } catch (error) { + console.error("Failed to get video path:", error); + return { success: false, message: "Failed to get video path", error: String(error) }; + } + }); - ipcMain.handle('get-recorded-video-path', async () => { - try { - const recordingsDir = await getRecordingsDir() - const entries = await fs.readdir(recordingsDir, { withFileTypes: true }) - const candidates = await Promise.all( - entries - .filter((entry) => entry.isFile() && /^recording-\d+\.(webm|mov|mp4)$/i.test(entry.name)) - .map(async (entry) => { - const fullPath = path.join(recordingsDir, entry.name) - const stat = await fs.stat(fullPath).catch(() => null) - return stat ? { path: fullPath, mtimeMs: stat.mtimeMs } : null - }), - ) - const sortedCandidates = candidates - .filter((candidate): candidate is { path: string; mtimeMs: number } => candidate !== null) - .sort((left, right) => right.mtimeMs - left.mtimeMs) + ipcMain.handle("set-recording-state", (_, recording: boolean) => { + if (recording) { + stopCursorCapture(); + stopInteractionCapture(); + startWindowBoundsCapture(); + void startNativeCursorMonitor(); + setIsCursorCaptureActive(true); + setActiveCursorSamples([]); + setPendingCursorSamples([]); + setCursorCaptureStartTimeMs(Date.now()); + setLinuxCursorScreenPoint(null); + setLastLeftClick(null); + sampleCursorPoint(); + startCursorSampling(); + void startInteractionCapture(); + } else { + setIsCursorCaptureActive(false); + stopCursorCapture(); + stopInteractionCapture(); + stopWindowBoundsCapture(); + stopNativeCursorMonitor(); + showCursor(); + setLinuxCursorScreenPoint(null); + snapshotCursorTelemetryForPersistence(); + setActiveCursorSamples([]); + } - for (const candidate of sortedCandidates) { - try { - await validateRecordedVideo(candidate.path) - return { success: true, path: candidate.path } - } catch (error) { - console.warn("Skipping unusable recovered recording candidate:", candidate.path, error) - } - } + const source = selectedSource || { name: "Screen" }; + BrowserWindow.getAllWindows().forEach((window) => { + if (!window.isDestroyed()) { + window.webContents.send("recording-state-changed", { + recording, + sourceName: source.name, + }); + } + }); - if (sortedCandidates.length === 0) { - return { success: false, message: 'No recorded video found' } - } + if (onRecordingStateChange) { + onRecordingStateChange(recording, source.name); + } + }); - return { success: false, message: 'No usable recorded video found' } - } catch (error) { - console.error('Failed to get video path:', error) - return { success: false, message: 'Failed to get video path', error: String(error) } - } - }) + ipcMain.handle("get-cursor-telemetry", async (_, videoPath?: string) => { + const targetVideoPath = normalizeVideoSourcePath(videoPath ?? currentVideoPath); + if (!targetVideoPath) { + return { success: true, samples: [] }; + } - ipcMain.handle('set-recording-state', (_, recording: boolean) => { - if (recording) { - stopCursorCapture() - stopInteractionCapture() - startWindowBoundsCapture() - void startNativeCursorMonitor() - setIsCursorCaptureActive(true) - setActiveCursorSamples([]) - setPendingCursorSamples([]) - setCursorCaptureStartTimeMs(Date.now()) - setLinuxCursorScreenPoint(null) - setLastLeftClick(null) - sampleCursorPoint() - startCursorSampling() - void startInteractionCapture() - } else { - setIsCursorCaptureActive(false) - stopCursorCapture() - stopInteractionCapture() - stopWindowBoundsCapture() - stopNativeCursorMonitor() - showCursor() - setLinuxCursorScreenPoint(null) - snapshotCursorTelemetryForPersistence() - setActiveCursorSamples([]) - } - - const source = selectedSource || { name: 'Screen' } - BrowserWindow.getAllWindows().forEach((window) => { - if (!window.isDestroyed()) { - window.webContents.send('recording-state-changed', { - recording, - sourceName: source.name, - }) - } - }) - - if (onRecordingStateChange) { - onRecordingStateChange(recording, source.name) - } - }) - - ipcMain.handle('get-cursor-telemetry', async (_, videoPath?: string) => { - const targetVideoPath = normalizeVideoSourcePath(videoPath ?? currentVideoPath) - if (!targetVideoPath) { - return { success: true, samples: [] } - } - - const telemetryPath = getTelemetryPathForVideo(targetVideoPath) - try { - const content = await fs.readFile(telemetryPath, 'utf-8') - const parsed = JSON.parse(content) - const rawSamples = Array.isArray(parsed) - ? parsed - : (Array.isArray(parsed?.samples) ? parsed.samples : []) - - const samples: CursorTelemetryPoint[] = rawSamples - .filter((sample: unknown) => Boolean(sample && typeof sample === 'object')) - .map((sample: unknown) => { - const point = sample as Partial - return { - timeMs: typeof point.timeMs === 'number' && Number.isFinite(point.timeMs) ? Math.max(0, point.timeMs) : 0, - cx: typeof point.cx === 'number' && Number.isFinite(point.cx) ? clamp(point.cx, 0, 1) : 0.5, - cy: typeof point.cy === 'number' && Number.isFinite(point.cy) ? clamp(point.cy, 0, 1) : 0.5, - interactionType: point.interactionType === 'click' - || point.interactionType === 'double-click' - || point.interactionType === 'right-click' - || point.interactionType === 'middle-click' - || point.interactionType === 'move' - || point.interactionType === 'mouseup' - ? point.interactionType - : undefined, - cursorType: point.cursorType === 'arrow' - || point.cursorType === 'text' - || point.cursorType === 'pointer' - || point.cursorType === 'crosshair' - || point.cursorType === 'open-hand' - || point.cursorType === 'closed-hand' - || point.cursorType === 'resize-ew' - || point.cursorType === 'resize-ns' - || point.cursorType === 'not-allowed' - ? point.cursorType - : undefined, - } - }) - .sort((a: CursorTelemetryPoint, b: CursorTelemetryPoint) => a.timeMs - b.timeMs) - - return { success: true, samples } - } catch (error) { - const nodeError = error as NodeJS.ErrnoException - if (nodeError.code === 'ENOENT') { - return { success: true, samples: [] } - } - console.error('Failed to load cursor telemetry:', error) - return { success: false, message: 'Failed to load cursor telemetry', error: String(error), samples: [] } - } - }) + const telemetryPath = getTelemetryPathForVideo(targetVideoPath); + try { + const content = await fs.readFile(telemetryPath, "utf-8"); + const parsed = JSON.parse(content); + const rawSamples = Array.isArray(parsed) + ? parsed + : Array.isArray(parsed?.samples) + ? parsed.samples + : []; + const samples: CursorTelemetryPoint[] = rawSamples + .filter((sample: unknown) => Boolean(sample && typeof sample === "object")) + .map((sample: unknown) => { + const point = sample as Partial; + return { + timeMs: + typeof point.timeMs === "number" && Number.isFinite(point.timeMs) + ? Math.max(0, point.timeMs) + : 0, + cx: + typeof point.cx === "number" && Number.isFinite(point.cx) + ? clamp(point.cx, 0, 1) + : 0.5, + cy: + typeof point.cy === "number" && Number.isFinite(point.cy) + ? clamp(point.cy, 0, 1) + : 0.5, + interactionType: + point.interactionType === "click" || + point.interactionType === "double-click" || + point.interactionType === "right-click" || + point.interactionType === "middle-click" || + point.interactionType === "move" || + point.interactionType === "mouseup" + ? point.interactionType + : undefined, + cursorType: + point.cursorType === "arrow" || + point.cursorType === "text" || + point.cursorType === "pointer" || + point.cursorType === "crosshair" || + point.cursorType === "open-hand" || + point.cursorType === "closed-hand" || + point.cursorType === "resize-ew" || + point.cursorType === "resize-ns" || + point.cursorType === "not-allowed" + ? point.cursorType + : undefined, + }; + }) + .sort((a: CursorTelemetryPoint, b: CursorTelemetryPoint) => a.timeMs - b.timeMs); + return { success: true, samples }; + } catch (error) { + const nodeError = error as NodeJS.ErrnoException; + if (nodeError.code === "ENOENT") { + return { success: true, samples: [] }; + } + console.error("Failed to load cursor telemetry:", error); + return { + success: false, + message: "Failed to load cursor telemetry", + error: String(error), + samples: [], + }; + } + }); } diff --git a/electron/ipc/state.ts b/electron/ipc/state.ts index 7fa98176..b1d809b8 100644 --- a/electron/ipc/state.ts +++ b/electron/ipc/state.ts @@ -44,6 +44,7 @@ export let windowsCaptureStopRequested = false; export let windowsCapturePaused = false; export let windowsSystemAudioPath: string | null = null; export let windowsMicAudioPath: string | null = null; +export let windowsOrphanedMicAudioPath: string | null = null; export let windowsPendingVideoPath: string | null = null; // ── Diagnostics ─────────────────────────────────────────────────────────────── @@ -90,7 +91,11 @@ export let cachedNativeMacWindowSources: import("./types").NativeMacWindowSource export let cachedNativeMacWindowSourcesAtMs = 0; // ── Native video export ─────────────────────────────────────────────────────── -export let cachedNativeVideoEncoder: { ffmpegPath: string; encodingMode: string; encoderName: string } | null = null; +export let cachedNativeVideoEncoder: { + ffmpegPath: string; + encodingMode: string; + encoderName: string; +} | null = null; // ── Native helper migration ─────────────────────────────────────────────────── export let nativeHelperMigrationPromise: Promise | null = null; @@ -102,68 +107,179 @@ export type { CursorInteractionType, CursorTelemetryPoint }; // TypeScript exported `let` can be reassigned by the owning module but importers // cannot assign to them directly. Provide simple setters for cross-module writes. -export function setSelectedSource(v: SelectedSource | null) { selectedSource = v; } -export function setCurrentProjectPath(v: string | null) { currentProjectPath = v; } -export function setCurrentVideoPath(v: string | null) { currentVideoPath = v; } -export function setCurrentRecordingSession(v: RecordingSessionData | null) { currentRecordingSession = v; } +export function setSelectedSource(v: SelectedSource | null) { + selectedSource = v; +} +export function setCurrentProjectPath(v: string | null) { + currentProjectPath = v; +} +export function setCurrentVideoPath(v: string | null) { + currentVideoPath = v; +} +export function setCurrentRecordingSession(v: RecordingSessionData | null) { + currentRecordingSession = v; +} -export function setNativeScreenRecordingActive(v: boolean) { nativeScreenRecordingActive = v; } -export function setNativeCaptureProcess(v: ChildProcessWithoutNullStreams | null) { nativeCaptureProcess = v; } -export function setNativeCaptureOutputBuffer(v: string) { nativeCaptureOutputBuffer = v; } -export function setNativeCaptureTargetPath(v: string | null) { nativeCaptureTargetPath = v; } -export function setNativeCaptureStopRequested(v: boolean) { nativeCaptureStopRequested = v; } -export function setNativeCaptureSystemAudioPath(v: string | null) { nativeCaptureSystemAudioPath = v; } -export function setNativeCaptureMicrophonePath(v: string | null) { nativeCaptureMicrophonePath = v; } -export function setNativeCapturePaused(v: boolean) { nativeCapturePaused = v; } +export function setNativeScreenRecordingActive(v: boolean) { + nativeScreenRecordingActive = v; +} +export function setNativeCaptureProcess(v: ChildProcessWithoutNullStreams | null) { + nativeCaptureProcess = v; +} +export function setNativeCaptureOutputBuffer(v: string) { + nativeCaptureOutputBuffer = v; +} +export function setNativeCaptureTargetPath(v: string | null) { + nativeCaptureTargetPath = v; +} +export function setNativeCaptureStopRequested(v: boolean) { + nativeCaptureStopRequested = v; +} +export function setNativeCaptureSystemAudioPath(v: string | null) { + nativeCaptureSystemAudioPath = v; +} +export function setNativeCaptureMicrophonePath(v: string | null) { + nativeCaptureMicrophonePath = v; +} +export function setNativeCapturePaused(v: boolean) { + nativeCapturePaused = v; +} -export function setNativeCursorMonitorProcess(v: ChildProcessWithoutNullStreams | null) { nativeCursorMonitorProcess = v; } -export function setNativeCursorMonitorOutputBuffer(v: string) { nativeCursorMonitorOutputBuffer = v; } +export function setNativeCursorMonitorProcess(v: ChildProcessWithoutNullStreams | null) { + nativeCursorMonitorProcess = v; +} +export function setNativeCursorMonitorOutputBuffer(v: string) { + nativeCursorMonitorOutputBuffer = v; +} -export function setWindowsCaptureProcess(v: ChildProcessWithoutNullStreams | null) { windowsCaptureProcess = v; } -export function setWindowsCaptureOutputBuffer(v: string) { windowsCaptureOutputBuffer = v; } -export function setWindowsCaptureTargetPath(v: string | null) { windowsCaptureTargetPath = v; } -export function setWindowsNativeCaptureActive(v: boolean) { windowsNativeCaptureActive = v; } -export function setWindowsCaptureStopRequested(v: boolean) { windowsCaptureStopRequested = v; } -export function setWindowsCapturePaused(v: boolean) { windowsCapturePaused = v; } -export function setWindowsSystemAudioPath(v: string | null) { windowsSystemAudioPath = v; } -export function setWindowsMicAudioPath(v: string | null) { windowsMicAudioPath = v; } -export function setWindowsPendingVideoPath(v: string | null) { windowsPendingVideoPath = v; } +export function setWindowsCaptureProcess(v: ChildProcessWithoutNullStreams | null) { + windowsCaptureProcess = v; +} +export function setWindowsCaptureOutputBuffer(v: string) { + windowsCaptureOutputBuffer = v; +} +export function setWindowsCaptureTargetPath(v: string | null) { + windowsCaptureTargetPath = v; +} +export function setWindowsNativeCaptureActive(v: boolean) { + windowsNativeCaptureActive = v; +} +export function setWindowsCaptureStopRequested(v: boolean) { + windowsCaptureStopRequested = v; +} +export function setWindowsCapturePaused(v: boolean) { + windowsCapturePaused = v; +} +export function setWindowsSystemAudioPath(v: string | null) { + windowsSystemAudioPath = v; +} +export function setWindowsMicAudioPath(v: string | null) { + windowsMicAudioPath = v; +} +export function setWindowsOrphanedMicAudioPath(v: string | null) { + windowsOrphanedMicAudioPath = v; +} +export function setWindowsPendingVideoPath(v: string | null) { + windowsPendingVideoPath = v; +} -export function setLastNativeCaptureDiagnostics(v: NativeCaptureDiagnostics | null) { lastNativeCaptureDiagnostics = v; } +export function setLastNativeCaptureDiagnostics(v: NativeCaptureDiagnostics | null) { + lastNativeCaptureDiagnostics = v; +} -export function setFfmpegScreenRecordingActive(v: boolean) { ffmpegScreenRecordingActive = v; } -export function setFfmpegCaptureProcess(v: ChildProcessWithoutNullStreams | null) { ffmpegCaptureProcess = v; } -export function setFfmpegCaptureOutputBuffer(v: string) { ffmpegCaptureOutputBuffer = v; } -export function setFfmpegCaptureTargetPath(v: string | null) { ffmpegCaptureTargetPath = v; } +export function setFfmpegScreenRecordingActive(v: boolean) { + ffmpegScreenRecordingActive = v; +} +export function setFfmpegCaptureProcess(v: ChildProcessWithoutNullStreams | null) { + ffmpegCaptureProcess = v; +} +export function setFfmpegCaptureOutputBuffer(v: string) { + ffmpegCaptureOutputBuffer = v; +} +export function setFfmpegCaptureTargetPath(v: string | null) { + ffmpegCaptureTargetPath = v; +} -export function setCustomRecordingsDir(v: string | null) { customRecordingsDir = v; } -export function setRecordingsDirLoaded(v: boolean) { recordingsDirLoaded = v; } +export function setCustomRecordingsDir(v: string | null) { + customRecordingsDir = v; +} +export function setRecordingsDirLoaded(v: boolean) { + recordingsDirLoaded = v; +} -export function setCachedSystemCursorAssets(v: Record | null) { cachedSystemCursorAssets = v; } -export function setCachedSystemCursorAssetsSourceMtimeMs(v: number | null) { cachedSystemCursorAssetsSourceMtimeMs = v; } +export function setCachedSystemCursorAssets(v: Record | null) { + cachedSystemCursorAssets = v; +} +export function setCachedSystemCursorAssetsSourceMtimeMs(v: number | null) { + cachedSystemCursorAssetsSourceMtimeMs = v; +} -export function setCountdownTimer(v: ReturnType | null) { countdownTimer = v; } -export function setCountdownCancelled(v: boolean) { countdownCancelled = v; } -export function setCountdownInProgress(v: boolean) { countdownInProgress = v; } -export function setCountdownRemaining(v: number | null) { countdownRemaining = v; } +export function setCountdownTimer(v: ReturnType | null) { + countdownTimer = v; +} +export function setCountdownCancelled(v: boolean) { + countdownCancelled = v; +} +export function setCountdownInProgress(v: boolean) { + countdownInProgress = v; +} +export function setCountdownRemaining(v: number | null) { + countdownRemaining = v; +} -export function setCurrentCursorVisualType(v: CursorVisualType | undefined) { currentCursorVisualType = v; } +export function setCurrentCursorVisualType(v: CursorVisualType | undefined) { + currentCursorVisualType = v; +} -export function setCursorCaptureInterval(v: NodeJS.Timeout | null) { cursorCaptureInterval = v; } -export function setCursorCaptureStartTimeMs(v: number) { cursorCaptureStartTimeMs = v; } -export function setActiveCursorSamples(v: CursorTelemetryPoint[]) { activeCursorSamples = v; } -export function setPendingCursorSamples(v: CursorTelemetryPoint[]) { pendingCursorSamples = v; } -export function setIsCursorCaptureActive(v: boolean) { isCursorCaptureActive = v; } -export function setInteractionCaptureCleanup(v: (() => void) | null) { interactionCaptureCleanup = v; } -export function setHasLoggedInteractionHookFailure(v: boolean) { hasLoggedInteractionHookFailure = v; } -export function setLastLeftClick(v: { timeMs: number; cx: number; cy: number } | null) { lastLeftClick = v; } -export function setLinuxCursorScreenPoint(v: { x: number; y: number; updatedAt: number } | null) { linuxCursorScreenPoint = v; } -export function setSelectedWindowBounds(v: WindowBounds | null) { selectedWindowBounds = v; } -export function setWindowBoundsCaptureInterval(v: NodeJS.Timeout | null) { windowBoundsCaptureInterval = v; } +export function setCursorCaptureInterval(v: NodeJS.Timeout | null) { + cursorCaptureInterval = v; +} +export function setCursorCaptureStartTimeMs(v: number) { + cursorCaptureStartTimeMs = v; +} +export function setActiveCursorSamples(v: CursorTelemetryPoint[]) { + activeCursorSamples = v; +} +export function setPendingCursorSamples(v: CursorTelemetryPoint[]) { + pendingCursorSamples = v; +} +export function setIsCursorCaptureActive(v: boolean) { + isCursorCaptureActive = v; +} +export function setInteractionCaptureCleanup(v: (() => void) | null) { + interactionCaptureCleanup = v; +} +export function setHasLoggedInteractionHookFailure(v: boolean) { + hasLoggedInteractionHookFailure = v; +} +export function setLastLeftClick(v: { timeMs: number; cx: number; cy: number } | null) { + lastLeftClick = v; +} +export function setLinuxCursorScreenPoint(v: { x: number; y: number; updatedAt: number } | null) { + linuxCursorScreenPoint = v; +} +export function setSelectedWindowBounds(v: WindowBounds | null) { + selectedWindowBounds = v; +} +export function setWindowBoundsCaptureInterval(v: NodeJS.Timeout | null) { + windowBoundsCaptureInterval = v; +} -export function setCachedNativeMacWindowSources(v: import("./types").NativeMacWindowSource[] | null) { cachedNativeMacWindowSources = v; } -export function setCachedNativeMacWindowSourcesAtMs(v: number) { cachedNativeMacWindowSourcesAtMs = v; } +export function setCachedNativeMacWindowSources( + v: import("./types").NativeMacWindowSource[] | null, +) { + cachedNativeMacWindowSources = v; +} +export function setCachedNativeMacWindowSourcesAtMs(v: number) { + cachedNativeMacWindowSourcesAtMs = v; +} -export function setCachedNativeVideoEncoder(v: { ffmpegPath: string; encodingMode: string; encoderName: string } | null) { cachedNativeVideoEncoder = v; } +export function setCachedNativeVideoEncoder( + v: { ffmpegPath: string; encodingMode: string; encoderName: string } | null, +) { + cachedNativeVideoEncoder = v; +} -export function setNativeHelperMigrationPromise(v: Promise | null) { nativeHelperMigrationPromise = v; } +export function setNativeHelperMigrationPromise(v: Promise | null) { + nativeHelperMigrationPromise = v; +} diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index fc3c3914..7342ed93 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -63,6 +63,7 @@ import { VideoExporter, } from "@/lib/exporter"; import { resolveMediaElementSource } from "@/lib/exporter/localMediaSource"; +import { resolveSourceAudioFallbackPaths } from "@/lib/exporter/sourceAudioFallback"; import { clampMediaTimeToDuration, estimateCompanionAudioStartDelaySeconds, @@ -157,8 +158,8 @@ import { extendAutoFullTrackClip, type FigureData, getClipSourceEndMs, - type PlaybackSpeed, type Padding, + type PlaybackSpeed, type SpeedRegion, type TrimRegion, type WebcamOverlaySettings, @@ -239,6 +240,7 @@ async function writeSmokeExportReport( } const DEFAULT_MP4_EXPORT_FRAME_RATE: ExportMp4FrameRate = 30; +const SOURCE_AUDIO_FALLBACK_TOAST_ID = "source-audio-fallback-error"; function getEncodingModeBitrateMultiplier(encodingMode: ExportEncodingMode): number { switch (encodingMode) { @@ -750,7 +752,9 @@ export default function VideoEditor() { } context.imageSmoothingEnabled = true; context.imageSmoothingQuality = "high"; - const editorBgHsl = getComputedStyle(document.documentElement).getPropertyValue("--editor-bg").trim(); + const editorBgHsl = getComputedStyle(document.documentElement) + .getPropertyValue("--editor-bg") + .trim(); context.fillStyle = editorBgHsl ? `hsl(${editorBgHsl})` : "#111113"; context.fillRect(0, 0, targetWidth, targetHeight); @@ -786,7 +790,9 @@ export default function VideoEditor() { padding, cropRegion, webcam, - webcamUrl: resolvedWebcamVideoUrl ?? (webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null), + webcamUrl: + resolvedWebcamVideoUrl ?? + (webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null), videoWidth: previewVideo.videoWidth, videoHeight: previewVideo.videoHeight, annotationRegions, @@ -1203,7 +1209,12 @@ export default function VideoEditor() { () => videoSourcePath ?? (videoPath ? fromFileUrl(videoPath) : null), [videoPath, videoSourcePath], ); - const hasSourceAudioFallback = sourceAudioFallbackPaths.length > 0; + const { hasEmbeddedSourceAudio, externalAudioPaths: previewSourceAudioFallbackPaths } = useMemo( + () => resolveSourceAudioFallbackPaths(currentSourcePath, sourceAudioFallbackPaths), + [currentSourcePath, sourceAudioFallbackPaths], + ); + const shouldMutePreviewVideo = + !hasEmbeddedSourceAudio && previewSourceAudioFallbackPaths.length > 0; useEffect(() => { let cancelled = false; @@ -1222,10 +1233,26 @@ export default function VideoEditor() { if (cancelled) { return; } - setSourceAudioFallbackPaths(result.success ? (result.paths ?? []) : []); - } catch { + if (!result.success) { + setSourceAudioFallbackPaths([]); + toast.warning( + result.error + ? `Could not load companion audio sources: ${summarizeErrorMessage(result.error)}` + : "Could not load companion audio sources. Playback and export may miss microphone audio.", + { id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 }, + ); + return; + } + + toast.dismiss(SOURCE_AUDIO_FALLBACK_TOAST_ID); + setSourceAudioFallbackPaths(result.paths ?? []); + } catch (error) { if (!cancelled) { setSourceAudioFallbackPaths([]); + toast.warning( + `Could not load companion audio sources: ${summarizeErrorMessage(String(error))}`, + { id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 }, + ); } } })(); @@ -2134,7 +2161,7 @@ export default function VideoEditor() { currentSourcePath, currentPersistedEditorState, lastSavedSnapshot?.projectId ?? null, - ); + ); const fileNameBase = currentSourcePath @@ -2249,7 +2276,7 @@ export default function VideoEditor() { currentSourcePath, currentPersistedEditorState, lastSavedSnapshot?.projectId ?? null, - ); + ); const thumbnailDataUrl = await captureProjectThumbnail(); const result = await window.electronAPI.saveProjectFileNamed( projectData, @@ -2949,7 +2976,9 @@ export default function VideoEditor() { regions.filter( (region) => !removedSegments.some( - (segment) => region.startMs < segment.endMs && region.endMs > segment.startMs, + (segment) => + region.startMs < segment.endMs && + region.endMs > segment.startMs, ), ); setZoomRegions((prev) => removeTrimmedRegions(prev)); @@ -3455,7 +3484,7 @@ export default function VideoEditor() { useEffect(() => { let cancelled = false; const existing = sourceAudioElementsRef.current; - const currentIds = new Set(sourceAudioFallbackPaths); + const currentIds = new Set(previewSourceAudioFallbackPaths); for (const [id, audio] of existing) { if (!currentIds.has(id)) { @@ -3468,7 +3497,7 @@ export default function VideoEditor() { } } - for (const audioPath of sourceAudioFallbackPaths) { + for (const audioPath of previewSourceAudioFallbackPaths) { let audio = existing.get(audioPath); if (!audio) { audio = new Audio(); @@ -3484,34 +3513,53 @@ export default function VideoEditor() { sourceAudioElementResourcesRef.current.set(audioPath, audioPath); void (async () => { - const resolved = await resolveMediaElementSource(audioPath); - const latestAudio = existing.get(audioPath); + try { + const resolved = await resolveMediaElementSource(audioPath); + const latestAudio = existing.get(audioPath); - if ( - cancelled || - latestAudio !== audio || - sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath - ) { - resolved.revoke(); - return; + if ( + cancelled || + latestAudio !== audio || + sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath + ) { + resolved.revoke(); + return; + } + + sourceAudioElementRevokersRef.current.set(audioPath, resolved.revoke); + latestAudio.src = resolved.src; + } catch (error) { + if (cancelled) { + return; + } + + sourceAudioElementRevokersRef.current.get(audioPath)?.(); + sourceAudioElementRevokersRef.current.delete(audioPath); + sourceAudioElementResourcesRef.current.delete(audioPath); + const latestAudio = existing.get(audioPath); + if (latestAudio === audio) { + latestAudio.pause(); + latestAudio.src = ""; + } + toast.warning( + `Could not load companion audio source: ${summarizeErrorMessage(getErrorMessage(error))}`, + { id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 }, + ); } - - sourceAudioElementRevokersRef.current.set(audioPath, resolved.revoke); - latestAudio.src = resolved.src; })(); } audio.volume = Math.max(0, Math.min(1, previewVolume)); } - if (sourceAudioFallbackPaths.length === 0) { + if (previewSourceAudioFallbackPaths.length === 0) { lastSourceAudioSyncTimeRef.current = null; } return () => { cancelled = true; }; - }, [previewVolume, sourceAudioFallbackPaths]); + }, [previewSourceAudioFallbackPaths, previewVolume]); useEffect(() => { return () => { @@ -3579,7 +3627,7 @@ export default function VideoEditor() { }, [isPlaying, currentTime, audioRegions, speedRegions]); useEffect(() => { - if (sourceAudioFallbackPaths.length === 0) { + if (previewSourceAudioFallbackPaths.length === 0) { lastSourceAudioSyncTimeRef.current = null; return; } @@ -3631,7 +3679,7 @@ export default function VideoEditor() { } lastSourceAudioSyncTimeRef.current = currentTime; - }, [currentTime, duration, isPlaying, sourceAudioFallbackPaths, speedRegions]); + }, [currentTime, duration, isPlaying, previewSourceAudioFallbackPaths, speedRegions]); const showExportSuccessToast = useCallback((filePath: string) => { toast.success(`Exported successfully to ${filePath}`, { @@ -3759,7 +3807,9 @@ export default function VideoEditor() { videoPadding: padding, cropRegion, webcam, - webcamUrl: resolvedWebcamVideoUrl ?? (webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null), + webcamUrl: + resolvedWebcamVideoUrl ?? + (webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null), annotationRegions, autoCaptions, autoCaptionSettings, @@ -3928,7 +3978,9 @@ export default function VideoEditor() { padding, cropRegion, webcam, - webcamUrl: resolvedWebcamVideoUrl ?? (webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null), + webcamUrl: + resolvedWebcamVideoUrl ?? + (webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null), annotationRegions, autoCaptions, autoCaptionSettings, @@ -4682,7 +4734,10 @@ export default function VideoEditor() {

{isRenderingAudio ? (

- {t("editor.export.processingAudioEdits", "Processing audio with speed/overlay edits")} + {t( + "editor.export.processingAudioEdits", + "Processing audio with speed/overlay edits", + )}

) : exportRenderSpeedLabel ? (

@@ -5180,7 +5235,7 @@ export default function VideoEditor() { cursorClickBounceDuration } cursorSway={cursorSway} - volume={hasSourceAudioFallback ? 0 : previewVolume} + volume={shouldMutePreviewVideo ? 0 : previewVolume} /> diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 29f5bcb7..1b56dad9 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -33,6 +33,10 @@ const WEBCAM_WIDTH = 1280; const WEBCAM_HEIGHT = 720; const WEBCAM_FRAME_RATE = 30; const WEBCAM_SUFFIX = "-webcam"; +const SOURCE_AUDIO_MUX_TOAST_ID = "recording-audio-mux-warning"; +const MICROPHONE_FALLBACK_TOAST_ID = "recording-microphone-fallback"; +const MICROPHONE_FALLBACK_ERROR_TOAST_ID = "recording-microphone-fallback-error"; +const MICROPHONE_SIDECAR_ERROR_TOAST_ID = "recording-microphone-sidecar-error"; const LINUX_PORTAL_SOURCE: ProcessedDesktopSource = { id: "screen:linux-portal", name: "Linux Portal", @@ -76,6 +80,36 @@ type UseScreenRecorderReturn = { setCountdownDelay: (delay: number) => void; }; +function getErrorMessage(error: unknown) { + if (error instanceof Error && error.message) { + return error.message; + } + + if (typeof error === "string" && error.trim().length > 0) { + return error; + } + + if (typeof error === "object" && error !== null) { + try { + const serialized = JSON.stringify(error); + if (serialized && serialized !== "{}") { + return serialized; + } + } catch { + // Ignore stringify failures and fall through to a generic message. + } + + if (typeof (error as { toString?: () => string }).toString === "function") { + const stringified = (error as { toString: () => string }).toString(); + if (stringified && stringified !== "[object Object]") { + return stringified; + } + } + } + + return "An unexpected error occurred"; +} + export function useScreenRecorder(): UseScreenRecorderReturn { const [recording, setRecording] = useState(false); const [paused, setPaused] = useState(false); @@ -446,9 +480,25 @@ export function useScreenRecorder(): UseScreenRecorderReturn { try { const arrayBuffer = await micFallbackBlob.arrayBuffer(); - await window.electronAPI.storeMicrophoneSidecar(arrayBuffer, finalPath); + const result = await window.electronAPI.storeMicrophoneSidecar( + arrayBuffer, + finalPath, + ); + if (!result.success) { + const errorMessage = + result.error || "Failed to save the fallback microphone audio track"; + console.warn("Failed to store microphone sidecar:", errorMessage); + toast.error( + `${errorMessage}. Recording was saved without the fallback microphone track.`, + { id: MICROPHONE_SIDECAR_ERROR_TOAST_ID, duration: 10000 }, + ); + } } catch (error) { console.warn("Failed to store microphone sidecar:", error); + toast.error( + `${getErrorMessage(error)}. Recording was saved without the fallback microphone track.`, + { id: MICROPHONE_SIDECAR_ERROR_TOAST_ID, duration: 10000 }, + ); } }, [], @@ -678,13 +728,24 @@ export function useScreenRecorder(): UseScreenRecorderReturn { await window.electronAPI.muxNativeWindowsRecording(pauseSegments); if (!muxResult?.success || !muxResult.path) { void logNativeCaptureDiagnostics("mux-native-windows-recording"); - const failureMessage = await buildNativeCaptureFailureMessage( - "mux-native-windows-recording", + if (!muxResult?.path) { + const failureMessage = await buildNativeCaptureFailureMessage( + "mux-native-windows-recording", + muxResult?.message || + "Failed to finalize the Windows recording, so the editor was not opened.", + ); + await notifyRecordingFinalizationFailure(failureMessage); + return; + } + + const warningMessage = + muxResult?.error || muxResult?.message || - "Failed to finalize the Windows recording, so the editor was not opened.", + "Failed to finish the native Windows audio mux"; + toast.warning( + `${warningMessage}. Recording was saved, but audio playback or export may be incomplete.`, + { id: SOURCE_AUDIO_MUX_TOAST_ID, duration: 10000 }, ); - await notifyRecordingFinalizationFailure(failureMessage); - return; } finalPath = muxResult.path; } @@ -976,6 +1037,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn { // When native mic capture is unavailable (macOS < 14), record mic // via browser getUserMedia so it can be saved as a sidecar file. if (nativeResult.microphoneFallbackRequired && microphoneEnabled) { + void logNativeCaptureDiagnostics("start-browser-microphone-fallback"); + toast.warning( + "Native microphone capture is unavailable. Using browser microphone fallback for this recording.", + { id: MICROPHONE_FALLBACK_TOAST_ID, duration: 8000 }, + ); try { const micStream = await navigator.mediaDevices.getUserMedia({ audio: microphoneDeviceId @@ -1005,6 +1071,16 @@ export function useScreenRecorder(): UseScreenRecorderReturn { micFallbackRecorder.current = recorder; } catch (micError) { console.warn("Browser microphone fallback failed:", micError); + const permissionDenied = + micError instanceof DOMException && + (micError.name === "NotAllowedError" || + micError.name === "SecurityError"); + toast.error( + permissionDenied + ? "Microphone permission denied. Recording will continue without microphone audio." + : `${getErrorMessage(micError)}. Recording will continue without microphone audio.`, + { id: MICROPHONE_FALLBACK_ERROR_TOAST_ID, duration: 10000 }, + ); } } diff --git a/src/lib/exporter/audioEncoder.test.ts b/src/lib/exporter/audioEncoder.test.ts new file mode 100644 index 00000000..a39e9358 --- /dev/null +++ b/src/lib/exporter/audioEncoder.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it, vi } from "vitest"; + +import { AudioProcessor } from "./audioEncoder"; + +type OfflineRenderTestHarness = AudioProcessor & { + decodeAudioFromUrl(url: string): Promise; + getMediaDurationSec(url: string): Promise; + loadAudioFileDemuxer(audioPath: string): Promise; + prepareOfflineRender( + videoUrl: string, + trimRegions: never[], + speedRegions: never[], + audioRegions: never[], + sourceAudioFallbackPaths: string[], + ): Promise<{ + mainBuffer: AudioBuffer | null; + companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number }>; + }>; + renderAndMuxOfflineAudio( + videoUrl: string, + trimRegions: never[], + speedRegions: never[], + audioRegions: never[], + sourceAudioFallbackPaths: string[], + muxer: unknown, + ): Promise; +}; + +describe("AudioProcessor offline render preparation", () => { + it("keeps embedded source audio separate from external companion sidecars", async () => { + const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness; + const mainBuffer = { duration: 10, numberOfChannels: 2 } as AudioBuffer; + const micBuffer = { duration: 9.5, numberOfChannels: 1 } as AudioBuffer; + + const decodeAudioFromUrl = vi + .spyOn(processor, "decodeAudioFromUrl") + .mockImplementation(async (url: string) => { + if (url === "file:///tmp/recording.mp4") { + return mainBuffer; + } + if (url === "/tmp/recording.mic.wav") { + return micBuffer; + } + return null; + }); + vi.spyOn(processor, "getMediaDurationSec").mockResolvedValue(10); + + const prepared = await processor.prepareOfflineRender( + "file:///tmp/recording.mp4", + [], + [], + [], + ["/tmp/recording.mp4", "/tmp/recording.mic.wav"], + ); + + expect(prepared.mainBuffer).toBe(mainBuffer); + expect(prepared.companionEntries).toHaveLength(1); + expect(prepared.companionEntries[0]?.buffer).toBe(micBuffer); + expect(decodeAudioFromUrl).toHaveBeenCalledWith("file:///tmp/recording.mp4"); + expect(decodeAudioFromUrl).toHaveBeenCalledWith("/tmp/recording.mic.wav"); + expect(decodeAudioFromUrl).not.toHaveBeenCalledWith("/tmp/recording.mp4"); + }); + + it("does not treat a single embedded fallback path as an external sidecar", async () => { + const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness; + const loadAudioFileDemuxer = vi.spyOn(processor, "loadAudioFileDemuxer"); + const renderAndMuxOfflineAudio = vi + .spyOn(processor, "renderAndMuxOfflineAudio") + .mockResolvedValue(); + + await processor.process( + null, + {} as never, + "file:///tmp/recording.mp4", + [], + [], + undefined, + [], + ["/tmp/recording.mp4"], + ); + + expect(loadAudioFileDemuxer).not.toHaveBeenCalled(); + expect(renderAndMuxOfflineAudio).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index a5bb79d5..150fbfea 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -5,11 +5,10 @@ import type { SpeedRegion, TrimRegion, } from "@/components/video-editor/types"; -import { - estimateCompanionAudioStartDelaySeconds, -} from "@/lib/mediaTiming"; +import { estimateCompanionAudioStartDelaySeconds } from "@/lib/mediaTiming"; import { resolveMediaElementSource } from "./localMediaSource"; import type { VideoMuxer } from "./muxer"; +import { resolveSourceAudioFallbackPaths } from "./sourceAudioFallback"; const AUDIO_BITRATE = 128_000; const DECODE_BACKPRESSURE_LIMIT = 20; @@ -36,20 +35,20 @@ interface PreparedOfflineRender { } export async function isAacAudioEncodingSupported( - sampleRate = 48_000, - numberOfChannels = 2, + sampleRate = 48_000, + numberOfChannels = 2, ): Promise { - try { - const support = await AudioEncoder.isConfigSupported({ - codec: MP4_AUDIO_CODEC, - sampleRate, - numberOfChannels, - bitrate: AUDIO_BITRATE, - }); - return support.supported === true; - } catch { - return false; - } + try { + const support = await AudioEncoder.isConfigSupported({ + codec: MP4_AUDIO_CODEC, + sampleRate, + numberOfChannels, + bitrate: AUDIO_BITRATE, + }); + return support.supported === true; + } catch { + return false; + } } type TrimLikeRegion = TrimRegion | ClipRegion; @@ -161,12 +160,19 @@ export class AudioProcessor { (audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0, ) : []; + const { hasEmbeddedSourceAudio, externalAudioPaths } = resolveSourceAudioFallbackPaths( + videoUrl, + sortedSourceAudioFallbackPaths, + ); + const needsSourceAudioMixing = + externalAudioPaths.length > 1 || + (hasEmbeddedSourceAudio && externalAudioPaths.length > 0); // When speed edits, audio regions, or multiple audio sources need mixing, use offline AudioContext pipeline. if ( sortedSpeedRegions.length > 0 || sortedAudioRegions.length > 0 || - sortedSourceAudioFallbackPaths.length > 1 + needsSourceAudioMixing ) { await this.renderAndMuxOfflineAudio( videoUrl, @@ -180,10 +186,8 @@ export class AudioProcessor { } // Single sidecar audio with no speed/audio edits: demux directly (skips slow real-time rendering). - if (sortedSourceAudioFallbackPaths.length === 1) { - const sidecarDemuxer = await this.loadAudioFileDemuxer( - sortedSourceAudioFallbackPaths[0], - ); + if (!hasEmbeddedSourceAudio && externalAudioPaths.length === 1) { + const sidecarDemuxer = await this.loadAudioFileDemuxer(externalAudioPaths[0]); if (sidecarDemuxer) { try { await this.processTrimOnlyAudio(sidecarDemuxer, muxer, sortedTrims); @@ -205,7 +209,7 @@ export class AudioProcessor { sortedTrims, [], [], - sortedSourceAudioFallbackPaths, + externalAudioPaths, muxer, ); return; @@ -547,23 +551,23 @@ export class AudioProcessor { if (this.cancelled) throw new Error("Export cancelled"); this.onProgress?.(0); - const hasExternalSources = sourceAudioFallbackPaths.length > 0; + const { externalAudioPaths } = resolveSourceAudioFallbackPaths( + videoUrl, + sourceAudioFallbackPaths, + ); - // Decode primary audio source (streaming decode with bulk fallback) - const mainBuffer = !hasExternalSources - ? await this.decodeAudioFromUrl(videoUrl) - : null; + // Decode embedded source audio separately from companion sidecars. + const mainBuffer = await this.decodeAudioFromUrl(videoUrl); if (this.cancelled) throw new Error("Export cancelled"); // Decode companion / sidecar audio files const companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number }> = []; - for (const audioPath of sourceAudioFallbackPaths) { + for (const audioPath of externalAudioPaths) { if (this.cancelled) throw new Error("Export cancelled"); const buffer = await this.decodeAudioFromUrl(audioPath); if (!buffer) continue; - const refDuration = - mainBuffer?.duration ?? (await this.getMediaDurationSec(videoUrl)); + const refDuration = mainBuffer?.duration ?? (await this.getMediaDurationSec(videoUrl)); companionEntries.push({ buffer, startDelaySec: estimateCompanionAudioStartDelaySeconds( @@ -593,7 +597,7 @@ export class AudioProcessor { let sourceDurationSec: number; if (mainBuffer) { sourceDurationSec = mainBuffer.duration; - } else if (hasExternalSources || regionEntries.length > 0) { + } else if (externalAudioPaths.length > 0 || regionEntries.length > 0) { sourceDurationSec = await this.getMediaDurationSec(videoUrl); } else { sourceDurationSec = primaryBuffer?.duration ?? 0; @@ -659,10 +663,7 @@ export class AudioProcessor { pendingMuxing = pendingMuxing .then(async () => { if (this.cancelled) return; - await muxer.addAudioChunk( - chunk, - !wroteFirstChunk ? meta : undefined, - ); + await muxer.addAudioChunk(chunk, !wroteFirstChunk ? meta : undefined); wroteFirstChunk = true; }) .catch((error) => { @@ -706,27 +707,17 @@ export class AudioProcessor { // Render timeline to a WAV blob for the native/FFmpeg export path. // Processes in chunks to avoid holding the entire output in memory. - private async renderToWavBlobChunked( - prepared: PreparedOfflineRender, - ): Promise { + private async renderToWavBlobChunked(prepared: PreparedOfflineRender): Promise { const totalOutputSec = Math.max(prepared.outputDurationMs / 1000, 0.01); const totalFrames = Math.ceil(totalOutputSec * OFFLINE_AUDIO_SAMPLE_RATE); const numChannels = prepared.numChannels; - const header = this.createWavHeader( - OFFLINE_AUDIO_SAMPLE_RATE, - numChannels, - totalFrames, - ); + const header = this.createWavHeader(OFFLINE_AUDIO_SAMPLE_RATE, numChannels, totalFrames); const pcmParts: ArrayBuffer[] = [header]; - await this.renderChunked( - prepared, - totalOutputSec, - async (rendered) => { - pcmParts.push(...this.audioBufferToPcmParts(rendered)); - }, - ); + await this.renderChunked(prepared, totalOutputSec, async (rendered) => { + pcmParts.push(...this.audioBufferToPcmParts(rendered)); + }); return new Blob(pcmParts, { type: "audio/wav" }); } @@ -747,10 +738,7 @@ export class AudioProcessor { const chunkCount = Math.ceil(totalOutputSec / OFFLINE_CHUNK_DURATION_SEC); for (let i = 0; i < chunkCount && !this.cancelled; i++) { - const chunkSec = Math.min( - OFFLINE_CHUNK_DURATION_SEC, - totalOutputSec - outputOffsetSec, - ); + const chunkSec = Math.min(OFFLINE_CHUNK_DURATION_SEC, totalOutputSec - outputOffsetSec); const chunkFrames = Math.ceil(chunkSec * OFFLINE_AUDIO_SAMPLE_RATE); const offlineCtx = new OfflineAudioContext( @@ -833,10 +821,7 @@ export class AudioProcessor { localEndSec = chunkDurationSec; } - const duration = Math.min( - localEndSec - localStartSec, - buffer.duration - bufferOffsetSec, - ); + const duration = Math.min(localEndSec - localStartSec, buffer.duration - bufferOffsetSec); if (duration <= 0.001) return; const gainNode = ctx.createGain(); @@ -869,10 +854,7 @@ export class AudioProcessor { const planarData = new Float32Array(frameCount * numChannels); for (let ch = 0; ch < numChannels; ch++) { const channelData = buffer.getChannelData(ch); - planarData.set( - channelData.subarray(offset, offset + frameCount), - ch * frameCount, - ); + planarData.set(channelData.subarray(offset, offset + frameCount), ch * frameCount); } const audioData = new AudioData({ @@ -880,19 +862,14 @@ export class AudioProcessor { sampleRate, numberOfFrames: frameCount, numberOfChannels: numChannels, - timestamp: Math.round( - (offset / sampleRate + timestampOffsetSec) * 1_000_000, - ), + timestamp: Math.round((offset / sampleRate + timestampOffsetSec) * 1_000_000), data: planarData, }); encoder.encode(audioData); audioData.close(); - while ( - encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT && - !this.cancelled - ) { + while (encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT && !this.cancelled) { await new Promise((r) => setTimeout(r, 1)); } } @@ -929,18 +906,13 @@ export class AudioProcessor { type: blob.type || "video/mp4", }); - const wasmUrl = new URL( - "./wasm/web-demuxer.wasm", - window.location.href, - ).href; + const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href; demuxer = new WebDemuxer({ wasmFilePath: wasmUrl }); await demuxer.load(file); let audioConfig: AudioDecoderConfig; try { - audioConfig = (await demuxer.getDecoderConfig( - "audio", - )) as AudioDecoderConfig; + audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig; } catch { return null; // No audio track } @@ -949,10 +921,7 @@ export class AudioProcessor { const numChannels = Math.min(audioConfig.numberOfChannels || 2, 2); // Accumulate decoded PCM per channel - const channelChunks: Float32Array[][] = Array.from( - { length: numChannels }, - () => [], - ); + const channelChunks: Float32Array[][] = Array.from({ length: numChannels }, () => []); let totalFrames = 0; let decodeError: Error | null = null; @@ -960,10 +929,7 @@ export class AudioProcessor { output: (data: AudioData) => { try { const frames = data.numberOfFrames; - const dataChannels = Math.min( - data.numberOfChannels, - numChannels, - ); + const dataChannels = Math.min(data.numberOfChannels, numChannels); const format = data.format; if (format?.includes("planar")) { @@ -973,9 +939,7 @@ export class AudioProcessor { }); const bytes = new ArrayBuffer(size); data.copyTo(bytes, { planeIndex: ch }); - channelChunks[ch].push( - this.rawToFloat32(bytes, format, frames), - ); + channelChunks[ch].push(this.rawToFloat32(bytes, format, frames)); } } else if (format) { // Interleaved format — deinterleave into per-channel arrays. @@ -993,8 +957,7 @@ export class AudioProcessor { for (let ch = 0; ch < dataChannels; ch++) { const chData = new Float32Array(frames); for (let i = 0; i < frames; i++) { - chData[i] = - interleaved[i * srcChannels + ch]; + chData[i] = interleaved[i * srcChannels + ch]; } channelChunks[ch].push(chData); } @@ -1011,18 +974,14 @@ export class AudioProcessor { } }, error: (err: DOMException) => { - decodeError = new Error( - `Streaming audio decode error: ${err.message}`, - ); + decodeError = new Error(`Streaming audio decode error: ${err.message}`); }, }); decoder.configure(audioConfig); const audioStream = demuxer.read("audio"); - const reader = ( - audioStream as ReadableStream - ).getReader(); + const reader = (audioStream as ReadableStream).getReader(); try { while (!this.cancelled) { @@ -1032,10 +991,7 @@ export class AudioProcessor { decoder.decode(chunk); - while ( - decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT && - !this.cancelled - ) { + while (decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT && !this.cancelled) { if (decodeError) throw decodeError; await new Promise((r) => setTimeout(r, 1)); } @@ -1085,11 +1041,7 @@ export class AudioProcessor { } // Convert raw bytes from AudioData to Float32Array based on the sample format. - private rawToFloat32( - bytes: ArrayBuffer, - format: string, - sampleCount: number, - ): Float32Array { + private rawToFloat32(bytes: ArrayBuffer, format: string, sampleCount: number): Float32Array { if (format.startsWith("f32")) { return new Float32Array(bytes); } @@ -1122,10 +1074,7 @@ export class AudioProcessor { } // Bulk decode fallback: loads entire file into memory and uses decodeAudioData. - private async bulkDecodeFromUrl( - url: string, - sampleRate: number, - ): Promise { + private async bulkDecodeFromUrl(url: string, sampleRate: number): Promise { try { const source = await resolveMediaElementSource(url); try { @@ -1197,8 +1146,7 @@ export class AudioProcessor { boundaries.add(sourceDurationMs); for (const trim of trimRegions) { - if (trim.startMs >= 0 && trim.startMs <= sourceDurationMs) - boundaries.add(trim.startMs); + if (trim.startMs >= 0 && trim.startMs <= sourceDurationMs) boundaries.add(trim.startMs); if (trim.endMs >= 0 && trim.endMs <= sourceDurationMs) boundaries.add(trim.endMs); } for (const speed of speedRegions) { @@ -1234,10 +1182,7 @@ export class AudioProcessor { } // Map a source-timeline timestamp to the corresponding output-timeline timestamp. - private sourceTimeToOutputTime( - sourceMs: number, - slices: TimelineSlice[], - ): number { + private sourceTimeToOutputTime(sourceMs: number, slices: TimelineSlice[]): number { let outputMs = 0; for (const slice of slices) { @@ -1303,8 +1248,7 @@ export class AudioProcessor { // Calculate output position (global then chunk-local) let localOutputStartSec = outputOffsetSec + trimmedFromStartSec / slice.speed - chunkOutputStartSec; - let localOutputEndSec = - localOutputStartSec + effectiveSourceDurationSec / slice.speed; + let localOutputEndSec = localOutputStartSec + effectiveSourceDurationSec / slice.speed; // Skip if entirely outside chunk window if (localOutputEndSec <= 0 || localOutputStartSec >= chunkDurationSec) { @@ -1401,11 +1345,7 @@ export class AudioProcessor { for (let i = 0; i < chunkFrames; i++) { for (let ch = 0; ch < numChannels; ch++) { const sample = Math.max(-1, Math.min(1, channels[ch][frameOffset + i])); - view.setInt16( - byteOffset, - sample < 0 ? sample * 0x8000 : sample * 0x7fff, - true, - ); + view.setInt16(byteOffset, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true); byteOffset += 2; } } diff --git a/src/lib/exporter/mediaResource.test.ts b/src/lib/exporter/mediaResource.test.ts new file mode 100644 index 00000000..48161745 --- /dev/null +++ b/src/lib/exporter/mediaResource.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; + +import { getLocalFilePathFromResource, getResourceFileName } from "./mediaResource"; + +describe("getLocalFilePathFromResource", () => { + it("extracts the path from file URLs", () => { + expect(getLocalFilePathFromResource("file:///tmp/example%20video.mp4")).toBe( + "/tmp/example video.mp4", + ); + }); + + it("extracts the approved file path from loopback media server URLs", () => { + expect( + getLocalFilePathFromResource( + "http://127.0.0.1:4321/video?path=%2Ftmp%2Fexample%20video.mp4", + ), + ).toBe("/tmp/example video.mp4"); + }); + + it("does not treat arbitrary remote URLs as local files", () => { + expect(getLocalFilePathFromResource("https://example.com/video.mp4")).toBeNull(); + }); +}); + +describe("getResourceFileName", () => { + it("uses the source file name for loopback media server URLs", () => { + expect( + getResourceFileName( + "http://127.0.0.1:4321/video?path=%2Ftmp%2Fexample%20video.mp4", + "fallback.mp4", + ), + ).toBe("example video.mp4"); + }); +}); \ No newline at end of file diff --git a/src/lib/exporter/mediaResource.ts b/src/lib/exporter/mediaResource.ts new file mode 100644 index 00000000..83e385eb --- /dev/null +++ b/src/lib/exporter/mediaResource.ts @@ -0,0 +1,111 @@ +const LOOPBACK_MEDIA_SERVER_HOSTS = new Set(["127.0.0.1", "localhost", "::1", "[::1]"]); + +export function isAbsoluteLocalPath(resource: string): boolean { + return ( + resource.startsWith("/") || + /^[A-Za-z]:[\\/]/.test(resource) || + /^\\\\[^\\]+\\[^\\]+/.test(resource) + ); +} + +function fromFileUrl(resource: string): string { + try { + const url = new URL(resource); + const pathname = decodeURIComponent(url.pathname); + + if (url.host && url.host !== "localhost") { + const uncPath = `//${url.host}${pathname.startsWith("/") ? pathname : `/${pathname}`}`; + return uncPath.replace(/\//g, "\\"); + } + + if (/^\/[A-Za-z]:/.test(pathname)) { + return pathname.slice(1); + } + + return pathname; + } catch { + const rawFallbackPath = resource.replace(/^file:\/\//i, ""); + let fallbackPath = rawFallbackPath; + try { + fallbackPath = decodeURIComponent(rawFallbackPath); + } catch { + // Keep raw best-effort path if percent decoding fails. + } + return fallbackPath.replace(/^\/([A-Za-z]:)/, "$1"); + } +} + +function getLoopbackMediaServerPath(resource: string): string | null { + try { + const url = new URL(resource); + if (url.pathname !== "/video") { + return null; + } + + if (!/^https?:$/i.test(url.protocol)) { + return null; + } + + if (!LOOPBACK_MEDIA_SERVER_HOSTS.has(url.hostname.toLowerCase())) { + return null; + } + + const pathParam = url.searchParams.get("path"); + if (!pathParam) { + return null; + } + + if (/^file:\/\//i.test(pathParam)) { + return fromFileUrl(pathParam); + } + + return isAbsoluteLocalPath(pathParam) ? pathParam : null; + } catch { + return null; + } +} + +function getPathBaseName(filePath: string): string { + const normalized = filePath.replace(/\\/g, "/"); + const segments = normalized.split("/").filter(Boolean); + return segments[segments.length - 1] ?? ""; +} + +export function getLocalFilePathFromResource(resource: string): string | null { + if (!resource) { + return null; + } + + if (/^file:\/\//i.test(resource)) { + return fromFileUrl(resource); + } + + const mediaServerPath = getLoopbackMediaServerPath(resource); + if (mediaServerPath) { + return mediaServerPath; + } + + return isAbsoluteLocalPath(resource) ? resource : null; +} + +export function getResourceFileName(resource: string, fallback: string): string { + const localFilePath = getLocalFilePathFromResource(resource); + if (localFilePath) { + const fileName = getPathBaseName(localFilePath); + if (fileName) { + return fileName; + } + } + + try { + const url = new URL(resource); + const fileName = getPathBaseName(decodeURIComponent(url.pathname)); + if (fileName) { + return fileName; + } + } catch { + // Ignore parse errors and fall back to the provided default. + } + + return fallback; +} \ No newline at end of file diff --git a/src/lib/exporter/sourceAudioFallback.test.ts b/src/lib/exporter/sourceAudioFallback.test.ts new file mode 100644 index 00000000..aa28f1d6 --- /dev/null +++ b/src/lib/exporter/sourceAudioFallback.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; + +import { resolveSourceAudioFallbackPaths } from "./sourceAudioFallback"; + +describe("resolveSourceAudioFallbackPaths", () => { + it("treats the video file path as embedded source audio when present in the fallback list", () => { + const videoPath = "/tmp/recording.mp4"; + + expect( + resolveSourceAudioFallbackPaths(videoPath, [videoPath, "/tmp/recording.mic.wav"]), + ).toEqual({ + hasEmbeddedSourceAudio: true, + externalAudioPaths: ["/tmp/recording.mic.wav"], + }); + }); + + it("keeps all fallback paths external when the video has no embedded source audio", () => { + expect( + resolveSourceAudioFallbackPaths("/tmp/recording.mp4", [ + "/tmp/recording.system.wav", + "/tmp/recording.mic.wav", + ]), + ).toEqual({ + hasEmbeddedSourceAudio: false, + externalAudioPaths: ["/tmp/recording.system.wav", "/tmp/recording.mic.wav"], + }); + }); + + it("matches embedded source audio when the video resource is a file URL", () => { + expect( + resolveSourceAudioFallbackPaths("file:///tmp/recording.mp4", [ + "/tmp/recording.mp4", + "/tmp/recording.mic.wav", + ]), + ).toEqual({ + hasEmbeddedSourceAudio: true, + externalAudioPaths: ["/tmp/recording.mic.wav"], + }); + }); + + it("normalizes Windows file URLs and local paths when checking embedded audio", () => { + expect( + resolveSourceAudioFallbackPaths("file:///C:/Users/Egg/Videos/recording.mp4", [ + "C:\\Users\\Egg\\Videos\\recording.mp4", + "C:\\Users\\Egg\\Videos\\recording.mic.wav", + ]), + ).toEqual({ + hasEmbeddedSourceAudio: true, + externalAudioPaths: ["C:\\Users\\Egg\\Videos\\recording.mic.wav"], + }); + }); + + it("matches Windows paths case-insensitively for embedded audio detection", () => { + expect( + resolveSourceAudioFallbackPaths("file:///C:/Users/Egg/Videos/recording.mp4", [ + "c:\\users\\egg\\videos\\recording.mp4", + "c:\\users\\egg\\videos\\recording.mic.wav", + ]), + ).toEqual({ + hasEmbeddedSourceAudio: true, + externalAudioPaths: ["c:\\users\\egg\\videos\\recording.mic.wav"], + }); + }); +}); diff --git a/src/lib/exporter/sourceAudioFallback.ts b/src/lib/exporter/sourceAudioFallback.ts new file mode 100644 index 00000000..6e1fcca4 --- /dev/null +++ b/src/lib/exporter/sourceAudioFallback.ts @@ -0,0 +1,48 @@ +import { getLocalFilePathFromResource } from "./mediaResource"; + +function normalizeSourceAudioFallbackPath(resourceOrPath: string): string | null { + if (typeof resourceOrPath !== "string") { + return null; + } + + const resolvedPath = getLocalFilePathFromResource(resourceOrPath) ?? resourceOrPath; + const trimmedPath = resolvedPath.trim(); + if (!trimmedPath) { + return null; + } + + const isWindowsPath = + /^[A-Za-z]:[\\/]/.test(trimmedPath) || /^\\\\[^\\]+\\[^\\]+/.test(trimmedPath); + if (isWindowsPath) { + return trimmedPath.replace(/\//g, "\\").toLowerCase(); + } + + return trimmedPath.replace(/\\/g, "/"); +} + +export function resolveSourceAudioFallbackPaths( + videoResource: string | null | undefined, + sourceAudioFallbackPaths: string[] | null | undefined, +) { + const normalizedPaths = (sourceAudioFallbackPaths ?? []) + .filter((audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0) + .map((audioPath) => ({ + audioPath, + normalizedPath: normalizeSourceAudioFallbackPath(audioPath), + })); + const localVideoSourcePath = videoResource + ? normalizeSourceAudioFallbackPath(videoResource) + : null; + const hasEmbeddedSourceAudio = + Boolean(localVideoSourcePath) && + normalizedPaths.some(({ normalizedPath }) => normalizedPath === localVideoSourcePath); + + return { + hasEmbeddedSourceAudio, + externalAudioPaths: hasEmbeddedSourceAudio + ? normalizedPaths + .filter(({ normalizedPath }) => normalizedPath !== localVideoSourcePath) + .map(({ audioPath }) => audioPath) + : normalizedPaths.map(({ audioPath }) => audioPath), + }; +}