From 3d6668fd60aea8523558b49683f10f025dd46e73 Mon Sep 17 00:00:00 2001 From: wiiiii123 Date: Fri, 8 May 2026 01:35:17 +0700 Subject: [PATCH] fix(recording): account for mic pause gaps --- electron/ipc/recording/diagnostics.test.ts | 67 ++++++++ electron/ipc/recording/diagnostics.ts | 86 ++++++++++ electron/ipc/register/recording.ts | 59 ++++++- src/hooks/useScreenRecorder.test.ts | 5 + src/hooks/useScreenRecorder.ts | 173 ++++++++++++++++----- 5 files changed, 352 insertions(+), 38 deletions(-) diff --git a/electron/ipc/recording/diagnostics.test.ts b/electron/ipc/recording/diagnostics.test.ts index 903c7722..21192e91 100644 --- a/electron/ipc/recording/diagnostics.test.ts +++ b/electron/ipc/recording/diagnostics.test.ts @@ -304,6 +304,73 @@ describe("getCompanionAudioFallbackPaths", () => { await expect(getCompanionAudioStartDelayMs(micPath)).resolves.toBeNull(); }); + it("classifies wall-clock mic chunk gaps covered by pause intervals", async () => { + const { summarizeMicrophoneChunkTiming } = await import("./diagnostics"); + + expect( + summarizeMicrophoneChunkTiming( + [ + { + index: 0, + size: 1024, + elapsedMs: 250, + deltaMs: null, + recordedElapsedMs: 250, + recordedDeltaMs: null, + }, + { + index: 1, + size: 1024, + elapsedMs: 8250, + deltaMs: 8000, + recordedElapsedMs: 500, + recordedDeltaMs: 250, + }, + ], + [{ startElapsedMs: 250, endElapsedMs: 8250, durationMs: 7750 }], + 250, + ), + ).toMatchObject({ + status: "pause-accounted", + wallClockGapCount: 1, + recordedGapCount: 0, + pausedDurationMs: 7750, + }); + }); + + it("flags recorded mic chunk gaps that remain after pause accounting", async () => { + const { summarizeMicrophoneChunkTiming } = await import("./diagnostics"); + + expect( + summarizeMicrophoneChunkTiming( + [ + { + index: 0, + size: 1024, + elapsedMs: 250, + deltaMs: null, + recordedElapsedMs: 250, + recordedDeltaMs: null, + }, + { + index: 1, + size: 1024, + elapsedMs: 2500, + deltaMs: 2250, + recordedElapsedMs: 2500, + recordedDeltaMs: 2250, + }, + ], + [], + 250, + ), + ).toMatchObject({ + status: "needs-review", + wallClockGapCount: 1, + recordedGapCount: 1, + }); + }); + it("rejects tiny MP4 container-only outputs before they reach the editor", async () => { const videoPath = path.join(tempRoot, "recording-123.mp4"); await fs.writeFile(videoPath, Buffer.alloc(261)); diff --git a/electron/ipc/recording/diagnostics.ts b/electron/ipc/recording/diagnostics.ts index 081e2412..b8fb6860 100644 --- a/electron/ipc/recording/diagnostics.ts +++ b/electron/ipc/recording/diagnostics.ts @@ -16,6 +16,21 @@ type CompanionAudioTimingMetadata = { startDelayMs?: number; }; +export type MicrophoneChunkTimingEvent = { + index: number; + size: number; + elapsedMs: number; + deltaMs: number | null; + recordedElapsedMs?: number; + recordedDeltaMs?: number | null; +}; + +export type MicrophonePauseInterval = { + startElapsedMs: number; + endElapsedMs?: number; + durationMs?: number; +}; + export type VideoStreamDurationProbe = { durationSeconds: number | null; frameCount: number | null; @@ -102,6 +117,77 @@ export function getRecordingAudioMuxTimeoutMs(durationSeconds: number) { ); } +function getFiniteNonNegativeNumber(value: unknown) { + return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null; +} + +export function summarizeMicrophoneChunkTiming( + chunkEvents: MicrophoneChunkTimingEvent[] | null | undefined, + pauseIntervals: MicrophonePauseInterval[] | null | undefined = [], + timesliceMs = 250, +) { + const normalizedTimesliceMs = + Number.isFinite(timesliceMs) && timesliceMs > 0 ? Math.round(timesliceMs) : 250; + const thresholdMs = Math.max(600, Math.round(normalizedTimesliceMs * 2.5)); + const events = Array.isArray(chunkEvents) ? chunkEvents : []; + const pauses = Array.isArray(pauseIntervals) ? pauseIntervals : []; + const pausedDurationMs = pauses.reduce((total, interval) => { + const durationMs = getFiniteNonNegativeNumber(interval.durationMs); + return durationMs === null ? total : total + Math.round(durationMs); + }, 0); + + const wallClockGaps = events + .filter((event) => (event.deltaMs ?? 0) > thresholdMs) + .map((event) => ({ + index: event.index, + deltaMs: Math.round(event.deltaMs ?? 0), + elapsedMs: Math.round(event.elapsedMs), + })); + const recordedGaps = events + .filter((event) => { + const recordedDeltaMs = + getFiniteNonNegativeNumber(event.recordedDeltaMs) ?? event.deltaMs ?? 0; + return recordedDeltaMs > thresholdMs; + }) + .map((event) => ({ + index: event.index, + deltaMs: Math.round( + getFiniteNonNegativeNumber(event.recordedDeltaMs) ?? event.deltaMs ?? 0, + ), + recordedElapsedMs: Math.round(event.recordedElapsedMs ?? event.elapsedMs), + })); + const maxWallClockDeltaMs = events.reduce( + (maxDeltaMs, event) => Math.max(maxDeltaMs, event.deltaMs ?? 0), + 0, + ); + const maxRecordedDeltaMs = events.reduce((maxDeltaMs, event) => { + const recordedDeltaMs = + getFiniteNonNegativeNumber(event.recordedDeltaMs) ?? event.deltaMs ?? 0; + return Math.max(maxDeltaMs, recordedDeltaMs); + }, 0); + const status = + recordedGaps.length > 0 + ? "needs-review" + : wallClockGaps.length > 0 && pausedDurationMs > 0 + ? "pause-accounted" + : "ok"; + + return { + status, + timesliceMs: normalizedTimesliceMs, + thresholdMs, + eventCount: events.length, + pauseIntervalCount: pauses.length, + pausedDurationMs, + maxWallClockDeltaMs: Math.round(maxWallClockDeltaMs), + maxRecordedDeltaMs: Math.round(maxRecordedDeltaMs), + wallClockGapCount: wallClockGaps.length, + recordedGapCount: recordedGaps.length, + wallClockGaps: wallClockGaps.slice(0, 10), + recordedGaps: recordedGaps.slice(0, 10), + }; +} + /** Probe the duration of a media file (in seconds) using the container header. */ export async function probeMediaDurationSeconds(filePath: string): Promise { const ffmpegPath = getFfmpegBinaryPath(); diff --git a/electron/ipc/register/recording.ts b/electron/ipc/register/recording.ts index 60db86ab..9321f5b8 100644 --- a/electron/ipc/register/recording.ts +++ b/electron/ipc/register/recording.ts @@ -45,8 +45,11 @@ import { import { getCompanionAudioFallbackInfo, getFileSizeIfPresent, + type MicrophoneChunkTimingEvent, + type MicrophonePauseInterval, type RecordingDiagnosticsSnapshot, recordNativeCaptureDiagnostics, + summarizeMicrophoneChunkTiming, validateRecordedVideo, writeRecordingDiagnosticsSnapshot, } from "../recording/diagnostics"; @@ -189,7 +192,7 @@ function pickPrimitiveRecord(value: unknown) { return entries.length > 0 ? Object.fromEntries(entries) : null; } -function pickMicrophoneChunkEvents(value: unknown) { +function pickMicrophoneChunkEvents(value: unknown): MicrophoneChunkTimingEvent[] | null { if (!Array.isArray(value)) { return null; } @@ -200,7 +203,7 @@ function pickMicrophoneChunkEvents(value: unknown) { return null; } - const { index, size, elapsedMs, deltaMs } = event; + const { index, size, elapsedMs, deltaMs, recordedElapsedMs, recordedDeltaMs } = event; if ( typeof index !== "number" || typeof size !== "number" || @@ -214,6 +217,15 @@ function pickMicrophoneChunkEvents(value: unknown) { size: Math.round(size), elapsedMs: Math.round(elapsedMs), deltaMs: typeof deltaMs === "number" ? Math.round(deltaMs) : null, + ...(typeof recordedElapsedMs === "number" && + Number.isFinite(recordedElapsedMs) && + recordedElapsedMs >= 0 + ? { recordedElapsedMs: Math.round(recordedElapsedMs) } + : {}), + recordedDeltaMs: + typeof recordedDeltaMs === "number" && Number.isFinite(recordedDeltaMs) + ? Math.max(0, Math.round(recordedDeltaMs)) + : null, }; }) .filter((event): event is NonNullable => event !== null); @@ -221,6 +233,37 @@ function pickMicrophoneChunkEvents(value: unknown) { return events.length > 0 ? events : null; } +function pickMicrophonePauseIntervals(value: unknown): MicrophonePauseInterval[] | null { + if (!Array.isArray(value)) { + return null; + } + + const intervals = value + .map((interval) => { + if (!isRecord(interval) || typeof interval.startElapsedMs !== "number") { + return null; + } + + const startElapsedMs = Math.max(0, Math.round(interval.startElapsedMs)); + return { + startElapsedMs, + ...(typeof interval.endElapsedMs === "number" && + Number.isFinite(interval.endElapsedMs) && + interval.endElapsedMs >= startElapsedMs + ? { endElapsedMs: Math.round(interval.endElapsedMs) } + : {}), + ...(typeof interval.durationMs === "number" && + Number.isFinite(interval.durationMs) && + interval.durationMs >= 0 + ? { durationMs: Math.round(interval.durationMs) } + : {}), + }; + }) + .filter((interval): interval is NonNullable => interval !== null); + + return intervals.length > 0 ? intervals : null; +} + function pickAudioInputDevices(value: unknown) { if (!Array.isArray(value)) { return null; @@ -1415,6 +1458,7 @@ export function registerRecordingHandlers( audioInputDevices?: unknown; mediaRecorder?: unknown; chunkEvents?: unknown; + pauseIntervals?: unknown; }, ) => { const baseName = videoPath.replace(/\.[^.]+$/, ""); @@ -1478,6 +1522,15 @@ export function registerRecordingHandlers( } : null; const chunkEvents = pickMicrophoneChunkEvents(options?.chunkEvents); + const pauseIntervals = pickMicrophonePauseIntervals(options?.pauseIntervals); + const chunkTiming = + chunkEvents || pauseIntervals + ? summarizeMicrophoneChunkTiming( + chunkEvents, + pauseIntervals, + mediaRecorder?.timesliceMs, + ) + : null; const metadata = { ...(Number.isFinite(startDelayMs) && (startDelayMs ?? 0) >= 0 ? { startDelayMs: Math.round(startDelayMs ?? 0) } @@ -1500,6 +1553,8 @@ export function registerRecordingHandlers( ? { mediaRecorder } : {}), ...(chunkEvents ? { chunkEvents } : {}), + ...(pauseIntervals ? { pauseIntervals } : {}), + ...(chunkTiming ? { chunkTiming } : {}), }; if (Object.keys(metadata).length > 0) { try { diff --git a/src/hooks/useScreenRecorder.test.ts b/src/hooks/useScreenRecorder.test.ts index 6a613459..e98554ad 100644 --- a/src/hooks/useScreenRecorder.test.ts +++ b/src/hooks/useScreenRecorder.test.ts @@ -19,6 +19,7 @@ function createMockMediaRecorder(initialState: RecordingState = "inactive") { resume: vi.fn(() => { if (_state === "paused") _state = "recording"; }), + requestData: vi.fn(), stop: vi.fn(() => { _state = "inactive"; }), @@ -151,6 +152,7 @@ function pauseRecording( webcamRecorder.pause(); } if (micFallbackRecorder?.state === "recording") { + micFallbackRecorder.requestData(); micFallbackRecorder.pause(); } return true; @@ -206,6 +208,7 @@ async function pauseNativeRecording( webcamRecorder.pause(); } if (micFallbackRecorder?.state === "recording") { + micFallbackRecorder.requestData(); micFallbackRecorder.pause(); } @@ -430,6 +433,7 @@ describe("useScreenRecorder state machine", () => { const result = pauseRecording(recorder, true, false, true, null, micFallback); expect(result).toBe(true); + expect(micFallback.requestData).toHaveBeenCalled(); expect(micFallback.state).toBe("paused"); }); @@ -613,6 +617,7 @@ describe("useScreenRecorder state machine", () => { const pausedResult = await pauseNativeRecording(webcam, { success: true }, micFallback); expect(pausedResult).toBe(true); expect(webcam.state).toBe("paused"); + expect(micFallback.requestData).toHaveBeenCalled(); expect(micFallback.state).toBe("paused"); expect(recorder.pause).not.toHaveBeenCalled(); diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index d16683aa..3d7933ee 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -84,6 +84,13 @@ type MicrophoneFallbackChunkEvent = { size: number; elapsedMs: number; deltaMs: number | null; + recordedElapsedMs: number; + recordedDeltaMs: number | null; +}; +type MicrophoneFallbackPauseInterval = { + startElapsedMs: number; + endElapsedMs?: number; + durationMs?: number; }; type MicrophoneFallbackRecorderMetadata = { mimeType: string; @@ -99,6 +106,7 @@ type MicrophoneSidecarOptions = { audioInputDevices?: MicrophoneAudioInputDeviceSnapshot[]; mediaRecorder?: MicrophoneFallbackRecorderMetadata; chunkEvents?: MicrophoneFallbackChunkEvent[]; + pauseIntervals?: MicrophoneFallbackPauseInterval[]; }; const LINUX_PORTAL_SOURCE: ProcessedDesktopSource = { id: "screen:linux-portal", @@ -299,6 +307,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const micFallbackRecorderMetadata = useRef(null); const micFallbackChunkEvents = useRef([]); const micFallbackRecorderStartedAt = useRef(null); + const micFallbackPauseStartedAt = useRef(null); + const micFallbackPausedDurationMs = useRef(0); + const micFallbackPauseIntervals = useRef([]); const browserMicrophoneProfile = useRef( DEFAULT_BROWSER_MICROPHONE_PROFILE, ); @@ -391,6 +402,32 @@ export function useScreenRecorder(): UseScreenRecorderReturn { }); }, []); + const getMicFallbackRecordedElapsedMs = useCallback((now = performance.now()) => { + const startedAt = micFallbackRecorderStartedAt.current; + if (startedAt === null) { + return 0; + } + + const currentPauseDurationMs = + micFallbackPauseStartedAt.current === null + ? 0 + : Math.max(0, now - micFallbackPauseStartedAt.current); + return Math.max( + 0, + Math.round( + now - startedAt - micFallbackPausedDurationMs.current - currentPauseDurationMs, + ), + ); + }, []); + + const resetMicFallbackTimingDiagnostics = useCallback(() => { + micFallbackChunkEvents.current = []; + micFallbackRecorderStartedAt.current = null; + micFallbackPauseStartedAt.current = null; + micFallbackPausedDurationMs.current = 0; + micFallbackPauseIntervals.current = []; + }, []); + const preparePermissions = useCallback(async (options: { startup?: boolean } = {}) => { const platform = await window.electronAPI.getPlatform(); if (platform !== "darwin") { @@ -497,31 +534,40 @@ export function useScreenRecorder(): UseScreenRecorderReturn { micFallbackRequestedConstraints.current = null; micFallbackAudioInputDevices.current = null; micFallbackRecorderMetadata.current = null; - micFallbackChunkEvents.current = []; - micFallbackRecorderStartedAt.current = null; + resetMicFallbackTimingDiagnostics(); } - }, []); + }, [resetMicFallbackTimingDiagnostics]); - const appendMicFallbackChunk = useCallback((event: BlobEvent) => { - if (event.data.size <= 0) { - return; - } + const appendMicFallbackChunk = useCallback( + (event: BlobEvent) => { + if (event.data.size <= 0) { + return; + } - micFallbackChunks.current.push(event.data); - const startedAt = micFallbackRecorderStartedAt.current; - if (startedAt === null) { - return; - } + micFallbackChunks.current.push(event.data); + const startedAt = micFallbackRecorderStartedAt.current; + if (startedAt === null) { + return; + } - const elapsedMs = Math.max(0, Math.round(performance.now() - startedAt)); - const previous = micFallbackChunkEvents.current[micFallbackChunkEvents.current.length - 1]; - micFallbackChunkEvents.current.push({ - index: micFallbackChunkEvents.current.length, - size: event.data.size, - elapsedMs, - deltaMs: previous ? Math.max(0, elapsedMs - previous.elapsedMs) : null, - }); - }, []); + const now = performance.now(); + const elapsedMs = Math.max(0, Math.round(now - startedAt)); + const recordedElapsedMs = getMicFallbackRecordedElapsedMs(now); + const previous = + micFallbackChunkEvents.current[micFallbackChunkEvents.current.length - 1]; + micFallbackChunkEvents.current.push({ + index: micFallbackChunkEvents.current.length, + size: event.data.size, + elapsedMs, + deltaMs: previous ? Math.max(0, elapsedMs - previous.elapsedMs) : null, + recordedElapsedMs, + recordedDeltaMs: previous + ? Math.max(0, recordedElapsedMs - previous.recordedElapsedMs) + : null, + }); + }, + [getMicFallbackRecordedElapsedMs], + ); const resolveBrowserCaptureSource = useCallback(async (source: ProcessedDesktopSource) => { if (!source?.id?.startsWith("screen:")) { @@ -606,6 +652,27 @@ export function useScreenRecorder(): UseScreenRecorderReturn { [], ); + const closeMicFallbackPauseInterval = useCallback((now = performance.now()) => { + const pauseStartedAt = micFallbackPauseStartedAt.current; + if (pauseStartedAt === null) { + return; + } + + const durationMs = Math.max(0, Math.round(now - pauseStartedAt)); + micFallbackPausedDurationMs.current += durationMs; + const startedAt = micFallbackRecorderStartedAt.current ?? now; + const lastInterval = + micFallbackPauseIntervals.current[micFallbackPauseIntervals.current.length - 1]; + if (lastInterval && lastInterval.endElapsedMs === undefined) { + lastInterval.endElapsedMs = Math.max( + lastInterval.startElapsedMs, + Math.round(now - startedAt), + ); + lastInterval.durationMs = durationMs; + } + micFallbackPauseStartedAt.current = null; + }, []); + const stopMicFallbackRecorder = useCallback((): Promise => { return new Promise((resolve) => { const recorder = micFallbackRecorder.current; @@ -614,6 +681,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { resolve(null); return; } + closeMicFallbackPauseInterval(); recorder.ondataavailable = appendMicFallbackChunk; recorder.onstop = () => { const blob = @@ -628,7 +696,38 @@ export function useScreenRecorder(): UseScreenRecorderReturn { }; recorder.stop(); }); - }, [appendMicFallbackChunk]); + }, [appendMicFallbackChunk, closeMicFallbackPauseInterval]); + + const pauseMicFallbackRecorder = useCallback(() => { + const recorder = micFallbackRecorder.current; + if (recorder?.state !== "recording") { + return; + } + + try { + recorder.requestData(); + } catch (error) { + console.warn("Failed to flush microphone fallback chunk before pause:", error); + } + + recorder.pause(); + const now = performance.now(); + const startedAt = micFallbackRecorderStartedAt.current ?? now; + micFallbackPauseStartedAt.current = now; + micFallbackPauseIntervals.current.push({ + startElapsedMs: Math.max(0, Math.round(now - startedAt)), + }); + }, []); + + const resumeMicFallbackRecorder = useCallback(() => { + const recorder = micFallbackRecorder.current; + if (recorder?.state !== "paused") { + return; + } + + closeMicFallbackPauseInterval(); + recorder.resume(); + }, [closeMicFallbackPauseInterval]); const storeMicrophoneSidecar = useCallback( async ( @@ -644,7 +743,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { micFallbackRequestedConstraints.current = null; micFallbackAudioInputDevices.current = null; micFallbackRecorderMetadata.current = null; - micFallbackChunkEvents.current = []; + resetMicFallbackTimingDiagnostics(); return; } @@ -679,6 +778,13 @@ export function useScreenRecorder(): UseScreenRecorderReturn { ...(micFallbackChunkEvents.current.length > 0 ? { chunkEvents: [...micFallbackChunkEvents.current] } : {}), + ...(micFallbackPauseIntervals.current.length > 0 + ? { + pauseIntervals: micFallbackPauseIntervals.current.map( + (interval) => ({ ...interval }), + ), + } + : {}), }; const result = await window.electronAPI.storeMicrophoneSidecar( arrayBuffer, @@ -706,10 +812,10 @@ export function useScreenRecorder(): UseScreenRecorderReturn { micFallbackRequestedConstraints.current = null; micFallbackAudioInputDevices.current = null; micFallbackRecorderMetadata.current = null; - micFallbackChunkEvents.current = []; + resetMicFallbackTimingDiagnostics(); } }, - [], + [resetMicFallbackTimingDiagnostics], ); const stopWebcamRecorder = useCallback(async () => { @@ -1306,7 +1412,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { audioBitsPerSecond: AUDIO_BITRATE_VOICE, timesliceMs: RECORDER_TIMESLICE_MS, }; - micFallbackChunkEvents.current = []; + resetMicFallbackTimingDiagnostics(); micFallbackRecorderStartedAt.current = performance.now(); recorder.ondataavailable = appendMicFallbackChunk; micFallbackStartDelayMs.current = Math.max( @@ -1321,8 +1427,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { micFallbackRequestedConstraints.current = null; micFallbackAudioInputDevices.current = null; micFallbackRecorderMetadata.current = null; - micFallbackChunkEvents.current = []; - micFallbackRecorderStartedAt.current = null; + resetMicFallbackTimingDiagnostics(); console.warn("Browser microphone fallback failed:", micError); const permissionDenied = micError instanceof DOMException && @@ -1667,9 +1772,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { if (webcamRecorder.current?.state === "recording") { webcamRecorder.current.pause(); } - if (micFallbackRecorder.current?.state === "recording") { - micFallbackRecorder.current.pause(); - } + pauseMicFallbackRecorder(); const boundaryMs = Date.now(); markRecordingPaused(boundaryMs); setPaused(true); @@ -1697,7 +1800,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } })(); } - }, [markRecordingPaused, paused, recording]); + }, [markRecordingPaused, pauseMicFallbackRecorder, paused, recording]); const resumeRecording = useCallback(() => { if (!recording || !paused) return; @@ -1715,9 +1818,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { if (webcamRecorder.current?.state === "paused") { webcamRecorder.current.resume(); } - if (micFallbackRecorder.current?.state === "paused") { - micFallbackRecorder.current.resume(); - } + resumeMicFallbackRecorder(); const boundaryMs = Date.now(); markRecordingResumed(boundaryMs); setPaused(false); @@ -1745,7 +1846,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } })(); } - }, [markRecordingResumed, paused, recording]); + }, [markRecordingResumed, paused, recording, resumeMicFallbackRecorder]); const cancelRecording = useCallback(() => { if (!recording) return;