fix(recording): account for mic pause gaps

This commit is contained in:
wiiiii123 committed 2026-05-08 01:35:17 +07:00
1 parent d5240170ce
commit 3d6668fd60
5 files changed
+352 -38

No files matched your search

@@ -304,6 +304,73 @@ describe("getCompanionAudioFallbackPaths", () => {
await expect(getCompanionAudioStartDelayMs(micPath)).resolves.toBeNull();
});
it("classifies wall-clock mic chunk gaps covered by pause intervals", async () => {
const { summarizeMicrophoneChunkTiming } = await import("./diagnostics");
expect(
summarizeMicrophoneChunkTiming(
[
{
index: 0,
size: 1024,
elapsedMs: 250,
deltaMs: null,
recordedElapsedMs: 250,
recordedDeltaMs: null,
},
{
index: 1,
size: 1024,
elapsedMs: 8250,
deltaMs: 8000,
recordedElapsedMs: 500,
recordedDeltaMs: 250,
},
],
[{ startElapsedMs: 250, endElapsedMs: 8250, durationMs: 7750 }],
250,
),
).toMatchObject({
status: "pause-accounted",
wallClockGapCount: 1,
recordedGapCount: 0,
pausedDurationMs: 7750,
});
});
it("flags recorded mic chunk gaps that remain after pause accounting", async () => {
const { summarizeMicrophoneChunkTiming } = await import("./diagnostics");
expect(
summarizeMicrophoneChunkTiming(
[
{
index: 0,
size: 1024,
elapsedMs: 250,
deltaMs: null,
recordedElapsedMs: 250,
recordedDeltaMs: null,
},
{
index: 1,
size: 1024,
elapsedMs: 2500,
deltaMs: 2250,
recordedElapsedMs: 2500,
recordedDeltaMs: 2250,
},
],
[],
250,
),
).toMatchObject({
status: "needs-review",
wallClockGapCount: 1,
recordedGapCount: 1,
});
});
it("rejects tiny MP4 container-only outputs before they reach the editor", async () => {
const videoPath = path.join(tempRoot, "recording-123.mp4");
await fs.writeFile(videoPath, Buffer.alloc(261));
+86
View File
@@ -16,6 +16,21 @@ type CompanionAudioTimingMetadata = {
startDelayMs?: number;
};
export type MicrophoneChunkTimingEvent = {
index: number;
size: number;
elapsedMs: number;
deltaMs: number | null;
recordedElapsedMs?: number;
recordedDeltaMs?: number | null;
};
export type MicrophonePauseInterval = {
startElapsedMs: number;
endElapsedMs?: number;
durationMs?: number;
};
export type VideoStreamDurationProbe = {
durationSeconds: number | null;
frameCount: number | null;
@@ -102,6 +117,77 @@ export function getRecordingAudioMuxTimeoutMs(durationSeconds: number) {
);
}
function getFiniteNonNegativeNumber(value: unknown) {
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null;
}
export function summarizeMicrophoneChunkTiming(
chunkEvents: MicrophoneChunkTimingEvent[] | null | undefined,
pauseIntervals: MicrophonePauseInterval[] | null | undefined = [],
timesliceMs = 250,
) {
const normalizedTimesliceMs =
Number.isFinite(timesliceMs) && timesliceMs > 0 ? Math.round(timesliceMs) : 250;
const thresholdMs = Math.max(600, Math.round(normalizedTimesliceMs * 2.5));
const events = Array.isArray(chunkEvents) ? chunkEvents : [];
const pauses = Array.isArray(pauseIntervals) ? pauseIntervals : [];
const pausedDurationMs = pauses.reduce((total, interval) => {
const durationMs = getFiniteNonNegativeNumber(interval.durationMs);
return durationMs === null ? total : total + Math.round(durationMs);
}, 0);
const wallClockGaps = events
.filter((event) => (event.deltaMs ?? 0) > thresholdMs)
.map((event) => ({
index: event.index,
deltaMs: Math.round(event.deltaMs ?? 0),
elapsedMs: Math.round(event.elapsedMs),
}));
const recordedGaps = events
.filter((event) => {
const recordedDeltaMs =
getFiniteNonNegativeNumber(event.recordedDeltaMs) ?? event.deltaMs ?? 0;
return recordedDeltaMs > thresholdMs;
})
.map((event) => ({
index: event.index,
deltaMs: Math.round(
getFiniteNonNegativeNumber(event.recordedDeltaMs) ?? event.deltaMs ?? 0,
),
recordedElapsedMs: Math.round(event.recordedElapsedMs ?? event.elapsedMs),
}));
const maxWallClockDeltaMs = events.reduce(
(maxDeltaMs, event) => Math.max(maxDeltaMs, event.deltaMs ?? 0),
0,
);
const maxRecordedDeltaMs = events.reduce((maxDeltaMs, event) => {
const recordedDeltaMs =
getFiniteNonNegativeNumber(event.recordedDeltaMs) ?? event.deltaMs ?? 0;
return Math.max(maxDeltaMs, recordedDeltaMs);
}, 0);
const status =
recordedGaps.length > 0
? "needs-review"
: wallClockGaps.length > 0 && pausedDurationMs > 0
? "pause-accounted"
: "ok";
return {
status,
timesliceMs: normalizedTimesliceMs,
thresholdMs,
eventCount: events.length,
pauseIntervalCount: pauses.length,
pausedDurationMs,
maxWallClockDeltaMs: Math.round(maxWallClockDeltaMs),
maxRecordedDeltaMs: Math.round(maxRecordedDeltaMs),
wallClockGapCount: wallClockGaps.length,
recordedGapCount: recordedGaps.length,
wallClockGaps: wallClockGaps.slice(0, 10),
recordedGaps: recordedGaps.slice(0, 10),
};
}
/** Probe the duration of a media file (in seconds) using the container header. */
export async function probeMediaDurationSeconds(filePath: string): Promise<number> {
const ffmpegPath = getFfmpegBinaryPath();
+57 -2
View File
@@ -45,8 +45,11 @@ import {
import {
getCompanionAudioFallbackInfo,
getFileSizeIfPresent,
type MicrophoneChunkTimingEvent,
type MicrophonePauseInterval,
type RecordingDiagnosticsSnapshot,
recordNativeCaptureDiagnostics,
summarizeMicrophoneChunkTiming,
validateRecordedVideo,
writeRecordingDiagnosticsSnapshot,
} from "../recording/diagnostics";
@@ -189,7 +192,7 @@ function pickPrimitiveRecord(value: unknown) {
return entries.length > 0 ? Object.fromEntries(entries) : null;
}
function pickMicrophoneChunkEvents(value: unknown) {
function pickMicrophoneChunkEvents(value: unknown): MicrophoneChunkTimingEvent[] | null {
if (!Array.isArray(value)) {
return null;
}
@@ -200,7 +203,7 @@ function pickMicrophoneChunkEvents(value: unknown) {
return null;
}
const { index, size, elapsedMs, deltaMs } = event;
const { index, size, elapsedMs, deltaMs, recordedElapsedMs, recordedDeltaMs } = event;
if (
typeof index !== "number" ||
typeof size !== "number" ||
@@ -214,6 +217,15 @@ function pickMicrophoneChunkEvents(value: unknown) {
size: Math.round(size),
elapsedMs: Math.round(elapsedMs),
deltaMs: typeof deltaMs === "number" ? Math.round(deltaMs) : null,
...(typeof recordedElapsedMs === "number" &&
Number.isFinite(recordedElapsedMs) &&
recordedElapsedMs >= 0
? { recordedElapsedMs: Math.round(recordedElapsedMs) }
: {}),
recordedDeltaMs:
typeof recordedDeltaMs === "number" && Number.isFinite(recordedDeltaMs)
? Math.max(0, Math.round(recordedDeltaMs))
: null,
};
})
.filter((event): event is NonNullable<typeof event> => event !== null);
@@ -221,6 +233,37 @@ function pickMicrophoneChunkEvents(value: unknown) {
return events.length > 0 ? events : null;
}
function pickMicrophonePauseIntervals(value: unknown): MicrophonePauseInterval[] | null {
if (!Array.isArray(value)) {
return null;
}
const intervals = value
.map((interval) => {
if (!isRecord(interval) || typeof interval.startElapsedMs !== "number") {
return null;
}
const startElapsedMs = Math.max(0, Math.round(interval.startElapsedMs));
return {
startElapsedMs,
...(typeof interval.endElapsedMs === "number" &&
Number.isFinite(interval.endElapsedMs) &&
interval.endElapsedMs >= startElapsedMs
? { endElapsedMs: Math.round(interval.endElapsedMs) }
: {}),
...(typeof interval.durationMs === "number" &&
Number.isFinite(interval.durationMs) &&
interval.durationMs >= 0
? { durationMs: Math.round(interval.durationMs) }
: {}),
};
})
.filter((interval): interval is NonNullable<typeof interval> => interval !== null);
return intervals.length > 0 ? intervals : null;
}
function pickAudioInputDevices(value: unknown) {
if (!Array.isArray(value)) {
return null;
@@ -1415,6 +1458,7 @@ export function registerRecordingHandlers(
audioInputDevices?: unknown;
mediaRecorder?: unknown;
chunkEvents?: unknown;
pauseIntervals?: unknown;
},
) => {
const baseName = videoPath.replace(/\.[^.]+$/, "");
@@ -1478,6 +1522,15 @@ export function registerRecordingHandlers(
}
: null;
const chunkEvents = pickMicrophoneChunkEvents(options?.chunkEvents);
const pauseIntervals = pickMicrophonePauseIntervals(options?.pauseIntervals);
const chunkTiming =
chunkEvents || pauseIntervals
? summarizeMicrophoneChunkTiming(
chunkEvents,
pauseIntervals,
mediaRecorder?.timesliceMs,
)
: null;
const metadata = {
...(Number.isFinite(startDelayMs) && (startDelayMs ?? 0) >= 0
? { startDelayMs: Math.round(startDelayMs ?? 0) }
@@ -1500,6 +1553,8 @@ export function registerRecordingHandlers(
? { mediaRecorder }
: {}),
...(chunkEvents ? { chunkEvents } : {}),
...(pauseIntervals ? { pauseIntervals } : {}),
...(chunkTiming ? { chunkTiming } : {}),
};
if (Object.keys(metadata).length > 0) {
try {
+5
View File
@@ -19,6 +19,7 @@ function createMockMediaRecorder(initialState: RecordingState = "inactive") {
resume: vi.fn(() => {
if (_state === "paused") _state = "recording";
}),
requestData: vi.fn(),
stop: vi.fn(() => {
_state = "inactive";
}),
@@ -151,6 +152,7 @@ function pauseRecording(
webcamRecorder.pause();
}
if (micFallbackRecorder?.state === "recording") {
micFallbackRecorder.requestData();
micFallbackRecorder.pause();
}
return true;
@@ -206,6 +208,7 @@ async function pauseNativeRecording(
webcamRecorder.pause();
}
if (micFallbackRecorder?.state === "recording") {
micFallbackRecorder.requestData();
micFallbackRecorder.pause();
}
@@ -430,6 +433,7 @@ describe("useScreenRecorder state machine", () => {
const result = pauseRecording(recorder, true, false, true, null, micFallback);
expect(result).toBe(true);
expect(micFallback.requestData).toHaveBeenCalled();
expect(micFallback.state).toBe("paused");
});
@@ -613,6 +617,7 @@ describe("useScreenRecorder state machine", () => {
const pausedResult = await pauseNativeRecording(webcam, { success: true }, micFallback);
expect(pausedResult).toBe(true);
expect(webcam.state).toBe("paused");
expect(micFallback.requestData).toHaveBeenCalled();
expect(micFallback.state).toBe("paused");
expect(recorder.pause).not.toHaveBeenCalled();
+137 -36
View File
@@ -84,6 +84,13 @@ type MicrophoneFallbackChunkEvent = {
size: number;
elapsedMs: number;
deltaMs: number | null;
recordedElapsedMs: number;
recordedDeltaMs: number | null;
};
type MicrophoneFallbackPauseInterval = {
startElapsedMs: number;
endElapsedMs?: number;
durationMs?: number;
};
type MicrophoneFallbackRecorderMetadata = {
mimeType: string;
@@ -99,6 +106,7 @@ type MicrophoneSidecarOptions = {
audioInputDevices?: MicrophoneAudioInputDeviceSnapshot[];
mediaRecorder?: MicrophoneFallbackRecorderMetadata;
chunkEvents?: MicrophoneFallbackChunkEvent[];
pauseIntervals?: MicrophoneFallbackPauseInterval[];
};
const LINUX_PORTAL_SOURCE: ProcessedDesktopSource = {
id: "screen:linux-portal",
@@ -299,6 +307,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
const micFallbackRecorderMetadata = useRef<MicrophoneFallbackRecorderMetadata | null>(null);
const micFallbackChunkEvents = useRef<MicrophoneFallbackChunkEvent[]>([]);
const micFallbackRecorderStartedAt = useRef<number | null>(null);
const micFallbackPauseStartedAt = useRef<number | null>(null);
const micFallbackPausedDurationMs = useRef(0);
const micFallbackPauseIntervals = useRef<MicrophoneFallbackPauseInterval[]>([]);
const browserMicrophoneProfile = useRef<BrowserMicrophoneProfile>(
DEFAULT_BROWSER_MICROPHONE_PROFILE,
);
@@ -391,6 +402,32 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
});
}, []);
const getMicFallbackRecordedElapsedMs = useCallback((now = performance.now()) => {
const startedAt = micFallbackRecorderStartedAt.current;
if (startedAt === null) {
return 0;
}
const currentPauseDurationMs =
micFallbackPauseStartedAt.current === null
? 0
: Math.max(0, now - micFallbackPauseStartedAt.current);
return Math.max(
0,
Math.round(
now - startedAt - micFallbackPausedDurationMs.current - currentPauseDurationMs,
),
);
}, []);
const resetMicFallbackTimingDiagnostics = useCallback(() => {
micFallbackChunkEvents.current = [];
micFallbackRecorderStartedAt.current = null;
micFallbackPauseStartedAt.current = null;
micFallbackPausedDurationMs.current = 0;
micFallbackPauseIntervals.current = [];
}, []);
const preparePermissions = useCallback(async (options: { startup?: boolean } = {}) => {
const platform = await window.electronAPI.getPlatform();
if (platform !== "darwin") {
@@ -497,31 +534,40 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
micFallbackRequestedConstraints.current = null;
micFallbackAudioInputDevices.current = null;
micFallbackRecorderMetadata.current = null;
micFallbackChunkEvents.current = [];
micFallbackRecorderStartedAt.current = null;
resetMicFallbackTimingDiagnostics();
}
}, []);
}, [resetMicFallbackTimingDiagnostics]);
const appendMicFallbackChunk = useCallback((event: BlobEvent) => {
if (event.data.size <= 0) {
return;
}
const appendMicFallbackChunk = useCallback(
(event: BlobEvent) => {
if (event.data.size <= 0) {
return;
}
micFallbackChunks.current.push(event.data);
const startedAt = micFallbackRecorderStartedAt.current;
if (startedAt === null) {
return;
}
micFallbackChunks.current.push(event.data);
const startedAt = micFallbackRecorderStartedAt.current;
if (startedAt === null) {
return;
}
const elapsedMs = Math.max(0, Math.round(performance.now() - startedAt));
const previous = micFallbackChunkEvents.current[micFallbackChunkEvents.current.length - 1];
micFallbackChunkEvents.current.push({
index: micFallbackChunkEvents.current.length,
size: event.data.size,
elapsedMs,
deltaMs: previous ? Math.max(0, elapsedMs - previous.elapsedMs) : null,
});
}, []);
const now = performance.now();
const elapsedMs = Math.max(0, Math.round(now - startedAt));
const recordedElapsedMs = getMicFallbackRecordedElapsedMs(now);
const previous =
micFallbackChunkEvents.current[micFallbackChunkEvents.current.length - 1];
micFallbackChunkEvents.current.push({
index: micFallbackChunkEvents.current.length,
size: event.data.size,
elapsedMs,
deltaMs: previous ? Math.max(0, elapsedMs - previous.elapsedMs) : null,
recordedElapsedMs,
recordedDeltaMs: previous
? Math.max(0, recordedElapsedMs - previous.recordedElapsedMs)
: null,
});
},
[getMicFallbackRecordedElapsedMs],
);
const resolveBrowserCaptureSource = useCallback(async (source: ProcessedDesktopSource) => {
if (!source?.id?.startsWith("screen:")) {
@@ -606,6 +652,27 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
[],
);
const closeMicFallbackPauseInterval = useCallback((now = performance.now()) => {
const pauseStartedAt = micFallbackPauseStartedAt.current;
if (pauseStartedAt === null) {
return;
}
const durationMs = Math.max(0, Math.round(now - pauseStartedAt));
micFallbackPausedDurationMs.current += durationMs;
const startedAt = micFallbackRecorderStartedAt.current ?? now;
const lastInterval =
micFallbackPauseIntervals.current[micFallbackPauseIntervals.current.length - 1];
if (lastInterval && lastInterval.endElapsedMs === undefined) {
lastInterval.endElapsedMs = Math.max(
lastInterval.startElapsedMs,
Math.round(now - startedAt),
);
lastInterval.durationMs = durationMs;
}
micFallbackPauseStartedAt.current = null;
}, []);
const stopMicFallbackRecorder = useCallback((): Promise<Blob | null> => {
return new Promise((resolve) => {
const recorder = micFallbackRecorder.current;
@@ -614,6 +681,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
resolve(null);
return;
}
closeMicFallbackPauseInterval();
recorder.ondataavailable = appendMicFallbackChunk;
recorder.onstop = () => {
const blob =
@@ -628,7 +696,38 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
};
recorder.stop();
});
}, [appendMicFallbackChunk]);
}, [appendMicFallbackChunk, closeMicFallbackPauseInterval]);
const pauseMicFallbackRecorder = useCallback(() => {
const recorder = micFallbackRecorder.current;
if (recorder?.state !== "recording") {
return;
}
try {
recorder.requestData();
} catch (error) {
console.warn("Failed to flush microphone fallback chunk before pause:", error);
}
recorder.pause();
const now = performance.now();
const startedAt = micFallbackRecorderStartedAt.current ?? now;
micFallbackPauseStartedAt.current = now;
micFallbackPauseIntervals.current.push({
startElapsedMs: Math.max(0, Math.round(now - startedAt)),
});
}, []);
const resumeMicFallbackRecorder = useCallback(() => {
const recorder = micFallbackRecorder.current;
if (recorder?.state !== "paused") {
return;
}
closeMicFallbackPauseInterval();
recorder.resume();
}, [closeMicFallbackPauseInterval]);
const storeMicrophoneSidecar = useCallback(
async (
@@ -644,7 +743,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
micFallbackRequestedConstraints.current = null;
micFallbackAudioInputDevices.current = null;
micFallbackRecorderMetadata.current = null;
micFallbackChunkEvents.current = [];
resetMicFallbackTimingDiagnostics();
return;
}
@@ -679,6 +778,13 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
...(micFallbackChunkEvents.current.length > 0
? { chunkEvents: [...micFallbackChunkEvents.current] }
: {}),
...(micFallbackPauseIntervals.current.length > 0
? {
pauseIntervals: micFallbackPauseIntervals.current.map(
(interval) => ({ ...interval }),
),
}
: {}),
};
const result = await window.electronAPI.storeMicrophoneSidecar(
arrayBuffer,
@@ -706,10 +812,10 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
micFallbackRequestedConstraints.current = null;
micFallbackAudioInputDevices.current = null;
micFallbackRecorderMetadata.current = null;
micFallbackChunkEvents.current = [];
resetMicFallbackTimingDiagnostics();
}
},
[],
[resetMicFallbackTimingDiagnostics],
);
const stopWebcamRecorder = useCallback(async () => {
@@ -1306,7 +1412,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
audioBitsPerSecond: AUDIO_BITRATE_VOICE,
timesliceMs: RECORDER_TIMESLICE_MS,
};
micFallbackChunkEvents.current = [];
resetMicFallbackTimingDiagnostics();
micFallbackRecorderStartedAt.current = performance.now();
recorder.ondataavailable = appendMicFallbackChunk;
micFallbackStartDelayMs.current = Math.max(
@@ -1321,8 +1427,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
micFallbackRequestedConstraints.current = null;
micFallbackAudioInputDevices.current = null;
micFallbackRecorderMetadata.current = null;
micFallbackChunkEvents.current = [];
micFallbackRecorderStartedAt.current = null;
resetMicFallbackTimingDiagnostics();
console.warn("Browser microphone fallback failed:", micError);
const permissionDenied =
micError instanceof DOMException &&
@@ -1667,9 +1772,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
if (webcamRecorder.current?.state === "recording") {
webcamRecorder.current.pause();
}
if (micFallbackRecorder.current?.state === "recording") {
micFallbackRecorder.current.pause();
}
pauseMicFallbackRecorder();
const boundaryMs = Date.now();
markRecordingPaused(boundaryMs);
setPaused(true);
@@ -1697,7 +1800,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
}
})();
}
}, [markRecordingPaused, paused, recording]);
}, [markRecordingPaused, pauseMicFallbackRecorder, paused, recording]);
const resumeRecording = useCallback(() => {
if (!recording || !paused) return;
@@ -1715,9 +1818,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
if (webcamRecorder.current?.state === "paused") {
webcamRecorder.current.resume();
}
if (micFallbackRecorder.current?.state === "paused") {
micFallbackRecorder.current.resume();
}
resumeMicFallbackRecorder();
const boundaryMs = Date.now();
markRecordingResumed(boundaryMs);
setPaused(false);
@@ -1745,7 +1846,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
}
})();
}
}, [markRecordingResumed, paused, recording]);
}, [markRecordingResumed, paused, recording, resumeMicFallbackRecorder]);
const cancelRecording = useCallback(() => {
if (!recording) return;