fix(captions): preserve fallback audio and speech between microphone words

electron/ipc/captions/generate.ts

@@ -340,7 +340,9 @@ export async function generateAutoCaptionsFromVideo(options: {
 	const candidates = await resolveCaptionAudioCandidates(options.videoPath);
 	const microphone = candidates.filter((source) => source.label === "microphone audio sidecar");
 	const system = candidates.filter((source) => source.label === "system audio sidecar");
-	const recording = candidates.filter((source) => source.label === "recording");
+	const secondary = candidates.filter(
+		(source) => !microphone.includes(source) && !system.includes(source),
+	);
 	if (microphone.length === 0) {
 		return generateCaptionsForSource({ ...options, candidates: [...system, ...candidates] });
 	}
@@ -355,7 +357,7 @@ export async function generateAutoCaptionsFromVideo(options: {
 		}
 	};
 	const micCues = await transcribeTrack(microphone);
-	const systemCues = await transcribeTrack([...system, ...recording]);
+	const systemCues = await transcribeTrack([...system, ...secondary]);
 	if (micCues === null && systemCues === null)
 		throw new NoCaptionAudioError("No audio could be extracted from the recording.");
 	return {

electron/ipc/captions/generation.test.ts

@@ -6,6 +6,7 @@ const mocks = vi.hoisted(() => ({
 	rm: vi.fn(),
 	companions: vi.fn(),
 	delay: vi.fn(),
+	session: vi.fn(),
 }));
 vi.mock("node:child_process", () => ({
 	execFile: (...args: unknown[]) => {
@@ -29,7 +30,7 @@ vi.mock("node:fs/promises", () => ({
 vi.mock("electron", () => ({ app: { getPath: () => "/tmp" } }));
 vi.mock("../ffmpeg/binary", () => ({ getFfmpegBinaryPath: () => "/ffmpeg" }));
 vi.mock("../paths/binaries", () => ({ getBundledWhisperExecutableCandidates: () => ["/whisper"] }));
-vi.mock("../project/session", () => ({ resolveRecordingSession: vi.fn().mockResolvedValue(null) }));
+vi.mock("../project/session", () => ({ resolveRecordingSession: mocks.session }));
 vi.mock("../recording/diagnostics", () => ({
 	getUsableCompanionAudioCandidates: mocks.companions,
 	getCompanionAudioStartDelayMs: mocks.delay,
@@ -55,6 +56,7 @@ const json = JSON.stringify({
 
 beforeEach(() => {
 	vi.clearAllMocks();
+	mocks.session.mockReset().mockResolvedValue(null);
 	mocks.companions.mockReset().mockResolvedValue([]);
 	mocks.delay.mockReset().mockResolvedValue(null);
 	mocks.exec.mockReset().mockResolvedValue({ stderr: "" });
@@ -135,3 +137,30 @@ describe("caption generation pipeline", () => {
 		expect((await generateAutoCaptionsFromVideo(options)).cues[0].text).toBe("Hello world.");
 	});
 });
+
+it("falls back to linked webcam audio when mic exists and other secondary sources fail", async () => {
+	mocks.session.mockResolvedValue({ webcamPath: "/webcam.mp4" });
+	mocks.companions.mockResolvedValue([
+		{
+			platform: "mac",
+			micPath: "/video.mic.wav",
+			systemPath: "/video.system.wav",
+			usablePaths: ["/video.mic.wav", "/video.system.wav"],
+		},
+	]);
+	mocks.exec.mockImplementation(async (file: string, args: string[]) => {
+		if (file === "/ffmpeg" && args.includes("pcm_s16le")) {
+			const source = args[args.indexOf("-i") + 1];
+			if (source === "/video.mp4" || source === "/video.system.wav")
+				throw new Error("No audio");
+		}
+		return { stderr: "" };
+	});
+	const result = await generateAutoCaptionsFromVideo(options);
+	expect(result.cues.length).toBeGreaterThan(0);
+	expect(
+		mocks.exec.mock.calls.some(
+			([file, args]) => file === "/ffmpeg" && args.includes("/webcam.mp4"),
+		),
+	).toBe(true);
+});

electron/ipc/captions/mergeSources.test.ts

@@ -73,3 +73,22 @@ it("preserves untimed speech next to timed words", async () => {
 	);
 	expect(result.map((c) => c.text).join(" ")).toContain("There is a timeline editor.");
 });
+
+it("preserves system speech between timed microphone words", () => {
+	const result = mergeCaptionSources(
+		[
+			{
+				id: "mic",
+				startMs: 0,
+				endMs: 3000,
+				text: "Hello again",
+				words: [
+					{ text: "Hello", startMs: 0, endMs: 600 },
+					{ text: "again", startMs: 2400, endMs: 3000 },
+				],
+			},
+		],
+		[{ id: "system", startMs: 1000, endMs: 1800, text: "In the gap" }],
+	);
+	expect(result.map((cue) => cue.text)).toContain("In the gap");
+});

electron/ipc/captions/mergeSources.ts

@@ -8,8 +8,9 @@ export function mergeCaptionSources(
 ): CaptionCuePayload[] {
 	// Sound-event labels are not competing microphone speech.
 	const micSpeech = microphone.filter((cue) => !/^(?:\s*[[(][^\])]+[\])]\s*)+$/.test(cue.text));
+	const micSpans = micSpeech.flatMap((cue) => (cue.words?.length ? cue.words : [cue]));
 	const overlapsMic = (startMs: number, endMs: number) =>
-		micSpeech.some((cue) => startMs < cue.endMs && endMs > cue.startMs);
+		micSpans.some((span) => startMs < span.endMs && endMs > span.startMs);
 	const systemCues: CaptionCuePayload[] = [];
 	for (const cue of system) {
 		if (!overlapsMic(cue.startMs, cue.endMs)) {

tests/ui/caption-speed.spec.ts

@@ -4,8 +4,11 @@ import { installDesktopBridge } from "./bridge";
 async function seek(page: Page, clip: Locator, timeMs: number) {
 	const end = Number(await clip.getAttribute("data-end-ms"));
 	const box = (await clip.locator(".timeline-block").boundingBox())!;
-	const row = (await page.locator('[data-timeline-row="row-clip"]').boundingBox())!;
-	await page.mouse.click(box.x + (box.width * timeMs) / end, row.y - 8);
+	const cap = (await page.getByTestId("playhead-cap").boundingBox())!;
+	await page.mouse.move(cap.x + cap.width / 2, cap.y + cap.height / 2);
+	await page.mouse.down();
+	await page.mouse.move(box.x + (box.width * timeMs) / end, cap.y + cap.height / 2);
+	await page.mouse.up();
 }
 
 test("captions render in preview and remain synchronized at 1x, 2x and 4x", async ({ page }) => {
@@ -100,6 +103,13 @@ test("captions render in preview and remain synchronized at 1x, 2x and 4x", asyn
 	await seek(page, clip, 0);
 	await page.getByRole("button", { name: "Play", exact: true }).click();
 	await expect(visibleCaption).toBeVisible();
+	await expect
+		.poll(() =>
+			page
+				.locator('video[aria-hidden="true"]')
+				.evaluate((video: HTMLVideoElement) => video.currentTime * 1000),
+		)
+		.toBeGreaterThan(sourceEnd);
 	await expect(visibleCaption).toHaveCount(0);
 	expect(errors).toEqual([]);
 });