From 3933531fd91f41b9d109646a33cb2bc8f995814b Mon Sep 17 00:00:00 2001 From: webadderall <131426131+webadderall@users.noreply.github.com> Date: Mon, 21 Sep 2026 08:50:23 +1000 Subject: [PATCH] fix(captions): preserve fallback audio and speech between microphone words --- electron/ipc/captions/generate.ts | 6 +++-- electron/ipc/captions/generation.test.ts | 31 +++++++++++++++++++++- electron/ipc/captions/mergeSources.test.ts | 19 +++++++++++++ electron/ipc/captions/mergeSources.ts | 3 ++- tests/ui/caption-speed.spec.ts | 14 ++++++++-- 5 files changed, 67 insertions(+), 6 deletions(-) diff --git a/electron/ipc/captions/generate.ts b/electron/ipc/captions/generate.ts index 2613d3f9..f394769a 100644 --- a/electron/ipc/captions/generate.ts +++ b/electron/ipc/captions/generate.ts @@ -340,7 +340,9 @@ export async function generateAutoCaptionsFromVideo(options: { const candidates = await resolveCaptionAudioCandidates(options.videoPath); const microphone = candidates.filter((source) => source.label === "microphone audio sidecar"); const system = candidates.filter((source) => source.label === "system audio sidecar"); - const recording = candidates.filter((source) => source.label === "recording"); + const secondary = candidates.filter( + (source) => !microphone.includes(source) && !system.includes(source), + ); if (microphone.length === 0) { return generateCaptionsForSource({ ...options, candidates: [...system, ...candidates] }); } @@ -355,7 +357,7 @@ export async function generateAutoCaptionsFromVideo(options: { } }; const micCues = await transcribeTrack(microphone); - const systemCues = await transcribeTrack([...system, ...recording]); + const systemCues = await transcribeTrack([...system, ...secondary]); if (micCues === null && systemCues === null) throw new NoCaptionAudioError("No audio could be extracted from the recording."); return { diff --git a/electron/ipc/captions/generation.test.ts b/electron/ipc/captions/generation.test.ts index 71807d80..70ba2658 100644 --- a/electron/ipc/captions/generation.test.ts +++ b/electron/ipc/captions/generation.test.ts @@ -6,6 +6,7 @@ const mocks = vi.hoisted(() => ({ rm: vi.fn(), companions: vi.fn(), delay: vi.fn(), + session: vi.fn(), })); vi.mock("node:child_process", () => ({ execFile: (...args: unknown[]) => { @@ -29,7 +30,7 @@ vi.mock("node:fs/promises", () => ({ vi.mock("electron", () => ({ app: { getPath: () => "/tmp" } })); vi.mock("../ffmpeg/binary", () => ({ getFfmpegBinaryPath: () => "/ffmpeg" })); vi.mock("../paths/binaries", () => ({ getBundledWhisperExecutableCandidates: () => ["/whisper"] })); -vi.mock("../project/session", () => ({ resolveRecordingSession: vi.fn().mockResolvedValue(null) })); +vi.mock("../project/session", () => ({ resolveRecordingSession: mocks.session })); vi.mock("../recording/diagnostics", () => ({ getUsableCompanionAudioCandidates: mocks.companions, getCompanionAudioStartDelayMs: mocks.delay, @@ -55,6 +56,7 @@ const json = JSON.stringify({ beforeEach(() => { vi.clearAllMocks(); + mocks.session.mockReset().mockResolvedValue(null); mocks.companions.mockReset().mockResolvedValue([]); mocks.delay.mockReset().mockResolvedValue(null); mocks.exec.mockReset().mockResolvedValue({ stderr: "" }); @@ -135,3 +137,30 @@ describe("caption generation pipeline", () => { expect((await generateAutoCaptionsFromVideo(options)).cues[0].text).toBe("Hello world."); }); }); + +it("falls back to linked webcam audio when mic exists and other secondary sources fail", async () => { + mocks.session.mockResolvedValue({ webcamPath: "/webcam.mp4" }); + mocks.companions.mockResolvedValue([ + { + platform: "mac", + micPath: "/video.mic.wav", + systemPath: "/video.system.wav", + usablePaths: ["/video.mic.wav", "/video.system.wav"], + }, + ]); + mocks.exec.mockImplementation(async (file: string, args: string[]) => { + if (file === "/ffmpeg" && args.includes("pcm_s16le")) { + const source = args[args.indexOf("-i") + 1]; + if (source === "/video.mp4" || source === "/video.system.wav") + throw new Error("No audio"); + } + return { stderr: "" }; + }); + const result = await generateAutoCaptionsFromVideo(options); + expect(result.cues.length).toBeGreaterThan(0); + expect( + mocks.exec.mock.calls.some( + ([file, args]) => file === "/ffmpeg" && args.includes("/webcam.mp4"), + ), + ).toBe(true); +}); diff --git a/electron/ipc/captions/mergeSources.test.ts b/electron/ipc/captions/mergeSources.test.ts index 41c1b914..4e7bb524 100644 --- a/electron/ipc/captions/mergeSources.test.ts +++ b/electron/ipc/captions/mergeSources.test.ts @@ -73,3 +73,22 @@ it("preserves untimed speech next to timed words", async () => { ); expect(result.map((c) => c.text).join(" ")).toContain("There is a timeline editor."); }); + +it("preserves system speech between timed microphone words", () => { + const result = mergeCaptionSources( + [ + { + id: "mic", + startMs: 0, + endMs: 3000, + text: "Hello again", + words: [ + { text: "Hello", startMs: 0, endMs: 600 }, + { text: "again", startMs: 2400, endMs: 3000 }, + ], + }, + ], + [{ id: "system", startMs: 1000, endMs: 1800, text: "In the gap" }], + ); + expect(result.map((cue) => cue.text)).toContain("In the gap"); +}); diff --git a/electron/ipc/captions/mergeSources.ts b/electron/ipc/captions/mergeSources.ts index 35f427d0..6d44622c 100644 --- a/electron/ipc/captions/mergeSources.ts +++ b/electron/ipc/captions/mergeSources.ts @@ -8,8 +8,9 @@ export function mergeCaptionSources( ): CaptionCuePayload[] { // Sound-event labels are not competing microphone speech. const micSpeech = microphone.filter((cue) => !/^(?:\s*[[(][^\])]+[\])]\s*)+$/.test(cue.text)); + const micSpans = micSpeech.flatMap((cue) => (cue.words?.length ? cue.words : [cue])); const overlapsMic = (startMs: number, endMs: number) => - micSpeech.some((cue) => startMs < cue.endMs && endMs > cue.startMs); + micSpans.some((span) => startMs < span.endMs && endMs > span.startMs); const systemCues: CaptionCuePayload[] = []; for (const cue of system) { if (!overlapsMic(cue.startMs, cue.endMs)) { diff --git a/tests/ui/caption-speed.spec.ts b/tests/ui/caption-speed.spec.ts index 812e833d..87fc629a 100644 --- a/tests/ui/caption-speed.spec.ts +++ b/tests/ui/caption-speed.spec.ts @@ -4,8 +4,11 @@ import { installDesktopBridge } from "./bridge"; async function seek(page: Page, clip: Locator, timeMs: number) { const end = Number(await clip.getAttribute("data-end-ms")); const box = (await clip.locator(".timeline-block").boundingBox())!; - const row = (await page.locator('[data-timeline-row="row-clip"]').boundingBox())!; - await page.mouse.click(box.x + (box.width * timeMs) / end, row.y - 8); + const cap = (await page.getByTestId("playhead-cap").boundingBox())!; + await page.mouse.move(cap.x + cap.width / 2, cap.y + cap.height / 2); + await page.mouse.down(); + await page.mouse.move(box.x + (box.width * timeMs) / end, cap.y + cap.height / 2); + await page.mouse.up(); } test("captions render in preview and remain synchronized at 1x, 2x and 4x", async ({ page }) => { @@ -100,6 +103,13 @@ test("captions render in preview and remain synchronized at 1x, 2x and 4x", asyn await seek(page, clip, 0); await page.getByRole("button", { name: "Play", exact: true }).click(); await expect(visibleCaption).toBeVisible(); + await expect + .poll(() => + page + .locator('video[aria-hidden="true"]') + .evaluate((video: HTMLVideoElement) => video.currentTime * 1000), + ) + .toBeGreaterThan(sourceEnd); await expect(visibleCaption).toHaveCount(0); expect(errors).toEqual([]); });