diff --git a/electron/ipc/captions/generate.ts b/electron/ipc/captions/generate.ts index 246c4f1a..2613d3f9 100644 --- a/electron/ipc/captions/generate.ts +++ b/electron/ipc/captions/generate.ts @@ -7,11 +7,16 @@ import { app } from "electron"; import { getFfmpegBinaryPath } from "../ffmpeg/binary"; import { getBundledWhisperExecutableCandidates } from "../paths/binaries"; import { resolveRecordingSession } from "../project/session"; -import { getUsableCompanionAudioCandidates } from "../recording/diagnostics"; +import { + getCompanionAudioStartDelayMs, + getUsableCompanionAudioCandidates, +} from "../recording/diagnostics"; import { normalizeVideoSourcePath } from "../utils"; -import { getCaptionCompanionAudioCandidates } from "./audioCandidates"; -import { parseSrtCues, parseWhisperJsonCues, shouldRetryWhisperWithoutJson } from "./parser"; +import { type CaptionAudioCandidate, getCaptionCompanionAudioCandidates } from "./audioCandidates"; +import { shouldRetryWhisperWithoutJson } from "./parser"; +import { readWhisperCaptionOutput } from "./output"; import { isMissingWindowsWhisperRuntimeDependency } from "./runtimeErrors"; +import { mergeCaptionSources } from "./mergeSources"; import { segmentCuesIntoPhrases } from "./segment"; import { parseSilenceIntervals, @@ -22,6 +27,8 @@ import { const execFileAsync = promisify(execFile); +class NoCaptionAudioError extends Error {} + async function executeWhisper(whisperExecutablePath: string, args: string[]) { try { await execFileAsync(whisperExecutablePath, args, { @@ -116,9 +123,15 @@ export async function resolveCaptionAudioCandidates(videoPath: string) { candidates.push({ path: normalizedCandidatePath, label }); }; - pushCandidate(videoPath, "recording"); const companionAudio = await getUsableCompanionAudioCandidates(videoPath); - for (const candidate of getCaptionCompanionAudioCandidates(companionAudio)) { + const companions = getCaptionCompanionAudioCandidates(companionAudio); + for (const candidate of companions.filter( + (candidate) => candidate.label === "microphone audio sidecar", + )) { + pushCandidate(candidate.path, candidate.label); + } + pushCandidate(videoPath, "recording"); + for (const candidate of companions) { pushCandidate(candidate.path, candidate.label); } @@ -132,8 +145,10 @@ export async function extractCaptionAudioSource(options: { videoPath: string; ffmpegPath: string; wavPath: string; + candidates?: CaptionAudioCandidate[]; }) { - const candidates = await resolveCaptionAudioCandidates(options.videoPath); + const candidates = + options.candidates ?? (await resolveCaptionAudioCandidates(options.videoPath)); const attemptedCandidates: Array<{ path: string; label: string; @@ -145,6 +160,9 @@ export async function extractCaptionAudioSource(options: { for (const candidate of candidates) { try { await ensureReadableFile(candidate.path); + const delayMs = candidate.label.endsWith("audio sidecar") + ? ((await getCompanionAudioStartDelayMs(candidate.path)) ?? 0) + : 0; await execFileAsync( options.ffmpegPath, [ @@ -154,6 +172,7 @@ export async function extractCaptionAudioSource(options: { "-map", "0:a:0", "-vn", + ...(delayMs > 0 ? ["-af", `adelay=${delayMs}:all=1`] : []), "-ac", "1", "-ar", @@ -181,7 +200,7 @@ export async function extractCaptionAudioSource(options: { attemptedCandidates, ); - throw new Error( + throw new NoCaptionAudioError( "No audio was found to transcribe in the saved recording file. Captions need an audio track. If this recording should have contained sound, the recording was saved without an audio stream.", ); } @@ -210,11 +229,12 @@ export async function detectSilenceIntervals(options: { return parseSilenceIntervals(stderr ?? ""); } -export async function generateAutoCaptionsFromVideo(options: { +async function generateCaptionsForSource(options: { videoPath: string; whisperExecutablePath?: string; whisperModelPath: string; language?: string; + candidates: CaptionAudioCandidate[]; }) { const ffmpegPath = getFfmpegBinaryPath(); const normalizedVideoPath = normalizeVideoSourcePath(options.videoPath); @@ -241,6 +261,7 @@ export async function generateAutoCaptionsFromVideo(options: { videoPath: normalizedVideoPath, ffmpegPath, wavPath, + candidates: options.candidates, }); const language = @@ -256,6 +277,9 @@ export async function generateAutoCaptionsFromVideo(options: { "-l", language, "-np", + // Do not carry a hallucinated phrase through later decoding windows. + "-mc", + "0", ]; let jsonEnabled = true; @@ -274,22 +298,7 @@ export async function generateAutoCaptionsFromVideo(options: { await executeWhisper(whisperExecutablePath, whisperBaseArgs); } - const timedCues = jsonEnabled - ? parseWhisperJsonCues(await fs.readFile(jsonPath, "utf-8")) - : []; - if (jsonEnabled && timedCues.length === 0) { - // JSON ran but yielded no word-timed cues (empty/malformed output). We fall back - // to SRT, which has no word timings — captions are then split by sentence text and - // silence rather than precise word timing. Surface it for diagnosis. - console.warn( - "[auto-captions] Whisper JSON produced no word-timed cues; falling back to SRT (no word timings).", - ); - } - const cues = - timedCues.length > 0 ? timedCues : parseSrtCues(await fs.readFile(srtPath, "utf-8")); - if (cues.length === 0) { - throw new Error("Whisper completed, but no caption cues were produced."); - } + const cues = await readWhisperCaptionOutput(outputBase, jsonEnabled); // Whisper cues run sentences together and don't break on pauses. Re-segment them // into one caption per sentence/phrase using Whisper's own word stream (punctuation @@ -321,3 +330,36 @@ export async function generateAutoCaptionsFromVideo(options: { ]); } } + +export async function generateAutoCaptionsFromVideo(options: { + videoPath: string; + whisperExecutablePath?: string; + whisperModelPath: string; + language?: string; +}) { + const candidates = await resolveCaptionAudioCandidates(options.videoPath); + const microphone = candidates.filter((source) => source.label === "microphone audio sidecar"); + const system = candidates.filter((source) => source.label === "system audio sidecar"); + const recording = candidates.filter((source) => source.label === "recording"); + if (microphone.length === 0) { + return generateCaptionsForSource({ ...options, candidates: [...system, ...candidates] }); + } + // Decode independently so simultaneous voices do not confuse recognition. The + // sidecar replaces embedded system audio to avoid transcribing it twice. + const transcribeTrack = async (sources: CaptionAudioCandidate[]) => { + try { + return (await generateCaptionsForSource({ ...options, candidates: sources })).cues; + } catch (error) { + if (!(error instanceof NoCaptionAudioError)) throw error; + return null; + } + }; + const micCues = await transcribeTrack(microphone); + const systemCues = await transcribeTrack([...system, ...recording]); + if (micCues === null && systemCues === null) + throw new NoCaptionAudioError("No audio could be extracted from the recording."); + return { + cues: mergeCaptionSources(micCues ?? [], systemCues ?? []), + audioSourceLabel: "microphone and system audio", + }; +} diff --git a/electron/ipc/captions/generation.test.ts b/electron/ipc/captions/generation.test.ts new file mode 100644 index 00000000..71807d80 --- /dev/null +++ b/electron/ipc/captions/generation.test.ts @@ -0,0 +1,137 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + exec: vi.fn(), + readFile: vi.fn(), + rm: vi.fn(), + companions: vi.fn(), + delay: vi.fn(), +})); +vi.mock("node:child_process", () => ({ + execFile: (...args: unknown[]) => { + const callback = args.pop() as (error: Error | null, result: unknown) => void; + Promise.resolve() + .then(() => mocks.exec(...args)) + .then( + (result) => callback(null, result ?? { stderr: "" }), + (error) => callback(error, undefined), + ); + }, + spawnSync: vi.fn(), +})); +vi.mock("node:fs/promises", () => ({ + default: { + access: vi.fn().mockResolvedValue(undefined), + readFile: mocks.readFile, + rm: mocks.rm, + }, +})); +vi.mock("electron", () => ({ app: { getPath: () => "/tmp" } })); +vi.mock("../ffmpeg/binary", () => ({ getFfmpegBinaryPath: () => "/ffmpeg" })); +vi.mock("../paths/binaries", () => ({ getBundledWhisperExecutableCandidates: () => ["/whisper"] })); +vi.mock("../project/session", () => ({ resolveRecordingSession: vi.fn().mockResolvedValue(null) })); +vi.mock("../recording/diagnostics", () => ({ + getUsableCompanionAudioCandidates: mocks.companions, + getCompanionAudioStartDelayMs: mocks.delay, +})); +vi.mock("../utils", () => ({ normalizeVideoSourcePath: (value: string) => value })); + +import { generateAutoCaptionsFromVideo } from "./generate"; + +const options = { videoPath: "/video.mp4", whisperModelPath: "/model.bin" }; +const srt = "1\n00:00:00,000 --> 00:00:02,000\nHello world.\n"; +const json = JSON.stringify({ + transcription: [ + { + offsets: { from: 0, to: 2000 }, + text: "Hello world.", + tokens: [ + { text: "Hello", offsets: { from: 0, to: 900 } }, + { text: " world.", offsets: { from: 900, to: 2000 } }, + ], + }, + ], +}); + +beforeEach(() => { + vi.clearAllMocks(); + mocks.companions.mockReset().mockResolvedValue([]); + mocks.delay.mockReset().mockResolvedValue(null); + mocks.exec.mockReset().mockResolvedValue({ stderr: "" }); + mocks.readFile + .mockReset() + .mockImplementation(async (file: string) => (file.endsWith(".json") ? json : srt)); + mocks.rm.mockResolvedValue(undefined); +}); + +const whisperCalls = () => mocks.exec.mock.calls.filter(([file]) => file === "/whisper"); + +describe("caption generation pipeline", () => { + it("transcribes both sidecars independently, preserving the microphone delay", async () => { + mocks.companions.mockResolvedValue([ + { + platform: "mac", + micPath: "/video.mic.wav", + systemPath: "/video.system.wav", + usablePaths: ["/video.system.wav", "/video.mic.wav"], + }, + ]); + mocks.delay.mockResolvedValue(450); + const result = await generateAutoCaptionsFromVideo(options); + expect(result.audioSourceLabel).toBe("microphone and system audio"); + expect(whisperCalls()).toHaveLength(2); + expect(mocks.exec.mock.calls.some(([, args]) => args.includes("/video.system.wav"))).toBe( + true, + ); + expect(mocks.exec.mock.calls[0][1]).toEqual( + expect.arrayContaining(["-i", "/video.mic.wav", "-af", "adelay=450:all=1"]), + ); + }); + + it("falls back to embedded audio when the microphone cannot be extracted", async () => { + mocks.companions.mockResolvedValue([ + { + platform: "mac", + micPath: "/video.mic.wav", + systemPath: "/video.system.wav", + usablePaths: ["/video.mic.wav"], + }, + ]); + mocks.exec.mockImplementation(async (_file: string, args: string[]) => { + if (args.includes("/video.mic.wav")) throw new Error("unreadable microphone"); + return { stderr: "" }; + }); + expect((await generateAutoCaptionsFromVideo(options)).cues[0].text).toBe("Hello world."); + }); + + it("extracts mono audio, transcribes, segments word timings and cleans up", async () => { + const result = await generateAutoCaptionsFromVideo(options); + expect(result.cues[0].text).toBe("Hello world."); + expect(result.cues[0].words).toHaveLength(2); + expect(mocks.exec.mock.calls[0][1]).toEqual( + expect.arrayContaining(["-ac", "1", "-ar", "16000"]), + ); + expect(whisperCalls()).toHaveLength(1); + expect(whisperCalls()[0][1]).toEqual(expect.arrayContaining(["-mc", "0"])); + expect(mocks.rm).toHaveBeenCalledTimes(3); + }); + + it("retries SRT only for a runtime that rejects full JSON", async () => { + mocks.exec.mockImplementation(async (file: string, args: string[]) => { + if (file === "/whisper" && args.includes("-ojf")) + throw new Error("unknown argument: -ojf"); + return { stderr: "" }; + }); + expect((await generateAutoCaptionsFromVideo(options)).cues).toHaveLength(1); + expect(whisperCalls()).toHaveLength(2); + expect(whisperCalls()[1][1]).not.toContain("-ojf"); + }); + + it("keeps transcription when optional silence detection fails", async () => { + mocks.exec.mockImplementation(async (_file: string, args: string[]) => { + if (args.includes("-af")) throw new Error("silence filter failed"); + return { stderr: "" }; + }); + expect((await generateAutoCaptionsFromVideo(options)).cues[0].text).toBe("Hello world."); + }); +}); diff --git a/electron/ipc/captions/mergeSources.test.ts b/electron/ipc/captions/mergeSources.test.ts new file mode 100644 index 00000000..41c1b914 --- /dev/null +++ b/electron/ipc/captions/mergeSources.test.ts @@ -0,0 +1,75 @@ +import { expect, it } from "vitest"; +import { mergeCaptionSources } from "./mergeSources"; +import { parseWhisperJsonCues } from "./parser"; + +it("keeps system speech when the microphone has no speech", () => { + const system = [{ id: "s", startMs: 0, endMs: 1000, text: "System speech" }]; + expect(mergeCaptionSources([], system)[0].text).toBe("System speech"); +}); +it("keeps both tracks, choosing mic only for overlapping words", () => { + const mic = [{ id: "m", startMs: 1000, endMs: 2000, text: "Mic speech" }]; + const system = [ + { + id: "s", + startMs: 0, + endMs: 3000, + text: "Before conflict after", + words: [ + { text: "Before", startMs: 0, endMs: 900 }, + { text: "conflict", startMs: 1100, endMs: 1900, leadingSpace: true }, + { text: "after", startMs: 2100, endMs: 3000, leadingSpace: true }, + ], + }, + ]; + expect(mergeCaptionSources(mic, system).map((cue) => cue.text)).toEqual([ + "Before", + "Mic speech", + "after", + ]); +}); +it("retains real Whisper word timing through control tokens and punctuation", () => { + const cues = parseWhisperJsonCues( + JSON.stringify({ + transcription: [ + { + offsets: { from: 0, to: 1000 }, + text: "Hello.", + tokens: [ + { text: "[_BEG_]", offsets: { from: 0, to: 0 } }, + { text: " Hello", offsets: { from: 0, to: 900 } }, + { text: ".", offsets: { from: 900, to: 900 } }, + { text: "[_TT_50]", offsets: { from: 1000, to: 1000 } }, + ], + }, + ], + }), + ); + expect(cues[0].words).toEqual([{ text: "Hello.", startMs: 0, endMs: 900 }]); +}); + +it("does not let mic sound labels replace system speech", () => { + expect( + mergeCaptionSources( + [{ id: "m", startMs: 0, endMs: 2000, text: "[Coughing]" }], + [{ id: "s", startMs: 0, endMs: 2000, text: "The spoken paragraph." }], + ).map((c) => c.text), + ).toEqual(["The spoken paragraph."]); +}); + +it("preserves untimed speech next to timed words", async () => { + const { segmentCuesIntoPhrases } = await import("./segment"); + const result = segmentCuesIntoPhrases( + [ + { + id: "a", + startMs: 0, + endMs: 1000, + text: "First.", + words: [{ text: "First.", startMs: 0, endMs: 1000 }], + }, + { id: "b", startMs: 1000, endMs: 3000, text: "There is a timeline editor." }, + ], + [], + ); + expect(result.map((c) => c.text).join(" ")).toContain("There is a timeline editor."); +}); diff --git a/electron/ipc/captions/mergeSources.ts b/electron/ipc/captions/mergeSources.ts new file mode 100644 index 00000000..c2057162 --- /dev/null +++ b/electron/ipc/captions/mergeSources.ts @@ -0,0 +1,46 @@ +import type { CaptionCuePayload, CaptionWordPayload } from "../types"; +import { buildCaptionTextFromWords } from "./parser"; + +/** Keep both speakers, with microphone speech taking priority during overlap. */ +export function mergeCaptionSources( + microphone: CaptionCuePayload[], + system: CaptionCuePayload[], +): CaptionCuePayload[] { + // Sound-event labels are not competing microphone speech. + const micSpeech = microphone.filter((cue) => !/^(?:\s*[\[(][^\])]+[\])]\s*)+$/.test(cue.text)); + const overlapsMic = (startMs: number, endMs: number) => + micSpeech.some((cue) => startMs < cue.endMs && endMs > cue.startMs); + const systemCues: CaptionCuePayload[] = []; + for (const cue of system) { + if (!overlapsMic(cue.startMs, cue.endMs)) { + systemCues.push(cue); + continue; + } + // Preserve words outside the conflict, rather than dropping a whole paragraph. + let run: CaptionWordPayload[] = []; + const flush = () => { + if (run.length === 0) return; + systemCues.push({ + id: cue.id, + startMs: run[0].startMs, + endMs: run[run.length - 1].endMs, + text: buildCaptionTextFromWords(run), + words: run, + }); + run = []; + }; + for (const word of cue.words ?? []) { + if (overlapsMic(word.startMs, word.endMs)) flush(); + else run.push(word); + } + flush(); + } + const micEvents = microphone.filter( + (cue) => + !micSpeech.includes(cue) && + !system.some((other) => cue.startMs < other.endMs && cue.endMs > other.startMs), + ); + return [...micSpeech, ...micEvents, ...systemCues] + .sort((a, b) => a.startMs - b.startMs || a.endMs - b.endMs) + .map((cue, index) => ({ ...cue, id: `caption-${index + 1}` })); +} diff --git a/electron/ipc/captions/output.test.ts b/electron/ipc/captions/output.test.ts new file mode 100644 index 00000000..8b000966 --- /dev/null +++ b/electron/ipc/captions/output.test.ts @@ -0,0 +1,51 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, expect, it } from "vitest"; +import { readWhisperCaptionOutput } from "./output"; +let directory: string; +let output: string; +beforeEach(async () => { + directory = await fs.mkdtemp(path.join(os.tmpdir(), "recordly-caption-output-")); + output = path.join(directory, "speech"); + await fs.writeFile(`${output}.srt`, "1\n00:00:01,200 --> 00:00:02,400\nFallback speech\n"); +}); +afterEach(async () => { + await fs.rm(directory, { recursive: true, force: true }); +}); +it.each([ + undefined, + "invalid json", + '{"transcription":[]}', +])("uses SRT when JSON is unusable: %s", async (json) => { + if (json !== undefined) await fs.writeFile(`${output}.json`, json); + const cues = await readWhisperCaptionOutput(output, true); + expect(cues).toEqual([ + { id: "caption-1", startMs: 1200, endMs: 2400, text: "Fallback speech" }, + ]); +}); +it("prefers valid JSON word timing", async () => { + await fs.writeFile( + `${output}.json`, + JSON.stringify({ + transcription: [ + { + offsets: { from: 1200, to: 2400 }, + text: "Timed speech", + tokens: [ + { text: "Timed", offsets: { from: 1200, to: 1800 } }, + { text: " speech", offsets: { from: 1800, to: 2400 } }, + ], + }, + ], + }), + ); + const cues = await readWhisperCaptionOutput(output, true); + expect(cues[0].text).toBe("Timed speech"); + expect(cues[0].words).toHaveLength(2); + expect(cues[0].words?.[1].startMs).toBe(1800); +}); +it("still surfaces missing subtitle output", async () => { + await fs.rm(`${output}.srt`); + await expect(readWhisperCaptionOutput(output, false)).rejects.toMatchObject({ code: "ENOENT" }); +}); diff --git a/electron/ipc/captions/output.ts b/electron/ipc/captions/output.ts new file mode 100644 index 00000000..a48f43f0 --- /dev/null +++ b/electron/ipc/captions/output.ts @@ -0,0 +1,18 @@ +import fs from "node:fs/promises"; +import { parseSrtCues, parseWhisperJsonCues } from "./parser"; + +/** Older Whisper builds may emit SRT without the requested word-timing JSON. */ +export async function readWhisperCaptionOutput(outputBase: string, jsonEnabled: boolean) { + if (jsonEnabled) { + let json = ""; + try { + json = await fs.readFile(`${outputBase}.json`, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + const timed = json ? parseWhisperJsonCues(json) : []; + if (timed.length) return timed; + console.warn("[auto-captions] No usable JSON timing output; falling back to SRT."); + } + return parseSrtCues(await fs.readFile(`${outputBase}.srt`, "utf8")); +} diff --git a/electron/ipc/captions/parser.ts b/electron/ipc/captions/parser.ts index ff4c62ce..7ab21318 100644 --- a/electron/ipc/captions/parser.ts +++ b/electron/ipc/captions/parser.ts @@ -31,7 +31,7 @@ export function parseWhisperJsonWords(tokens: unknown): CaptionWordPayload[] { const tokenData = token as WhisperJsonToken; const tokenText = typeof tokenData.text === "string" ? tokenData.text : ""; - if (!tokenText) { + if (!tokenText || /^\[_[^\]]+\]$/.test(tokenText)) { continue; } @@ -49,11 +49,13 @@ export function parseWhisperJsonWords(tokens: unknown): CaptionWordPayload[] { continue; } - if (tokenStartMs == null || tokenEndMs == null || tokenEndMs <= tokenStartMs) { - return []; - } - const previousWord = words.length > 0 ? words[words.length - 1] : null; + if (previousWord && !nextLeadingSpace && /^\p{P}+$/u.test(part)) { + previousWord.text += part; + continue; + } + if (tokenStartMs == null || tokenEndMs == null || tokenEndMs <= tokenStartMs) return []; + if (!previousWord || nextLeadingSpace) { words.push({ text: part, @@ -179,5 +181,7 @@ export function parseSrtCues(content: string): CaptionCuePayload[] { export function shouldRetryWhisperWithoutJson(error: unknown): boolean { const message = error instanceof Error ? error.message : String(error); - return /unknown argument|output-json-full|output-json|ojf|\boj\b/i.test(message); + return /(?:unknown|unrecognized|unsupported) (?:argument|option)[:\s=]+['"]?--?(?:ojf|output-json-full)\b/i.test( + message, + ); } diff --git a/electron/ipc/captions/segment.ts b/electron/ipc/captions/segment.ts index 60407a00..b5528447 100644 --- a/electron/ipc/captions/segment.ts +++ b/electron/ipc/captions/segment.ts @@ -374,6 +374,17 @@ export function segmentCuesIntoPhrases( return []; } + // Keep untimed segments separate: merging them into timed segments would + // rebuild the text from words and silently discard the untimed speech. + if (!hasWordTimings(cues) && cues.some((cue) => (cue.words?.length ?? 0) > 0)) { + const segmented = cues.flatMap((cue) => segmentCuesIntoPhrases([cue], silences, options)); + segmented.sort((a, b) => a.startMs - b.startMs || a.endMs - b.endMs); + for (let index = 0; index < segmented.length - 1; index++) { + segmented[index].endMs = Math.min(segmented[index].endMs, segmented[index + 1].startMs); + } + return renumberCues(segmented.filter((cue) => cue.endMs > cue.startMs)); + } + // No word timings (SRT path): the silence-only segmenter trims/merges by acoustic // silence but can't see sentence boundaries, so a continuous paragraph would collapse // into one caption. Re-segment by silence first, then split each cue on its sentence diff --git a/src/components/video-editor/CaptionListPanel.tsx b/src/components/video-editor/CaptionListPanel.tsx index 58e38eb9..f3aa6f7f 100644 --- a/src/components/video-editor/CaptionListPanel.tsx +++ b/src/components/video-editor/CaptionListPanel.tsx @@ -113,6 +113,7 @@ function CaptionEditor({ {t("captions.editor.text", "Text")}