fix(captions): prevent repeated sound labels and preserve speech from both sources

electron/ipc/captions/generate.ts

@@ -7,11 +7,16 @@ import { app } from "electron";
 import { getFfmpegBinaryPath } from "../ffmpeg/binary";
 import { getBundledWhisperExecutableCandidates } from "../paths/binaries";
 import { resolveRecordingSession } from "../project/session";
-import { getUsableCompanionAudioCandidates } from "../recording/diagnostics";
+import {
+	getCompanionAudioStartDelayMs,
+	getUsableCompanionAudioCandidates,
+} from "../recording/diagnostics";
 import { normalizeVideoSourcePath } from "../utils";
-import { getCaptionCompanionAudioCandidates } from "./audioCandidates";
-import { parseSrtCues, parseWhisperJsonCues, shouldRetryWhisperWithoutJson } from "./parser";
+import { type CaptionAudioCandidate, getCaptionCompanionAudioCandidates } from "./audioCandidates";
+import { shouldRetryWhisperWithoutJson } from "./parser";
+import { readWhisperCaptionOutput } from "./output";
 import { isMissingWindowsWhisperRuntimeDependency } from "./runtimeErrors";
+import { mergeCaptionSources } from "./mergeSources";
 import { segmentCuesIntoPhrases } from "./segment";
 import {
 	parseSilenceIntervals,
@@ -22,6 +27,8 @@ import {
 
 const execFileAsync = promisify(execFile);
 
+class NoCaptionAudioError extends Error {}
+
 async function executeWhisper(whisperExecutablePath: string, args: string[]) {
 	try {
 		await execFileAsync(whisperExecutablePath, args, {
@@ -116,9 +123,15 @@ export async function resolveCaptionAudioCandidates(videoPath: string) {
 		candidates.push({ path: normalizedCandidatePath, label });
 	};
 
-	pushCandidate(videoPath, "recording");
 	const companionAudio = await getUsableCompanionAudioCandidates(videoPath);
-	for (const candidate of getCaptionCompanionAudioCandidates(companionAudio)) {
+	const companions = getCaptionCompanionAudioCandidates(companionAudio);
+	for (const candidate of companions.filter(
+		(candidate) => candidate.label === "microphone audio sidecar",
+	)) {
+		pushCandidate(candidate.path, candidate.label);
+	}
+	pushCandidate(videoPath, "recording");
+	for (const candidate of companions) {
 		pushCandidate(candidate.path, candidate.label);
 	}
 
@@ -132,8 +145,10 @@ export async function extractCaptionAudioSource(options: {
 	videoPath: string;
 	ffmpegPath: string;
 	wavPath: string;
+	candidates?: CaptionAudioCandidate[];
 }) {
-	const candidates = await resolveCaptionAudioCandidates(options.videoPath);
+	const candidates =
+		options.candidates ?? (await resolveCaptionAudioCandidates(options.videoPath));
 	const attemptedCandidates: Array<{
 		path: string;
 		label: string;
@@ -145,6 +160,9 @@ export async function extractCaptionAudioSource(options: {
 	for (const candidate of candidates) {
 		try {
 			await ensureReadableFile(candidate.path);
+			const delayMs = candidate.label.endsWith("audio sidecar")
+				? ((await getCompanionAudioStartDelayMs(candidate.path)) ?? 0)
+				: 0;
 			await execFileAsync(
 				options.ffmpegPath,
 				[
@@ -154,6 +172,7 @@ export async function extractCaptionAudioSource(options: {
 					"-map",
 					"0:a:0",
 					"-vn",
+					...(delayMs > 0 ? ["-af", `adelay=${delayMs}:all=1`] : []),
 					"-ac",
 					"1",
 					"-ar",
@@ -181,7 +200,7 @@ export async function extractCaptionAudioSource(options: {
 		attemptedCandidates,
 	);
 
-	throw new Error(
+	throw new NoCaptionAudioError(
 		"No audio was found to transcribe in the saved recording file. Captions need an audio track. If this recording should have contained sound, the recording was saved without an audio stream.",
 	);
 }
@@ -210,11 +229,12 @@ export async function detectSilenceIntervals(options: {
 	return parseSilenceIntervals(stderr ?? "");
 }
 
-export async function generateAutoCaptionsFromVideo(options: {
+async function generateCaptionsForSource(options: {
 	videoPath: string;
 	whisperExecutablePath?: string;
 	whisperModelPath: string;
 	language?: string;
+	candidates: CaptionAudioCandidate[];
 }) {
 	const ffmpegPath = getFfmpegBinaryPath();
 	const normalizedVideoPath = normalizeVideoSourcePath(options.videoPath);
@@ -241,6 +261,7 @@ export async function generateAutoCaptionsFromVideo(options: {
 			videoPath: normalizedVideoPath,
 			ffmpegPath,
 			wavPath,
+			candidates: options.candidates,
 		});
 
 		const language =
@@ -256,6 +277,9 @@ export async function generateAutoCaptionsFromVideo(options: {
 			"-l",
 			language,
 			"-np",
+			// Do not carry a hallucinated phrase through later decoding windows.
+			"-mc",
+			"0",
 		];
 
 		let jsonEnabled = true;
@@ -274,22 +298,7 @@ export async function generateAutoCaptionsFromVideo(options: {
 			await executeWhisper(whisperExecutablePath, whisperBaseArgs);
 		}
 
-		const timedCues = jsonEnabled
-			? parseWhisperJsonCues(await fs.readFile(jsonPath, "utf-8"))
-			: [];
-		if (jsonEnabled && timedCues.length === 0) {
-			// JSON ran but yielded no word-timed cues (empty/malformed output). We fall back
-			// to SRT, which has no word timings — captions are then split by sentence text and
-			// silence rather than precise word timing. Surface it for diagnosis.
-			console.warn(
-				"[auto-captions] Whisper JSON produced no word-timed cues; falling back to SRT (no word timings).",
-			);
-		}
-		const cues =
-			timedCues.length > 0 ? timedCues : parseSrtCues(await fs.readFile(srtPath, "utf-8"));
-		if (cues.length === 0) {
-			throw new Error("Whisper completed, but no caption cues were produced.");
-		}
+		const cues = await readWhisperCaptionOutput(outputBase, jsonEnabled);
 
 		// Whisper cues run sentences together and don't break on pauses. Re-segment them
 		// into one caption per sentence/phrase using Whisper's own word stream (punctuation
@@ -321,3 +330,36 @@ export async function generateAutoCaptionsFromVideo(options: {
 		]);
 	}
 }
+
+export async function generateAutoCaptionsFromVideo(options: {
+	videoPath: string;
+	whisperExecutablePath?: string;
+	whisperModelPath: string;
+	language?: string;
+}) {
+	const candidates = await resolveCaptionAudioCandidates(options.videoPath);
+	const microphone = candidates.filter((source) => source.label === "microphone audio sidecar");
+	const system = candidates.filter((source) => source.label === "system audio sidecar");
+	const recording = candidates.filter((source) => source.label === "recording");
+	if (microphone.length === 0) {
+		return generateCaptionsForSource({ ...options, candidates: [...system, ...candidates] });
+	}
+	// Decode independently so simultaneous voices do not confuse recognition. The
+	// sidecar replaces embedded system audio to avoid transcribing it twice.
+	const transcribeTrack = async (sources: CaptionAudioCandidate[]) => {
+		try {
+			return (await generateCaptionsForSource({ ...options, candidates: sources })).cues;
+		} catch (error) {
+			if (!(error instanceof NoCaptionAudioError)) throw error;
+			return null;
+		}
+	};
+	const micCues = await transcribeTrack(microphone);
+	const systemCues = await transcribeTrack([...system, ...recording]);
+	if (micCues === null && systemCues === null)
+		throw new NoCaptionAudioError("No audio could be extracted from the recording.");
+	return {
+		cues: mergeCaptionSources(micCues ?? [], systemCues ?? []),
+		audioSourceLabel: "microphone and system audio",
+	};
+}

electron/ipc/captions/generation.test.ts

@@ -0,0 +1,137 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+	exec: vi.fn(),
+	readFile: vi.fn(),
+	rm: vi.fn(),
+	companions: vi.fn(),
+	delay: vi.fn(),
+}));
+vi.mock("node:child_process", () => ({
+	execFile: (...args: unknown[]) => {
+		const callback = args.pop() as (error: Error | null, result: unknown) => void;
+		Promise.resolve()
+			.then(() => mocks.exec(...args))
+			.then(
+				(result) => callback(null, result ?? { stderr: "" }),
+				(error) => callback(error, undefined),
+			);
+	},
+	spawnSync: vi.fn(),
+}));
+vi.mock("node:fs/promises", () => ({
+	default: {
+		access: vi.fn().mockResolvedValue(undefined),
+		readFile: mocks.readFile,
+		rm: mocks.rm,
+	},
+}));
+vi.mock("electron", () => ({ app: { getPath: () => "/tmp" } }));
+vi.mock("../ffmpeg/binary", () => ({ getFfmpegBinaryPath: () => "/ffmpeg" }));
+vi.mock("../paths/binaries", () => ({ getBundledWhisperExecutableCandidates: () => ["/whisper"] }));
+vi.mock("../project/session", () => ({ resolveRecordingSession: vi.fn().mockResolvedValue(null) }));
+vi.mock("../recording/diagnostics", () => ({
+	getUsableCompanionAudioCandidates: mocks.companions,
+	getCompanionAudioStartDelayMs: mocks.delay,
+}));
+vi.mock("../utils", () => ({ normalizeVideoSourcePath: (value: string) => value }));
+
+import { generateAutoCaptionsFromVideo } from "./generate";
+
+const options = { videoPath: "/video.mp4", whisperModelPath: "/model.bin" };
+const srt = "1\n00:00:00,000 --> 00:00:02,000\nHello world.\n";
+const json = JSON.stringify({
+	transcription: [
+		{
+			offsets: { from: 0, to: 2000 },
+			text: "Hello world.",
+			tokens: [
+				{ text: "Hello", offsets: { from: 0, to: 900 } },
+				{ text: " world.", offsets: { from: 900, to: 2000 } },
+			],
+		},
+	],
+});
+
+beforeEach(() => {
+	vi.clearAllMocks();
+	mocks.companions.mockReset().mockResolvedValue([]);
+	mocks.delay.mockReset().mockResolvedValue(null);
+	mocks.exec.mockReset().mockResolvedValue({ stderr: "" });
+	mocks.readFile
+		.mockReset()
+		.mockImplementation(async (file: string) => (file.endsWith(".json") ? json : srt));
+	mocks.rm.mockResolvedValue(undefined);
+});
+
+const whisperCalls = () => mocks.exec.mock.calls.filter(([file]) => file === "/whisper");
+
+describe("caption generation pipeline", () => {
+	it("transcribes both sidecars independently, preserving the microphone delay", async () => {
+		mocks.companions.mockResolvedValue([
+			{
+				platform: "mac",
+				micPath: "/video.mic.wav",
+				systemPath: "/video.system.wav",
+				usablePaths: ["/video.system.wav", "/video.mic.wav"],
+			},
+		]);
+		mocks.delay.mockResolvedValue(450);
+		const result = await generateAutoCaptionsFromVideo(options);
+		expect(result.audioSourceLabel).toBe("microphone and system audio");
+		expect(whisperCalls()).toHaveLength(2);
+		expect(mocks.exec.mock.calls.some(([, args]) => args.includes("/video.system.wav"))).toBe(
+			true,
+		);
+		expect(mocks.exec.mock.calls[0][1]).toEqual(
+			expect.arrayContaining(["-i", "/video.mic.wav", "-af", "adelay=450:all=1"]),
+		);
+	});
+
+	it("falls back to embedded audio when the microphone cannot be extracted", async () => {
+		mocks.companions.mockResolvedValue([
+			{
+				platform: "mac",
+				micPath: "/video.mic.wav",
+				systemPath: "/video.system.wav",
+				usablePaths: ["/video.mic.wav"],
+			},
+		]);
+		mocks.exec.mockImplementation(async (_file: string, args: string[]) => {
+			if (args.includes("/video.mic.wav")) throw new Error("unreadable microphone");
+			return { stderr: "" };
+		});
+		expect((await generateAutoCaptionsFromVideo(options)).cues[0].text).toBe("Hello world.");
+	});
+
+	it("extracts mono audio, transcribes, segments word timings and cleans up", async () => {
+		const result = await generateAutoCaptionsFromVideo(options);
+		expect(result.cues[0].text).toBe("Hello world.");
+		expect(result.cues[0].words).toHaveLength(2);
+		expect(mocks.exec.mock.calls[0][1]).toEqual(
+			expect.arrayContaining(["-ac", "1", "-ar", "16000"]),
+		);
+		expect(whisperCalls()).toHaveLength(1);
+		expect(whisperCalls()[0][1]).toEqual(expect.arrayContaining(["-mc", "0"]));
+		expect(mocks.rm).toHaveBeenCalledTimes(3);
+	});
+
+	it("retries SRT only for a runtime that rejects full JSON", async () => {
+		mocks.exec.mockImplementation(async (file: string, args: string[]) => {
+			if (file === "/whisper" && args.includes("-ojf"))
+				throw new Error("unknown argument: -ojf");
+			return { stderr: "" };
+		});
+		expect((await generateAutoCaptionsFromVideo(options)).cues).toHaveLength(1);
+		expect(whisperCalls()).toHaveLength(2);
+		expect(whisperCalls()[1][1]).not.toContain("-ojf");
+	});
+
+	it("keeps transcription when optional silence detection fails", async () => {
+		mocks.exec.mockImplementation(async (_file: string, args: string[]) => {
+			if (args.includes("-af")) throw new Error("silence filter failed");
+			return { stderr: "" };
+		});
+		expect((await generateAutoCaptionsFromVideo(options)).cues[0].text).toBe("Hello world.");
+	});
+});

electron/ipc/captions/mergeSources.test.ts

@@ -0,0 +1,75 @@
+import { expect, it } from "vitest";
+import { mergeCaptionSources } from "./mergeSources";
+import { parseWhisperJsonCues } from "./parser";
+
+it("keeps system speech when the microphone has no speech", () => {
+	const system = [{ id: "s", startMs: 0, endMs: 1000, text: "System speech" }];
+	expect(mergeCaptionSources([], system)[0].text).toBe("System speech");
+});
+it("keeps both tracks, choosing mic only for overlapping words", () => {
+	const mic = [{ id: "m", startMs: 1000, endMs: 2000, text: "Mic speech" }];
+	const system = [
+		{
+			id: "s",
+			startMs: 0,
+			endMs: 3000,
+			text: "Before conflict after",
+			words: [
+				{ text: "Before", startMs: 0, endMs: 900 },
+				{ text: "conflict", startMs: 1100, endMs: 1900, leadingSpace: true },
+				{ text: "after", startMs: 2100, endMs: 3000, leadingSpace: true },
+			],
+		},
+	];
+	expect(mergeCaptionSources(mic, system).map((cue) => cue.text)).toEqual([
+		"Before",
+		"Mic speech",
+		"after",
+	]);
+});
+it("retains real Whisper word timing through control tokens and punctuation", () => {
+	const cues = parseWhisperJsonCues(
+		JSON.stringify({
+			transcription: [
+				{
+					offsets: { from: 0, to: 1000 },
+					text: "Hello.",
+					tokens: [
+						{ text: "[_BEG_]", offsets: { from: 0, to: 0 } },
+						{ text: " Hello", offsets: { from: 0, to: 900 } },
+						{ text: ".", offsets: { from: 900, to: 900 } },
+						{ text: "[_TT_50]", offsets: { from: 1000, to: 1000 } },
+					],
+				},
+			],
+		}),
+	);
+	expect(cues[0].words).toEqual([{ text: "Hello.", startMs: 0, endMs: 900 }]);
+});
+
+it("does not let mic sound labels replace system speech", () => {
+	expect(
+		mergeCaptionSources(
+			[{ id: "m", startMs: 0, endMs: 2000, text: "[Coughing]" }],
+			[{ id: "s", startMs: 0, endMs: 2000, text: "The spoken paragraph." }],
+		).map((c) => c.text),
+	).toEqual(["The spoken paragraph."]);
+});
+
+it("preserves untimed speech next to timed words", async () => {
+	const { segmentCuesIntoPhrases } = await import("./segment");
+	const result = segmentCuesIntoPhrases(
+		[
+			{
+				id: "a",
+				startMs: 0,
+				endMs: 1000,
+				text: "First.",
+				words: [{ text: "First.", startMs: 0, endMs: 1000 }],
+			},
+			{ id: "b", startMs: 1000, endMs: 3000, text: "There is a timeline editor." },
+		],
+		[],
+	);
+	expect(result.map((c) => c.text).join(" ")).toContain("There is a timeline editor.");
+});

electron/ipc/captions/mergeSources.ts

@@ -0,0 +1,46 @@
+import type { CaptionCuePayload, CaptionWordPayload } from "../types";
+import { buildCaptionTextFromWords } from "./parser";
+
+/** Keep both speakers, with microphone speech taking priority during overlap. */
+export function mergeCaptionSources(
+	microphone: CaptionCuePayload[],
+	system: CaptionCuePayload[],
+): CaptionCuePayload[] {
+	// Sound-event labels are not competing microphone speech.
+	const micSpeech = microphone.filter((cue) => !/^(?:\s*[\[(][^\])]+[\])]\s*)+$/.test(cue.text));
+	const overlapsMic = (startMs: number, endMs: number) =>
+		micSpeech.some((cue) => startMs < cue.endMs && endMs > cue.startMs);
+	const systemCues: CaptionCuePayload[] = [];
+	for (const cue of system) {
+		if (!overlapsMic(cue.startMs, cue.endMs)) {
+			systemCues.push(cue);
+			continue;
+		}
+		// Preserve words outside the conflict, rather than dropping a whole paragraph.
+		let run: CaptionWordPayload[] = [];
+		const flush = () => {
+			if (run.length === 0) return;
+			systemCues.push({
+				id: cue.id,
+				startMs: run[0].startMs,
+				endMs: run[run.length - 1].endMs,
+				text: buildCaptionTextFromWords(run),
+				words: run,
+			});
+			run = [];
+		};
+		for (const word of cue.words ?? []) {
+			if (overlapsMic(word.startMs, word.endMs)) flush();
+			else run.push(word);
+		}
+		flush();
+	}
+	const micEvents = microphone.filter(
+		(cue) =>
+			!micSpeech.includes(cue) &&
+			!system.some((other) => cue.startMs < other.endMs && cue.endMs > other.startMs),
+	);
+	return [...micSpeech, ...micEvents, ...systemCues]
+		.sort((a, b) => a.startMs - b.startMs || a.endMs - b.endMs)
+		.map((cue, index) => ({ ...cue, id: `caption-${index + 1}` }));
+}

electron/ipc/captions/output.test.ts

@@ -0,0 +1,51 @@
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, beforeEach, expect, it } from "vitest";
+import { readWhisperCaptionOutput } from "./output";
+let directory: string;
+let output: string;
+beforeEach(async () => {
+	directory = await fs.mkdtemp(path.join(os.tmpdir(), "recordly-caption-output-"));
+	output = path.join(directory, "speech");
+	await fs.writeFile(`${output}.srt`, "1\n00:00:01,200 --> 00:00:02,400\nFallback speech\n");
+});
+afterEach(async () => {
+	await fs.rm(directory, { recursive: true, force: true });
+});
+it.each([
+	undefined,
+	"invalid json",
+	'{"transcription":[]}',
+])("uses SRT when JSON is unusable: %s", async (json) => {
+	if (json !== undefined) await fs.writeFile(`${output}.json`, json);
+	const cues = await readWhisperCaptionOutput(output, true);
+	expect(cues).toEqual([
+		{ id: "caption-1", startMs: 1200, endMs: 2400, text: "Fallback speech" },
+	]);
+});
+it("prefers valid JSON word timing", async () => {
+	await fs.writeFile(
+		`${output}.json`,
+		JSON.stringify({
+			transcription: [
+				{
+					offsets: { from: 1200, to: 2400 },
+					text: "Timed speech",
+					tokens: [
+						{ text: "Timed", offsets: { from: 1200, to: 1800 } },
+						{ text: " speech", offsets: { from: 1800, to: 2400 } },
+					],
+				},
+			],
+		}),
+	);
+	const cues = await readWhisperCaptionOutput(output, true);
+	expect(cues[0].text).toBe("Timed speech");
+	expect(cues[0].words).toHaveLength(2);
+	expect(cues[0].words?.[1].startMs).toBe(1800);
+});
+it("still surfaces missing subtitle output", async () => {
+	await fs.rm(`${output}.srt`);
+	await expect(readWhisperCaptionOutput(output, false)).rejects.toMatchObject({ code: "ENOENT" });
+});

electron/ipc/captions/output.ts

@@ -0,0 +1,18 @@
+import fs from "node:fs/promises";
+import { parseSrtCues, parseWhisperJsonCues } from "./parser";
+
+/** Older Whisper builds may emit SRT without the requested word-timing JSON. */
+export async function readWhisperCaptionOutput(outputBase: string, jsonEnabled: boolean) {
+	if (jsonEnabled) {
+		let json = "";
+		try {
+			json = await fs.readFile(`${outputBase}.json`, "utf8");
+		} catch (error) {
+			if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+		}
+		const timed = json ? parseWhisperJsonCues(json) : [];
+		if (timed.length) return timed;
+		console.warn("[auto-captions] No usable JSON timing output; falling back to SRT.");
+	}
+	return parseSrtCues(await fs.readFile(`${outputBase}.srt`, "utf8"));
+}

electron/ipc/captions/parser.ts

@@ -31,7 +31,7 @@ export function parseWhisperJsonWords(tokens: unknown): CaptionWordPayload[] {
 
 		const tokenData = token as WhisperJsonToken;
 		const tokenText = typeof tokenData.text === "string" ? tokenData.text : "";
-		if (!tokenText) {
+		if (!tokenText || /^\[_[^\]]+\]$/.test(tokenText)) {
 			continue;
 		}
 
@@ -49,11 +49,13 @@ export function parseWhisperJsonWords(tokens: unknown): CaptionWordPayload[] {
 				continue;
 			}
 
-			if (tokenStartMs == null || tokenEndMs == null || tokenEndMs <= tokenStartMs) {
-				return [];
-			}
-
 			const previousWord = words.length > 0 ? words[words.length - 1] : null;
+			if (previousWord && !nextLeadingSpace && /^\p{P}+$/u.test(part)) {
+				previousWord.text += part;
+				continue;
+			}
+			if (tokenStartMs == null || tokenEndMs == null || tokenEndMs <= tokenStartMs) return [];
+
 			if (!previousWord || nextLeadingSpace) {
 				words.push({
 					text: part,
@@ -179,5 +181,7 @@ export function parseSrtCues(content: string): CaptionCuePayload[] {
 
 export function shouldRetryWhisperWithoutJson(error: unknown): boolean {
 	const message = error instanceof Error ? error.message : String(error);
-	return /unknown argument|output-json-full|output-json|ojf|\boj\b/i.test(message);
+	return /(?:unknown|unrecognized|unsupported) (?:argument|option)[:\s=]+['"]?--?(?:ojf|output-json-full)\b/i.test(
+		message,
+	);
 }

electron/ipc/captions/segment.ts

@@ -374,6 +374,17 @@ export function segmentCuesIntoPhrases(
 		return [];
 	}
 
+	// Keep untimed segments separate: merging them into timed segments would
+	// rebuild the text from words and silently discard the untimed speech.
+	if (!hasWordTimings(cues) && cues.some((cue) => (cue.words?.length ?? 0) > 0)) {
+		const segmented = cues.flatMap((cue) => segmentCuesIntoPhrases([cue], silences, options));
+		segmented.sort((a, b) => a.startMs - b.startMs || a.endMs - b.endMs);
+		for (let index = 0; index < segmented.length - 1; index++) {
+			segmented[index].endMs = Math.min(segmented[index].endMs, segmented[index + 1].startMs);
+		}
+		return renumberCues(segmented.filter((cue) => cue.endMs > cue.startMs));
+	}
+
 	// No word timings (SRT path): the silence-only segmenter trims/merges by acoustic
 	// silence but can't see sentence boundaries, so a continuous paragraph would collapse
 	// into one caption. Re-segment by silence first, then split each cue on its sentence

src/components/video-editor/CaptionListPanel.tsx

@@ -113,6 +113,7 @@ function CaptionEditor({
 					{t("captions.editor.text", "Text")}
 				</span>
 				<TextArea
+					data-caption-text-editor
 					value={draftText}
 					rows={3}
 					// Freshly-added captions start empty — focus the field so the

src/components/video-editor/captions/useAutoCaptionController.test.ts

@@ -0,0 +1,86 @@
+import { beforeEach, afterEach, expect, it, vi } from "vitest";
+import { useAutoCaptionController } from "./useAutoCaptionController";
+const state = vi.hoisted(() => ({ refs: [] as { current: unknown }[], index: 0 }));
+vi.mock("react", () => ({
+	useCallback: (callback: unknown) => callback,
+	useEffect: () => {},
+	useRef: (current: unknown) =>
+		state.refs[state.index++] ?? (state.refs[state.index - 1] = { current }),
+}));
+vi.mock("@/components/ui/toast", () => ({
+	toast: { error: vi.fn(), info: vi.fn(), success: vi.fn() },
+}));
+beforeEach(() => {
+	state.refs = [];
+	state.index = 0;
+});
+afterEach(() => vi.unstubAllGlobals());
+function setup() {
+	let complete!: (result: unknown) => void;
+	const generateAutoCaptions = vi.fn(
+		() =>
+			new Promise((resolve) => {
+				complete = resolve;
+			}),
+	);
+	vi.stubGlobal("window", { electronAPI: { generateAutoCaptions } });
+	const params = {
+		videoSourcePath: "/recordings/a.mp4",
+		videoPath: "file:///recordings/a.mp4",
+		whisperModelPath: "/models/small.bin",
+		isGeneratingCaptions: false,
+		autoCaptionSettings: { language: "en" },
+		setIsGeneratingCaptions: vi.fn(),
+		setAutoCaptions: vi.fn(),
+		setAutoCaptionSettings: vi.fn(),
+		syncActiveVideoSource: vi.fn(),
+		t: (key: string) => key,
+	} as unknown as Parameters<typeof useAutoCaptionController>[0];
+	const render = () => {
+		state.index = 0;
+		return useAutoCaptionController(params);
+	};
+	return {
+		params,
+		render,
+		generateAutoCaptions,
+		complete: (result: unknown) => complete(result),
+	};
+}
+const cues = [{ id: "generated", startMs: 0, endMs: 1000, text: "Hello" }];
+it("deduplicates requests and enables captions without resetting the active session", async () => {
+	const test = setup();
+	const controller = test.render();
+	const pending = controller.handleGenerateAutoCaptions();
+	await controller.handleGenerateAutoCaptions();
+	expect(test.generateAutoCaptions).toHaveBeenCalledTimes(1);
+	test.complete({ success: true, cues });
+	await pending;
+	expect(test.params.setAutoCaptions).toHaveBeenCalledWith(cues);
+	expect(test.params.setAutoCaptionSettings).toHaveBeenCalledOnce();
+	expect(test.params.syncActiveVideoSource).not.toHaveBeenCalled();
+	expect(test.params.setIsGeneratingCaptions).toHaveBeenLastCalledWith(false);
+});
+it("discards results after switching recordings", async () => {
+	const test = setup();
+	const pending = test.render().handleGenerateAutoCaptions();
+	test.params.videoSourcePath = "/recordings/b.mp4";
+	test.render();
+	test.complete({ success: true, cues });
+	await pending;
+	expect(test.params.setAutoCaptions).not.toHaveBeenCalled();
+	expect(test.params.setIsGeneratingCaptions).toHaveBeenLastCalledWith(false);
+});
+it("preserves existing captions on failure and permits retry", async () => {
+	const test = setup();
+	const controller = test.render();
+	const pending = controller.handleGenerateAutoCaptions();
+	test.complete({ success: false, error: "Transcription failed" });
+	await pending;
+	expect(test.params.setAutoCaptions).not.toHaveBeenCalled();
+	const retry = controller.handleGenerateAutoCaptions();
+	test.complete({ success: true, cues });
+	await retry;
+	expect(test.generateAutoCaptions).toHaveBeenCalledTimes(2);
+	expect(test.params.setAutoCaptions).toHaveBeenCalledWith(cues);
+});

src/components/video-editor/captions/useAutoCaptionController.ts

@@ -61,6 +61,8 @@ export function useAutoCaptionController({
 	syncActiveVideoSource,
 }: UseAutoCaptionControllerParams) {
 	const captionGenerationInFlightRef = useRef(false);
+	const activeSourceRef = useRef({ videoSourcePath, videoPath });
+	activeSourceRef.current = { videoSourcePath, videoPath };
 
 	useEffect(() => {
 		const unsubscribe = window.electronAPI.onWhisperSmallModelDownloadProgress((state) => {
@@ -158,6 +160,10 @@ export function useAutoCaptionController({
 		captionGenerationInFlightRef.current = true;
 		setIsGeneratingCaptions(true);
 		try {
+			if (!whisperModelPath) {
+				toast.error("Select a Whisper model or download the small model first");
+				return;
+			}
 			let sourcePath = resolveAutoCaptionSourcePath({ videoSourcePath, videoPath });
 			if (!sourcePath) {
 				const sessionResult = await window.electronAPI.getCurrentRecordingSession?.();
@@ -176,22 +182,22 @@ export function useAutoCaptionController({
 				toast.error("No source video is loaded");
 				return;
 			}
-			await syncActiveVideoSource(sourcePath, webcamSourcePath);
+			// Refreshing an unchanged session revokes active companion-audio URLs.
 			if (sourcePath !== videoSourcePath) {
+				await syncActiveVideoSource(sourcePath, webcamSourcePath);
 				setVideoSourcePath(sourcePath);
 				setVideoPath(await resolveVideoUrl(sourcePath));
 			}
-			if (!whisperModelPath) {
-				toast.error("Select a Whisper model or download the small model first");
-				return;
-			}
-
 			const result = await window.electronAPI.generateAutoCaptions({
 				videoPath: sourcePath,
 				whisperExecutablePath: whisperExecutablePath ?? undefined,
 				whisperModelPath,
 				language: autoCaptionSettings.language,
 			});
+			if (resolveAutoCaptionSourcePath(activeSourceRef.current) !== sourcePath) {
+				toast.info("Recording changed; generated captions were not applied.");
+				return;
+			}
 			if (!result.success || !result.cues) {
 				const errorMessage = result.error ? getErrorMessage(result.error) : result.message;
 				toast.error(errorMessage || "Failed to generate captions");

tests/ui/caption-speed.spec.ts

@@ -0,0 +1,105 @@
+import { expect, test, type Page, type Locator } from "@playwright/test";
+import { installDesktopBridge } from "./bridge";
+
+async function seek(page: Page, clip: Locator, timeMs: number) {
+	const end = Number(await clip.getAttribute("data-end-ms"));
+	const box = (await clip.locator(".timeline-block").boundingBox())!;
+	const row = (await page.locator('[data-timeline-row="row-clip"]').boundingBox())!;
+	await page.mouse.click(box.x + (box.width * timeMs) / end, row.y - 8);
+}
+
+test("captions render in preview and remain synchronized at 1x, 2x and 4x", async ({ page }) => {
+	test.setTimeout(60000);
+	const errors: string[] = [];
+	page.on("pageerror", (error) => errors.push(error.message));
+	await installDesktopBridge(page, "filmstrip.mp4");
+	await page.goto("/?windowType=editor");
+	const clip = page.locator('[data-variant="clip"]');
+	await expect(clip).toHaveAttribute("data-end-ms", "6000", { timeout: 20000 });
+	await expect(clip.locator("img").first()).toBeVisible({ timeout: 20000 });
+	await expect(page.getByLabel("Loading preview")).toHaveCount(0);
+	await page.getByRole("radio", { name: "Captions", exact: true }).click();
+	const show = page.getByRole("switch", { name: "Show", exact: true });
+	await page
+		.locator('[data-slot="switch"]')
+		.filter({ has: show })
+		.locator('[data-slot="switch-control"]')
+		.click();
+	const strip = (await page.locator("[data-caption-add-target]").boundingBox())!;
+	await page.mouse.click(strip.x + strip.width / 6, strip.y + 8);
+	const caption = page.locator('[data-variant="caption"]');
+	await expect(caption).toHaveCount(1);
+	const text = page.getByRole("textbox", { name: "Text", exact: true });
+	await text.fill("Caption speed check");
+	await text.press("Enter");
+	const sourceStart = Number(await caption.getAttribute("data-start-ms"));
+	const sourceEnd = Number(await caption.getAttribute("data-end-ms"));
+	const visibleCaption = page.getByRole("button", { name: "Edit current caption", exact: true });
+	// Double-clicking a timeline caption immediately focuses its text editor.
+	await caption.dblclick();
+	await expect(text).toBeFocused();
+	await text.fill("Caption speed check");
+	await text.press("Enter");
+	const selectionColors = await caption.locator(".timeline-block").evaluate((node) => {
+		const probe = document.createElement("div");
+		probe.style.borderColor = "var(--accent)";
+		document.body.append(probe);
+		const expected = getComputedStyle(probe).borderColor;
+		probe.remove();
+		return { actual: getComputedStyle(node).borderColor, expected };
+	});
+	expect(selectionColors.actual).toBe(selectionColors.expected);
+	await seek(page, clip, (sourceStart + sourceEnd) / 2);
+	await visibleCaption.dblclick();
+	const inline = page.getByRole("textbox", { name: "Edit current caption", exact: true });
+	await expect(inline).toBeFocused();
+	await inline.fill("Direct caption edit");
+	await inline.press("Tab");
+	await expect(visibleCaption).toHaveText("Direct caption edit");
+	await visibleCaption.dblclick();
+	await inline.fill("Discard this draft");
+	await inline.press("Escape");
+	await expect(visibleCaption).toHaveText("Direct caption edit");
+	await visibleCaption.dblclick();
+	await inline.fill("Caption speed check");
+	await inline.press("Tab");
+	const video = page.locator('video[aria-hidden="true"]');
+	for (const speed of [1, 2, 4]) {
+		await clip.click({ position: { x: 80, y: 20 } });
+		const control = page.getByRole("slider", { name: "Speed", exact: true });
+		await control.focus();
+		while (Number(await control.inputValue()) < speed) await control.press("ArrowRight");
+		await expect(control).toHaveValue(String(speed));
+		await expect(clip).toHaveAttribute("data-end-ms", String(6000 / speed));
+		await expect
+			.poll(async () => Number(await caption.getAttribute("data-start-ms")))
+			.toBeCloseTo(sourceStart / speed, 0);
+		await expect
+			.poll(async () => Number(await caption.getAttribute("data-end-ms")))
+			.toBeCloseTo(sourceEnd / speed, 0);
+		await seek(page, clip, (sourceStart + sourceEnd) / 2 / speed);
+		await expect(visibleCaption).toBeVisible();
+		await expect(visibleCaption).toHaveText("Caption speed check");
+		await expect
+			.poll(() => video.evaluate((node: HTMLVideoElement) => node.currentTime))
+			.toBeCloseTo((sourceStart + sourceEnd) / 2000, 1);
+		await expect
+			.poll(() =>
+				visibleCaption.evaluate((node) =>
+					Number(getComputedStyle(node.parentElement!).opacity),
+				),
+			)
+			.toBeGreaterThan(0.9);
+		if (speed === 2) await page.screenshot({ path: "test-results/captions-at-2x.png" });
+		await seek(page, clip, (sourceStart - 300) / speed);
+		await expect(visibleCaption).toHaveCount(0);
+		await seek(page, clip, (sourceEnd + 300) / speed);
+		await expect(visibleCaption).toHaveCount(0);
+	}
+	// Exercise moving playback as well as paused seeks.
+	await seek(page, clip, 0);
+	await page.getByRole("button", { name: "Play", exact: true }).click();
+	await expect(visibleCaption).toBeVisible();
+	await expect(visibleCaption).toHaveCount(0);
+	expect(errors).toEqual([]);
+});