feat(captions): segment captions by sentence using punctuation and pauses

electron/ipc/captions/generate.ts

@@ -9,9 +9,9 @@ import { getBundledWhisperExecutableCandidates } from "../paths/binaries";
 import { resolveRecordingSession } from "../project/session";
 import { normalizeVideoSourcePath } from "../utils";
 import { parseSrtCues, parseWhisperJsonCues, shouldRetryWhisperWithoutJson } from "./parser";
+import { segmentCuesIntoPhrases } from "./segment";
 import {
 	parseSilenceIntervals,
-	resegmentCuesBySilence,
 	SILENCE_DETECT_MIN_S,
 	SILENCE_NOISE_DB,
 	type SilenceInterval,
@@ -259,19 +259,28 @@ export async function generateAutoCaptionsFromVideo(options: {
 		const timedCues = jsonEnabled
 			? parseWhisperJsonCues(await fs.readFile(jsonPath, "utf-8"))
 			: [];
+		if (jsonEnabled && timedCues.length === 0) {
+			// JSON ran but yielded no word-timed cues (empty/malformed output). We fall back
+			// to SRT, which has no word timings — captions are then split by sentence text and
+			// silence rather than precise word timing. Surface it for diagnosis.
+			console.warn(
+				"[auto-captions] Whisper JSON produced no word-timed cues; falling back to SRT (no word timings).",
+			);
+		}
 		const cues =
 			timedCues.length > 0 ? timedCues : parseSrtCues(await fs.readFile(srtPath, "utf-8"));
 		if (cues.length === 0) {
 			throw new Error("Whisper completed, but no caption cues were produced.");
 		}
 
-		// Whisper cues span silence and don't break on pauses. Re-segment them against
-		// ground-truth acoustic silence (ffmpeg `silencedetect`) so captions only cover
-		// real speech. Failure here must not block caption generation — fall back to raw.
+		// Whisper cues run sentences together and don't break on pauses. Re-segment them
+		// into one caption per sentence/phrase using Whisper's own word stream (punctuation
+		// + pauses), backed by ground-truth acoustic silence (ffmpeg `silencedetect`).
+		// Failure here must not block caption generation — fall back to raw.
 		let cuesToReturn = cues;
 		try {
 			const silences = await detectSilenceIntervals({ ffmpegPath, wavPath });
-			const resegmented = resegmentCuesBySilence(cues, silences);
+			const resegmented = segmentCuesIntoPhrases(cues, silences);
 			if (resegmented.length > 0) {
 				cuesToReturn = resegmented;
 			}

electron/ipc/captions/segment.test.ts

@@ -0,0 +1,347 @@
+import { describe, expect, it } from "vitest";
+import type { CaptionCuePayload } from "../types";
+import { endsSentence, segmentCuesIntoPhrases } from "./segment";
+
+describe("endsSentence", () => {
+	it("detects terminal punctuation, ignoring trailing closers", () => {
+		expect(endsSentence("world.")).toBe(true);
+		expect(endsSentence("you?")).toBe(true);
+		expect(endsSentence("stop!")).toBe(true);
+		expect(endsSentence("wait…")).toBe(true);
+		expect(endsSentence('said."')).toBe(true);
+		expect(endsSentence("done,")).toBe(false);
+		expect(endsSentence("hello")).toBe(false);
+	});
+
+	it("does not treat common abbreviations or initialisms as a sentence end", () => {
+		expect(endsSentence("Mr.")).toBe(false);
+		expect(endsSentence("Dr.")).toBe(false);
+		expect(endsSentence("etc.")).toBe(false);
+		expect(endsSentence("e.g.")).toBe(false);
+		expect(endsSentence("U.S.")).toBe(false);
+		expect(endsSentence("J.")).toBe(false);
+		// real sentence ends are unaffected
+		expect(endsSentence("Smith.")).toBe(true);
+		expect(endsSentence("5.")).toBe(true);
+		// `?`/`!` after an abbreviation-like token still end the sentence
+		expect(endsSentence("really?")).toBe(true);
+	});
+});
+
+describe("segmentCuesIntoPhrases", () => {
+	it("splits back-to-back sentences with no pause into separate captions", () => {
+		const cues: CaptionCuePayload[] = [
+			{
+				id: "caption-1",
+				startMs: 0,
+				endMs: 1_800,
+				text: "Hello world. How are you?",
+				words: [
+					{ text: "Hello", startMs: 0, endMs: 400 },
+					{ text: "world.", startMs: 400, endMs: 800, leadingSpace: true },
+					{ text: "How", startMs: 800, endMs: 1_100, leadingSpace: true },
+					{ text: "are", startMs: 1_100, endMs: 1_400, leadingSpace: true },
+					{ text: "you?", startMs: 1_400, endMs: 1_800, leadingSpace: true },
+				],
+			},
+		];
+		const result = segmentCuesIntoPhrases(cues, []);
+		expect(result).toHaveLength(2);
+		expect(result[0].text).toBe("Hello world.");
+		expect(result[1].text).toBe("How are you?");
+		expect(result[0].endMs).toBeLessThanOrEqual(result[1].startMs);
+	});
+
+	it("does not leak the first word of the next sentence into the previous caption", () => {
+		const cues: CaptionCuePayload[] = [
+			{
+				id: "caption-1",
+				startMs: 0,
+				endMs: 1_800,
+				text: "Hello world. How are you?",
+				words: [
+					{ text: "Hello", startMs: 0, endMs: 400 },
+					{ text: "world.", startMs: 400, endMs: 800, leadingSpace: true },
+					{ text: "How", startMs: 800, endMs: 1_100, leadingSpace: true },
+					{ text: "are", startMs: 1_100, endMs: 1_400, leadingSpace: true },
+					{ text: "you?", startMs: 1_400, endMs: 1_800, leadingSpace: true },
+				],
+			},
+		];
+		const result = segmentCuesIntoPhrases(cues, []);
+		expect(result[1].words?.[0].text).toBe("How");
+		expect(result[1].words?.[0].startMs).toBe(800);
+		// the first word of a phrase has no leading space
+		expect(result[1].words?.[0].leadingSpace).toBeUndefined();
+	});
+
+	it("merges rapid-fire short sentences into one caption (word-timed)", () => {
+		const cues: CaptionCuePayload[] = [
+			{
+				id: "caption-1",
+				startMs: 0,
+				endMs: 800,
+				text: "Okay. Great.",
+				words: [
+					{ text: "Okay.", startMs: 0, endMs: 400 },
+					{ text: "Great.", startMs: 400, endMs: 800, leadingSpace: true },
+				],
+			},
+		];
+		const result = segmentCuesIntoPhrases(cues, []);
+		expect(result).toHaveLength(1);
+		expect(result[0].text).toBe("Okay. Great.");
+		expect(result[0].words?.map((word) => word.text)).toEqual(["Okay.", "Great."]);
+	});
+
+	it("merges rapid-fire short sentences into one caption (word-less)", () => {
+		const cues: CaptionCuePayload[] = [
+			{ id: "caption-1", startMs: 0, endMs: 400, text: "Okay." },
+			{ id: "caption-2", startMs: 400, endMs: 800, text: "Great." },
+		];
+		const result = segmentCuesIntoPhrases(cues, []);
+		expect(result).toHaveLength(1);
+		expect(result[0].text).toBe("Okay. Great.");
+	});
+
+	it("does not merge a short sentence into a full-length one", () => {
+		const cues: CaptionCuePayload[] = [
+			{
+				id: "caption-1",
+				startMs: 0,
+				endMs: 2_400,
+				text: "Okay. This is a full length sentence that should stand alone.",
+				words: [
+					{ text: "Okay.", startMs: 0, endMs: 400 },
+					{ text: "This", startMs: 400, endMs: 700, leadingSpace: true },
+					{ text: "is", startMs: 700, endMs: 900, leadingSpace: true },
+					{ text: "a", startMs: 900, endMs: 1_050, leadingSpace: true },
+					{ text: "full", startMs: 1_050, endMs: 1_350, leadingSpace: true },
+					{ text: "length", startMs: 1_350, endMs: 1_650, leadingSpace: true },
+					{ text: "sentence", startMs: 1_650, endMs: 1_950, leadingSpace: true },
+					{ text: "that", startMs: 1_950, endMs: 2_100, leadingSpace: true },
+					{ text: "should", startMs: 2_100, endMs: 2_250, leadingSpace: true },
+					{ text: "stand", startMs: 2_250, endMs: 2_350, leadingSpace: true },
+					{ text: "alone.", startMs: 2_350, endMs: 2_400, leadingSpace: true },
+				],
+			},
+		];
+		const result = segmentCuesIntoPhrases(cues, []);
+		expect(result).toHaveLength(2);
+		expect(result[0].text).toBe("Okay.");
+		expect(result[1].text).toBe("This is a full length sentence that should stand alone.");
+	});
+
+	it("keeps a comma/clause inside one caption", () => {
+		const cues: CaptionCuePayload[] = [
+			{
+				id: "caption-1",
+				startMs: 0,
+				endMs: 1_500,
+				text: "When I'm done, I start.",
+				words: [
+					{ text: "When", startMs: 0, endMs: 300 },
+					{ text: "I'm", startMs: 300, endMs: 600, leadingSpace: true },
+					{ text: "done,", startMs: 600, endMs: 900, leadingSpace: true },
+					{ text: "I", startMs: 900, endMs: 1_100, leadingSpace: true },
+					{ text: "start.", startMs: 1_100, endMs: 1_500, leadingSpace: true },
+				],
+			},
+		];
+		const result = segmentCuesIntoPhrases(cues, []);
+		expect(result).toHaveLength(1);
+		expect(result[0].text).toBe("When I'm done, I start.");
+	});
+
+	it("does not split on an abbreviation, only on the real sentence end", () => {
+		const cues: CaptionCuePayload[] = [
+			{
+				id: "caption-1",
+				startMs: 0,
+				endMs: 1_800,
+				text: "Hello Mr. Smith. How are you?",
+				words: [
+					{ text: "Hello", startMs: 0, endMs: 300 },
+					{ text: "Mr.", startMs: 300, endMs: 600, leadingSpace: true },
+					{ text: "Smith.", startMs: 600, endMs: 900, leadingSpace: true },
+					{ text: "How", startMs: 900, endMs: 1_200, leadingSpace: true },
+					{ text: "are", startMs: 1_200, endMs: 1_500, leadingSpace: true },
+					{ text: "you?", startMs: 1_500, endMs: 1_800, leadingSpace: true },
+				],
+			},
+		];
+		const result = segmentCuesIntoPhrases(cues, []);
+		expect(result).toHaveLength(2);
+		expect(result[0].text).toBe("Hello Mr. Smith.");
+		expect(result[1].text).toBe("How are you?");
+	});
+
+	it("splits on a real pause even without punctuation", () => {
+		const cues: CaptionCuePayload[] = [
+			{
+				id: "caption-1",
+				startMs: 0,
+				endMs: 2_000,
+				text: "one two three four",
+				words: [
+					{ text: "one", startMs: 0, endMs: 300 },
+					{ text: "two", startMs: 300, endMs: 600, leadingSpace: true },
+					// 800ms gap (>= 700ms pause) -> phrase boundary
+					{ text: "three", startMs: 1_400, endMs: 1_700, leadingSpace: true },
+					{ text: "four", startMs: 1_700, endMs: 2_000, leadingSpace: true },
+				],
+			},
+		];
+		const result = segmentCuesIntoPhrases(cues, []);
+		expect(result).toHaveLength(2);
+		expect(result[0].text).toBe("one two");
+		expect(result[1].text).toBe("three four");
+	});
+
+	it("splits on a long acoustic silence in a word gap even when the word gap is short", () => {
+		const cues: CaptionCuePayload[] = [
+			{
+				id: "caption-1",
+				startMs: 0,
+				endMs: 2_600,
+				text: "alpha beta",
+				words: [
+					{ text: "alpha", startMs: 0, endMs: 600 },
+					{ text: "beta", startMs: 2_200, endMs: 2_600, leadingSpace: true },
+				],
+			},
+		];
+		// pauseMs high so the word gap (1600ms) wouldn't split on its own; the 1500ms
+		// silence sitting in the gap is what forces the break. beta starts after the
+		// silence ends, so it is not dropped as a hallucination.
+		const result = segmentCuesIntoPhrases(cues, [{ startMs: 600, endMs: 2_100 }], {
+			pauseMs: 5_000,
+		});
+		expect(result).toHaveLength(2);
+		expect(result[0].text).toBe("alpha");
+		expect(result[1].text).toBe("beta");
+	});
+
+	it("does not split that same input when there is no silence and the gap is below the pause", () => {
+		const cues: CaptionCuePayload[] = [
+			{
+				id: "caption-1",
+				startMs: 0,
+				endMs: 2_600,
+				text: "alpha beta",
+				words: [
+					{ text: "alpha", startMs: 0, endMs: 600 },
+					{ text: "beta", startMs: 2_200, endMs: 2_600, leadingSpace: true },
+				],
+			},
+		];
+		const result = segmentCuesIntoPhrases(cues, [], { pauseMs: 5_000 });
+		expect(result).toHaveLength(1);
+		expect(result[0].text).toBe("alpha beta");
+	});
+
+	it("splits a runaway phrase with no punctuation and no pause via the safety cap", () => {
+		const words = Array.from({ length: 12 }, (_, index) => ({
+			text: `w${index}`,
+			startMs: index * 300,
+			endMs: index * 300 + 300,
+			...(index > 0 ? { leadingSpace: true } : {}),
+		}));
+		const cues: CaptionCuePayload[] = [
+			{
+				id: "caption-1",
+				startMs: 0,
+				endMs: 3_600,
+				text: words.map((w) => w.text).join(" "),
+				words,
+			},
+		];
+		const result = segmentCuesIntoPhrases(cues, [], { maxPhraseMs: 2_000 });
+		expect(result.length).toBeGreaterThan(1);
+		expect(result.map((cue) => cue.text).join(" ")).toBe(words.map((w) => w.text).join(" "));
+	});
+
+	it("drops words that fall entirely inside a long silence (hallucination)", () => {
+		const cues: CaptionCuePayload[] = [
+			{
+				id: "caption-1",
+				startMs: 5_000,
+				endMs: 6_000,
+				text: "Thank you.",
+				words: [
+					{ text: "Thank", startMs: 5_000, endMs: 5_500 },
+					{ text: "you.", startMs: 5_500, endMs: 6_000, leadingSpace: true },
+				],
+			},
+		];
+		const result = segmentCuesIntoPhrases(cues, [{ startMs: 4_000, endMs: 9_000 }]);
+		expect(result).toHaveLength(0);
+	});
+
+	it("produces sorted, non-overlapping cues with sequential ids", () => {
+		const cues: CaptionCuePayload[] = [
+			{
+				id: "caption-1",
+				startMs: 0,
+				endMs: 2_400,
+				text: "First phrase here. Second phrase here. Third one.",
+				words: [
+					{ text: "First", startMs: 0, endMs: 300 },
+					{ text: "phrase", startMs: 300, endMs: 600, leadingSpace: true },
+					{ text: "here.", startMs: 600, endMs: 900, leadingSpace: true },
+					{ text: "Second", startMs: 900, endMs: 1_200, leadingSpace: true },
+					{ text: "phrase", startMs: 1_200, endMs: 1_500, leadingSpace: true },
+					{ text: "here.", startMs: 1_500, endMs: 1_800, leadingSpace: true },
+					{ text: "Third", startMs: 1_800, endMs: 2_100, leadingSpace: true },
+					{ text: "one.", startMs: 2_100, endMs: 2_400, leadingSpace: true },
+				],
+			},
+		];
+		const result = segmentCuesIntoPhrases(cues, []);
+		expect(result).toHaveLength(3);
+		for (let index = 0; index < result.length - 1; index += 1) {
+			expect(result[index].startMs).toBeLessThanOrEqual(result[index + 1].startMs);
+			expect(result[index].endMs).toBeLessThanOrEqual(result[index + 1].startMs);
+		}
+		expect(result.map((cue) => cue.id)).toEqual(
+			result.map((_, index) => `caption-${index + 1}`),
+		);
+	});
+
+	it("keeps a single-sentence word-less cue intact (legacy silence trim preserved)", () => {
+		const cues: CaptionCuePayload[] = [
+			{ id: "caption-1", startMs: 0, endMs: 13_920, text: "Hello, this is a test" },
+		];
+		// One sentence -> unchanged: trim leading silence, keep text (legacy behavior).
+		const result = segmentCuesIntoPhrases(cues, [{ startMs: 0, endMs: 3_000 }]);
+		expect(result).toHaveLength(1);
+		expect(result[0].startMs).toBe(3_000 - 80);
+		expect(result[0].endMs).toBe(13_920 + 80);
+		expect(result[0].text).toBe("Hello, this is a test");
+	});
+
+	it("splits a continuous word-less paragraph into one caption per sentence", () => {
+		// The reported bug: no word timings (SRT path) + continuous speech collapsed into one
+		// caption. It must now split on sentence punctuation even without word timings.
+		const cues: CaptionCuePayload[] = [
+			{ id: "caption-1", startMs: 0, endMs: 3_000, text: "This is one." },
+			{ id: "caption-2", startMs: 3_000, endMs: 6_000, text: "This is two." },
+			{ id: "caption-3", startMs: 6_000, endMs: 9_000, text: "This is three." },
+		];
+		const result = segmentCuesIntoPhrases(cues, []);
+		expect(result).toHaveLength(3);
+		expect(result.map((cue) => cue.text)).toEqual([
+			"This is one.",
+			"This is two.",
+			"This is three.",
+		]);
+		for (let index = 0; index < result.length - 1; index += 1) {
+			expect(result[index].endMs).toBeLessThanOrEqual(result[index + 1].startMs);
+		}
+		expect(result.map((cue) => cue.id)).toEqual(["caption-1", "caption-2", "caption-3"]);
+	});
+
+	it("returns an empty array for empty input", () => {
+		expect(segmentCuesIntoPhrases([], [])).toEqual([]);
+	});
+});

electron/ipc/captions/segment.ts

@@ -0,0 +1,451 @@
+import type { CaptionCuePayload, CaptionWordPayload } from "../types";
+import { buildCaptionTextFromWords } from "./parser";
+import { padSpans, resegmentCuesBySilence, type SilenceInterval } from "./silence";
+
+/**
+ * Phrase-aware caption segmentation.
+ *
+ * Whisper breaks speech on its own internal boundaries — not on sentences — so a single
+ * cue can run two phrases together, and re-segmenting purely on acoustic silence merges
+ * back-to-back sentences and misassigns boundary words. We instead walk Whisper's own
+ * word stream (which carries punctuation) and start a new caption at a real boundary:
+ *   - the end of a sentence (`.`, `?`, `!`, `…`), or
+ *   - a real pause — a large gap between two consecutive words, or a long ffmpeg
+ *     `silencedetect` interval sitting in that gap.
+ *
+ * Because every break happens *between two consecutive words*, a word can never leak into
+ * the wrong caption (the failure mode of center-time region assignment). Commas/clauses
+ * stay inside a caption and a whole sentence is allowed to be one caption; only a high
+ * safety cap splits a runaway phrase with no punctuation and no pause.
+ *
+ * When the transcript has no word timings (SRT fallback) we first re-segment by acoustic
+ * silence (`resegmentCuesBySilence`) and then split each cue on its sentence punctuation, so
+ * a continuous paragraph still becomes one caption per sentence (timing is proportional).
+ */
+
+/** A gap (ms) between two consecutive words this long or longer starts a new phrase. */
+const DEFAULT_PHRASE_PAUSE_MS = 700;
+/** An ffmpeg `silencedetect` interval this long inside a word gap also starts a new phrase. */
+const DEFAULT_SPLIT_SILENCE_MS = 1_500;
+/** Padding (ms) kept around each phrase so captions don't feel clipped. */
+const DEFAULT_EDGE_PAD_MS = 80;
+/** Safety cap: a phrase with no sentence end and no pause is split once it gets this long. */
+const DEFAULT_MAX_PHRASE_MS = 12_000;
+
+/**
+ * A caption shorter than this (ms) is "too quick" and may be merged with an adjacent short
+ * caption so rapid-fire one-word sentences ("Okay." "Great.") don't each flash by alone.
+ */
+const DEFAULT_MIN_CAPTION_MS = 800;
+/** Only merge short captions separated by at most this gap (ms) — never across a real pause. */
+const DEFAULT_MERGE_GAP_MS = 400;
+/** A merged short-caption run never grows past this duration (ms) or character count. */
+const DEFAULT_MAX_MERGED_MS = 2_500;
+const DEFAULT_MAX_MERGED_CHARS = 80;
+
+export interface SegmentOptions {
+	/** Word gap (ms) that splits one phrase into two. Lower = more, shorter captions. */
+	pauseMs?: number;
+	/** Minimum acoustic silence (ms) inside a word gap that also splits a phrase. */
+	splitSilenceMs?: number;
+	/** Padding (ms) kept around each phrase. */
+	edgePadMs?: number;
+	/** Safety cap (ms) that splits a punctuation-less, pause-less runaway phrase. */
+	maxPhraseMs?: number;
+	/** A caption shorter than this (ms) may be merged with an adjacent short caption. */
+	minCaptionMs?: number;
+	/** Only merge short captions separated by at most this gap (ms). */
+	mergeGapMs?: number;
+	/** A merged short-caption run never grows past this duration (ms). */
+	maxMergedMs?: number;
+	/** A merged short-caption run never grows past this character count. */
+	maxMergedChars?: number;
+}
+
+interface CaptionPiece {
+	startMs: number;
+	endMs: number;
+	text: string;
+	words: CaptionWordPayload[];
+}
+
+const SENTENCE_END = /[.?!…。！？]$/;
+/** Closing quotes/brackets that can trail terminal punctuation, e.g. `said."` */
+const TRAILING_CLOSERS = /[)\]}"'”’»」』）】］｝>]+$/u;
+
+/**
+ * Unambiguous English titles that take a trailing period mid-sentence. Kept deliberately
+ * short: only words that are never themselves a sentence (so we don't suppress a real
+ * break — e.g. "no" is excluded because "No." is a valid sentence). Dotted initialisms
+ * like "e.g."/"U.S."/"a.m." are handled by the regex below, not this list. `?`/`!`/`…`
+ * always end a sentence. For non-English audio this simply never matches.
+ */
+const ABBREVIATIONS = new Set(["mr", "mrs", "ms", "dr", "prof", "sr", "jr", "st", "vs", "etc"]);
+
+/** A trailing period belongs to an abbreviation/initialism rather than ending a sentence. */
+function isAbbreviation(text: string): boolean {
+	const trimmed = text.trim().replace(TRAILING_CLOSERS, "").trim();
+	if (!trimmed.endsWith(".")) {
+		return false; // only a plain period can be an abbreviation marker
+	}
+	const core = trimmed.slice(0, -1).toLowerCase();
+	if (core.length === 0) {
+		return false;
+	}
+	// Single-letter initial ("J.", "U.") or dotted initialism ("U.S.", "e.g.", "a.m.").
+	if (/^[a-z]$/.test(core) || /^[a-z](\.[a-z])+$/.test(core)) {
+		return true;
+	}
+	return ABBREVIATIONS.has(core);
+}
+
+/**
+ * True when a word's text ends a sentence, ignoring trailing closing quotes/brackets and
+ * common abbreviations (so "Mr. Smith" or "e.g." don't start a new caption).
+ */
+export function endsSentence(text: string): boolean {
+	const trimmed = text.trim().replace(TRAILING_CLOSERS, "").trim();
+	if (!SENTENCE_END.test(trimmed)) {
+		return false;
+	}
+	return !isAbbreviation(text);
+}
+
+/** Every cue carries usable word timing, so we can segment on the word stream. */
+function hasWordTimings(cues: CaptionCuePayload[]): boolean {
+	return cues.length > 0 && cues.every((cue) => Array.isArray(cue.words) && cue.words.length > 0);
+}
+
+/** Flatten all cues' words into one time-ordered stream, spacing across cue joins. */
+function flattenWords(cues: CaptionCuePayload[]): CaptionWordPayload[] {
+	const stream: CaptionWordPayload[] = [];
+	for (const cue of cues) {
+		const words = (cue.words ?? []) as CaptionWordPayload[];
+		words.forEach((word, index) => {
+			// A new cue continues the speech, so its first word leads with a space.
+			const leadingSpace =
+				stream.length > 0 && (index === 0 ? true : word.leadingSpace !== false);
+			stream.push({
+				text: word.text,
+				startMs: word.startMs,
+				endMs: word.endMs,
+				...(leadingSpace ? { leadingSpace: true } : {}),
+			});
+		});
+	}
+	return stream.sort((left, right) => left.startMs - right.startMs || left.endMs - right.endMs);
+}
+
+/** Drop words that sit entirely inside a long detected silence (Whisper hallucinations). */
+function dropHallucinations(
+	words: CaptionWordPayload[],
+	silences: SilenceInterval[],
+	splitSilenceMs: number,
+): CaptionWordPayload[] {
+	const longSilences = silences.filter(
+		(silence) => silence.endMs - silence.startMs >= splitSilenceMs,
+	);
+	if (longSilences.length === 0) {
+		return words;
+	}
+	return words.filter(
+		(word) =>
+			!longSilences.some(
+				(silence) => silence.startMs <= word.startMs && word.endMs <= silence.endMs,
+			),
+	);
+}
+
+/** A long silence interval overlaps the gap between two consecutive words. */
+function silenceInGap(
+	gapStartMs: number,
+	gapEndMs: number,
+	silences: SilenceInterval[],
+	splitSilenceMs: number,
+): boolean {
+	return silences.some(
+		(silence) =>
+			silence.endMs - silence.startMs >= splitSilenceMs &&
+			silence.startMs < gapEndMs &&
+			silence.endMs > gapStartMs,
+	);
+}
+
+/** Reset the first word's leading space so a phrase reads as its own line. */
+function normalizePhraseWords(words: CaptionWordPayload[]): CaptionWordPayload[] {
+	return words.map((word, index) => {
+		if (index === 0 && word.leadingSpace) {
+			const { leadingSpace: _leadingSpace, ...rest } = word;
+			return rest;
+		}
+		return word;
+	});
+}
+
+/** Group a cue's words into runs that each end on a sentence boundary. */
+function groupWordsBySentence(words: CaptionWordPayload[]): CaptionWordPayload[][] {
+	const groups: CaptionWordPayload[][] = [];
+	let current: CaptionWordPayload[] = [];
+	words.forEach((word, index) => {
+		current.push(word);
+		if (endsSentence(word.text) && index < words.length - 1) {
+			groups.push(current);
+			current = [];
+		}
+	});
+	if (current.length > 0) {
+		groups.push(current);
+	}
+	return groups;
+}
+
+/**
+ * Split a word-less cue's text into one cue per sentence, distributing the cue's time span
+ * across sentences by character length. Used on the fallback (no word timing) path so a
+ * continuous paragraph still becomes one caption per sentence.
+ */
+function splitTextBySentence(cue: CaptionCuePayload): CaptionCuePayload[] {
+	const tokens = cue.text.trim().split(/\s+/).filter(Boolean);
+	if (tokens.length <= 1) {
+		return [cue];
+	}
+
+	const groups: string[][] = [];
+	let current: string[] = [];
+	tokens.forEach((token, index) => {
+		current.push(token);
+		if (endsSentence(token) && index < tokens.length - 1) {
+			groups.push(current);
+			current = [];
+		}
+	});
+	if (current.length > 0) {
+		groups.push(current);
+	}
+	if (groups.length <= 1) {
+		return [cue];
+	}
+
+	const texts = groups.map((group) => group.join(" "));
+	const totalChars = texts.reduce((sum, text) => sum + text.length, 0) || 1;
+	const spanMs = Math.max(1, cue.endMs - cue.startMs);
+	let cursorMs = cue.startMs;
+	return texts.map((text, index) => {
+		const startMs = cursorMs;
+		const endMs =
+			index === texts.length - 1
+				? cue.endMs
+				: Math.min(
+						cue.endMs - 1,
+						Math.round(startMs + (spanMs * text.length) / totalChars),
+					);
+		cursorMs = Math.max(startMs + 1, endMs);
+		return { id: cue.id, startMs, endMs: Math.max(startMs + 1, endMs), text };
+	});
+}
+
+/** Split one re-segmented cue into one cue per sentence (by words if present, else text). */
+function splitCueBySentences(cue: CaptionCuePayload): CaptionCuePayload[] {
+	const words = Array.isArray(cue.words) ? (cue.words as CaptionWordPayload[]) : [];
+	if (words.length === 0) {
+		return splitTextBySentence(cue);
+	}
+
+	const groups = groupWordsBySentence(words);
+	if (groups.length <= 1) {
+		return [cue];
+	}
+	return groups.map((group) => {
+		const phraseWords = normalizePhraseWords(group);
+		return {
+			id: cue.id,
+			startMs: phraseWords[0].startMs,
+			endMs: phraseWords[phraseWords.length - 1].endMs,
+			text: buildCaptionTextFromWords(phraseWords),
+			words: phraseWords,
+		};
+	});
+}
+
+/** Concatenate two adjacent cues into one, joining words (with a space) when both have them. */
+function mergeTwoCues(left: CaptionCuePayload, right: CaptionCuePayload): CaptionCuePayload {
+	const leftWords = Array.isArray(left.words) ? (left.words as CaptionWordPayload[]) : [];
+	const rightWords = Array.isArray(right.words) ? (right.words as CaptionWordPayload[]) : [];
+	if (leftWords.length > 0 && rightWords.length > 0) {
+		// The right cue's first word started its own phrase (no leading space) — restore it.
+		const joined = normalizePhraseWords([
+			...leftWords,
+			...rightWords.map((word, index) =>
+				index === 0 ? { ...word, leadingSpace: true } : word,
+			),
+		]);
+		return {
+			id: left.id,
+			startMs: left.startMs,
+			endMs: right.endMs,
+			text: buildCaptionTextFromWords(joined),
+			words: joined,
+		};
+	}
+	return {
+		id: left.id,
+		startMs: left.startMs,
+		endMs: right.endMs,
+		text: `${left.text} ${right.text}`.trim(),
+	};
+}
+
+interface MergeOptions {
+	minCaptionMs: number;
+	mergeGapMs: number;
+	maxMergedMs: number;
+	maxMergedChars: number;
+}
+
+/**
+ * Merge adjacent captions that are BOTH short and rapid-fire (tiny gap), so quick one-word
+ * sentences like "Okay." "Great." read as one caption instead of flashing by individually.
+ * Only merges when both sides are short, so a short caption never absorbs a full-length one,
+ * and never across a real pause or past the size caps.
+ */
+function mergeShortAdjacentCaptions(
+	cues: CaptionCuePayload[],
+	options: MergeOptions,
+): CaptionCuePayload[] {
+	if (cues.length <= 1) {
+		return cues;
+	}
+
+	const merged: CaptionCuePayload[] = [];
+	let group = cues[0];
+	for (let index = 1; index < cues.length; index += 1) {
+		const next = cues[index];
+		const groupDurationMs = group.endMs - group.startMs;
+		const nextDurationMs = next.endMs - next.startMs;
+		const gapMs = next.startMs - group.endMs;
+		const combinedDurationMs = next.endMs - group.startMs;
+		const combinedChars = group.text.length + next.text.length + 1;
+
+		const canMerge =
+			groupDurationMs < options.minCaptionMs &&
+			nextDurationMs < options.minCaptionMs &&
+			gapMs <= options.mergeGapMs &&
+			combinedDurationMs <= options.maxMergedMs &&
+			combinedChars <= options.maxMergedChars;
+
+		if (canMerge) {
+			group = mergeTwoCues(group, next);
+		} else {
+			merged.push(group);
+			group = next;
+		}
+	}
+	merged.push(group);
+	return merged;
+}
+
+/** Assign sequential, stable ids to the final cue list. */
+function renumberCues(cues: CaptionCuePayload[]): CaptionCuePayload[] {
+	return cues.map((cue, index) => ({ ...cue, id: `caption-${index + 1}` }));
+}
+
+/**
+ * Re-segment Whisper cues into one caption per sentence/phrase, then merge rapid-fire short
+ * sentences back together. Returns sorted, non-overlapping cues with fresh ids. Falls back to
+ * silence-only re-segmentation (plus sentence splitting) when the transcript has no word timings.
+ */
+export function segmentCuesIntoPhrases(
+	cues: CaptionCuePayload[],
+	silences: SilenceInterval[],
+	options: SegmentOptions = {},
+): CaptionCuePayload[] {
+	const pauseMs = options.pauseMs ?? DEFAULT_PHRASE_PAUSE_MS;
+	const splitSilenceMs = options.splitSilenceMs ?? DEFAULT_SPLIT_SILENCE_MS;
+	const edgePadMs = options.edgePadMs ?? DEFAULT_EDGE_PAD_MS;
+	const maxPhraseMs = options.maxPhraseMs ?? DEFAULT_MAX_PHRASE_MS;
+	const mergeOptions: MergeOptions = {
+		minCaptionMs: options.minCaptionMs ?? DEFAULT_MIN_CAPTION_MS,
+		mergeGapMs: options.mergeGapMs ?? DEFAULT_MERGE_GAP_MS,
+		maxMergedMs: options.maxMergedMs ?? DEFAULT_MAX_MERGED_MS,
+		maxMergedChars: options.maxMergedChars ?? DEFAULT_MAX_MERGED_CHARS,
+	};
+
+	if (cues.length === 0) {
+		return [];
+	}
+
+	// No word timings (SRT path): the silence-only segmenter trims/merges by acoustic
+	// silence but can't see sentence boundaries, so a continuous paragraph would collapse
+	// into one caption. Re-segment by silence first, then split each cue on its sentence
+	// punctuation so we still get one caption per sentence.
+	if (!hasWordTimings(cues)) {
+		const base = resegmentCuesBySilence(cues, silences, { splitSilenceMs, edgePadMs });
+		const sentences = base.flatMap(splitCueBySentences);
+		return renumberCues(mergeShortAdjacentCaptions(sentences, mergeOptions));
+	}
+
+	const sortedCues = [...cues].sort(
+		(left, right) => left.startMs - right.startMs || left.endMs - right.endMs,
+	);
+	const stream = dropHallucinations(flattenWords(sortedCues), silences, splitSilenceMs);
+	if (stream.length === 0) {
+		return [];
+	}
+
+	const phrases: CaptionWordPayload[][] = [];
+	let current: CaptionWordPayload[] = [];
+
+	for (let index = 0; index < stream.length; index += 1) {
+		const word = stream[index];
+		current.push(word);
+
+		const next = stream[index + 1];
+		if (!next) {
+			break;
+		}
+
+		const gapMs = next.startMs - word.endMs;
+		const phraseDurationMs = word.endMs - current[0].startMs;
+		const shouldBreak =
+			endsSentence(word.text) ||
+			gapMs >= pauseMs ||
+			silenceInGap(word.endMs, next.startMs, silences, splitSilenceMs) ||
+			phraseDurationMs >= maxPhraseMs;
+
+		if (shouldBreak) {
+			phrases.push(current);
+			current = [];
+		}
+	}
+	if (current.length > 0) {
+		phrases.push(current);
+	}
+
+	const pieces: CaptionPiece[] = phrases
+		.map((words) => {
+			const phraseWords = normalizePhraseWords(words);
+			return {
+				startMs: phraseWords[0].startMs,
+				endMs: phraseWords[phraseWords.length - 1].endMs,
+				text: buildCaptionTextFromWords(phraseWords),
+				words: phraseWords,
+			};
+		})
+		.filter((piece) => piece.text.trim().length > 0);
+
+	if (pieces.length === 0) {
+		return [];
+	}
+
+	pieces.sort((left, right) => left.startMs - right.startMs || left.endMs - right.endMs);
+	padSpans(pieces, edgePadMs);
+
+	const sentenceCues: CaptionCuePayload[] = pieces.map((piece) => ({
+		id: "",
+		startMs: piece.startMs,
+		endMs: piece.endMs,
+		text: piece.text,
+		...(piece.words.length > 0 ? { words: piece.words } : {}),
+	}));
+	return renumberCues(mergeShortAdjacentCaptions(sentenceCues, mergeOptions));
+}

electron/ipc/captions/silence.ts

@@ -171,7 +171,7 @@ function splitTextProportionally(text: string, overlaps: Span[]): string[] {
 }
 
 /** Pad spans toward neighbors by at most half the silent gap, so cues never overlap. */
-function padSpans(spans: Span[], edgePadMs: number): void {
+export function padSpans(spans: Span[], edgePadMs: number): void {
 	for (let index = 0; index < spans.length; index += 1) {
 		const prevEndMs = index > 0 ? spans[index - 1].endMs : null;
 		const nextStartMs = index < spans.length - 1 ? spans[index + 1].startMs : null;

src/components/video-editor/captionLayout.test.ts

@@ -0,0 +1,41 @@
+import { describe, expect, it } from "vitest";
+import { flattenCaptionWords } from "./captionLayout";
+import type { CaptionCue } from "./types";
+
+describe("flattenCaptionWords", () => {
+	it("forces a break at every cue boundary so each phrase shows on its own", () => {
+		const cues: CaptionCue[] = [
+			{
+				id: "a",
+				startMs: 0,
+				endMs: 1_000,
+				text: "hello world",
+				words: [
+					{ text: "hello", startMs: 0, endMs: 500 },
+					{ text: "world", startMs: 500, endMs: 1_000, leadingSpace: true },
+				],
+			},
+			{
+				// back-to-back with cue "a" (no gap) — would previously be re-packed by width
+				id: "b",
+				startMs: 1_000,
+				endMs: 2_000,
+				text: "next one",
+				words: [
+					{ text: "next", startMs: 1_000, endMs: 1_500 },
+					{ text: "one", startMs: 1_500, endMs: 2_000, leadingSpace: true },
+				],
+			},
+		];
+
+		const flattened = flattenCaptionWords(cues);
+		const firstWordOfSecondCue = flattened.find(
+			(word) => word.cueId === "b" && word.cueWordIndex === 0,
+		);
+
+		expect(firstWordOfSecondCue?.forcedBreakBefore).toBe(true);
+		expect(firstWordOfSecondCue?.leadingSpace).toBe(false);
+		// the very first word of the first cue never forces a break
+		expect(flattened[0].forcedBreakBefore).toBe(false);
+	});
+});

src/components/video-editor/captionLayout.ts

@@ -179,8 +179,10 @@ export function flattenCaptionWords(cues: CaptionCue[]) {
 		const cueDuration = Math.max(1, cue.endMs - cue.startMs);
 		const fallbackWordDuration = cueDuration / sourceWords.length;
 		const previousCue = cueIndex > 0 ? cues[cueIndex - 1] : null;
-		const shouldForceCueBreak =
-			previousCue !== null && cue.startMs - previousCue.endMs >= CAPTION_BLOCK_GAP_BREAK_MS;
+		// Each cue is its own phrase, so always start a new line/page at a cue boundary.
+		// Otherwise back-to-back phrases (a small gap) get re-packed together by width and
+		// their boundary disappears on screen — we want one phrase shown at a time.
+		const shouldForceCueBreak = previousCue !== null;
 
 		sourceWords.forEach((word, wordIndex) => {
 			const fallbackStartMs = cue.startMs + fallbackWordDuration * wordIndex;