diff --git a/electron/ipc/captions/generate.ts b/electron/ipc/captions/generate.ts index 2cab7319..038d1f87 100644 --- a/electron/ipc/captions/generate.ts +++ b/electron/ipc/captions/generate.ts @@ -9,9 +9,9 @@ import { getBundledWhisperExecutableCandidates } from "../paths/binaries"; import { resolveRecordingSession } from "../project/session"; import { normalizeVideoSourcePath } from "../utils"; import { parseSrtCues, parseWhisperJsonCues, shouldRetryWhisperWithoutJson } from "./parser"; +import { segmentCuesIntoPhrases } from "./segment"; import { parseSilenceIntervals, - resegmentCuesBySilence, SILENCE_DETECT_MIN_S, SILENCE_NOISE_DB, type SilenceInterval, @@ -259,19 +259,28 @@ export async function generateAutoCaptionsFromVideo(options: { const timedCues = jsonEnabled ? parseWhisperJsonCues(await fs.readFile(jsonPath, "utf-8")) : []; + if (jsonEnabled && timedCues.length === 0) { + // JSON ran but yielded no word-timed cues (empty/malformed output). We fall back + // to SRT, which has no word timings — captions are then split by sentence text and + // silence rather than precise word timing. Surface it for diagnosis. + console.warn( + "[auto-captions] Whisper JSON produced no word-timed cues; falling back to SRT (no word timings).", + ); + } const cues = timedCues.length > 0 ? timedCues : parseSrtCues(await fs.readFile(srtPath, "utf-8")); if (cues.length === 0) { throw new Error("Whisper completed, but no caption cues were produced."); } - // Whisper cues span silence and don't break on pauses. Re-segment them against - // ground-truth acoustic silence (ffmpeg `silencedetect`) so captions only cover - // real speech. Failure here must not block caption generation — fall back to raw. + // Whisper cues run sentences together and don't break on pauses. Re-segment them + // into one caption per sentence/phrase using Whisper's own word stream (punctuation + // + pauses), backed by ground-truth acoustic silence (ffmpeg `silencedetect`). + // Failure here must not block caption generation — fall back to raw. let cuesToReturn = cues; try { const silences = await detectSilenceIntervals({ ffmpegPath, wavPath }); - const resegmented = resegmentCuesBySilence(cues, silences); + const resegmented = segmentCuesIntoPhrases(cues, silences); if (resegmented.length > 0) { cuesToReturn = resegmented; } diff --git a/electron/ipc/captions/segment.test.ts b/electron/ipc/captions/segment.test.ts new file mode 100644 index 00000000..94344d59 --- /dev/null +++ b/electron/ipc/captions/segment.test.ts @@ -0,0 +1,347 @@ +import { describe, expect, it } from "vitest"; +import type { CaptionCuePayload } from "../types"; +import { endsSentence, segmentCuesIntoPhrases } from "./segment"; + +describe("endsSentence", () => { + it("detects terminal punctuation, ignoring trailing closers", () => { + expect(endsSentence("world.")).toBe(true); + expect(endsSentence("you?")).toBe(true); + expect(endsSentence("stop!")).toBe(true); + expect(endsSentence("wait…")).toBe(true); + expect(endsSentence('said."')).toBe(true); + expect(endsSentence("done,")).toBe(false); + expect(endsSentence("hello")).toBe(false); + }); + + it("does not treat common abbreviations or initialisms as a sentence end", () => { + expect(endsSentence("Mr.")).toBe(false); + expect(endsSentence("Dr.")).toBe(false); + expect(endsSentence("etc.")).toBe(false); + expect(endsSentence("e.g.")).toBe(false); + expect(endsSentence("U.S.")).toBe(false); + expect(endsSentence("J.")).toBe(false); + // real sentence ends are unaffected + expect(endsSentence("Smith.")).toBe(true); + expect(endsSentence("5.")).toBe(true); + // `?`/`!` after an abbreviation-like token still end the sentence + expect(endsSentence("really?")).toBe(true); + }); +}); + +describe("segmentCuesIntoPhrases", () => { + it("splits back-to-back sentences with no pause into separate captions", () => { + const cues: CaptionCuePayload[] = [ + { + id: "caption-1", + startMs: 0, + endMs: 1_800, + text: "Hello world. How are you?", + words: [ + { text: "Hello", startMs: 0, endMs: 400 }, + { text: "world.", startMs: 400, endMs: 800, leadingSpace: true }, + { text: "How", startMs: 800, endMs: 1_100, leadingSpace: true }, + { text: "are", startMs: 1_100, endMs: 1_400, leadingSpace: true }, + { text: "you?", startMs: 1_400, endMs: 1_800, leadingSpace: true }, + ], + }, + ]; + const result = segmentCuesIntoPhrases(cues, []); + expect(result).toHaveLength(2); + expect(result[0].text).toBe("Hello world."); + expect(result[1].text).toBe("How are you?"); + expect(result[0].endMs).toBeLessThanOrEqual(result[1].startMs); + }); + + it("does not leak the first word of the next sentence into the previous caption", () => { + const cues: CaptionCuePayload[] = [ + { + id: "caption-1", + startMs: 0, + endMs: 1_800, + text: "Hello world. How are you?", + words: [ + { text: "Hello", startMs: 0, endMs: 400 }, + { text: "world.", startMs: 400, endMs: 800, leadingSpace: true }, + { text: "How", startMs: 800, endMs: 1_100, leadingSpace: true }, + { text: "are", startMs: 1_100, endMs: 1_400, leadingSpace: true }, + { text: "you?", startMs: 1_400, endMs: 1_800, leadingSpace: true }, + ], + }, + ]; + const result = segmentCuesIntoPhrases(cues, []); + expect(result[1].words?.[0].text).toBe("How"); + expect(result[1].words?.[0].startMs).toBe(800); + // the first word of a phrase has no leading space + expect(result[1].words?.[0].leadingSpace).toBeUndefined(); + }); + + it("merges rapid-fire short sentences into one caption (word-timed)", () => { + const cues: CaptionCuePayload[] = [ + { + id: "caption-1", + startMs: 0, + endMs: 800, + text: "Okay. Great.", + words: [ + { text: "Okay.", startMs: 0, endMs: 400 }, + { text: "Great.", startMs: 400, endMs: 800, leadingSpace: true }, + ], + }, + ]; + const result = segmentCuesIntoPhrases(cues, []); + expect(result).toHaveLength(1); + expect(result[0].text).toBe("Okay. Great."); + expect(result[0].words?.map((word) => word.text)).toEqual(["Okay.", "Great."]); + }); + + it("merges rapid-fire short sentences into one caption (word-less)", () => { + const cues: CaptionCuePayload[] = [ + { id: "caption-1", startMs: 0, endMs: 400, text: "Okay." }, + { id: "caption-2", startMs: 400, endMs: 800, text: "Great." }, + ]; + const result = segmentCuesIntoPhrases(cues, []); + expect(result).toHaveLength(1); + expect(result[0].text).toBe("Okay. Great."); + }); + + it("does not merge a short sentence into a full-length one", () => { + const cues: CaptionCuePayload[] = [ + { + id: "caption-1", + startMs: 0, + endMs: 2_400, + text: "Okay. This is a full length sentence that should stand alone.", + words: [ + { text: "Okay.", startMs: 0, endMs: 400 }, + { text: "This", startMs: 400, endMs: 700, leadingSpace: true }, + { text: "is", startMs: 700, endMs: 900, leadingSpace: true }, + { text: "a", startMs: 900, endMs: 1_050, leadingSpace: true }, + { text: "full", startMs: 1_050, endMs: 1_350, leadingSpace: true }, + { text: "length", startMs: 1_350, endMs: 1_650, leadingSpace: true }, + { text: "sentence", startMs: 1_650, endMs: 1_950, leadingSpace: true }, + { text: "that", startMs: 1_950, endMs: 2_100, leadingSpace: true }, + { text: "should", startMs: 2_100, endMs: 2_250, leadingSpace: true }, + { text: "stand", startMs: 2_250, endMs: 2_350, leadingSpace: true }, + { text: "alone.", startMs: 2_350, endMs: 2_400, leadingSpace: true }, + ], + }, + ]; + const result = segmentCuesIntoPhrases(cues, []); + expect(result).toHaveLength(2); + expect(result[0].text).toBe("Okay."); + expect(result[1].text).toBe("This is a full length sentence that should stand alone."); + }); + + it("keeps a comma/clause inside one caption", () => { + const cues: CaptionCuePayload[] = [ + { + id: "caption-1", + startMs: 0, + endMs: 1_500, + text: "When I'm done, I start.", + words: [ + { text: "When", startMs: 0, endMs: 300 }, + { text: "I'm", startMs: 300, endMs: 600, leadingSpace: true }, + { text: "done,", startMs: 600, endMs: 900, leadingSpace: true }, + { text: "I", startMs: 900, endMs: 1_100, leadingSpace: true }, + { text: "start.", startMs: 1_100, endMs: 1_500, leadingSpace: true }, + ], + }, + ]; + const result = segmentCuesIntoPhrases(cues, []); + expect(result).toHaveLength(1); + expect(result[0].text).toBe("When I'm done, I start."); + }); + + it("does not split on an abbreviation, only on the real sentence end", () => { + const cues: CaptionCuePayload[] = [ + { + id: "caption-1", + startMs: 0, + endMs: 1_800, + text: "Hello Mr. Smith. How are you?", + words: [ + { text: "Hello", startMs: 0, endMs: 300 }, + { text: "Mr.", startMs: 300, endMs: 600, leadingSpace: true }, + { text: "Smith.", startMs: 600, endMs: 900, leadingSpace: true }, + { text: "How", startMs: 900, endMs: 1_200, leadingSpace: true }, + { text: "are", startMs: 1_200, endMs: 1_500, leadingSpace: true }, + { text: "you?", startMs: 1_500, endMs: 1_800, leadingSpace: true }, + ], + }, + ]; + const result = segmentCuesIntoPhrases(cues, []); + expect(result).toHaveLength(2); + expect(result[0].text).toBe("Hello Mr. Smith."); + expect(result[1].text).toBe("How are you?"); + }); + + it("splits on a real pause even without punctuation", () => { + const cues: CaptionCuePayload[] = [ + { + id: "caption-1", + startMs: 0, + endMs: 2_000, + text: "one two three four", + words: [ + { text: "one", startMs: 0, endMs: 300 }, + { text: "two", startMs: 300, endMs: 600, leadingSpace: true }, + // 800ms gap (>= 700ms pause) -> phrase boundary + { text: "three", startMs: 1_400, endMs: 1_700, leadingSpace: true }, + { text: "four", startMs: 1_700, endMs: 2_000, leadingSpace: true }, + ], + }, + ]; + const result = segmentCuesIntoPhrases(cues, []); + expect(result).toHaveLength(2); + expect(result[0].text).toBe("one two"); + expect(result[1].text).toBe("three four"); + }); + + it("splits on a long acoustic silence in a word gap even when the word gap is short", () => { + const cues: CaptionCuePayload[] = [ + { + id: "caption-1", + startMs: 0, + endMs: 2_600, + text: "alpha beta", + words: [ + { text: "alpha", startMs: 0, endMs: 600 }, + { text: "beta", startMs: 2_200, endMs: 2_600, leadingSpace: true }, + ], + }, + ]; + // pauseMs high so the word gap (1600ms) wouldn't split on its own; the 1500ms + // silence sitting in the gap is what forces the break. beta starts after the + // silence ends, so it is not dropped as a hallucination. + const result = segmentCuesIntoPhrases(cues, [{ startMs: 600, endMs: 2_100 }], { + pauseMs: 5_000, + }); + expect(result).toHaveLength(2); + expect(result[0].text).toBe("alpha"); + expect(result[1].text).toBe("beta"); + }); + + it("does not split that same input when there is no silence and the gap is below the pause", () => { + const cues: CaptionCuePayload[] = [ + { + id: "caption-1", + startMs: 0, + endMs: 2_600, + text: "alpha beta", + words: [ + { text: "alpha", startMs: 0, endMs: 600 }, + { text: "beta", startMs: 2_200, endMs: 2_600, leadingSpace: true }, + ], + }, + ]; + const result = segmentCuesIntoPhrases(cues, [], { pauseMs: 5_000 }); + expect(result).toHaveLength(1); + expect(result[0].text).toBe("alpha beta"); + }); + + it("splits a runaway phrase with no punctuation and no pause via the safety cap", () => { + const words = Array.from({ length: 12 }, (_, index) => ({ + text: `w${index}`, + startMs: index * 300, + endMs: index * 300 + 300, + ...(index > 0 ? { leadingSpace: true } : {}), + })); + const cues: CaptionCuePayload[] = [ + { + id: "caption-1", + startMs: 0, + endMs: 3_600, + text: words.map((w) => w.text).join(" "), + words, + }, + ]; + const result = segmentCuesIntoPhrases(cues, [], { maxPhraseMs: 2_000 }); + expect(result.length).toBeGreaterThan(1); + expect(result.map((cue) => cue.text).join(" ")).toBe(words.map((w) => w.text).join(" ")); + }); + + it("drops words that fall entirely inside a long silence (hallucination)", () => { + const cues: CaptionCuePayload[] = [ + { + id: "caption-1", + startMs: 5_000, + endMs: 6_000, + text: "Thank you.", + words: [ + { text: "Thank", startMs: 5_000, endMs: 5_500 }, + { text: "you.", startMs: 5_500, endMs: 6_000, leadingSpace: true }, + ], + }, + ]; + const result = segmentCuesIntoPhrases(cues, [{ startMs: 4_000, endMs: 9_000 }]); + expect(result).toHaveLength(0); + }); + + it("produces sorted, non-overlapping cues with sequential ids", () => { + const cues: CaptionCuePayload[] = [ + { + id: "caption-1", + startMs: 0, + endMs: 2_400, + text: "First phrase here. Second phrase here. Third one.", + words: [ + { text: "First", startMs: 0, endMs: 300 }, + { text: "phrase", startMs: 300, endMs: 600, leadingSpace: true }, + { text: "here.", startMs: 600, endMs: 900, leadingSpace: true }, + { text: "Second", startMs: 900, endMs: 1_200, leadingSpace: true }, + { text: "phrase", startMs: 1_200, endMs: 1_500, leadingSpace: true }, + { text: "here.", startMs: 1_500, endMs: 1_800, leadingSpace: true }, + { text: "Third", startMs: 1_800, endMs: 2_100, leadingSpace: true }, + { text: "one.", startMs: 2_100, endMs: 2_400, leadingSpace: true }, + ], + }, + ]; + const result = segmentCuesIntoPhrases(cues, []); + expect(result).toHaveLength(3); + for (let index = 0; index < result.length - 1; index += 1) { + expect(result[index].startMs).toBeLessThanOrEqual(result[index + 1].startMs); + expect(result[index].endMs).toBeLessThanOrEqual(result[index + 1].startMs); + } + expect(result.map((cue) => cue.id)).toEqual( + result.map((_, index) => `caption-${index + 1}`), + ); + }); + + it("keeps a single-sentence word-less cue intact (legacy silence trim preserved)", () => { + const cues: CaptionCuePayload[] = [ + { id: "caption-1", startMs: 0, endMs: 13_920, text: "Hello, this is a test" }, + ]; + // One sentence -> unchanged: trim leading silence, keep text (legacy behavior). + const result = segmentCuesIntoPhrases(cues, [{ startMs: 0, endMs: 3_000 }]); + expect(result).toHaveLength(1); + expect(result[0].startMs).toBe(3_000 - 80); + expect(result[0].endMs).toBe(13_920 + 80); + expect(result[0].text).toBe("Hello, this is a test"); + }); + + it("splits a continuous word-less paragraph into one caption per sentence", () => { + // The reported bug: no word timings (SRT path) + continuous speech collapsed into one + // caption. It must now split on sentence punctuation even without word timings. + const cues: CaptionCuePayload[] = [ + { id: "caption-1", startMs: 0, endMs: 3_000, text: "This is one." }, + { id: "caption-2", startMs: 3_000, endMs: 6_000, text: "This is two." }, + { id: "caption-3", startMs: 6_000, endMs: 9_000, text: "This is three." }, + ]; + const result = segmentCuesIntoPhrases(cues, []); + expect(result).toHaveLength(3); + expect(result.map((cue) => cue.text)).toEqual([ + "This is one.", + "This is two.", + "This is three.", + ]); + for (let index = 0; index < result.length - 1; index += 1) { + expect(result[index].endMs).toBeLessThanOrEqual(result[index + 1].startMs); + } + expect(result.map((cue) => cue.id)).toEqual(["caption-1", "caption-2", "caption-3"]); + }); + + it("returns an empty array for empty input", () => { + expect(segmentCuesIntoPhrases([], [])).toEqual([]); + }); +}); diff --git a/electron/ipc/captions/segment.ts b/electron/ipc/captions/segment.ts new file mode 100644 index 00000000..912f089c --- /dev/null +++ b/electron/ipc/captions/segment.ts @@ -0,0 +1,451 @@ +import type { CaptionCuePayload, CaptionWordPayload } from "../types"; +import { buildCaptionTextFromWords } from "./parser"; +import { padSpans, resegmentCuesBySilence, type SilenceInterval } from "./silence"; + +/** + * Phrase-aware caption segmentation. + * + * Whisper breaks speech on its own internal boundaries — not on sentences — so a single + * cue can run two phrases together, and re-segmenting purely on acoustic silence merges + * back-to-back sentences and misassigns boundary words. We instead walk Whisper's own + * word stream (which carries punctuation) and start a new caption at a real boundary: + * - the end of a sentence (`.`, `?`, `!`, `…`), or + * - a real pause — a large gap between two consecutive words, or a long ffmpeg + * `silencedetect` interval sitting in that gap. + * + * Because every break happens *between two consecutive words*, a word can never leak into + * the wrong caption (the failure mode of center-time region assignment). Commas/clauses + * stay inside a caption and a whole sentence is allowed to be one caption; only a high + * safety cap splits a runaway phrase with no punctuation and no pause. + * + * When the transcript has no word timings (SRT fallback) we first re-segment by acoustic + * silence (`resegmentCuesBySilence`) and then split each cue on its sentence punctuation, so + * a continuous paragraph still becomes one caption per sentence (timing is proportional). + */ + +/** A gap (ms) between two consecutive words this long or longer starts a new phrase. */ +const DEFAULT_PHRASE_PAUSE_MS = 700; +/** An ffmpeg `silencedetect` interval this long inside a word gap also starts a new phrase. */ +const DEFAULT_SPLIT_SILENCE_MS = 1_500; +/** Padding (ms) kept around each phrase so captions don't feel clipped. */ +const DEFAULT_EDGE_PAD_MS = 80; +/** Safety cap: a phrase with no sentence end and no pause is split once it gets this long. */ +const DEFAULT_MAX_PHRASE_MS = 12_000; + +/** + * A caption shorter than this (ms) is "too quick" and may be merged with an adjacent short + * caption so rapid-fire one-word sentences ("Okay." "Great.") don't each flash by alone. + */ +const DEFAULT_MIN_CAPTION_MS = 800; +/** Only merge short captions separated by at most this gap (ms) — never across a real pause. */ +const DEFAULT_MERGE_GAP_MS = 400; +/** A merged short-caption run never grows past this duration (ms) or character count. */ +const DEFAULT_MAX_MERGED_MS = 2_500; +const DEFAULT_MAX_MERGED_CHARS = 80; + +export interface SegmentOptions { + /** Word gap (ms) that splits one phrase into two. Lower = more, shorter captions. */ + pauseMs?: number; + /** Minimum acoustic silence (ms) inside a word gap that also splits a phrase. */ + splitSilenceMs?: number; + /** Padding (ms) kept around each phrase. */ + edgePadMs?: number; + /** Safety cap (ms) that splits a punctuation-less, pause-less runaway phrase. */ + maxPhraseMs?: number; + /** A caption shorter than this (ms) may be merged with an adjacent short caption. */ + minCaptionMs?: number; + /** Only merge short captions separated by at most this gap (ms). */ + mergeGapMs?: number; + /** A merged short-caption run never grows past this duration (ms). */ + maxMergedMs?: number; + /** A merged short-caption run never grows past this character count. */ + maxMergedChars?: number; +} + +interface CaptionPiece { + startMs: number; + endMs: number; + text: string; + words: CaptionWordPayload[]; +} + +const SENTENCE_END = /[.?!…。！？]$/; +/** Closing quotes/brackets that can trail terminal punctuation, e.g. `said."` */ +const TRAILING_CLOSERS = /[)\]}"'”’»」』）】］｝>]+$/u; + +/** + * Unambiguous English titles that take a trailing period mid-sentence. Kept deliberately + * short: only words that are never themselves a sentence (so we don't suppress a real + * break — e.g. "no" is excluded because "No." is a valid sentence). Dotted initialisms + * like "e.g."/"U.S."/"a.m." are handled by the regex below, not this list. `?`/`!`/`…` + * always end a sentence. For non-English audio this simply never matches. + */ +const ABBREVIATIONS = new Set(["mr", "mrs", "ms", "dr", "prof", "sr", "jr", "st", "vs", "etc"]); + +/** A trailing period belongs to an abbreviation/initialism rather than ending a sentence. */ +function isAbbreviation(text: string): boolean { + const trimmed = text.trim().replace(TRAILING_CLOSERS, "").trim(); + if (!trimmed.endsWith(".")) { + return false; // only a plain period can be an abbreviation marker + } + const core = trimmed.slice(0, -1).toLowerCase(); + if (core.length === 0) { + return false; + } + // Single-letter initial ("J.", "U.") or dotted initialism ("U.S.", "e.g.", "a.m."). + if (/^[a-z]$/.test(core) || /^[a-z](\.[a-z])+$/.test(core)) { + return true; + } + return ABBREVIATIONS.has(core); +} + +/** + * True when a word's text ends a sentence, ignoring trailing closing quotes/brackets and + * common abbreviations (so "Mr. Smith" or "e.g." don't start a new caption). + */ +export function endsSentence(text: string): boolean { + const trimmed = text.trim().replace(TRAILING_CLOSERS, "").trim(); + if (!SENTENCE_END.test(trimmed)) { + return false; + } + return !isAbbreviation(text); +} + +/** Every cue carries usable word timing, so we can segment on the word stream. */ +function hasWordTimings(cues: CaptionCuePayload[]): boolean { + return cues.length > 0 && cues.every((cue) => Array.isArray(cue.words) && cue.words.length > 0); +} + +/** Flatten all cues' words into one time-ordered stream, spacing across cue joins. */ +function flattenWords(cues: CaptionCuePayload[]): CaptionWordPayload[] { + const stream: CaptionWordPayload[] = []; + for (const cue of cues) { + const words = (cue.words ?? []) as CaptionWordPayload[]; + words.forEach((word, index) => { + // A new cue continues the speech, so its first word leads with a space. + const leadingSpace = + stream.length > 0 && (index === 0 ? true : word.leadingSpace !== false); + stream.push({ + text: word.text, + startMs: word.startMs, + endMs: word.endMs, + ...(leadingSpace ? { leadingSpace: true } : {}), + }); + }); + } + return stream.sort((left, right) => left.startMs - right.startMs || left.endMs - right.endMs); +} + +/** Drop words that sit entirely inside a long detected silence (Whisper hallucinations). */ +function dropHallucinations( + words: CaptionWordPayload[], + silences: SilenceInterval[], + splitSilenceMs: number, +): CaptionWordPayload[] { + const longSilences = silences.filter( + (silence) => silence.endMs - silence.startMs >= splitSilenceMs, + ); + if (longSilences.length === 0) { + return words; + } + return words.filter( + (word) => + !longSilences.some( + (silence) => silence.startMs <= word.startMs && word.endMs <= silence.endMs, + ), + ); +} + +/** A long silence interval overlaps the gap between two consecutive words. */ +function silenceInGap( + gapStartMs: number, + gapEndMs: number, + silences: SilenceInterval[], + splitSilenceMs: number, +): boolean { + return silences.some( + (silence) => + silence.endMs - silence.startMs >= splitSilenceMs && + silence.startMs < gapEndMs && + silence.endMs > gapStartMs, + ); +} + +/** Reset the first word's leading space so a phrase reads as its own line. */ +function normalizePhraseWords(words: CaptionWordPayload[]): CaptionWordPayload[] { + return words.map((word, index) => { + if (index === 0 && word.leadingSpace) { + const { leadingSpace: _leadingSpace, ...rest } = word; + return rest; + } + return word; + }); +} + +/** Group a cue's words into runs that each end on a sentence boundary. */ +function groupWordsBySentence(words: CaptionWordPayload[]): CaptionWordPayload[][] { + const groups: CaptionWordPayload[][] = []; + let current: CaptionWordPayload[] = []; + words.forEach((word, index) => { + current.push(word); + if (endsSentence(word.text) && index < words.length - 1) { + groups.push(current); + current = []; + } + }); + if (current.length > 0) { + groups.push(current); + } + return groups; +} + +/** + * Split a word-less cue's text into one cue per sentence, distributing the cue's time span + * across sentences by character length. Used on the fallback (no word timing) path so a + * continuous paragraph still becomes one caption per sentence. + */ +function splitTextBySentence(cue: CaptionCuePayload): CaptionCuePayload[] { + const tokens = cue.text.trim().split(/\s+/).filter(Boolean); + if (tokens.length <= 1) { + return [cue]; + } + + const groups: string[][] = []; + let current: string[] = []; + tokens.forEach((token, index) => { + current.push(token); + if (endsSentence(token) && index < tokens.length - 1) { + groups.push(current); + current = []; + } + }); + if (current.length > 0) { + groups.push(current); + } + if (groups.length <= 1) { + return [cue]; + } + + const texts = groups.map((group) => group.join(" ")); + const totalChars = texts.reduce((sum, text) => sum + text.length, 0) || 1; + const spanMs = Math.max(1, cue.endMs - cue.startMs); + let cursorMs = cue.startMs; + return texts.map((text, index) => { + const startMs = cursorMs; + const endMs = + index === texts.length - 1 + ? cue.endMs + : Math.min( + cue.endMs - 1, + Math.round(startMs + (spanMs * text.length) / totalChars), + ); + cursorMs = Math.max(startMs + 1, endMs); + return { id: cue.id, startMs, endMs: Math.max(startMs + 1, endMs), text }; + }); +} + +/** Split one re-segmented cue into one cue per sentence (by words if present, else text). */ +function splitCueBySentences(cue: CaptionCuePayload): CaptionCuePayload[] { + const words = Array.isArray(cue.words) ? (cue.words as CaptionWordPayload[]) : []; + if (words.length === 0) { + return splitTextBySentence(cue); + } + + const groups = groupWordsBySentence(words); + if (groups.length <= 1) { + return [cue]; + } + return groups.map((group) => { + const phraseWords = normalizePhraseWords(group); + return { + id: cue.id, + startMs: phraseWords[0].startMs, + endMs: phraseWords[phraseWords.length - 1].endMs, + text: buildCaptionTextFromWords(phraseWords), + words: phraseWords, + }; + }); +} + +/** Concatenate two adjacent cues into one, joining words (with a space) when both have them. */ +function mergeTwoCues(left: CaptionCuePayload, right: CaptionCuePayload): CaptionCuePayload { + const leftWords = Array.isArray(left.words) ? (left.words as CaptionWordPayload[]) : []; + const rightWords = Array.isArray(right.words) ? (right.words as CaptionWordPayload[]) : []; + if (leftWords.length > 0 && rightWords.length > 0) { + // The right cue's first word started its own phrase (no leading space) — restore it. + const joined = normalizePhraseWords([ + ...leftWords, + ...rightWords.map((word, index) => + index === 0 ? { ...word, leadingSpace: true } : word, + ), + ]); + return { + id: left.id, + startMs: left.startMs, + endMs: right.endMs, + text: buildCaptionTextFromWords(joined), + words: joined, + }; + } + return { + id: left.id, + startMs: left.startMs, + endMs: right.endMs, + text: `${left.text} ${right.text}`.trim(), + }; +} + +interface MergeOptions { + minCaptionMs: number; + mergeGapMs: number; + maxMergedMs: number; + maxMergedChars: number; +} + +/** + * Merge adjacent captions that are BOTH short and rapid-fire (tiny gap), so quick one-word + * sentences like "Okay." "Great." read as one caption instead of flashing by individually. + * Only merges when both sides are short, so a short caption never absorbs a full-length one, + * and never across a real pause or past the size caps. + */ +function mergeShortAdjacentCaptions( + cues: CaptionCuePayload[], + options: MergeOptions, +): CaptionCuePayload[] { + if (cues.length <= 1) { + return cues; + } + + const merged: CaptionCuePayload[] = []; + let group = cues[0]; + for (let index = 1; index < cues.length; index += 1) { + const next = cues[index]; + const groupDurationMs = group.endMs - group.startMs; + const nextDurationMs = next.endMs - next.startMs; + const gapMs = next.startMs - group.endMs; + const combinedDurationMs = next.endMs - group.startMs; + const combinedChars = group.text.length + next.text.length + 1; + + const canMerge = + groupDurationMs < options.minCaptionMs && + nextDurationMs < options.minCaptionMs && + gapMs <= options.mergeGapMs && + combinedDurationMs <= options.maxMergedMs && + combinedChars <= options.maxMergedChars; + + if (canMerge) { + group = mergeTwoCues(group, next); + } else { + merged.push(group); + group = next; + } + } + merged.push(group); + return merged; +} + +/** Assign sequential, stable ids to the final cue list. */ +function renumberCues(cues: CaptionCuePayload[]): CaptionCuePayload[] { + return cues.map((cue, index) => ({ ...cue, id: `caption-${index + 1}` })); +} + +/** + * Re-segment Whisper cues into one caption per sentence/phrase, then merge rapid-fire short + * sentences back together. Returns sorted, non-overlapping cues with fresh ids. Falls back to + * silence-only re-segmentation (plus sentence splitting) when the transcript has no word timings. + */ +export function segmentCuesIntoPhrases( + cues: CaptionCuePayload[], + silences: SilenceInterval[], + options: SegmentOptions = {}, +): CaptionCuePayload[] { + const pauseMs = options.pauseMs ?? DEFAULT_PHRASE_PAUSE_MS; + const splitSilenceMs = options.splitSilenceMs ?? DEFAULT_SPLIT_SILENCE_MS; + const edgePadMs = options.edgePadMs ?? DEFAULT_EDGE_PAD_MS; + const maxPhraseMs = options.maxPhraseMs ?? DEFAULT_MAX_PHRASE_MS; + const mergeOptions: MergeOptions = { + minCaptionMs: options.minCaptionMs ?? DEFAULT_MIN_CAPTION_MS, + mergeGapMs: options.mergeGapMs ?? DEFAULT_MERGE_GAP_MS, + maxMergedMs: options.maxMergedMs ?? DEFAULT_MAX_MERGED_MS, + maxMergedChars: options.maxMergedChars ?? DEFAULT_MAX_MERGED_CHARS, + }; + + if (cues.length === 0) { + return []; + } + + // No word timings (SRT path): the silence-only segmenter trims/merges by acoustic + // silence but can't see sentence boundaries, so a continuous paragraph would collapse + // into one caption. Re-segment by silence first, then split each cue on its sentence + // punctuation so we still get one caption per sentence. + if (!hasWordTimings(cues)) { + const base = resegmentCuesBySilence(cues, silences, { splitSilenceMs, edgePadMs }); + const sentences = base.flatMap(splitCueBySentences); + return renumberCues(mergeShortAdjacentCaptions(sentences, mergeOptions)); + } + + const sortedCues = [...cues].sort( + (left, right) => left.startMs - right.startMs || left.endMs - right.endMs, + ); + const stream = dropHallucinations(flattenWords(sortedCues), silences, splitSilenceMs); + if (stream.length === 0) { + return []; + } + + const phrases: CaptionWordPayload[][] = []; + let current: CaptionWordPayload[] = []; + + for (let index = 0; index < stream.length; index += 1) { + const word = stream[index]; + current.push(word); + + const next = stream[index + 1]; + if (!next) { + break; + } + + const gapMs = next.startMs - word.endMs; + const phraseDurationMs = word.endMs - current[0].startMs; + const shouldBreak = + endsSentence(word.text) || + gapMs >= pauseMs || + silenceInGap(word.endMs, next.startMs, silences, splitSilenceMs) || + phraseDurationMs >= maxPhraseMs; + + if (shouldBreak) { + phrases.push(current); + current = []; + } + } + if (current.length > 0) { + phrases.push(current); + } + + const pieces: CaptionPiece[] = phrases + .map((words) => { + const phraseWords = normalizePhraseWords(words); + return { + startMs: phraseWords[0].startMs, + endMs: phraseWords[phraseWords.length - 1].endMs, + text: buildCaptionTextFromWords(phraseWords), + words: phraseWords, + }; + }) + .filter((piece) => piece.text.trim().length > 0); + + if (pieces.length === 0) { + return []; + } + + pieces.sort((left, right) => left.startMs - right.startMs || left.endMs - right.endMs); + padSpans(pieces, edgePadMs); + + const sentenceCues: CaptionCuePayload[] = pieces.map((piece) => ({ + id: "", + startMs: piece.startMs, + endMs: piece.endMs, + text: piece.text, + ...(piece.words.length > 0 ? { words: piece.words } : {}), + })); + return renumberCues(mergeShortAdjacentCaptions(sentenceCues, mergeOptions)); +} diff --git a/electron/ipc/captions/silence.ts b/electron/ipc/captions/silence.ts index 2ddb7faa..c8750ca1 100644 --- a/electron/ipc/captions/silence.ts +++ b/electron/ipc/captions/silence.ts @@ -171,7 +171,7 @@ function splitTextProportionally(text: string, overlaps: Span[]): string[] { } /** Pad spans toward neighbors by at most half the silent gap, so cues never overlap. */ -function padSpans(spans: Span[], edgePadMs: number): void { +export function padSpans(spans: Span[], edgePadMs: number): void { for (let index = 0; index < spans.length; index += 1) { const prevEndMs = index > 0 ? spans[index - 1].endMs : null; const nextStartMs = index < spans.length - 1 ? spans[index + 1].startMs : null; diff --git a/src/components/video-editor/captionLayout.test.ts b/src/components/video-editor/captionLayout.test.ts new file mode 100644 index 00000000..9cdc733b --- /dev/null +++ b/src/components/video-editor/captionLayout.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vitest"; +import { flattenCaptionWords } from "./captionLayout"; +import type { CaptionCue } from "./types"; + +describe("flattenCaptionWords", () => { + it("forces a break at every cue boundary so each phrase shows on its own", () => { + const cues: CaptionCue[] = [ + { + id: "a", + startMs: 0, + endMs: 1_000, + text: "hello world", + words: [ + { text: "hello", startMs: 0, endMs: 500 }, + { text: "world", startMs: 500, endMs: 1_000, leadingSpace: true }, + ], + }, + { + // back-to-back with cue "a" (no gap) — would previously be re-packed by width + id: "b", + startMs: 1_000, + endMs: 2_000, + text: "next one", + words: [ + { text: "next", startMs: 1_000, endMs: 1_500 }, + { text: "one", startMs: 1_500, endMs: 2_000, leadingSpace: true }, + ], + }, + ]; + + const flattened = flattenCaptionWords(cues); + const firstWordOfSecondCue = flattened.find( + (word) => word.cueId === "b" && word.cueWordIndex === 0, + ); + + expect(firstWordOfSecondCue?.forcedBreakBefore).toBe(true); + expect(firstWordOfSecondCue?.leadingSpace).toBe(false); + // the very first word of the first cue never forces a break + expect(flattened[0].forcedBreakBefore).toBe(false); + }); +}); diff --git a/src/components/video-editor/captionLayout.ts b/src/components/video-editor/captionLayout.ts index 8b2faad2..adacaae6 100644 --- a/src/components/video-editor/captionLayout.ts +++ b/src/components/video-editor/captionLayout.ts @@ -179,8 +179,10 @@ export function flattenCaptionWords(cues: CaptionCue[]) { const cueDuration = Math.max(1, cue.endMs - cue.startMs); const fallbackWordDuration = cueDuration / sourceWords.length; const previousCue = cueIndex > 0 ? cues[cueIndex - 1] : null; - const shouldForceCueBreak = - previousCue !== null && cue.startMs - previousCue.endMs >= CAPTION_BLOCK_GAP_BREAK_MS; + // Each cue is its own phrase, so always start a new line/page at a cue boundary. + // Otherwise back-to-back phrases (a small gap) get re-packed together by width and + // their boundary disappears on screen — we want one phrase shown at a time. + const shouldForceCueBreak = previousCue !== null; sourceWords.forEach((word, wordIndex) => { const fallbackStartMs = cue.startMs + fallbackWordDuration * wordIndex;