From 4d51b177ae6323ebd96ea7dfc9e720aae9952ecc Mon Sep 17 00:00:00 2001 From: webadderall <131426131+webadderall@users.noreply.github.com> Date: Sun, 22 Mar 2026 18:32:55 +1100 Subject: [PATCH] feat: add caption layout and styling engine Introduce the timed caption block layout engine used by preview and export rendering, including line breaking, page building, animation state, width-relative scaling, and merged-cue spacing fixes. --- src/components/video-editor/captionLayout.ts | 463 +++++++++++++++++++ src/components/video-editor/captionStyle.ts | 68 +++ 2 files changed, 531 insertions(+) create mode 100644 src/components/video-editor/captionLayout.ts create mode 100644 src/components/video-editor/captionStyle.ts diff --git a/src/components/video-editor/captionLayout.ts b/src/components/video-editor/captionLayout.ts new file mode 100644 index 00000000..ca23ac1c --- /dev/null +++ b/src/components/video-editor/captionLayout.ts @@ -0,0 +1,463 @@ +import type { + AutoCaptionAnimation, + AutoCaptionSettings, + CaptionCue, + CaptionCueWord, +} from "./types"; + +export type CaptionWordState = "spoken" | "active" | "upcoming"; + +export interface CaptionWordLayout { + text: string; + index: number; + forcedBreakBefore: boolean; + leadingSpace: boolean; + startMs: number; + endMs: number; + state: CaptionWordState; +} + +export interface CaptionLineLayout { + words: CaptionWordLayout[]; + width: number; + startWordIndex: number; + endWordIndex: number; +} + +interface CaptionPageLayout { + lines: CaptionLineLayout[]; + startMs: number; + endMs: number; +} + +export interface ActiveCaptionLayout { + cue: CaptionCue; + blockKey: string; + visibleLines: CaptionLineLayout[]; + hasWordTimings: boolean; + activeWordIndex: number; + activeWordProgress: number; + visiblePageIndex: number; + opacity: number; + translateY: number; + scale: number; +} + +type CaptionSourceWord = { + cueId: string; + text: string; + forcedBreakBefore: boolean; + leadingSpace?: boolean; + startMs?: number; + endMs?: number; +}; + +const CAPTION_ENTER_MS = 180; +const CAPTION_EXIT_MS = 140; +const CAPTION_BLOCK_GAP_BREAK_MS = 500; + +function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)); +} + +function clamp01(value: number) { + return clamp(value, 0, 1); +} + +function splitCaptionWordsFromText(text: string) { + const sourceLines = text + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line.length > 0); + const words: CaptionSourceWord[] = []; + + sourceLines.forEach((line, lineIndex) => { + line.split(/\s+/).filter(Boolean).forEach((word, wordIndex) => { + words.push({ + cueId: "", + text: word, + forcedBreakBefore: lineIndex > 0 && wordIndex === 0, + }); + }); + }); + + return words; +} + +function splitCaptionWords(cue: CaptionCue) { + if (Array.isArray(cue.words) && cue.words.length > 0) { + return cue.words + .filter((word): word is CaptionCueWord => Boolean(word && typeof word.text === "string")) + .map((word) => ({ + cueId: cue.id, + text: word.text.trim(), + forcedBreakBefore: false, + leadingSpace: Boolean(word.leadingSpace), + startMs: word.startMs, + endMs: word.endMs, + })) + .filter((word) => word.text.length > 0); + } + + return splitCaptionWordsFromText(cue.text); +} + +function getActiveCaptionCue(cues: CaptionCue[], timeMs: number) { + for (const cue of cues) { + if (timeMs >= cue.startMs && timeMs <= cue.endMs) { + return cue; + } + } + + return null; +} + +function flattenCaptionWords(cues: CaptionCue[]) { + const flattened: Array<{ + cueId: string; + text: string; + forcedBreakBefore: boolean; + leadingSpace: boolean; + startMs: number; + endMs: number; + hasRealTiming: boolean; + }> = []; + + cues.forEach((cue, cueIndex) => { + const sourceWords = splitCaptionWords(cue); + if (sourceWords.length === 0) { + return; + } + + const cueHasRealWordTimings = sourceWords.every( + (word) => isFiniteNumber(word.startMs) && isFiniteNumber(word.endMs) && word.endMs > word.startMs, + ); + const cueDuration = Math.max(1, cue.endMs - cue.startMs); + const fallbackWordDuration = cueDuration / sourceWords.length; + const previousCue = cueIndex > 0 ? cues[cueIndex - 1] : null; + const shouldForceCueBreak = + previousCue !== null && cue.startMs - previousCue.endMs >= CAPTION_BLOCK_GAP_BREAK_MS; + + sourceWords.forEach((word, wordIndex) => { + const fallbackStartMs = cue.startMs + fallbackWordDuration * wordIndex; + const fallbackEndMs = + wordIndex === sourceWords.length - 1 + ? cue.endMs + : cue.startMs + fallbackWordDuration * (wordIndex + 1); + const startsMergedCue = wordIndex === 0 && !word.forcedBreakBefore && !shouldForceCueBreak; + const leadingSpace = wordIndex === 0 + ? flattened.length > 0 && startsMergedCue + : (word.leadingSpace ?? true); + + flattened.push({ + cueId: cue.id, + text: word.text, + forcedBreakBefore: + word.forcedBreakBefore || (wordIndex === 0 && shouldForceCueBreak), + leadingSpace, + startMs: cueHasRealWordTimings ? word.startMs! : fallbackStartMs, + endMs: cueHasRealWordTimings ? word.endMs! : Math.max(fallbackStartMs + 1, fallbackEndMs), + hasRealTiming: cueHasRealWordTimings, + }); + }); + }); + + return flattened; +} + +function getCaptionAnimationState( + animationStyle: AutoCaptionAnimation, + enterProgress: number, + exitProgress: number, +) { + const visibility = Math.min(enterProgress, exitProgress); + + switch (animationStyle) { + case "none": + return { + opacity: 1, + translateY: 0, + scale: 1, + }; + case "fade": + return { + opacity: 0.3 + visibility * 0.7, + translateY: 0, + scale: 1, + }; + case "rise": + return { + opacity: 0.25 + visibility * 0.75, + translateY: (1 - visibility) * 18, + scale: 0.985 + visibility * 0.015, + }; + case "pop": + default: + return { + opacity: 0.35 + visibility * 0.65, + translateY: (1 - visibility) * 8, + scale: 0.94 + visibility * 0.06, + }; + } +} + +function buildCaptionPages(options: { + lines: CaptionLineLayout[]; + words: CaptionWordLayout[]; + maxRows: number; + hasWordTimings: boolean; + cue: CaptionCue; +}) { + const pages: CaptionPageLayout[] = []; + const totalPages = Math.max(1, Math.ceil(options.lines.length / options.maxRows)); + let lineIndex = 0; + let remainingLines = options.lines.length; + + for (let pageIndex = 0; pageIndex < totalPages; pageIndex += 1) { + const remainingPages = totalPages - pageIndex; + const pageSize = Math.min( + options.maxRows, + Math.max(1, Math.ceil(remainingLines / remainingPages)), + ); + const pageLines = options.lines.slice(lineIndex, lineIndex + pageSize); + if (pageLines.length === 0) { + continue; + } + + lineIndex += pageLines.length; + remainingLines -= pageLines.length; + + if (options.hasWordTimings) { + const firstWord = options.words[pageLines[0].startWordIndex]; + const lastWord = options.words[pageLines[pageLines.length - 1].endWordIndex]; + pages.push({ + lines: pageLines, + startMs: firstWord?.startMs ?? options.cue.startMs, + endMs: lastWord?.endMs ?? options.cue.endMs, + }); + continue; + } + + pages.push({ + lines: pageLines, + startMs: 0, + endMs: 0, + }); + } + + if (!options.hasWordTimings) { + const totalWords = Math.max(1, options.words.length); + const cueDuration = Math.max(1, options.cue.endMs - options.cue.startMs); + let elapsedMs = options.cue.startMs; + + pages.forEach((page, pageIndex) => { + const pageWordCount = page.lines.reduce((count, line) => count + line.words.length, 0); + const proportionalDuration = Math.round((cueDuration * pageWordCount) / totalWords); + const nextElapsed = pageIndex === pages.length - 1 + ? options.cue.endMs + : Math.min(options.cue.endMs, elapsedMs + Math.max(1, proportionalDuration)); + + page.startMs = elapsedMs; + page.endMs = Math.max(elapsedMs + 1, nextElapsed); + elapsedMs = page.endMs; + }); + } + + for (let index = 0; index < pages.length - 1; index += 1) { + pages[index].endMs = Math.max(pages[index].startMs + 1, pages[index + 1].startMs); + } + + if (pages.length > 0) { + pages[pages.length - 1].endMs = Math.max(pages[pages.length - 1].startMs + 1, options.cue.endMs); + } + + return pages; +} + +function buildCaptionLines(options: { + words: CaptionWordLayout[]; + maxWidthPx: number; + measureText: (text: string) => number; +}) { + const wordCount = options.words.length; + const segmentWidths = options.words.map((word, index) => { + if (index === 0) { + return options.measureText(word.text); + } + + return options.measureText(`${word.leadingSpace ? " " : ""}${word.text}`); + }); + const plainWordWidths = options.words.map((word) => options.measureText(word.text)); + const bestCost = new Array(wordCount + 1).fill(Number.POSITIVE_INFINITY); + const nextBreak = new Array(wordCount).fill(wordCount); + + bestCost[wordCount] = 0; + + for (let startIndex = wordCount - 1; startIndex >= 0; startIndex -= 1) { + let lineWidth = plainWordWidths[startIndex]; + + for (let endIndex = startIndex; endIndex < wordCount; endIndex += 1) { + if (endIndex > startIndex) { + if (options.words[endIndex].forcedBreakBefore) { + break; + } + + lineWidth += segmentWidths[endIndex]; + } + + if (lineWidth > options.maxWidthPx && endIndex > startIndex) { + break; + } + + const slack = Math.max(0, options.maxWidthPx - lineWidth); + const fullness = options.maxWidthPx <= 0 ? 1 : lineWidth / options.maxWidthPx; + const isLastLine = endIndex === wordCount - 1; + const linePenalty = isLastLine + ? slack * slack * 0.18 + : slack * slack + (fullness < 0.72 ? (0.72 - fullness) * 26000 : 0); + const candidateCost = linePenalty + bestCost[endIndex + 1]; + + if (candidateCost < bestCost[startIndex]) { + bestCost[startIndex] = candidateCost; + nextBreak[startIndex] = endIndex + 1; + } + } + } + + const lines: CaptionLineLayout[] = []; + let startIndex = 0; + + while (startIndex < wordCount) { + const endIndexExclusive = Math.max(startIndex + 1, nextBreak[startIndex]); + const lineWords = options.words.slice(startIndex, endIndexExclusive).map((word, index) => ({ + ...word, + leadingSpace: index === 0 ? false : word.leadingSpace, + })); + const width = lineWords.reduce( + (total, word, index) => total + options.measureText(`${index === 0 ? "" : word.leadingSpace ? " " : ""}${word.text}`), + 0, + ); + + lines.push({ + words: lineWords, + width, + startWordIndex: lineWords[0].index, + endWordIndex: lineWords[lineWords.length - 1].index, + }); + + startIndex = endIndexExclusive; + } + + return lines; +} + +function getVisibleCaptionPageIndex(pages: CaptionPageLayout[], timeMs: number) { + for (let index = 0; index < pages.length; index += 1) { + if (timeMs >= pages[index].startMs && timeMs <= pages[index].endMs) { + return index; + } + } + + return -1; +} + +export function buildActiveCaptionLayout(options: { + cues: CaptionCue[]; + timeMs: number; + settings: AutoCaptionSettings; + maxWidthPx: number; + measureText: (text: string) => number; +}) { + const sourceWords = flattenCaptionWords(options.cues); + if (sourceWords.length === 0) { + return null; + } + + const hasWordTimings = sourceWords.every((word) => word.hasRealTiming); + + let activeWordIndex = -1; + if (hasWordTimings) { + activeWordIndex = sourceWords.findIndex((word) => options.timeMs >= word.startMs && options.timeMs < word.endMs); + if (activeWordIndex < 0) { + activeWordIndex = sourceWords.findIndex((word) => options.timeMs < word.startMs); + activeWordIndex = activeWordIndex < 0 ? sourceWords.length - 1 : clamp(activeWordIndex - 1, 0, sourceWords.length - 1); + } + } + const maxRows = clamp(Math.round(options.settings.maxRows || 1), 1, 4); + + const words: CaptionWordLayout[] = sourceWords.map((word, index) => { + return { + text: word.text, + index, + forcedBreakBefore: word.forcedBreakBefore, + leadingSpace: word.leadingSpace, + startMs: word.startMs, + endMs: word.endMs, + state: !hasWordTimings + ? "spoken" + : index < activeWordIndex + ? "spoken" + : index === activeWordIndex + ? "active" + : "upcoming", + }; + }); + + const lines = buildCaptionLines({ + words, + maxWidthPx: options.maxWidthPx, + measureText: options.measureText, + }); + + const pages = buildCaptionPages({ + lines, + words, + maxRows, + hasWordTimings: true, + cue: { + id: sourceWords[0].cueId, + startMs: sourceWords[0].startMs, + endMs: sourceWords[sourceWords.length - 1].endMs, + text: "", + }, + }); + const visiblePageIndex = getVisibleCaptionPageIndex(pages, options.timeMs); + if (visiblePageIndex < 0) { + return null; + } + const visiblePage = pages[visiblePageIndex] ?? null; + const visibleLines = visiblePage?.lines ?? lines.slice(0, maxRows); + const activeWord = activeWordIndex >= 0 ? words[activeWordIndex] : null; + const activeWordProgress = activeWord + ? clamp01((options.timeMs - activeWord.startMs) / Math.max(1, activeWord.endMs - activeWord.startMs)) + : 0; + const animationStartMs = visiblePage?.startMs ?? sourceWords[0].startMs; + const animationEndMs = visiblePage?.endMs ?? sourceWords[sourceWords.length - 1].endMs; + const pageStartWordIndex = visibleLines[0]?.startWordIndex ?? 0; + const pageCueId = sourceWords[pageStartWordIndex]?.cueId ?? sourceWords[0].cueId; + const activeCue = getActiveCaptionCue(options.cues, options.timeMs) ?? options.cues.find((candidate) => candidate.id === pageCueId) ?? options.cues[0]; + const enterProgress = clamp01((options.timeMs - animationStartMs) / CAPTION_ENTER_MS); + const exitProgress = clamp01((animationEndMs - options.timeMs) / CAPTION_EXIT_MS); + const animation = getCaptionAnimationState( + options.settings.animationStyle, + enterProgress, + exitProgress, + ); + + return { + cue: activeCue, + blockKey: `${Math.round(animationStartMs)}-${Math.round(animationEndMs)}`, + visibleLines, + hasWordTimings, + activeWordIndex, + activeWordProgress, + visiblePageIndex, + opacity: animation.opacity, + translateY: animation.translateY, + scale: animation.scale, + } satisfies ActiveCaptionLayout; +} + +function isFiniteNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value); +} \ No newline at end of file diff --git a/src/components/video-editor/captionStyle.ts b/src/components/video-editor/captionStyle.ts new file mode 100644 index 00000000..47953092 --- /dev/null +++ b/src/components/video-editor/captionStyle.ts @@ -0,0 +1,68 @@ +import type { CaptionWordState } from "./captionLayout"; +import { DEFAULT_AUTO_CAPTION_SETTINGS } from "./types"; + +export const CAPTION_FONT_WEIGHT = 400; +export const CAPTION_LINE_HEIGHT = 1.32; + +const DEFAULT_CAPTION_REFERENCE_WIDTH = + 1920 * (DEFAULT_AUTO_CAPTION_SETTINGS.maxWidth / 100); + +export function getCaptionTargetWidth(containerWidth: number, maxWidthPercent: number) { + return Math.max(1, containerWidth * (maxWidthPercent / 100)); +} + +export function getCaptionScaledFontSize( + fontSize: number, + containerWidth: number, + maxWidthPercent: number, +) { + return Math.max( + 14, + fontSize * (getCaptionTargetWidth(containerWidth, maxWidthPercent) / DEFAULT_CAPTION_REFERENCE_WIDTH), + ); +} + +export function getCaptionPadding(fontSize: number) { + return { + x: fontSize * 1.1, + y: fontSize * 0.78, + }; +} + +export function getCaptionScaledRadius(radius: number, fontSize: number) { + const baseline = Math.max(1, DEFAULT_AUTO_CAPTION_SETTINGS.fontSize); + return Math.max(0, radius * (fontSize / baseline)); +} + +export function getCaptionTextMaxWidth(containerWidth: number, maxWidthPercent: number, fontSize: number) { + const padding = getCaptionPadding(fontSize); + return Math.max(fontSize * 4, getCaptionTargetWidth(containerWidth, maxWidthPercent) - padding.x * 2); +} + +export function getCaptionWordVisualState(hasWordTimings: boolean, state: CaptionWordState) { + if (!hasWordTimings) { + return { + isInactive: false, + opacity: 1, + }; + } + + switch (state) { + case "upcoming": + return { + isInactive: true, + opacity: 0.82, + }; + case "spoken": + return { + isInactive: false, + opacity: 0.72, + }; + case "active": + default: + return { + isInactive: false, + opacity: 1, + }; + } +} \ No newline at end of file