From 97a76ee34fb2ffab4c90068d7d687e1d78700348 Mon Sep 17 00:00:00 2001 From: webadderall <131426131+webadderall@users.noreply.github.com> Date: Sun, 22 Mar 2026 18:33:27 +1100 Subject: [PATCH] feat: add caption controls to the settings panel Add caption generation controls, caption appearance settings, cursor style selection, and related editor controls to the settings sidebar. --- src/components/video-editor/SettingsPanel.tsx | 493 +++++++++++++++++- 1 file changed, 472 insertions(+), 21 deletions(-) diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx index 8c1cf2a8..be5806f8 100644 --- a/src/components/video-editor/SettingsPanel.tsx +++ b/src/components/video-editor/SettingsPanel.tsx @@ -1,4 +1,6 @@ import { LayoutGroup } from "motion/react"; +import minimalCursorUrl from "../../../Minimal Cursor.svg"; +import tahoeCursorUrl from "../../assets/cursors/Cursor=Default.svg"; import { Palette, Trash2, @@ -9,7 +11,15 @@ import { AnimatePresence, motion } from "motion/react"; import { useEffect, useMemo, useRef, useState } from "react"; import { toast } from "sonner"; import { Button } from "@/components/ui/button"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; import { Switch } from "@/components/ui/switch"; +import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group"; import { getAssetPath, getRenderableAssetUrl } from "@/lib/assetPath"; import { cn } from "@/lib/utils"; import { BUILT_IN_WALLPAPERS, getAvailableWallpapers } from "@/lib/wallpapers"; @@ -20,16 +30,22 @@ import { AnnotationSettingsPanel } from "./AnnotationSettingsPanel"; import { loadEditorPreferences, saveEditorPreferences } from "./editorPreferences"; import { SliderControl } from "./SliderControl"; import type { + AutoCaptionAnimation, + AutoCaptionSettings, + CaptionCue, AnnotationRegion, AnnotationType, CropRegion, + CursorStyle, FigureData, PlaybackSpeed, WebcamPositionPreset, WebcamOverlaySettings, ZoomDepth, + ZoomTransitionEasing, } from "./types"; import { + DEFAULT_AUTO_CAPTION_SETTINGS, DEFAULT_WEBCAM_CORNER_RADIUS, DEFAULT_WEBCAM_MARGIN, DEFAULT_WEBCAM_POSITION_PRESET, @@ -43,6 +59,7 @@ import { DEFAULT_CURSOR_CLICK_BOUNCE_DURATION, DEFAULT_CURSOR_MOTION_BLUR, DEFAULT_CURSOR_SIZE, + DEFAULT_CURSOR_STYLE, DEFAULT_CURSOR_SMOOTHING, DEFAULT_CURSOR_SWAY, DEFAULT_ZOOM_MOTION_BLUR, @@ -50,6 +67,7 @@ import { } from "./types"; import { getWebcamPositionForPreset, resolveWebcamCorner } from "./webcamOverlay"; import { fromCursorSwaySliderValue, toCursorSwaySliderValue } from "./videoPlayback/cursorSway"; +import { uploadedCursorAssets, UPLOADED_CURSOR_SAMPLE_SIZE } from "./videoPlayback/uploadedCursorAssets"; const GRADIENTS = [ "linear-gradient( 111.6deg, rgba(114,167,232,1) 9.4%, rgba(253,129,82,1) 43.9%, rgba(253,129,82,1) 54.8%, rgba(249,202,86,1) 86.3% )", @@ -78,8 +96,15 @@ const GRADIENTS = [ "linear-gradient(to right, #0acffe 0%, #495aff 100%)", ]; +const CAPTION_ANIMATION_OPTIONS: Array<{ value: AutoCaptionAnimation; label: string }> = [ + { value: "none", label: "Off" }, + { value: "fade", label: "Fade" }, + { value: "rise", label: "Rise" }, + { value: "pop", label: "Pop" }, +]; + type BackgroundTab = "image" | "color" | "gradient"; -export type EditorEffectSection = "scene" | "cursor" | "webcam" | "zoom" | "frame" | "crop"; +export type EditorEffectSection = "scene" | "cursor" | "captions" | "webcam" | "zoom" | "frame" | "crop"; function isHexWallpaper(value: string): boolean { return /^#(?:[0-9a-f]{3}){1,2}$/i.test(value); @@ -124,10 +149,28 @@ interface SettingsPanelProps { onZoomMotionBlurChange?: (amount: number) => void; connectZooms?: boolean; onConnectZoomsChange?: (enabled: boolean) => void; + zoomInDurationMs?: number; + onZoomInDurationMsChange?: (duration: number) => void; + zoomInOverlapMs?: number; + onZoomInOverlapMsChange?: (duration: number) => void; + zoomOutDurationMs?: number; + onZoomOutDurationMsChange?: (duration: number) => void; + connectedZoomGapMs?: number; + onConnectedZoomGapMsChange?: (duration: number) => void; + connectedZoomDurationMs?: number; + onConnectedZoomDurationMsChange?: (duration: number) => void; + zoomInEasing?: ZoomTransitionEasing; + onZoomInEasingChange?: (easing: ZoomTransitionEasing) => void; + zoomOutEasing?: ZoomTransitionEasing; + onZoomOutEasingChange?: (easing: ZoomTransitionEasing) => void; + connectedZoomEasing?: ZoomTransitionEasing; + onConnectedZoomEasingChange?: (easing: ZoomTransitionEasing) => void; showCursor?: boolean; onShowCursorChange?: (enabled: boolean) => void; loopCursor?: boolean; onLoopCursorChange?: (enabled: boolean) => void; + cursorStyle?: CursorStyle; + onCursorStyleChange?: (style: CursorStyle) => void; cursorSize?: number; onCursorSizeChange?: (size: number) => void; cursorSmoothing?: number; @@ -159,6 +202,19 @@ interface SettingsPanelProps { onAnnotationStyleChange?: (id: string, style: Partial) => void; onAnnotationFigureDataChange?: (id: string, figureData: FigureData) => void; onAnnotationDelete?: (id: string) => void; + autoCaptions?: CaptionCue[]; + autoCaptionSettings?: AutoCaptionSettings; + whisperExecutablePath?: string | null; + whisperModelPath?: string | null; + whisperModelDownloadStatus?: "idle" | "downloading" | "downloaded" | "error"; + whisperModelDownloadProgress?: number; + isGeneratingCaptions?: boolean; + onAutoCaptionSettingsChange?: (settings: AutoCaptionSettings) => void; + onPickWhisperExecutable?: () => void; + onGenerateAutoCaptions?: () => void; + onClearAutoCaptions?: () => void; + onDownloadWhisperSmallModel?: () => void; + onDeleteWhisperSmallModel?: () => void; selectedSpeedId?: string | null; selectedSpeedValue?: PlaybackSpeed | null; onSpeedChange?: (speed: PlaybackSpeed) => void; @@ -191,6 +247,199 @@ const WEBCAM_POSITION_PRESETS: Array<{ { preset: "bottom-right", label: "↘" }, ]; +const CURSOR_STYLE_OPTIONS: Array<{ value: CursorStyle; label: string }> = [ + { value: "tahoe", label: "Tahoe" }, + { value: "dot", label: "Dot" }, + { value: "figma", label: "Minimal" }, + { value: "mono", label: "Inverted" }, +]; + +const CAPTION_LANGUAGE_OPTIONS = [ + { value: "auto", label: "Auto Detect" }, + { value: "en", label: "English" }, + { value: "es", label: "Spanish" }, + { value: "fr", label: "French" }, + { value: "de", label: "German" }, + { value: "it", label: "Italian" }, + { value: "pt", label: "Portuguese" }, + { value: "zh", label: "Chinese" }, + { value: "ja", label: "Japanese" }, + { value: "ko", label: "Korean" }, +] as const; + +function loadPreviewImage(url: string) { + return new Promise((resolve, reject) => { + const image = new Image(); + image.onload = () => resolve(image); + image.onerror = () => reject(new Error(`Failed to load preview asset: ${url}`)); + image.src = url; + }); +} + +function trimCanvasToAlpha( + canvas: HTMLCanvasElement, + hotspot?: { x: number; y: number }, +) { + const ctx = canvas.getContext("2d"); + if (!ctx) { + return { + dataUrl: canvas.toDataURL("image/png"), + width: canvas.width, + height: canvas.height, + hotspot, + }; + } + + const { width, height } = canvas; + const imageData = ctx.getImageData(0, 0, width, height); + const { data } = imageData; + let minX = width; + let minY = height; + let maxX = -1; + let maxY = -1; + + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) { + const alpha = data[(y * width + x) * 4 + 3]; + if (alpha === 0) { + continue; + } + + minX = Math.min(minX, x); + minY = Math.min(minY, y); + maxX = Math.max(maxX, x); + maxY = Math.max(maxY, y); + } + } + + if (maxX < minX || maxY < minY) { + return { + dataUrl: canvas.toDataURL("image/png"), + width, + height, + hotspot, + }; + } + + const croppedWidth = maxX - minX + 1; + const croppedHeight = maxY - minY + 1; + const croppedCanvas = document.createElement("canvas"); + croppedCanvas.width = croppedWidth; + croppedCanvas.height = croppedHeight; + const croppedCtx = croppedCanvas.getContext("2d")!; + croppedCtx.drawImage(canvas, minX, minY, croppedWidth, croppedHeight, 0, 0, croppedWidth, croppedHeight); + + return { + dataUrl: croppedCanvas.toDataURL("image/png"), + width: croppedWidth, + height: croppedHeight, + hotspot: hotspot + ? { + x: hotspot.x - minX, + y: hotspot.y - minY, + } + : undefined, + }; +} + +async function createTrimmedSvgPreview( + url: string, + sampleSize: number, + trim?: { x: number; y: number; width: number; height: number }, +) { + const image = await loadPreviewImage(url); + const sourceCanvas = document.createElement("canvas"); + sourceCanvas.width = sampleSize; + sourceCanvas.height = sampleSize; + const sourceCtx = sourceCanvas.getContext("2d")!; + sourceCtx.drawImage(image, 0, 0, sampleSize, sampleSize); + + if (trim) { + const croppedCanvas = document.createElement("canvas"); + croppedCanvas.width = trim.width; + croppedCanvas.height = trim.height; + const croppedCtx = croppedCanvas.getContext("2d")!; + croppedCtx.drawImage( + sourceCanvas, + trim.x, + trim.y, + trim.width, + trim.height, + 0, + 0, + trim.width, + trim.height, + ); + return croppedCanvas.toDataURL("image/png"); + } + + return trimCanvasToAlpha(sourceCanvas).dataUrl; +} + +async function createInvertedPreview(url: string) { + const image = await loadPreviewImage(url); + const canvas = document.createElement("canvas"); + canvas.width = image.naturalWidth; + canvas.height = image.naturalHeight; + const ctx = canvas.getContext("2d")!; + ctx.drawImage(image, 0, 0); + const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); + const { data } = imageData; + for (let index = 0; index < data.length; index += 4) { + if (data[index + 3] === 0) { + continue; + } + data[index] = 255 - data[index]; + data[index + 1] = 255 - data[index + 1]; + data[index + 2] = 255 - data[index + 2]; + } + ctx.putImageData(imageData, 0, 0); + return canvas.toDataURL("image/png"); +} + +function CursorStylePreview({ + style, + previewUrls, +}: { + style: CursorStyle; + previewUrls: Partial>; +}) { + if (style === "tahoe") { + return ( + + ); + } + + if (style === "figma") { + return ( + + ); + } + + if (style === "dot") { + return ; + } + + return ( + + ); +} + export function SettingsPanel({ panelMode = "editor", activeEffectSection: activeEffectSectionProp, @@ -208,12 +457,12 @@ export function SettingsPanel({ onBackgroundBlurChange, zoomMotionBlur = 0, onZoomMotionBlurChange, - connectZooms = true, - onConnectZoomsChange, showCursor = false, onShowCursorChange, loopCursor = false, onLoopCursorChange, + cursorStyle = DEFAULT_CURSOR_STYLE, + onCursorStyleChange, cursorSize = 5, onCursorSizeChange, cursorSmoothing = 2, @@ -245,6 +494,17 @@ export function SettingsPanel({ onAnnotationStyleChange, onAnnotationFigureDataChange, onAnnotationDelete, + autoCaptions = [], + autoCaptionSettings = DEFAULT_AUTO_CAPTION_SETTINGS, + whisperModelPath, + whisperModelDownloadStatus = "idle", + whisperModelDownloadProgress = 0, + isGeneratingCaptions = false, + onAutoCaptionSettingsChange, + onGenerateAutoCaptions, + onClearAutoCaptions, + onDownloadWhisperSmallModel, + onDeleteWhisperSmallModel, selectedSpeedId, selectedSpeedValue, onSpeedChange, @@ -270,6 +530,13 @@ export function SettingsPanel({ () => builtInWallpapers.map((wallpaper) => wallpaper.publicPath), [builtInWallpapers], ); + const captionCueCount = autoCaptions.length; + const updateAutoCaptionSettings = (partial: Partial) => { + onAutoCaptionSettingsChange?.({ + ...autoCaptionSettings, + ...partial, + }); + }; useEffect(() => { let mounted = true; @@ -329,6 +596,46 @@ export function SettingsPanel({ const defaultWebcam = initialEditorPreferences.webcam; const [internalActiveEffectSection] = useState("scene"); const activeEffectSection = activeEffectSectionProp ?? internalActiveEffectSection; + const [cursorPreviewUrls, setCursorPreviewUrls] = useState>>({}); + + useEffect(() => { + let cancelled = false; + + void (async () => { + try { + const tahoeAsset = uploadedCursorAssets.arrow; + const tahoePreview = tahoeAsset + ? await createTrimmedSvgPreview( + tahoeAsset.url, + UPLOADED_CURSOR_SAMPLE_SIZE, + tahoeAsset.trim, + ) + : tahoeCursorUrl; + const minimalPreview = await createTrimmedSvgPreview(minimalCursorUrl, 512); + const invertedPreview = await createInvertedPreview(tahoePreview); + + if (!cancelled) { + setCursorPreviewUrls({ + tahoe: tahoePreview, + figma: minimalPreview, + mono: invertedPreview, + }); + } + } catch { + if (!cancelled) { + setCursorPreviewUrls({ + tahoe: tahoeCursorUrl, + figma: minimalCursorUrl, + mono: tahoeCursorUrl, + }); + } + } + })(); + + return () => { + cancelled = true; + }; + }, []); useEffect(() => { setBackgroundTab(getBackgroundTabForWallpaper(selected)); @@ -494,12 +801,12 @@ export function SettingsPanel({ const resetZoomSection = () => { onZoomMotionBlurChange?.(initialEditorPreferences.zoomMotionBlur); - onConnectZoomsChange?.(initialEditorPreferences.connectZooms); }; const resetCursorSection = () => { onShowCursorChange?.(initialEditorPreferences.showCursor); onLoopCursorChange?.(initialEditorPreferences.loopCursor); + onCursorStyleChange?.(initialEditorPreferences.cursorStyle); onCursorSizeChange?.(initialEditorPreferences.cursorSize); onCursorSmoothingChange?.(initialEditorPreferences.cursorSmoothing); onCursorMotionBlurChange?.(initialEditorPreferences.cursorMotionBlur); @@ -828,7 +1135,7 @@ export function SettingsPanel({
- Zoom + {tSettings("sections.zoom", "Zoom")}
-
- {tSettings("effects.connectZooms")} - -
); + const captionsSectionContent = ( +
+
+
+ {tSettings("sections.captions", "Captions")} + +
+
+ {tSettings("captions.enabled", "Show")} + updateAutoCaptionSettings({ enabled })} + className="data-[state=checked]:bg-[#2563EB] scale-75" + /> +
+
+ +
+
+
{tSettings("captions.language", "Language")}
+ +
+
+
+ {whisperModelDownloadStatus === "downloading" ? ( + + ) : whisperModelPath ? ( + + ) : ( + + )} + +
+
+
+ + {isGeneratingCaptions ? ( +
+
+ {tSettings("captions.generatingStatus", "Generating captions. This can take a moment.")} +
+
+
+ ) : null} +
+ {whisperModelDownloadStatus === "downloading" ? ( +
+
+
+ ) : null} +
+ +
+
+
{tSettings("captions.animation", "Animation")}
+ +
+ +
{tSettings("captions.fontSettings", "Font Settings")}
+ updateAutoCaptionSettings({ fontSize: value })} formatValue={(value) => `${Math.round(value)}px`} parseInput={(text) => parseFloat(text.replace(/px$/, ""))} /> + updateAutoCaptionSettings({ maxRows: Math.round(value) })} formatValue={(value) => `${Math.round(value)}`} parseInput={(text) => parseFloat(text)} /> + updateAutoCaptionSettings({ bottomOffset: value })} formatValue={(value) => `${Math.round(value)}%`} parseInput={(text) => parseFloat(text.replace(/%$/, ""))} /> + updateAutoCaptionSettings({ maxWidth: value })} formatValue={(value) => `${Math.round(value)}%`} parseInput={(text) => parseFloat(text.replace(/%$/, ""))} /> + updateAutoCaptionSettings({ boxRadius: value })} formatValue={(value) => `${Number.isInteger(value) ? value.toFixed(0) : value.toFixed(1)}px`} parseInput={(text) => parseFloat(text.replace(/px$/, ""))} /> + updateAutoCaptionSettings({ backgroundOpacity: value })} formatValue={(value) => `${Math.round(value * 100)}%`} parseInput={(text) => parseFloat(text.replace(/%$/, "")) / 100} /> +
+
+ ); + const effectSectionContent = (() => { + const sceneSectionContent = ( +
+ {backgroundSettingsContent} + {zoomSectionContent} + {frameSectionContent} + {cropSectionContent} +
+ ); + switch (activeEffectSection) { case "scene": + return sceneSectionContent; case "zoom": + return sceneSectionContent; case "frame": + return sceneSectionContent; case "crop": - return ( -
- {backgroundSettingsContent} - {zoomSectionContent} - {frameSectionContent} - {cropSectionContent} -
- ); + return sceneSectionContent; + case "captions": + return captionsSectionContent; case "cursor": return (
@@ -941,6 +1362,36 @@ export function SettingsPanel({
+
+ { + if (value) { + onCursorStyleChange?.(value as CursorStyle); + } + }} + className="grid grid-cols-4 gap-2" + aria-label={tSettings("effects.cursorStyle", "Cursor Style")} + > + {CURSOR_STYLE_OPTIONS.map((option) => ( + +
+
+ +
+
+ + ))} + +
onCursorSizeChange?.(v)} formatValue={(v) => `${v.toFixed(2)}×`} parseInput={(text) => parseFloat(text.replace(/×$/, ""))} /> onCursorSmoothingChange?.(v)} formatValue={(v) => (v <= 0 ? tSettings("effects.off") : v.toFixed(2))} parseInput={(text) => parseFloat(text)} /> onCursorMotionBlurChange?.(v)} formatValue={(v) => `${v.toFixed(2)}×`} parseInput={(text) => parseFloat(text.replace(/×$/, ""))} />