From 9341a5295ef325bbbdb04872b4c5f143093049d1 Mon Sep 17 00:00:00 2001 From: webadderall <131426131+webadderall@users.noreply.github.com> Date: Sun, 22 Mar 2026 18:33:43 +1100 Subject: [PATCH] feat: render captions in preview playback Render active captions in the live preview with responsive layout, crop-aware aspect ratio handling, and supporting playback layout refinements. --- src/components/video-editor/VideoPlayback.tsx | 291 +++++++++++++++++- .../video-editor/videoPlayback/layoutUtils.ts | 1 - .../videoPlayback/zoomRegionUtils.ts | 19 +- 3 files changed, 302 insertions(+), 9 deletions(-) diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx index b852489a..0e8fe195 100644 --- a/src/components/video-editor/VideoPlayback.tsx +++ b/src/components/video-editor/VideoPlayback.tsx @@ -25,6 +25,8 @@ import { } from "pixi.js"; import { MotionBlurFilter } from "pixi-filters/motion-blur"; import { + type AutoCaptionSettings, + type CaptionCue, ZOOM_DEPTH_SCALES, type ZoomRegion, type ZoomFocus, @@ -33,7 +35,9 @@ import { type SpeedRegion, type AnnotationRegion, type CursorTelemetryPoint, + type CursorStyle, type WebcamOverlaySettings, + type ZoomTransitionEasing, } from "./types"; import { DEFAULT_FOCUS, @@ -45,6 +49,18 @@ import { PixiCursorOverlay, preloadCursorAssets, } from "./videoPlayback/cursorRenderer"; +import { + buildActiveCaptionLayout, +} from "./captionLayout"; +import { + CAPTION_FONT_WEIGHT, + CAPTION_LINE_HEIGHT, + getCaptionPadding, + getCaptionScaledRadius, + getCaptionScaledFontSize, + getCaptionTextMaxWidth, + getCaptionWordVisualState, +} from "./captionStyle"; import { clamp01 } from "./videoPlayback/mathUtils"; import { findDominantRegion } from "./videoPlayback/zoomRegionUtils"; import { clampFocusToStage as clampFocusToStageUtil } from "./videoPlayback/focusUtils"; @@ -70,10 +86,19 @@ import { DEFAULT_CURSOR_SIZE, DEFAULT_CURSOR_SMOOTHING, DEFAULT_CURSOR_SWAY, + DEFAULT_CONNECTED_ZOOM_DURATION_MS, + DEFAULT_CONNECTED_ZOOM_EASING, + DEFAULT_CONNECTED_ZOOM_GAP_MS, DEFAULT_WEBCAM_CORNER_RADIUS, DEFAULT_WEBCAM_REACT_TO_ZOOM, DEFAULT_WEBCAM_SHADOW, DEFAULT_WEBCAM_SIZE, + DEFAULT_ZOOM_IN_DURATION_MS, + DEFAULT_ZOOM_IN_EASING, + DEFAULT_ZOOM_IN_OVERLAP_MS, + DEFAULT_ZOOM_OUT_DURATION_MS, + DEFAULT_ZOOM_OUT_EASING, + getDefaultCaptionFontFamily, } from "./types"; import { getWebcamOverlayPosition, getWebcamOverlaySizePx } from "./webcamOverlay"; import { getSquircleSvgPath } from "@/lib/geometry/squircle"; @@ -100,6 +125,26 @@ function createPlaybackAnimationState(): PlaybackAnimationState { }; } +function getEffectiveNativeAspectRatio( + dimensions: { width: number; height: number } | null | undefined, + cropRegion?: import("./types").CropRegion, +): number { + if (!dimensions || dimensions.height <= 0 || dimensions.width <= 0) { + return 16 / 9; + } + + const cropWidth = cropRegion?.width ?? 1; + const cropHeight = cropRegion?.height ?? 1; + const effectiveWidth = dimensions.width * cropWidth; + const effectiveHeight = dimensions.height * cropHeight; + + if (effectiveWidth <= 0 || effectiveHeight <= 0) { + return dimensions.width / dimensions.height; + } + + return effectiveWidth / effectiveHeight; +} + interface VideoPlaybackProps { videoPath: string; onDurationChange: (duration: number) => void; @@ -118,6 +163,14 @@ interface VideoPlaybackProps { backgroundBlur?: number; zoomMotionBlur?: number; connectZooms?: boolean; + zoomInDurationMs?: number; + zoomInOverlapMs?: number; + zoomOutDurationMs?: number; + connectedZoomGapMs?: number; + connectedZoomDurationMs?: number; + zoomInEasing?: ZoomTransitionEasing; + zoomOutEasing?: ZoomTransitionEasing; + connectedZoomEasing?: ZoomTransitionEasing; borderRadius?: number; padding?: number; cropRegion?: import("./types").CropRegion; @@ -127,6 +180,8 @@ interface VideoPlaybackProps { speedRegions?: SpeedRegion[]; aspectRatio: AspectRatio; annotationRegions?: AnnotationRegion[]; + autoCaptions?: CaptionCue[]; + autoCaptionSettings?: AutoCaptionSettings; selectedAnnotationId?: string | null; onSelectAnnotation?: (id: string | null) => void; onAnnotationPositionChange?: ( @@ -139,6 +194,7 @@ interface VideoPlaybackProps { ) => void; cursorTelemetry?: CursorTelemetryPoint[]; showCursor?: boolean; + cursorStyle?: CursorStyle; cursorSize?: number; cursorSmoothing?: number; cursorMotionBlur?: number; @@ -179,6 +235,14 @@ const VideoPlayback = forwardRef( backgroundBlur = 0, zoomMotionBlur = 0, connectZooms = true, + zoomInDurationMs = DEFAULT_ZOOM_IN_DURATION_MS, + zoomInOverlapMs = DEFAULT_ZOOM_IN_OVERLAP_MS, + zoomOutDurationMs = DEFAULT_ZOOM_OUT_DURATION_MS, + connectedZoomGapMs = DEFAULT_CONNECTED_ZOOM_GAP_MS, + connectedZoomDurationMs = DEFAULT_CONNECTED_ZOOM_DURATION_MS, + zoomInEasing = DEFAULT_ZOOM_IN_EASING, + zoomOutEasing = DEFAULT_ZOOM_OUT_EASING, + connectedZoomEasing = DEFAULT_CONNECTED_ZOOM_EASING, borderRadius = 0, padding = 50, cropRegion, @@ -188,12 +252,15 @@ const VideoPlayback = forwardRef( speedRegions = [], aspectRatio, annotationRegions = [], + autoCaptions = [], + autoCaptionSettings, selectedAnnotationId, onSelectAnnotation, onAnnotationPositionChange, onAnnotationSizeChange, cursorTelemetry = [], showCursor = false, + cursorStyle = "tahoe", cursorSize = DEFAULT_CURSOR_SIZE, cursorSmoothing = DEFAULT_CURSOR_SMOOTHING, cursorMotionBlur = DEFAULT_CURSOR_MOTION_BLUR, @@ -219,6 +286,7 @@ const VideoPlayback = forwardRef( const webcamVideoRef = useRef(null); const webcamBubbleRef = useRef(null); const webcamBubbleInnerRef = useRef(null); + const captionBoxRef = useRef(null); const currentTimeRef = useRef(0); const zoomRegionsRef = useRef([]); const selectedZoomIdRef = useRef(null); @@ -259,16 +327,95 @@ const VideoPlayback = forwardRef( const lastWebcamSyncTimeRef = useRef(null); const zoomMotionBlurRef = useRef(zoomMotionBlur); const connectZoomsRef = useRef(connectZooms); + const zoomInDurationMsRef = useRef(zoomInDurationMs); + const zoomInOverlapMsRef = useRef(zoomInOverlapMs); + const zoomOutDurationMsRef = useRef(zoomOutDurationMs); + const connectedZoomGapMsRef = useRef(connectedZoomGapMs); + const connectedZoomDurationMsRef = useRef(connectedZoomDurationMs); + const zoomInEasingRef = useRef(zoomInEasing); + const zoomOutEasingRef = useRef(zoomOutEasing); + const connectedZoomEasingRef = useRef(connectedZoomEasing); const videoReadyRafRef = useRef(null); const cursorOverlayRef = useRef(null); const cursorTelemetryRef = useRef([]); const showCursorRef = useRef(showCursor); const cursorSizeRef = useRef(cursorSize); + const cursorStyleRef = useRef(cursorStyle); const cursorSmoothingRef = useRef(cursorSmoothing); const cursorMotionBlurRef = useRef(cursorMotionBlur); const cursorClickBounceRef = useRef(cursorClickBounce); const cursorClickBounceDurationRef = useRef(cursorClickBounceDuration); const cursorSwayRef = useRef(cursorSway); + + const activeCaptionLayout = useMemo(() => { + if (!autoCaptionSettings?.enabled || autoCaptions.length === 0 || typeof document === "undefined") { + return null; + } + + const overlayWidth = overlayRef.current?.clientWidth || 960; + const fontSize = getCaptionScaledFontSize( + autoCaptionSettings.fontSize, + overlayWidth, + autoCaptionSettings.maxWidth, + ); + const maxTextWidthPx = getCaptionTextMaxWidth( + overlayWidth, + autoCaptionSettings.maxWidth, + fontSize, + ); + const measurementCanvas = document.createElement("canvas"); + const measurementContext = measurementCanvas.getContext("2d"); + if (!measurementContext) { + return null; + } + + measurementContext.font = `${CAPTION_FONT_WEIGHT} ${fontSize}px ${getDefaultCaptionFontFamily()}`; + + return buildActiveCaptionLayout({ + cues: autoCaptions, + timeMs: Math.round(currentTime * 1000), + settings: autoCaptionSettings, + maxWidthPx: maxTextWidthPx, + measureText: (text) => measurementContext.measureText(text).width, + }); + }, [autoCaptionSettings, autoCaptions, currentTime]); + + useEffect(() => { + const captionBox = captionBoxRef.current; + if (!captionBox || !activeCaptionLayout || !autoCaptionSettings) { + if (captionBox) { + captionBox.style.clipPath = ""; + captionBox.style.removeProperty("-webkit-clip-path"); + } + return; + } + + const frame = requestAnimationFrame(() => { + const width = captionBox.offsetWidth; + const height = captionBox.offsetHeight; + if (width <= 0 || height <= 0) { + return; + } + + const fontSize = getCaptionScaledFontSize( + autoCaptionSettings.fontSize, + overlayRef.current?.clientWidth || 960, + autoCaptionSettings.maxWidth, + ); + + const squirclePath = getSquircleSvgPath({ + x: 0, + y: 0, + width, + height, + radius: getCaptionScaledRadius(autoCaptionSettings.boxRadius, fontSize), + }); + captionBox.style.clipPath = `path('${squirclePath}')`; + captionBox.style.setProperty("-webkit-clip-path", `path('${squirclePath}')`); + }); + + return () => cancelAnimationFrame(frame); + }, [activeCaptionLayout, autoCaptionSettings]); const motionBlurStateRef = useRef(createMotionBlurState()); const applyWebcamBubbleLayout = useCallback((zoomScale: number) => { @@ -620,6 +767,38 @@ const VideoPlayback = forwardRef( connectZoomsRef.current = connectZooms; }, [connectZooms]); + useEffect(() => { + zoomInDurationMsRef.current = zoomInDurationMs; + }, [zoomInDurationMs]); + + useEffect(() => { + zoomInOverlapMsRef.current = zoomInOverlapMs; + }, [zoomInOverlapMs]); + + useEffect(() => { + zoomOutDurationMsRef.current = zoomOutDurationMs; + }, [zoomOutDurationMs]); + + useEffect(() => { + connectedZoomGapMsRef.current = connectedZoomGapMs; + }, [connectedZoomGapMs]); + + useEffect(() => { + connectedZoomDurationMsRef.current = connectedZoomDurationMs; + }, [connectedZoomDurationMs]); + + useEffect(() => { + zoomInEasingRef.current = zoomInEasing; + }, [zoomInEasing]); + + useEffect(() => { + zoomOutEasingRef.current = zoomOutEasing; + }, [zoomOutEasing]); + + useEffect(() => { + connectedZoomEasingRef.current = connectedZoomEasing; + }, [connectedZoomEasing]); + useEffect(() => { cursorTelemetryRef.current = cursorTelemetry; }, [cursorTelemetry]); @@ -628,6 +807,10 @@ const VideoPlayback = forwardRef( showCursorRef.current = showCursor; }, [showCursor]); + useEffect(() => { + cursorStyleRef.current = cursorStyle; + }, [cursorStyle]); + useEffect(() => { cursorSizeRef.current = cursorSize; }, [cursorSize]); @@ -877,6 +1060,7 @@ const VideoPlayback = forwardRef( if (cursorOverlayEnabled) { const cursorOverlay = new PixiCursorOverlay({ dotRadius: DEFAULT_CURSOR_CONFIG.dotRadius * cursorSizeRef.current, + style: cursorStyleRef.current, smoothingFactor: cursorSmoothingRef.current, motionBlur: cursorMotionBlurRef.current, clickBounce: cursorClickBounceRef.current, @@ -1228,6 +1412,7 @@ const VideoPlayback = forwardRef( } overlay.setDotRadius(DEFAULT_CURSOR_CONFIG.dotRadius * cursorSize); + overlay.setStyle(cursorStyle); overlay.setSmoothingFactor(cursorSmoothing); overlay.setMotionBlur(cursorMotionBlur); overlay.setClickBounce(cursorClickBounce); @@ -1235,6 +1420,7 @@ const VideoPlayback = forwardRef( overlay.setSway(cursorSway); overlay.reset(); }, [ + cursorStyle, cursorSize, cursorSmoothing, cursorMotionBlur, @@ -1352,12 +1538,18 @@ const VideoPlayback = forwardRef( const nativeAspectRatio = (() => { const locked = lockedVideoDimensionsRef.current; - if (locked && locked.height > 0) { - return locked.width / locked.height; + if (locked) { + return getEffectiveNativeAspectRatio(locked, cropRegion); } const video = videoRef.current; - if (video && video.videoHeight > 0) { - return video.videoWidth / video.videoHeight; + if (video && video.videoHeight > 0 && video.videoWidth > 0) { + return getEffectiveNativeAspectRatio( + { + width: video.videoWidth, + height: video.videoHeight, + }, + cropRegion, + ); } return 16 / 9; })(); @@ -1429,6 +1621,97 @@ const VideoPlayback = forwardRef( ) : null} + {activeCaptionLayout && autoCaptionSettings ? ( +
+
+
+ {activeCaptionLayout.visibleLines.map((line) => ( +
+ {line.words.map((word) => { + const visualState = getCaptionWordVisualState( + activeCaptionLayout.hasWordTimings, + word.state, + ); + + return ( + + {`${word.leadingSpace ? " " : ""}${word.text}`} + + ); + })} +
+ ))} +
+
+
+ ) : null} {(() => { const filtered = (annotationRegions || []).filter( (annotation) => { diff --git a/src/components/video-editor/videoPlayback/layoutUtils.ts b/src/components/video-editor/videoPlayback/layoutUtils.ts index 77e6cd7f..ec1dd6af 100644 --- a/src/components/video-editor/videoPlayback/layoutUtils.ts +++ b/src/components/video-editor/videoPlayback/layoutUtils.ts @@ -71,7 +71,6 @@ export function layoutVideoContent(params: LayoutParams): LayoutResult | null { const scale = Math.min( maxDisplayWidth / croppedVideoWidth, maxDisplayHeight / croppedVideoHeight, - 1 ); videoSprite.scale.set(scale); diff --git a/src/components/video-editor/videoPlayback/zoomRegionUtils.ts b/src/components/video-editor/videoPlayback/zoomRegionUtils.ts index 0b9080f8..df321bcf 100644 --- a/src/components/video-editor/videoPlayback/zoomRegionUtils.ts +++ b/src/components/video-editor/videoPlayback/zoomRegionUtils.ts @@ -1,5 +1,7 @@ import type { ZoomFocus, ZoomRegion } from "../types"; -import { ZOOM_DEPTH_SCALES } from "../types"; +import { + ZOOM_DEPTH_SCALES, +} from "../types"; import { TRANSITION_WINDOW_MS, ZOOM_IN_TRANSITION_WINDOW_MS } from "./constants"; import { clampFocusToScale } from "./focusUtils"; import { clamp01, cubicBezier, easeOutScreenStudio } from "./mathUtils"; @@ -35,7 +37,10 @@ function easeConnectedPan(value: number) { return cubicBezier(0.1, 0.0, 0.2, 1.0, value); } -export function computeRegionStrength(region: ZoomRegion, timeMs: number) { +export function computeRegionStrength( + region: ZoomRegion, + timeMs: number, +) { const zoomInEnd = region.startMs + ZOOM_IN_OVERLAP_MS; const leadInStart = zoomInEnd - ZOOM_IN_TRANSITION_WINDOW_MS; const leadOutEnd = region.endMs + TRANSITION_WINDOW_MS; @@ -92,7 +97,11 @@ function getConnectedRegionPairs(regions: ZoomRegion[]) { return pairs; } -function getActiveRegion(regions: ZoomRegion[], timeMs: number, connectedPairs: ConnectedRegionPair[]) { +function getActiveRegion( + regions: ZoomRegion[], + timeMs: number, + connectedPairs: ConnectedRegionPair[], +) { const activeRegions = regions .map((region) => { const outgoingPair = connectedPairs.find((pair) => pair.currentRegion.id === region.id); @@ -198,7 +207,9 @@ export function findDominantRegion(regions: ZoomRegion[], timeMs: number, option blendedScale: number | null; transition: ConnectedPanTransition | null; } { - const connectedPairs = options.connectZooms ? getConnectedRegionPairs(regions) : []; + const connectedPairs = options.connectZooms + ? getConnectedRegionPairs(regions) + : []; if (options.connectZooms) { const connectedTransition = getConnectedRegionTransition(connectedPairs, timeMs);