diff --git a/src/components/video-editor/AnnotationOverlay.tsx b/src/components/video-editor/AnnotationOverlay.tsx index 4081cd21..dd96d32a 100644 --- a/src/components/video-editor/AnnotationOverlay.tsx +++ b/src/components/video-editor/AnnotationOverlay.tsx @@ -14,6 +14,7 @@ interface AnnotationOverlayProps { onClick: (id: string) => void; zIndex: number; isSelectedBoost: boolean; // Boost z-index when selected for easy editing + scale?: number; } export function AnnotationOverlay({ @@ -26,6 +27,7 @@ export function AnnotationOverlay({ onClick, zIndex, isSelectedBoost, + scale = 1, }: AnnotationOverlayProps) { const x = (annotation.position.x / 100) * containerWidth; const y = (annotation.position.y / 100) * containerHeight; @@ -143,6 +145,7 @@ export function AnnotationOverlay({ { isDraggingRef.current = true; }} diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx index b2da2194..729c2939 100644 --- a/src/components/video-editor/VideoPlayback.tsx +++ b/src/components/video-editor/VideoPlayback.tsx @@ -200,6 +200,12 @@ type CaptionEditSession = { draft: string; }; +type SceneTransformState = { + scale: number; + x: number; + y: number; +}; + function createPlaybackAnimationState(): PlaybackAnimationState { return { scale: 1, @@ -501,6 +507,12 @@ const VideoPlayback = forwardRef( null, ); const [frameUpdateCounter, setFrameUpdateCounter] = useState(0); + const [annotationSceneTransform, setAnnotationSceneTransform] = + useState({ + scale: 1, + x: 0, + y: 0, + }); useEffect(() => { let framesSignature = getRegisteredFramesSignature(); @@ -2324,6 +2336,21 @@ const VideoPlayback = forwardRef( state.x = appliedTransform.x; state.y = appliedTransform.y; state.appliedScale = appliedTransform.scale; + setAnnotationSceneTransform((current) => { + if ( + Math.abs(current.scale - appliedTransform.scale) < 0.001 && + Math.abs(current.x - appliedTransform.x) < 0.1 && + Math.abs(current.y - appliedTransform.y) < 0.1 + ) { + return current; + } + + return { + scale: appliedTransform.scale, + x: appliedTransform.x, + y: appliedTransform.y, + }; + }); }; const ticker = () => { @@ -3304,58 +3331,72 @@ const VideoPlayback = forwardRef( ) : null} - {(() => { - const filtered = (annotationRegions || []).filter((annotation) => { - if ( - typeof annotation.startMs !== "number" || - typeof annotation.endMs !== "number" - ) - return false; +
+ {(() => { + const filtered = (annotationRegions || []).filter((annotation) => { + if ( + typeof annotation.startMs !== "number" || + typeof annotation.endMs !== "number" + ) + return false; - if (annotation.id === selectedAnnotationId) return true; + if (annotation.id === selectedAnnotationId) return true; - const timeMs = Math.round(currentTime * 1000); - return timeMs >= annotation.startMs && timeMs <= annotation.endMs; - }); - - // Sort by z-index (lowest to highest) so higher z-index renders on top - const sorted = [...filtered].sort((a, b) => a.zIndex - b.zIndex); - - // Handle click-through cycling: when clicking same annotation, cycle to next - const handleAnnotationClick = (clickedId: string) => { - if (!onSelectAnnotation) return; - - // If clicking on already selected annotation and there are multiple overlapping - if (clickedId === selectedAnnotationId && sorted.length > 1) { - // Find current index and cycle to next - const currentIndex = sorted.findIndex( - (a) => a.id === clickedId, + const timeMs = Math.round(currentTime * 1000); + return ( + timeMs >= annotation.startMs && timeMs <= annotation.endMs ); - const nextIndex = (currentIndex + 1) % sorted.length; - onSelectAnnotation(sorted[nextIndex].id); - } else { - // First click or clicking different annotation - onSelectAnnotation(clickedId); - } - }; + }); - return sorted.map((annotation) => ( - - onAnnotationPositionChange?.(id, position) + // Sort by z-index (lowest to highest) so higher z-index renders on top + const sorted = [...filtered].sort((a, b) => a.zIndex - b.zIndex); + + // Handle click-through cycling: when clicking same annotation, cycle to next + const handleAnnotationClick = (clickedId: string) => { + if (!onSelectAnnotation) return; + + // If clicking on already selected annotation and there are multiple overlapping + if (clickedId === selectedAnnotationId && sorted.length > 1) { + // Find current index and cycle to next + const currentIndex = sorted.findIndex( + (a) => a.id === clickedId, + ); + const nextIndex = (currentIndex + 1) % sorted.length; + onSelectAnnotation(sorted[nextIndex].id); + } else { + // First click or clicking different annotation + onSelectAnnotation(clickedId); } - onSizeChange={(id, size) => onAnnotationSizeChange?.(id, size)} - onClick={handleAnnotationClick} - zIndex={annotation.zIndex} - isSelectedBoost={annotation.id === selectedAnnotationId} - /> - )); - })()} + }; + + return sorted.map((annotation) => ( + + onAnnotationPositionChange?.(id, position) + } + onSizeChange={(id, size) => + onAnnotationSizeChange?.(id, size) + } + onClick={handleAnnotationClick} + zIndex={annotation.zIndex} + isSelectedBoost={annotation.id === selectedAnnotationId} + scale={annotationSceneTransform.scale} + /> + )); + })()} +
)} {/* Keep the source video off-screen instead of display:none so the diff --git a/src/components/video-editor/captionEditing.test.ts b/src/components/video-editor/captionEditing.test.ts index 7cfc1831..48d3ce95 100644 --- a/src/components/video-editor/captionEditing.test.ts +++ b/src/components/video-editor/captionEditing.test.ts @@ -114,4 +114,31 @@ describe("captionEditing", () => { expect(updateCaptionCuesForEditedTarget(cues, visibleTarget, " \n\t ")).toBe(cues); }); + + it("keeps sound-effect style captions editable when word entries are blank", () => { + const cues: CaptionCue[] = [ + { + id: "sound-effect", + startMs: 1_000, + endMs: 2_000, + text: "clears throat", + words: [{ text: "", startMs: 1_000, endMs: 2_000 }], + }, + ]; + + const layout = buildActiveCaptionLayout({ + cues, + timeMs: 1_500, + settings: DEFAULT_AUTO_CAPTION_SETTINGS, + maxWidthPx: 500, + measureText: (text) => text.length * 10, + }); + + expect(layout?.editTarget.text).toBe("clears throat"); + + const updated = updateCaptionCuesForEditedTarget(cues, layout!.editTarget, "coughs"); + + expect(updated[0].text).toBe("coughs"); + expect(updated[0].words).toEqual([{ text: "coughs", startMs: 1_000, endMs: 2_000 }]); + }); }); diff --git a/src/components/video-editor/captionEditing.ts b/src/components/video-editor/captionEditing.ts index b5084426..9ff55bef 100644 --- a/src/components/video-editor/captionEditing.ts +++ b/src/components/video-editor/captionEditing.ts @@ -56,13 +56,19 @@ function buildCaptionWordsForEditedText( } function normalizeCaptionWords(cue: CaptionCue): CaptionCueWord[] { + const validSourceWords = Array.isArray(cue.words) + ? cue.words.filter((word): word is CaptionCueWord => + Boolean( + word && typeof word.text === "string" && normalizeCaptionEditText(word.text), + ), + ) + : []; const sourceWords = - Array.isArray(cue.words) && cue.words.length > 0 - ? cue.words + validSourceWords.length > 0 + ? validSourceWords : buildCaptionWordsForEditedText(cue.text, cue.startMs, cue.endMs); return sourceWords - .filter((word): word is CaptionCueWord => Boolean(word && typeof word.text === "string")) .map((word) => { const startMs = Math.max( cue.startMs, diff --git a/src/components/video-editor/captionLayout.ts b/src/components/video-editor/captionLayout.ts index 6c0d987e..9d383294 100644 --- a/src/components/video-editor/captionLayout.ts +++ b/src/components/video-editor/captionLayout.ts @@ -95,7 +95,7 @@ function splitCaptionWordsFromText(text: string) { function splitCaptionWords(cue: CaptionCue) { if (Array.isArray(cue.words) && cue.words.length > 0) { - return cue.words + const words = cue.words .filter((word): word is CaptionCueWord => Boolean(word && typeof word.text === "string"), ) @@ -109,6 +109,10 @@ function splitCaptionWords(cue: CaptionCue) { endMs: word.endMs, })) .filter((word) => word.text.length > 0); + + if (words.length > 0) { + return words; + } } return splitCaptionWordsFromText(cue.text).map((word) => ({ diff --git a/src/lib/exporter/annotationRenderer.ts b/src/lib/exporter/annotationRenderer.ts index a8822a81..bb06e605 100644 --- a/src/lib/exporter/annotationRenderer.ts +++ b/src/lib/exporter/annotationRenderer.ts @@ -8,6 +8,28 @@ export interface AnnotationRenderAssets { imageCache: Map; } +interface AnnotationSceneTransform { + scale: number; + x: number; + y: number; +} + +function transformAnnotationRect( + rect: { x: number; y: number; width: number; height: number }, + sceneTransform?: AnnotationSceneTransform, +) { + if (!sceneTransform) { + return rect; + } + + return { + x: rect.x * sceneTransform.scale + sceneTransform.x, + y: rect.y * sceneTransform.scale + sceneTransform.y, + width: rect.width * sceneTransform.scale, + height: rect.height * sceneTransform.scale, + }; +} + const annotationImagePromiseCache = new Map>(); let blurBufferCanvas: HTMLCanvasElement | null = null; @@ -334,6 +356,7 @@ export async function renderAnnotations( currentTimeMs: number, scaleFactor: number = 1.0, assets?: AnnotationRenderAssets, + sceneTransform?: AnnotationSceneTransform, ): Promise { const activeAnnotations = annotations.filter( (ann) => currentTimeMs >= ann.startMs && currentTimeMs <= ann.endMs, @@ -342,14 +365,21 @@ export async function renderAnnotations( const sortedAnnotations = [...activeAnnotations].sort((a, b) => a.zIndex - b.zIndex); for (const annotation of sortedAnnotations) { - const x = (annotation.position.x / 100) * canvasWidth; - const y = (annotation.position.y / 100) * canvasHeight; - const width = (annotation.size.width / 100) * canvasWidth; - const height = (annotation.size.height / 100) * canvasHeight; + const rect = transformAnnotationRect( + { + x: (annotation.position.x / 100) * canvasWidth, + y: (annotation.position.y / 100) * canvasHeight, + width: (annotation.size.width / 100) * canvasWidth, + height: (annotation.size.height / 100) * canvasHeight, + }, + sceneTransform, + ); + const { x, y, width, height } = rect; + const effectiveScaleFactor = scaleFactor * (sceneTransform?.scale ?? 1); switch (annotation.type) { case "text": - renderText(ctx, annotation, x, y, width, height, scaleFactor); + renderText(ctx, annotation, x, y, width, height, effectiveScaleFactor); break; case "image": @@ -367,20 +397,20 @@ export async function renderAnnotations( y, width, height, - scaleFactor, + effectiveScaleFactor, ); } break; case "blur": { const blurStrength = - (annotation.blurIntensity ?? BLUR_ANNOTATION_STRENGTH) * scaleFactor; + (annotation.blurIntensity ?? BLUR_ANNOTATION_STRENGTH) * effectiveScaleFactor; const padding = Math.ceil(blurStrength * 2); ctx.save(); ctx.beginPath(); - const borderRadius = (annotation.style.borderRadius ?? 0) * scaleFactor; + const borderRadius = (annotation.style.borderRadius ?? 0) * effectiveScaleFactor; ctx.roundRect(x, y, width, height, borderRadius); ctx.clip(); diff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.ts index 049c25bb..439cb4ae 100644 --- a/src/lib/exporter/frameRenderer.ts +++ b/src/lib/exporter/frameRenderer.ts @@ -5,8 +5,8 @@ import type { AnnotationRegion, AutoCaptionSettings, CaptionCue, - CursorClickEffectStyle, CropRegion, + CursorClickEffectStyle, CursorStyle, CursorTelemetryPoint, Padding, @@ -448,14 +448,14 @@ export class FrameRenderer { massMultiplier: this.config.cursorSpringMassMultiplier, }, motionBlur: this.config.cursorMotionBlur ?? 0, - clickEffect: - this.config.cursorClickEffect ?? DEFAULT_CURSOR_CONFIG.clickEffect, + clickEffect: this.config.cursorClickEffect ?? DEFAULT_CURSOR_CONFIG.clickEffect, clickEffectColor: this.config.cursorClickEffectColor ?? DEFAULT_CURSOR_CONFIG.clickEffectColor, clickEffectScale: this.config.cursorClickEffectScale ?? DEFAULT_CURSOR_CONFIG.clickEffectScale, clickEffectOpacity: - this.config.cursorClickEffectOpacity ?? DEFAULT_CURSOR_CONFIG.clickEffectOpacity, + this.config.cursorClickEffectOpacity ?? + DEFAULT_CURSOR_CONFIG.clickEffectOpacity, clickEffectDurationMs: this.config.cursorClickEffectDurationMs ?? DEFAULT_CURSOR_CONFIG.clickEffectDurationMs, @@ -1547,6 +1547,8 @@ export class FrameRenderer { this.config.height, temporalSnapshot.timeMs, scaleFactor, + undefined, + temporalSnapshot.sceneTransform, ); } @@ -1731,6 +1733,12 @@ export class FrameRenderer { this.config.height, timeMs, scaleFactor, + undefined, + { + scale: this.animationState.appliedScale, + x: this.animationState.x, + y: this.animationState.y, + }, ); } diff --git a/src/lib/exporter/modernFrameRenderer.ts b/src/lib/exporter/modernFrameRenderer.ts index 6ab918d8..ed6d0768 100644 --- a/src/lib/exporter/modernFrameRenderer.ts +++ b/src/lib/exporter/modernFrameRenderer.ts @@ -15,8 +15,8 @@ import type { AnnotationRegion, AutoCaptionSettings, CaptionCue, - CursorClickEffectStyle, CropRegion, + CursorClickEffectStyle, CursorStyle, CursorTelemetryPoint, Padding, @@ -598,8 +598,8 @@ export class FrameRenderer { this.webcamRootContainer.addChild(this.webcamContainer); this.webcamRootContainer.visible = false; + this.cameraContainer.addChild(this.annotationContainer); this.overlayContainer.addChild(this.webcamRootContainer); - this.overlayContainer.addChild(this.annotationContainer); this.overlayContainer.addChild(this.captionContainer); this.videoMaskGraphics = new Graphics(); @@ -622,14 +622,14 @@ export class FrameRenderer { massMultiplier: this.config.cursorSpringMassMultiplier, }, motionBlur: this.config.cursorMotionBlur ?? 0, - clickEffect: - this.config.cursorClickEffect ?? DEFAULT_CURSOR_CONFIG.clickEffect, + clickEffect: this.config.cursorClickEffect ?? DEFAULT_CURSOR_CONFIG.clickEffect, clickEffectColor: this.config.cursorClickEffectColor ?? DEFAULT_CURSOR_CONFIG.clickEffectColor, clickEffectScale: this.config.cursorClickEffectScale ?? DEFAULT_CURSOR_CONFIG.clickEffectScale, clickEffectOpacity: - this.config.cursorClickEffectOpacity ?? DEFAULT_CURSOR_CONFIG.clickEffectOpacity, + this.config.cursorClickEffectOpacity ?? + DEFAULT_CURSOR_CONFIG.clickEffectOpacity, clickEffectDurationMs: this.config.cursorClickEffectDurationMs ?? DEFAULT_CURSOR_CONFIG.clickEffectDurationMs, @@ -1586,6 +1586,11 @@ export class FrameRenderer { timeMs, this.annotationScaleFactor, this.annotationAssets ?? undefined, + { + scale: this.animationState.appliedScale, + x: this.animationState.x, + y: this.animationState.y, + }, ); this.drawCaptionOverlay(context); @@ -2549,15 +2554,10 @@ export class FrameRenderer { const usesDefaultCropRegion = isWebcamCropRegionDefault(this.config.webcam?.cropRegion); const needsCacheBackedSource = !usesDefaultCropRegion || - (typeof HTMLVideoElement !== "undefined" && - liveSource instanceof HTMLVideoElement); + (typeof HTMLVideoElement !== "undefined" && liveSource instanceof HTMLVideoElement); if (needsCacheBackedSource) { - this.refreshWebcamFrameCache( - liveSource, - liveSourceWidth, - liveSourceHeight, - ); + this.refreshWebcamFrameCache(liveSource, liveSourceWidth, liveSourceHeight); const cachedSource = this.getCachedWebcamRenderSource(); if (cachedSource) { this.setWebcamRenderMode("live");