Anchor annotations to scene zoom

src/components/video-editor/AnnotationOverlay.tsx

@@ -14,6 +14,7 @@ interface AnnotationOverlayProps {
 	onClick: (id: string) => void;
 	zIndex: number;
 	isSelectedBoost: boolean; // Boost z-index when selected for easy editing
+	scale?: number;
 }
 
 export function AnnotationOverlay({
@@ -26,6 +27,7 @@ export function AnnotationOverlay({
 	onClick,
 	zIndex,
 	isSelectedBoost,
+	scale = 1,
 }: AnnotationOverlayProps) {
 	const x = (annotation.position.x / 100) * containerWidth;
 	const y = (annotation.position.y / 100) * containerHeight;
@@ -143,6 +145,7 @@ export function AnnotationOverlay({
 		<Rnd
 			position={{ x, y }}
 			size={{ width, height }}
+			scale={scale}
 			onDragStart={() => {
 				isDraggingRef.current = true;
 			}}

src/components/video-editor/VideoPlayback.tsx

@@ -200,6 +200,12 @@ type CaptionEditSession = {
 	draft: string;
 };
 
+type SceneTransformState = {
+	scale: number;
+	x: number;
+	y: number;
+};
+
 function createPlaybackAnimationState(): PlaybackAnimationState {
 	return {
 		scale: 1,
@@ -501,6 +507,12 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
 			null,
 		);
 		const [frameUpdateCounter, setFrameUpdateCounter] = useState(0);
+		const [annotationSceneTransform, setAnnotationSceneTransform] =
+			useState<SceneTransformState>({
+				scale: 1,
+				x: 0,
+				y: 0,
+			});
 
 		useEffect(() => {
 			let framesSignature = getRegisteredFramesSignature();
@@ -2324,6 +2336,21 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
 				state.x = appliedTransform.x;
 				state.y = appliedTransform.y;
 				state.appliedScale = appliedTransform.scale;
+				setAnnotationSceneTransform((current) => {
+					if (
+						Math.abs(current.scale - appliedTransform.scale) < 0.001 &&
+						Math.abs(current.x - appliedTransform.x) < 0.1 &&
+						Math.abs(current.y - appliedTransform.y) < 0.1
+					) {
+						return current;
+					}
+
+					return {
+						scale: appliedTransform.scale,
+						x: appliedTransform.x,
+						y: appliedTransform.y,
+					};
+				});
 			};
 
 			const ticker = () => {
@@ -3304,58 +3331,72 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
 								</div>
 							</div>
 						) : null}
-						{(() => {
-							const filtered = (annotationRegions || []).filter((annotation) => {
-								if (
-									typeof annotation.startMs !== "number" ||
-									typeof annotation.endMs !== "number"
-								)
-									return false;
+						<div
+							className="absolute inset-0"
+							style={{
+								pointerEvents: "none",
+								transform: `translate(${annotationSceneTransform.x}px, ${annotationSceneTransform.y}px) scale(${annotationSceneTransform.scale})`,
+								transformOrigin: "top left",
+							}}
+						>
+							{(() => {
+								const filtered = (annotationRegions || []).filter((annotation) => {
+									if (
+										typeof annotation.startMs !== "number" ||
+										typeof annotation.endMs !== "number"
+									)
+										return false;
 
-								if (annotation.id === selectedAnnotationId) return true;
+									if (annotation.id === selectedAnnotationId) return true;
 
-								const timeMs = Math.round(currentTime * 1000);
-								return timeMs >= annotation.startMs && timeMs <= annotation.endMs;
-							});
-
-							// Sort by z-index (lowest to highest) so higher z-index renders on top
-							const sorted = [...filtered].sort((a, b) => a.zIndex - b.zIndex);
-
-							// Handle click-through cycling: when clicking same annotation, cycle to next
-							const handleAnnotationClick = (clickedId: string) => {
-								if (!onSelectAnnotation) return;
-
-								// If clicking on already selected annotation and there are multiple overlapping
-								if (clickedId === selectedAnnotationId && sorted.length > 1) {
-									// Find current index and cycle to next
-									const currentIndex = sorted.findIndex(
-										(a) => a.id === clickedId,
+									const timeMs = Math.round(currentTime * 1000);
+									return (
+										timeMs >= annotation.startMs && timeMs <= annotation.endMs
 									);
-									const nextIndex = (currentIndex + 1) % sorted.length;
-									onSelectAnnotation(sorted[nextIndex].id);
-								} else {
-									// First click or clicking different annotation
-									onSelectAnnotation(clickedId);
-								}
-							};
+								});
 
-							return sorted.map((annotation) => (
-								<AnnotationOverlay
-									key={annotation.id}
-									annotation={annotation}
-									isSelected={annotation.id === selectedAnnotationId}
-									containerWidth={overlayRef.current?.clientWidth || 800}
-									containerHeight={overlayRef.current?.clientHeight || 600}
-									onPositionChange={(id, position) =>
-										onAnnotationPositionChange?.(id, position)
+								// Sort by z-index (lowest to highest) so higher z-index renders on top
+								const sorted = [...filtered].sort((a, b) => a.zIndex - b.zIndex);
+
+								// Handle click-through cycling: when clicking same annotation, cycle to next
+								const handleAnnotationClick = (clickedId: string) => {
+									if (!onSelectAnnotation) return;
+
+									// If clicking on already selected annotation and there are multiple overlapping
+									if (clickedId === selectedAnnotationId && sorted.length > 1) {
+										// Find current index and cycle to next
+										const currentIndex = sorted.findIndex(
+											(a) => a.id === clickedId,
+										);
+										const nextIndex = (currentIndex + 1) % sorted.length;
+										onSelectAnnotation(sorted[nextIndex].id);
+									} else {
+										// First click or clicking different annotation
+										onSelectAnnotation(clickedId);
 									}
-									onSizeChange={(id, size) => onAnnotationSizeChange?.(id, size)}
-									onClick={handleAnnotationClick}
-									zIndex={annotation.zIndex}
-									isSelectedBoost={annotation.id === selectedAnnotationId}
-								/>
-							));
-						})()}
+								};
+
+								return sorted.map((annotation) => (
+									<AnnotationOverlay
+										key={annotation.id}
+										annotation={annotation}
+										isSelected={annotation.id === selectedAnnotationId}
+										containerWidth={overlayRef.current?.clientWidth || 800}
+										containerHeight={overlayRef.current?.clientHeight || 600}
+										onPositionChange={(id, position) =>
+											onAnnotationPositionChange?.(id, position)
+										}
+										onSizeChange={(id, size) =>
+											onAnnotationSizeChange?.(id, size)
+										}
+										onClick={handleAnnotationClick}
+										zIndex={annotation.zIndex}
+										isSelectedBoost={annotation.id === selectedAnnotationId}
+										scale={annotationSceneTransform.scale}
+									/>
+								));
+							})()}
+						</div>
 					</div>
 				)}
 				{/* Keep the source video off-screen instead of display:none so the

src/components/video-editor/captionEditing.test.ts

@@ -114,4 +114,31 @@ describe("captionEditing", () => {
 
 		expect(updateCaptionCuesForEditedTarget(cues, visibleTarget, " \n\t ")).toBe(cues);
 	});
+
+	it("keeps sound-effect style captions editable when word entries are blank", () => {
+		const cues: CaptionCue[] = [
+			{
+				id: "sound-effect",
+				startMs: 1_000,
+				endMs: 2_000,
+				text: "clears throat",
+				words: [{ text: "", startMs: 1_000, endMs: 2_000 }],
+			},
+		];
+
+		const layout = buildActiveCaptionLayout({
+			cues,
+			timeMs: 1_500,
+			settings: DEFAULT_AUTO_CAPTION_SETTINGS,
+			maxWidthPx: 500,
+			measureText: (text) => text.length * 10,
+		});
+
+		expect(layout?.editTarget.text).toBe("clears throat");
+
+		const updated = updateCaptionCuesForEditedTarget(cues, layout!.editTarget, "coughs");
+
+		expect(updated[0].text).toBe("coughs");
+		expect(updated[0].words).toEqual([{ text: "coughs", startMs: 1_000, endMs: 2_000 }]);
+	});
 });

src/components/video-editor/captionEditing.ts

@@ -56,13 +56,19 @@ function buildCaptionWordsForEditedText(
 }
 
 function normalizeCaptionWords(cue: CaptionCue): CaptionCueWord[] {
+	const validSourceWords = Array.isArray(cue.words)
+		? cue.words.filter((word): word is CaptionCueWord =>
+				Boolean(
+					word && typeof word.text === "string" && normalizeCaptionEditText(word.text),
+				),
+			)
+		: [];
 	const sourceWords =
-		Array.isArray(cue.words) && cue.words.length > 0
-			? cue.words
+		validSourceWords.length > 0
+			? validSourceWords
 			: buildCaptionWordsForEditedText(cue.text, cue.startMs, cue.endMs);
 
 	return sourceWords
-		.filter((word): word is CaptionCueWord => Boolean(word && typeof word.text === "string"))
 		.map((word) => {
 			const startMs = Math.max(
 				cue.startMs,

src/components/video-editor/captionLayout.ts

@@ -95,7 +95,7 @@ function splitCaptionWordsFromText(text: string) {
 
 function splitCaptionWords(cue: CaptionCue) {
 	if (Array.isArray(cue.words) && cue.words.length > 0) {
-		return cue.words
+		const words = cue.words
 			.filter((word): word is CaptionCueWord =>
 				Boolean(word && typeof word.text === "string"),
 			)
@@ -109,6 +109,10 @@ function splitCaptionWords(cue: CaptionCue) {
 				endMs: word.endMs,
 			}))
 			.filter((word) => word.text.length > 0);
+
+		if (words.length > 0) {
+			return words;
+		}
 	}
 
 	return splitCaptionWordsFromText(cue.text).map((word) => ({

src/lib/exporter/annotationRenderer.ts

@@ -8,6 +8,28 @@ export interface AnnotationRenderAssets {
 	imageCache: Map<string, HTMLImageElement>;
 }
 
+interface AnnotationSceneTransform {
+	scale: number;
+	x: number;
+	y: number;
+}
+
+function transformAnnotationRect(
+	rect: { x: number; y: number; width: number; height: number },
+	sceneTransform?: AnnotationSceneTransform,
+) {
+	if (!sceneTransform) {
+		return rect;
+	}
+
+	return {
+		x: rect.x * sceneTransform.scale + sceneTransform.x,
+		y: rect.y * sceneTransform.scale + sceneTransform.y,
+		width: rect.width * sceneTransform.scale,
+		height: rect.height * sceneTransform.scale,
+	};
+}
+
 const annotationImagePromiseCache = new Map<string, Promise<HTMLImageElement | null>>();
 
 let blurBufferCanvas: HTMLCanvasElement | null = null;
@@ -334,6 +356,7 @@ export async function renderAnnotations(
 	currentTimeMs: number,
 	scaleFactor: number = 1.0,
 	assets?: AnnotationRenderAssets,
+	sceneTransform?: AnnotationSceneTransform,
 ): Promise<void> {
 	const activeAnnotations = annotations.filter(
 		(ann) => currentTimeMs >= ann.startMs && currentTimeMs <= ann.endMs,
@@ -342,14 +365,21 @@ export async function renderAnnotations(
 	const sortedAnnotations = [...activeAnnotations].sort((a, b) => a.zIndex - b.zIndex);
 
 	for (const annotation of sortedAnnotations) {
-		const x = (annotation.position.x / 100) * canvasWidth;
-		const y = (annotation.position.y / 100) * canvasHeight;
-		const width = (annotation.size.width / 100) * canvasWidth;
-		const height = (annotation.size.height / 100) * canvasHeight;
+		const rect = transformAnnotationRect(
+			{
+				x: (annotation.position.x / 100) * canvasWidth,
+				y: (annotation.position.y / 100) * canvasHeight,
+				width: (annotation.size.width / 100) * canvasWidth,
+				height: (annotation.size.height / 100) * canvasHeight,
+			},
+			sceneTransform,
+		);
+		const { x, y, width, height } = rect;
+		const effectiveScaleFactor = scaleFactor * (sceneTransform?.scale ?? 1);
 
 		switch (annotation.type) {
 			case "text":
-				renderText(ctx, annotation, x, y, width, height, scaleFactor);
+				renderText(ctx, annotation, x, y, width, height, effectiveScaleFactor);
 				break;
 
 			case "image":
@@ -367,20 +397,20 @@ export async function renderAnnotations(
 						y,
 						width,
 						height,
-						scaleFactor,
+						effectiveScaleFactor,
 					);
 				}
 				break;
 
 			case "blur": {
 				const blurStrength =
-					(annotation.blurIntensity ?? BLUR_ANNOTATION_STRENGTH) * scaleFactor;
+					(annotation.blurIntensity ?? BLUR_ANNOTATION_STRENGTH) * effectiveScaleFactor;
 				const padding = Math.ceil(blurStrength * 2);
 
 				ctx.save();
 
 				ctx.beginPath();
-				const borderRadius = (annotation.style.borderRadius ?? 0) * scaleFactor;
+				const borderRadius = (annotation.style.borderRadius ?? 0) * effectiveScaleFactor;
 				ctx.roundRect(x, y, width, height, borderRadius);
 				ctx.clip();
 

src/lib/exporter/frameRenderer.ts

@@ -5,8 +5,8 @@ import type {
 	AnnotationRegion,
 	AutoCaptionSettings,
 	CaptionCue,
-	CursorClickEffectStyle,
 	CropRegion,
+	CursorClickEffectStyle,
 	CursorStyle,
 	CursorTelemetryPoint,
 	Padding,
@@ -448,14 +448,14 @@ export class FrameRenderer {
 					massMultiplier: this.config.cursorSpringMassMultiplier,
 				},
 				motionBlur: this.config.cursorMotionBlur ?? 0,
-				clickEffect:
-					this.config.cursorClickEffect ?? DEFAULT_CURSOR_CONFIG.clickEffect,
+				clickEffect: this.config.cursorClickEffect ?? DEFAULT_CURSOR_CONFIG.clickEffect,
 				clickEffectColor:
 					this.config.cursorClickEffectColor ?? DEFAULT_CURSOR_CONFIG.clickEffectColor,
 				clickEffectScale:
 					this.config.cursorClickEffectScale ?? DEFAULT_CURSOR_CONFIG.clickEffectScale,
 				clickEffectOpacity:
-					this.config.cursorClickEffectOpacity ?? DEFAULT_CURSOR_CONFIG.clickEffectOpacity,
+					this.config.cursorClickEffectOpacity ??
+					DEFAULT_CURSOR_CONFIG.clickEffectOpacity,
 				clickEffectDurationMs:
 					this.config.cursorClickEffectDurationMs ??
 					DEFAULT_CURSOR_CONFIG.clickEffectDurationMs,
@@ -1547,6 +1547,8 @@ export class FrameRenderer {
 					this.config.height,
 					temporalSnapshot.timeMs,
 					scaleFactor,
+					undefined,
+					temporalSnapshot.sceneTransform,
 				);
 			}
 
@@ -1731,6 +1733,12 @@ export class FrameRenderer {
 				this.config.height,
 				timeMs,
 				scaleFactor,
+				undefined,
+				{
+					scale: this.animationState.appliedScale,
+					x: this.animationState.x,
+					y: this.animationState.y,
+				},
 			);
 		}
 

src/lib/exporter/modernFrameRenderer.ts

@@ -15,8 +15,8 @@ import type {
 	AnnotationRegion,
 	AutoCaptionSettings,
 	CaptionCue,
-	CursorClickEffectStyle,
 	CropRegion,
+	CursorClickEffectStyle,
 	CursorStyle,
 	CursorTelemetryPoint,
 	Padding,
@@ -598,8 +598,8 @@ export class FrameRenderer {
 		this.webcamRootContainer.addChild(this.webcamContainer);
 		this.webcamRootContainer.visible = false;
 
+		this.cameraContainer.addChild(this.annotationContainer);
 		this.overlayContainer.addChild(this.webcamRootContainer);
-		this.overlayContainer.addChild(this.annotationContainer);
 		this.overlayContainer.addChild(this.captionContainer);
 
 		this.videoMaskGraphics = new Graphics();
@@ -622,14 +622,14 @@ export class FrameRenderer {
 					massMultiplier: this.config.cursorSpringMassMultiplier,
 				},
 				motionBlur: this.config.cursorMotionBlur ?? 0,
-				clickEffect:
-					this.config.cursorClickEffect ?? DEFAULT_CURSOR_CONFIG.clickEffect,
+				clickEffect: this.config.cursorClickEffect ?? DEFAULT_CURSOR_CONFIG.clickEffect,
 				clickEffectColor:
 					this.config.cursorClickEffectColor ?? DEFAULT_CURSOR_CONFIG.clickEffectColor,
 				clickEffectScale:
 					this.config.cursorClickEffectScale ?? DEFAULT_CURSOR_CONFIG.clickEffectScale,
 				clickEffectOpacity:
-					this.config.cursorClickEffectOpacity ?? DEFAULT_CURSOR_CONFIG.clickEffectOpacity,
+					this.config.cursorClickEffectOpacity ??
+					DEFAULT_CURSOR_CONFIG.clickEffectOpacity,
 				clickEffectDurationMs:
 					this.config.cursorClickEffectDurationMs ??
 					DEFAULT_CURSOR_CONFIG.clickEffectDurationMs,
@@ -1586,6 +1586,11 @@ export class FrameRenderer {
 			timeMs,
 			this.annotationScaleFactor,
 			this.annotationAssets ?? undefined,
+			{
+				scale: this.animationState.appliedScale,
+				x: this.animationState.x,
+				y: this.animationState.y,
+			},
 		);
 
 		this.drawCaptionOverlay(context);
@@ -2549,15 +2554,10 @@ export class FrameRenderer {
 			const usesDefaultCropRegion = isWebcamCropRegionDefault(this.config.webcam?.cropRegion);
 			const needsCacheBackedSource =
 				!usesDefaultCropRegion ||
-				(typeof HTMLVideoElement !== "undefined" &&
-					liveSource instanceof HTMLVideoElement);
+				(typeof HTMLVideoElement !== "undefined" && liveSource instanceof HTMLVideoElement);
 
 			if (needsCacheBackedSource) {
-				this.refreshWebcamFrameCache(
-					liveSource,
-					liveSourceWidth,
-					liveSourceHeight,
-				);
+				this.refreshWebcamFrameCache(liveSource, liveSourceWidth, liveSourceHeight);
 				const cachedSource = this.getCachedWebcamRenderSource();
 				if (cachedSource) {
 					this.setWebcamRenderMode("live");