feat: render captions in the export pipeline

Add canvas caption rendering during frame export, carry caption settings through GIF and MP4 export paths, and clean up muxer resources while exposing the saving phase in export progress.

src/lib/exporter/captionRenderer.ts

@@ -0,0 +1,104 @@
+import {
+  getDefaultCaptionFontFamily,
+  type AutoCaptionSettings,
+  type CaptionCue,
+} from "@/components/video-editor/types";
+import { drawSquircleOnCanvas } from "@/lib/geometry/squircle";
+import {
+  buildActiveCaptionLayout,
+} from "@/components/video-editor/captionLayout";
+import {
+  CAPTION_FONT_WEIGHT,
+  CAPTION_LINE_HEIGHT,
+  getCaptionPadding,
+  getCaptionScaledRadius,
+  getCaptionScaledFontSize,
+  getCaptionTextMaxWidth,
+  getCaptionWordVisualState,
+} from "@/components/video-editor/captionStyle";
+
+export function renderCaptions(
+  ctx: CanvasRenderingContext2D,
+  cues: CaptionCue[],
+  settings: AutoCaptionSettings,
+  width: number,
+  height: number,
+  timeMs: number,
+) {
+  if (!settings.enabled || cues.length === 0) {
+    return;
+  }
+
+  ctx.save();
+
+  const fontSize = getCaptionScaledFontSize(settings.fontSize, width, settings.maxWidth);
+  ctx.font = `${CAPTION_FONT_WEIGHT} ${fontSize}px ${getDefaultCaptionFontFamily()}`;
+  const padding = getCaptionPadding(fontSize);
+
+  const activeCaptionLayout = buildActiveCaptionLayout({
+    cues,
+    timeMs,
+    settings,
+    maxWidthPx: getCaptionTextMaxWidth(width, settings.maxWidth, fontSize),
+    measureText: (text) => ctx.measureText(text).width,
+  });
+  if (!activeCaptionLayout) {
+    ctx.restore();
+    return;
+  }
+
+  const lineHeight = fontSize * CAPTION_LINE_HEIGHT;
+  const paddingX = padding.x;
+  const paddingY = padding.y;
+  const textBlockHeight = activeCaptionLayout.visibleLines.length * lineHeight;
+  const boxHeight = textBlockHeight + paddingY * 2;
+  const centerX = width / 2;
+  const centerY = height - (height * settings.bottomOffset) / 100 - boxHeight / 2;
+  const maxMeasuredWidth = activeCaptionLayout.visibleLines.reduce(
+    (largest, line) => Math.max(largest, line.width),
+    0,
+  );
+  const boxWidth = Math.min(width * (settings.maxWidth / 100) + paddingX * 2, maxMeasuredWidth + paddingX * 2);
+
+  ctx.translate(centerX, centerY + activeCaptionLayout.translateY);
+  ctx.scale(activeCaptionLayout.scale, activeCaptionLayout.scale);
+  ctx.globalAlpha = activeCaptionLayout.opacity;
+
+  ctx.fillStyle = `rgba(0, 0, 0, ${settings.backgroundOpacity})`;
+  drawSquircleOnCanvas(ctx, {
+    x: -boxWidth / 2,
+    y: -boxHeight / 2,
+    width: boxWidth,
+    height: boxHeight,
+    radius: getCaptionScaledRadius(settings.boxRadius, fontSize),
+  });
+  ctx.fill();
+
+  ctx.textAlign = "left";
+  ctx.textBaseline = "middle";
+
+  activeCaptionLayout.visibleLines.forEach((line, lineIndex) => {
+    let cursorX = -line.width / 2;
+    const lineY = -boxHeight / 2 + paddingY + lineHeight * lineIndex + lineHeight / 2;
+
+    line.words.forEach((word) => {
+      const segmentText = `${word.leadingSpace ? " " : ""}${word.text}`;
+      const segmentWidth = ctx.measureText(segmentText).width;
+      const visualState = getCaptionWordVisualState(
+        activeCaptionLayout.hasWordTimings,
+        word.state,
+      );
+
+      ctx.save();
+      ctx.translate(cursorX, lineY);
+      ctx.fillStyle = visualState.isInactive ? settings.inactiveTextColor : settings.textColor;
+      ctx.globalAlpha = activeCaptionLayout.opacity * visualState.opacity;
+      ctx.fillText(segmentText, 0, 0);
+      ctx.restore();
+
+      cursorX += segmentWidth;
+    });
+  });
+
+  ctx.restore();
+}

src/lib/exporter/frameRenderer.ts

@@ -5,15 +5,20 @@ import {
   Graphics,
   BlurFilter,
   Texture,
+  TextureSource,
 } from "pixi.js";
 import { MotionBlurFilter } from "pixi-filters/motion-blur";
 import type {
+  AutoCaptionSettings,
+  CaptionCue,
   ZoomRegion,
   CropRegion,
   AnnotationRegion,
   SpeedRegion,
+  CursorStyle,
   CursorTelemetryPoint,
   WebcamOverlaySettings,
+  ZoomTransitionEasing,
 } from "@/components/video-editor/types";
 import { ZOOM_DEPTH_SCALES } from "@/components/video-editor/types";
 import { getAssetPath, getRenderableAssetUrl } from "@/lib/assetPath";
@@ -31,6 +36,7 @@ import {
   ZOOM_TRANSLATION_DEADZONE_PX,
 } from "@/components/video-editor/videoPlayback/constants";
 import { renderAnnotations } from "./annotationRenderer";
+import { renderCaptions } from "./captionRenderer";
 import {
   PixiCursorOverlay,
   DEFAULT_CURSOR_CONFIG,
@@ -52,6 +58,14 @@ interface FrameRenderConfig {
   backgroundBlur: number;
   zoomMotionBlur?: number;
   connectZooms?: boolean;
+  zoomInDurationMs?: number;
+  zoomInOverlapMs?: number;
+  zoomOutDurationMs?: number;
+  connectedZoomGapMs?: number;
+  connectedZoomDurationMs?: number;
+  zoomInEasing?: ZoomTransitionEasing;
+  zoomOutEasing?: ZoomTransitionEasing;
+  connectedZoomEasing?: ZoomTransitionEasing;
   borderRadius?: number;
   padding?: number;
   cropRegion: CropRegion;
@@ -60,11 +74,14 @@ interface FrameRenderConfig {
   videoWidth: number;
   videoHeight: number;
   annotationRegions?: AnnotationRegion[];
+  autoCaptions?: CaptionCue[];
+  autoCaptionSettings?: AutoCaptionSettings;
   speedRegions?: SpeedRegion[];
   previewWidth?: number;
   previewHeight?: number;
   cursorTelemetry?: CursorTelemetryPoint[];
   showCursor?: boolean;
+  cursorStyle?: CursorStyle;
   cursorSize?: number;
   cursorSmoothing?: number;
   cursorMotionBlur?: number;
@@ -103,6 +120,7 @@ export class FrameRenderer {
   private videoContainer: Container | null = null;
   private cursorContainer: Container | null = null;
   private videoSprite: Sprite | null = null;
+  private videoTextureSource: TextureSource<any> | null = null;
   private backgroundSprite: Sprite | null = null;
   private maskGraphics: Graphics | null = null;
   private blurFilter: BlurFilter | null = null;
@@ -190,6 +208,7 @@ export class FrameRenderer {
       this.cursorOverlay = new PixiCursorOverlay({
         dotRadius:
           DEFAULT_CURSOR_CONFIG.dotRadius * (this.config.cursorSize ?? 1.4),
+        style: this.config.cursorStyle ?? "tahoe",
         smoothingFactor:
           this.config.cursorSmoothing ?? DEFAULT_CURSOR_CONFIG.smoothingFactor,
         motionBlur: this.config.cursorMotionBlur ?? 0,
@@ -693,6 +712,7 @@ export class FrameRenderer {
     if (!this.videoSprite) {
       const texture = Texture.from(videoFrame as any);
       this.videoSprite = new Sprite(texture);
+      this.videoTextureSource = texture.source as TextureSource<any>;
       this.videoContainer.addChild(this.videoSprite);
       if (this.cursorOverlay && this.cursorContainer) {
         this.cursorContainer.addChild(this.cursorOverlay.container);
@@ -701,11 +721,9 @@ export class FrameRenderer {
         this.videoContainer.addChild(this.maskGraphics);
       }
     } else {
-      // Destroy old texture to avoid memory leaks, then create new one
-      const oldTexture = this.videoSprite.texture;
-      const newTexture = Texture.from(videoFrame as any);
-      this.videoSprite.texture = newTexture;
-      oldTexture.destroy(true);
+      this.videoTextureSource ??= this.videoSprite.texture.source as TextureSource<any>;
+      this.videoTextureSource.resource = videoFrame as any;
+      this.videoTextureSource.update();
     }
 
     // Apply layout
@@ -783,6 +801,22 @@ export class FrameRenderer {
         scaleFactor,
       );
     }
+
+    if (
+    this.config.autoCaptions &&
+    this.config.autoCaptions.length > 0 &&
+    this.config.autoCaptionSettings &&
+    this.compositeCtx
+  ) {
+    renderCaptions(
+    this.compositeCtx,
+    this.config.autoCaptions,
+    this.config.autoCaptionSettings,
+    this.config.width,
+    this.config.height,
+    timeMs,
+    );
+  }
   }
 
   private updateLayout(): void {
@@ -1220,6 +1254,7 @@ export class FrameRenderer {
       this.videoSprite.destroy({ texture: false, textureSource: false });
       videoTexture?.destroy(true);
       this.videoSprite = null;
+      this.videoTextureSource = null;
     }
     this.backgroundSprite = null;
     if (this.app) {

src/lib/exporter/gifExporter.ts

@@ -9,13 +9,17 @@ import type {
 import { StreamingVideoDecoder } from "./streamingDecoder";
 import { FrameRenderer } from "./frameRenderer";
 import type {
+  AutoCaptionSettings,
   ZoomRegion,
   CropRegion,
   TrimRegion,
   AnnotationRegion,
+  CaptionCue,
   SpeedRegion,
+  CursorStyle,
   CursorTelemetryPoint,
   WebcamOverlaySettings,
+  ZoomTransitionEasing,
 } from "@/components/video-editor/types";
 
 const GIF_WORKER_URL = new URL(
@@ -39,6 +43,14 @@ interface GifExporterConfig {
   backgroundBlur: number;
   zoomMotionBlur?: number;
   connectZooms?: boolean;
+  zoomInDurationMs?: number;
+  zoomInOverlapMs?: number;
+  zoomOutDurationMs?: number;
+  connectedZoomGapMs?: number;
+  connectedZoomDurationMs?: number;
+  zoomInEasing?: ZoomTransitionEasing;
+  zoomOutEasing?: ZoomTransitionEasing;
+  connectedZoomEasing?: ZoomTransitionEasing;
   borderRadius?: number;
   padding?: number;
   videoPadding?: number;
@@ -46,8 +58,11 @@ interface GifExporterConfig {
   webcam?: WebcamOverlaySettings;
   webcamUrl?: string | null;
   annotationRegions?: AnnotationRegion[];
+  autoCaptions?: CaptionCue[];
+  autoCaptionSettings?: AutoCaptionSettings;
   cursorTelemetry?: CursorTelemetryPoint[];
   showCursor?: boolean;
+  cursorStyle?: CursorStyle;
   cursorSize?: number;
   cursorSmoothing?: number;
   cursorMotionBlur?: number;
@@ -126,6 +141,14 @@ export class GifExporter {
         backgroundBlur: this.config.backgroundBlur,
         zoomMotionBlur: this.config.zoomMotionBlur,
         connectZooms: this.config.connectZooms,
+        zoomInDurationMs: this.config.zoomInDurationMs,
+        zoomInOverlapMs: this.config.zoomInOverlapMs,
+        zoomOutDurationMs: this.config.zoomOutDurationMs,
+    		connectedZoomGapMs: this.config.connectedZoomGapMs,
+    		connectedZoomDurationMs: this.config.connectedZoomDurationMs,
+    		zoomInEasing: this.config.zoomInEasing,
+    		zoomOutEasing: this.config.zoomOutEasing,
+    		connectedZoomEasing: this.config.connectedZoomEasing,
         borderRadius: this.config.borderRadius,
         padding: this.config.padding,
         cropRegion: this.config.cropRegion,
@@ -134,11 +157,14 @@ export class GifExporter {
         videoWidth: videoInfo.width,
         videoHeight: videoInfo.height,
         annotationRegions: this.config.annotationRegions,
+        autoCaptions: this.config.autoCaptions,
+        autoCaptionSettings: this.config.autoCaptionSettings,
         speedRegions: this.config.speedRegions,
         previewWidth: this.config.previewWidth,
         previewHeight: this.config.previewHeight,
         cursorTelemetry: this.config.cursorTelemetry,
         showCursor: this.config.showCursor,
+        cursorStyle: this.config.cursorStyle,
         cursorSize: this.config.cursorSize,
         cursorSmoothing: this.config.cursorSmoothing,
         cursorMotionBlur: this.config.cursorMotionBlur,

src/lib/exporter/muxer.ts

@@ -86,5 +86,12 @@ export class VideoMuxer {
     
     return new Blob([buffer], { type: 'video/mp4' });
   }
+
+  destroy(): void {
+    this.output = null;
+    this.videoSource = null;
+    this.audioSource = null;
+    this.target = null;
+  }
 }
 

src/lib/exporter/types.ts

@@ -11,7 +11,7 @@ export interface ExportProgress {
   totalFrames: number;
   percentage: number;
   estimatedTimeRemaining: number; // in seconds
-  phase?: 'extracting' | 'finalizing'; // Phase of export
+  phase?: 'extracting' | 'finalizing' | 'saving'; // Phase of export
   renderProgress?: number; // 0-100, progress of GIF rendering phase
 }
 

src/lib/exporter/videoExporter.ts

@@ -1,11 +1,15 @@
 import type {
+	AutoCaptionSettings,
 	AnnotationRegion,
 	AudioRegion,
+	CaptionCue,
 	CropRegion,
+	CursorStyle,
 	CursorTelemetryPoint,
 	SpeedRegion,
 	TrimRegion,
 	WebcamOverlaySettings,
+	ZoomTransitionEasing,
 	ZoomRegion,
 } from "@/components/video-editor/types";
 import { AudioProcessor } from "./audioEncoder";
@@ -25,6 +29,14 @@ interface VideoExporterConfig extends ExportConfig {
 	backgroundBlur: number;
 	zoomMotionBlur?: number;
 	connectZooms?: boolean;
+	zoomInDurationMs?: number;
+	zoomInOverlapMs?: number;
+	zoomOutDurationMs?: number;
+	connectedZoomGapMs?: number;
+	connectedZoomDurationMs?: number;
+	zoomInEasing?: ZoomTransitionEasing;
+	zoomOutEasing?: ZoomTransitionEasing;
+	connectedZoomEasing?: ZoomTransitionEasing;
 	borderRadius?: number;
 	padding?: number;
 	videoPadding?: number;
@@ -32,8 +44,11 @@ interface VideoExporterConfig extends ExportConfig {
 	webcam?: WebcamOverlaySettings;
 	webcamUrl?: string | null;
 	annotationRegions?: AnnotationRegion[];
+	autoCaptions?: CaptionCue[];
+	autoCaptionSettings?: AutoCaptionSettings;
 	cursorTelemetry?: CursorTelemetryPoint[];
 	showCursor?: boolean;
+	cursorStyle?: CursorStyle;
 	cursorSize?: number;
 	cursorSmoothing?: number;
 	cursorMotionBlur?: number;
@@ -87,6 +102,14 @@ export class VideoExporter {
 				backgroundBlur: this.config.backgroundBlur,
 				zoomMotionBlur: this.config.zoomMotionBlur,
 				connectZooms: this.config.connectZooms,
+				zoomInDurationMs: this.config.zoomInDurationMs,
+				zoomInOverlapMs: this.config.zoomInOverlapMs,
+				zoomOutDurationMs: this.config.zoomOutDurationMs,
+				connectedZoomGapMs: this.config.connectedZoomGapMs,
+				connectedZoomDurationMs: this.config.connectedZoomDurationMs,
+				zoomInEasing: this.config.zoomInEasing,
+				zoomOutEasing: this.config.zoomOutEasing,
+				connectedZoomEasing: this.config.connectedZoomEasing,
 				borderRadius: this.config.borderRadius,
 				padding: this.config.padding,
 				cropRegion: this.config.cropRegion,
@@ -95,11 +118,14 @@ export class VideoExporter {
 				videoWidth: videoInfo.width,
 				videoHeight: videoInfo.height,
 				annotationRegions: this.config.annotationRegions,
+				autoCaptions: this.config.autoCaptions,
+				autoCaptionSettings: this.config.autoCaptionSettings,
 				speedRegions: this.config.speedRegions,
 				previewWidth: this.config.previewWidth,
 				previewHeight: this.config.previewHeight,
 				cursorTelemetry: this.config.cursorTelemetry,
 				showCursor: this.config.showCursor,
+				cursorStyle: this.config.cursorStyle,
 				cursorSize: this.config.cursorSize,
 				cursorSmoothing: this.config.cursorSmoothing,
 				cursorMotionBlur: this.config.cursorMotionBlur,
@@ -440,6 +466,14 @@ export class VideoExporter {
 			this.renderer = null;
 		}
 
+		if (this.muxer) {
+			try {
+				this.muxer.destroy();
+			} catch (e) {
+				console.warn("Error destroying muxer:", e);
+			}
+		}
+
 		this.muxer = null;
 		this.audioProcessor = null;
 		this.encodeQueue = 0;