From 549871816b1f4d387348c865eadb433505ec6316 Mon Sep 17 00:00:00 2001 From: webadderall <131426131+webadderall@users.noreply.github.com> Date: Sun, 22 Mar 2026 18:33:26 +1100 Subject: [PATCH] feat: render captions in the export pipeline Add canvas caption rendering during frame export, carry caption settings through GIF and MP4 export paths, and clean up muxer resources while exposing the saving phase in export progress. --- src/lib/exporter/captionRenderer.ts | 104 ++++++++++++++++++++++++++++ src/lib/exporter/frameRenderer.ts | 45 ++++++++++-- src/lib/exporter/gifExporter.ts | 26 +++++++ src/lib/exporter/muxer.ts | 7 ++ src/lib/exporter/types.ts | 2 +- src/lib/exporter/videoExporter.ts | 34 +++++++++ 6 files changed, 212 insertions(+), 6 deletions(-) create mode 100644 src/lib/exporter/captionRenderer.ts diff --git a/src/lib/exporter/captionRenderer.ts b/src/lib/exporter/captionRenderer.ts new file mode 100644 index 00000000..8cbf4586 --- /dev/null +++ b/src/lib/exporter/captionRenderer.ts @@ -0,0 +1,104 @@ +import { + getDefaultCaptionFontFamily, + type AutoCaptionSettings, + type CaptionCue, +} from "@/components/video-editor/types"; +import { drawSquircleOnCanvas } from "@/lib/geometry/squircle"; +import { + buildActiveCaptionLayout, +} from "@/components/video-editor/captionLayout"; +import { + CAPTION_FONT_WEIGHT, + CAPTION_LINE_HEIGHT, + getCaptionPadding, + getCaptionScaledRadius, + getCaptionScaledFontSize, + getCaptionTextMaxWidth, + getCaptionWordVisualState, +} from "@/components/video-editor/captionStyle"; + +export function renderCaptions( + ctx: CanvasRenderingContext2D, + cues: CaptionCue[], + settings: AutoCaptionSettings, + width: number, + height: number, + timeMs: number, +) { + if (!settings.enabled || cues.length === 0) { + return; + } + + ctx.save(); + + const fontSize = getCaptionScaledFontSize(settings.fontSize, width, settings.maxWidth); + ctx.font = `${CAPTION_FONT_WEIGHT} ${fontSize}px ${getDefaultCaptionFontFamily()}`; + const padding = getCaptionPadding(fontSize); + + const activeCaptionLayout = buildActiveCaptionLayout({ + cues, + timeMs, + settings, + maxWidthPx: getCaptionTextMaxWidth(width, settings.maxWidth, fontSize), + measureText: (text) => ctx.measureText(text).width, + }); + if (!activeCaptionLayout) { + ctx.restore(); + return; + } + + const lineHeight = fontSize * CAPTION_LINE_HEIGHT; + const paddingX = padding.x; + const paddingY = padding.y; + const textBlockHeight = activeCaptionLayout.visibleLines.length * lineHeight; + const boxHeight = textBlockHeight + paddingY * 2; + const centerX = width / 2; + const centerY = height - (height * settings.bottomOffset) / 100 - boxHeight / 2; + const maxMeasuredWidth = activeCaptionLayout.visibleLines.reduce( + (largest, line) => Math.max(largest, line.width), + 0, + ); + const boxWidth = Math.min(width * (settings.maxWidth / 100) + paddingX * 2, maxMeasuredWidth + paddingX * 2); + + ctx.translate(centerX, centerY + activeCaptionLayout.translateY); + ctx.scale(activeCaptionLayout.scale, activeCaptionLayout.scale); + ctx.globalAlpha = activeCaptionLayout.opacity; + + ctx.fillStyle = `rgba(0, 0, 0, ${settings.backgroundOpacity})`; + drawSquircleOnCanvas(ctx, { + x: -boxWidth / 2, + y: -boxHeight / 2, + width: boxWidth, + height: boxHeight, + radius: getCaptionScaledRadius(settings.boxRadius, fontSize), + }); + ctx.fill(); + + ctx.textAlign = "left"; + ctx.textBaseline = "middle"; + + activeCaptionLayout.visibleLines.forEach((line, lineIndex) => { + let cursorX = -line.width / 2; + const lineY = -boxHeight / 2 + paddingY + lineHeight * lineIndex + lineHeight / 2; + + line.words.forEach((word) => { + const segmentText = `${word.leadingSpace ? " " : ""}${word.text}`; + const segmentWidth = ctx.measureText(segmentText).width; + const visualState = getCaptionWordVisualState( + activeCaptionLayout.hasWordTimings, + word.state, + ); + + ctx.save(); + ctx.translate(cursorX, lineY); + ctx.fillStyle = visualState.isInactive ? settings.inactiveTextColor : settings.textColor; + ctx.globalAlpha = activeCaptionLayout.opacity * visualState.opacity; + ctx.fillText(segmentText, 0, 0); + ctx.restore(); + + cursorX += segmentWidth; + }); + }); + + ctx.restore(); +} \ No newline at end of file diff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.ts index fb5d049f..be9d8ce3 100644 --- a/src/lib/exporter/frameRenderer.ts +++ b/src/lib/exporter/frameRenderer.ts @@ -5,15 +5,20 @@ import { Graphics, BlurFilter, Texture, + TextureSource, } from "pixi.js"; import { MotionBlurFilter } from "pixi-filters/motion-blur"; import type { + AutoCaptionSettings, + CaptionCue, ZoomRegion, CropRegion, AnnotationRegion, SpeedRegion, + CursorStyle, CursorTelemetryPoint, WebcamOverlaySettings, + ZoomTransitionEasing, } from "@/components/video-editor/types"; import { ZOOM_DEPTH_SCALES } from "@/components/video-editor/types"; import { getAssetPath, getRenderableAssetUrl } from "@/lib/assetPath"; @@ -31,6 +36,7 @@ import { ZOOM_TRANSLATION_DEADZONE_PX, } from "@/components/video-editor/videoPlayback/constants"; import { renderAnnotations } from "./annotationRenderer"; +import { renderCaptions } from "./captionRenderer"; import { PixiCursorOverlay, DEFAULT_CURSOR_CONFIG, @@ -52,6 +58,14 @@ interface FrameRenderConfig { backgroundBlur: number; zoomMotionBlur?: number; connectZooms?: boolean; + zoomInDurationMs?: number; + zoomInOverlapMs?: number; + zoomOutDurationMs?: number; + connectedZoomGapMs?: number; + connectedZoomDurationMs?: number; + zoomInEasing?: ZoomTransitionEasing; + zoomOutEasing?: ZoomTransitionEasing; + connectedZoomEasing?: ZoomTransitionEasing; borderRadius?: number; padding?: number; cropRegion: CropRegion; @@ -60,11 +74,14 @@ interface FrameRenderConfig { videoWidth: number; videoHeight: number; annotationRegions?: AnnotationRegion[]; + autoCaptions?: CaptionCue[]; + autoCaptionSettings?: AutoCaptionSettings; speedRegions?: SpeedRegion[]; previewWidth?: number; previewHeight?: number; cursorTelemetry?: CursorTelemetryPoint[]; showCursor?: boolean; + cursorStyle?: CursorStyle; cursorSize?: number; cursorSmoothing?: number; cursorMotionBlur?: number; @@ -103,6 +120,7 @@ export class FrameRenderer { private videoContainer: Container | null = null; private cursorContainer: Container | null = null; private videoSprite: Sprite | null = null; + private videoTextureSource: TextureSource | null = null; private backgroundSprite: Sprite | null = null; private maskGraphics: Graphics | null = null; private blurFilter: BlurFilter | null = null; @@ -190,6 +208,7 @@ export class FrameRenderer { this.cursorOverlay = new PixiCursorOverlay({ dotRadius: DEFAULT_CURSOR_CONFIG.dotRadius * (this.config.cursorSize ?? 1.4), + style: this.config.cursorStyle ?? "tahoe", smoothingFactor: this.config.cursorSmoothing ?? DEFAULT_CURSOR_CONFIG.smoothingFactor, motionBlur: this.config.cursorMotionBlur ?? 0, @@ -693,6 +712,7 @@ export class FrameRenderer { if (!this.videoSprite) { const texture = Texture.from(videoFrame as any); this.videoSprite = new Sprite(texture); + this.videoTextureSource = texture.source as TextureSource; this.videoContainer.addChild(this.videoSprite); if (this.cursorOverlay && this.cursorContainer) { this.cursorContainer.addChild(this.cursorOverlay.container); @@ -701,11 +721,9 @@ export class FrameRenderer { this.videoContainer.addChild(this.maskGraphics); } } else { - // Destroy old texture to avoid memory leaks, then create new one - const oldTexture = this.videoSprite.texture; - const newTexture = Texture.from(videoFrame as any); - this.videoSprite.texture = newTexture; - oldTexture.destroy(true); + this.videoTextureSource ??= this.videoSprite.texture.source as TextureSource; + this.videoTextureSource.resource = videoFrame as any; + this.videoTextureSource.update(); } // Apply layout @@ -783,6 +801,22 @@ export class FrameRenderer { scaleFactor, ); } + + if ( + this.config.autoCaptions && + this.config.autoCaptions.length > 0 && + this.config.autoCaptionSettings && + this.compositeCtx + ) { + renderCaptions( + this.compositeCtx, + this.config.autoCaptions, + this.config.autoCaptionSettings, + this.config.width, + this.config.height, + timeMs, + ); + } } private updateLayout(): void { @@ -1220,6 +1254,7 @@ export class FrameRenderer { this.videoSprite.destroy({ texture: false, textureSource: false }); videoTexture?.destroy(true); this.videoSprite = null; + this.videoTextureSource = null; } this.backgroundSprite = null; if (this.app) { diff --git a/src/lib/exporter/gifExporter.ts b/src/lib/exporter/gifExporter.ts index 8b0db134..08dffff3 100644 --- a/src/lib/exporter/gifExporter.ts +++ b/src/lib/exporter/gifExporter.ts @@ -9,13 +9,17 @@ import type { import { StreamingVideoDecoder } from "./streamingDecoder"; import { FrameRenderer } from "./frameRenderer"; import type { + AutoCaptionSettings, ZoomRegion, CropRegion, TrimRegion, AnnotationRegion, + CaptionCue, SpeedRegion, + CursorStyle, CursorTelemetryPoint, WebcamOverlaySettings, + ZoomTransitionEasing, } from "@/components/video-editor/types"; const GIF_WORKER_URL = new URL( @@ -39,6 +43,14 @@ interface GifExporterConfig { backgroundBlur: number; zoomMotionBlur?: number; connectZooms?: boolean; + zoomInDurationMs?: number; + zoomInOverlapMs?: number; + zoomOutDurationMs?: number; + connectedZoomGapMs?: number; + connectedZoomDurationMs?: number; + zoomInEasing?: ZoomTransitionEasing; + zoomOutEasing?: ZoomTransitionEasing; + connectedZoomEasing?: ZoomTransitionEasing; borderRadius?: number; padding?: number; videoPadding?: number; @@ -46,8 +58,11 @@ interface GifExporterConfig { webcam?: WebcamOverlaySettings; webcamUrl?: string | null; annotationRegions?: AnnotationRegion[]; + autoCaptions?: CaptionCue[]; + autoCaptionSettings?: AutoCaptionSettings; cursorTelemetry?: CursorTelemetryPoint[]; showCursor?: boolean; + cursorStyle?: CursorStyle; cursorSize?: number; cursorSmoothing?: number; cursorMotionBlur?: number; @@ -126,6 +141,14 @@ export class GifExporter { backgroundBlur: this.config.backgroundBlur, zoomMotionBlur: this.config.zoomMotionBlur, connectZooms: this.config.connectZooms, + zoomInDurationMs: this.config.zoomInDurationMs, + zoomInOverlapMs: this.config.zoomInOverlapMs, + zoomOutDurationMs: this.config.zoomOutDurationMs, + connectedZoomGapMs: this.config.connectedZoomGapMs, + connectedZoomDurationMs: this.config.connectedZoomDurationMs, + zoomInEasing: this.config.zoomInEasing, + zoomOutEasing: this.config.zoomOutEasing, + connectedZoomEasing: this.config.connectedZoomEasing, borderRadius: this.config.borderRadius, padding: this.config.padding, cropRegion: this.config.cropRegion, @@ -134,11 +157,14 @@ export class GifExporter { videoWidth: videoInfo.width, videoHeight: videoInfo.height, annotationRegions: this.config.annotationRegions, + autoCaptions: this.config.autoCaptions, + autoCaptionSettings: this.config.autoCaptionSettings, speedRegions: this.config.speedRegions, previewWidth: this.config.previewWidth, previewHeight: this.config.previewHeight, cursorTelemetry: this.config.cursorTelemetry, showCursor: this.config.showCursor, + cursorStyle: this.config.cursorStyle, cursorSize: this.config.cursorSize, cursorSmoothing: this.config.cursorSmoothing, cursorMotionBlur: this.config.cursorMotionBlur, diff --git a/src/lib/exporter/muxer.ts b/src/lib/exporter/muxer.ts index 5934af04..a052fec0 100644 --- a/src/lib/exporter/muxer.ts +++ b/src/lib/exporter/muxer.ts @@ -86,5 +86,12 @@ export class VideoMuxer { return new Blob([buffer], { type: 'video/mp4' }); } + + destroy(): void { + this.output = null; + this.videoSource = null; + this.audioSource = null; + this.target = null; + } } diff --git a/src/lib/exporter/types.ts b/src/lib/exporter/types.ts index 812d850d..a92d28db 100644 --- a/src/lib/exporter/types.ts +++ b/src/lib/exporter/types.ts @@ -11,7 +11,7 @@ export interface ExportProgress { totalFrames: number; percentage: number; estimatedTimeRemaining: number; // in seconds - phase?: 'extracting' | 'finalizing'; // Phase of export + phase?: 'extracting' | 'finalizing' | 'saving'; // Phase of export renderProgress?: number; // 0-100, progress of GIF rendering phase } diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index 12096587..435ac483 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -1,11 +1,15 @@ import type { + AutoCaptionSettings, AnnotationRegion, AudioRegion, + CaptionCue, CropRegion, + CursorStyle, CursorTelemetryPoint, SpeedRegion, TrimRegion, WebcamOverlaySettings, + ZoomTransitionEasing, ZoomRegion, } from "@/components/video-editor/types"; import { AudioProcessor } from "./audioEncoder"; @@ -25,6 +29,14 @@ interface VideoExporterConfig extends ExportConfig { backgroundBlur: number; zoomMotionBlur?: number; connectZooms?: boolean; + zoomInDurationMs?: number; + zoomInOverlapMs?: number; + zoomOutDurationMs?: number; + connectedZoomGapMs?: number; + connectedZoomDurationMs?: number; + zoomInEasing?: ZoomTransitionEasing; + zoomOutEasing?: ZoomTransitionEasing; + connectedZoomEasing?: ZoomTransitionEasing; borderRadius?: number; padding?: number; videoPadding?: number; @@ -32,8 +44,11 @@ interface VideoExporterConfig extends ExportConfig { webcam?: WebcamOverlaySettings; webcamUrl?: string | null; annotationRegions?: AnnotationRegion[]; + autoCaptions?: CaptionCue[]; + autoCaptionSettings?: AutoCaptionSettings; cursorTelemetry?: CursorTelemetryPoint[]; showCursor?: boolean; + cursorStyle?: CursorStyle; cursorSize?: number; cursorSmoothing?: number; cursorMotionBlur?: number; @@ -87,6 +102,14 @@ export class VideoExporter { backgroundBlur: this.config.backgroundBlur, zoomMotionBlur: this.config.zoomMotionBlur, connectZooms: this.config.connectZooms, + zoomInDurationMs: this.config.zoomInDurationMs, + zoomInOverlapMs: this.config.zoomInOverlapMs, + zoomOutDurationMs: this.config.zoomOutDurationMs, + connectedZoomGapMs: this.config.connectedZoomGapMs, + connectedZoomDurationMs: this.config.connectedZoomDurationMs, + zoomInEasing: this.config.zoomInEasing, + zoomOutEasing: this.config.zoomOutEasing, + connectedZoomEasing: this.config.connectedZoomEasing, borderRadius: this.config.borderRadius, padding: this.config.padding, cropRegion: this.config.cropRegion, @@ -95,11 +118,14 @@ export class VideoExporter { videoWidth: videoInfo.width, videoHeight: videoInfo.height, annotationRegions: this.config.annotationRegions, + autoCaptions: this.config.autoCaptions, + autoCaptionSettings: this.config.autoCaptionSettings, speedRegions: this.config.speedRegions, previewWidth: this.config.previewWidth, previewHeight: this.config.previewHeight, cursorTelemetry: this.config.cursorTelemetry, showCursor: this.config.showCursor, + cursorStyle: this.config.cursorStyle, cursorSize: this.config.cursorSize, cursorSmoothing: this.config.cursorSmoothing, cursorMotionBlur: this.config.cursorMotionBlur, @@ -440,6 +466,14 @@ export class VideoExporter { this.renderer = null; } + if (this.muxer) { + try { + this.muxer.destroy(); + } catch (e) { + console.warn("Error destroying muxer:", e); + } + } + this.muxer = null; this.audioProcessor = null; this.encodeQueue = 0;