diff --git a/src/lib/exporter/captionRenderer.ts b/src/lib/exporter/captionRenderer.ts new file mode 100644 index 00000000..8cbf4586 --- /dev/null +++ b/src/lib/exporter/captionRenderer.ts @@ -0,0 +1,104 @@ +import { + getDefaultCaptionFontFamily, + type AutoCaptionSettings, + type CaptionCue, +} from "@/components/video-editor/types"; +import { drawSquircleOnCanvas } from "@/lib/geometry/squircle"; +import { + buildActiveCaptionLayout, +} from "@/components/video-editor/captionLayout"; +import { + CAPTION_FONT_WEIGHT, + CAPTION_LINE_HEIGHT, + getCaptionPadding, + getCaptionScaledRadius, + getCaptionScaledFontSize, + getCaptionTextMaxWidth, + getCaptionWordVisualState, +} from "@/components/video-editor/captionStyle"; + +export function renderCaptions( + ctx: CanvasRenderingContext2D, + cues: CaptionCue[], + settings: AutoCaptionSettings, + width: number, + height: number, + timeMs: number, +) { + if (!settings.enabled || cues.length === 0) { + return; + } + + ctx.save(); + + const fontSize = getCaptionScaledFontSize(settings.fontSize, width, settings.maxWidth); + ctx.font = `${CAPTION_FONT_WEIGHT} ${fontSize}px ${getDefaultCaptionFontFamily()}`; + const padding = getCaptionPadding(fontSize); + + const activeCaptionLayout = buildActiveCaptionLayout({ + cues, + timeMs, + settings, + maxWidthPx: getCaptionTextMaxWidth(width, settings.maxWidth, fontSize), + measureText: (text) => ctx.measureText(text).width, + }); + if (!activeCaptionLayout) { + ctx.restore(); + return; + } + + const lineHeight = fontSize * CAPTION_LINE_HEIGHT; + const paddingX = padding.x; + const paddingY = padding.y; + const textBlockHeight = activeCaptionLayout.visibleLines.length * lineHeight; + const boxHeight = textBlockHeight + paddingY * 2; + const centerX = width / 2; + const centerY = height - (height * settings.bottomOffset) / 100 - boxHeight / 2; + const maxMeasuredWidth = activeCaptionLayout.visibleLines.reduce( + (largest, line) => Math.max(largest, line.width), + 0, + ); + const boxWidth = Math.min(width * (settings.maxWidth / 100) + paddingX * 2, maxMeasuredWidth + paddingX * 2); + + ctx.translate(centerX, centerY + activeCaptionLayout.translateY); + ctx.scale(activeCaptionLayout.scale, activeCaptionLayout.scale); + ctx.globalAlpha = activeCaptionLayout.opacity; + + ctx.fillStyle = `rgba(0, 0, 0, ${settings.backgroundOpacity})`; + drawSquircleOnCanvas(ctx, { + x: -boxWidth / 2, + y: -boxHeight / 2, + width: boxWidth, + height: boxHeight, + radius: getCaptionScaledRadius(settings.boxRadius, fontSize), + }); + ctx.fill(); + + ctx.textAlign = "left"; + ctx.textBaseline = "middle"; + + activeCaptionLayout.visibleLines.forEach((line, lineIndex) => { + let cursorX = -line.width / 2; + const lineY = -boxHeight / 2 + paddingY + lineHeight * lineIndex + lineHeight / 2; + + line.words.forEach((word) => { + const segmentText = `${word.leadingSpace ? " " : ""}${word.text}`; + const segmentWidth = ctx.measureText(segmentText).width; + const visualState = getCaptionWordVisualState( + activeCaptionLayout.hasWordTimings, + word.state, + ); + + ctx.save(); + ctx.translate(cursorX, lineY); + ctx.fillStyle = visualState.isInactive ? settings.inactiveTextColor : settings.textColor; + ctx.globalAlpha = activeCaptionLayout.opacity * visualState.opacity; + ctx.fillText(segmentText, 0, 0); + ctx.restore(); + + cursorX += segmentWidth; + }); + }); + + ctx.restore(); +} \ No newline at end of file diff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.ts index fb5d049f..be9d8ce3 100644 --- a/src/lib/exporter/frameRenderer.ts +++ b/src/lib/exporter/frameRenderer.ts @@ -5,15 +5,20 @@ import { Graphics, BlurFilter, Texture, + TextureSource, } from "pixi.js"; import { MotionBlurFilter } from "pixi-filters/motion-blur"; import type { + AutoCaptionSettings, + CaptionCue, ZoomRegion, CropRegion, AnnotationRegion, SpeedRegion, + CursorStyle, CursorTelemetryPoint, WebcamOverlaySettings, + ZoomTransitionEasing, } from "@/components/video-editor/types"; import { ZOOM_DEPTH_SCALES } from "@/components/video-editor/types"; import { getAssetPath, getRenderableAssetUrl } from "@/lib/assetPath"; @@ -31,6 +36,7 @@ import { ZOOM_TRANSLATION_DEADZONE_PX, } from "@/components/video-editor/videoPlayback/constants"; import { renderAnnotations } from "./annotationRenderer"; +import { renderCaptions } from "./captionRenderer"; import { PixiCursorOverlay, DEFAULT_CURSOR_CONFIG, @@ -52,6 +58,14 @@ interface FrameRenderConfig { backgroundBlur: number; zoomMotionBlur?: number; connectZooms?: boolean; + zoomInDurationMs?: number; + zoomInOverlapMs?: number; + zoomOutDurationMs?: number; + connectedZoomGapMs?: number; + connectedZoomDurationMs?: number; + zoomInEasing?: ZoomTransitionEasing; + zoomOutEasing?: ZoomTransitionEasing; + connectedZoomEasing?: ZoomTransitionEasing; borderRadius?: number; padding?: number; cropRegion: CropRegion; @@ -60,11 +74,14 @@ interface FrameRenderConfig { videoWidth: number; videoHeight: number; annotationRegions?: AnnotationRegion[]; + autoCaptions?: CaptionCue[]; + autoCaptionSettings?: AutoCaptionSettings; speedRegions?: SpeedRegion[]; previewWidth?: number; previewHeight?: number; cursorTelemetry?: CursorTelemetryPoint[]; showCursor?: boolean; + cursorStyle?: CursorStyle; cursorSize?: number; cursorSmoothing?: number; cursorMotionBlur?: number; @@ -103,6 +120,7 @@ export class FrameRenderer { private videoContainer: Container | null = null; private cursorContainer: Container | null = null; private videoSprite: Sprite | null = null; + private videoTextureSource: TextureSource | null = null; private backgroundSprite: Sprite | null = null; private maskGraphics: Graphics | null = null; private blurFilter: BlurFilter | null = null; @@ -190,6 +208,7 @@ export class FrameRenderer { this.cursorOverlay = new PixiCursorOverlay({ dotRadius: DEFAULT_CURSOR_CONFIG.dotRadius * (this.config.cursorSize ?? 1.4), + style: this.config.cursorStyle ?? "tahoe", smoothingFactor: this.config.cursorSmoothing ?? DEFAULT_CURSOR_CONFIG.smoothingFactor, motionBlur: this.config.cursorMotionBlur ?? 0, @@ -693,6 +712,7 @@ export class FrameRenderer { if (!this.videoSprite) { const texture = Texture.from(videoFrame as any); this.videoSprite = new Sprite(texture); + this.videoTextureSource = texture.source as TextureSource; this.videoContainer.addChild(this.videoSprite); if (this.cursorOverlay && this.cursorContainer) { this.cursorContainer.addChild(this.cursorOverlay.container); @@ -701,11 +721,9 @@ export class FrameRenderer { this.videoContainer.addChild(this.maskGraphics); } } else { - // Destroy old texture to avoid memory leaks, then create new one - const oldTexture = this.videoSprite.texture; - const newTexture = Texture.from(videoFrame as any); - this.videoSprite.texture = newTexture; - oldTexture.destroy(true); + this.videoTextureSource ??= this.videoSprite.texture.source as TextureSource; + this.videoTextureSource.resource = videoFrame as any; + this.videoTextureSource.update(); } // Apply layout @@ -783,6 +801,22 @@ export class FrameRenderer { scaleFactor, ); } + + if ( + this.config.autoCaptions && + this.config.autoCaptions.length > 0 && + this.config.autoCaptionSettings && + this.compositeCtx + ) { + renderCaptions( + this.compositeCtx, + this.config.autoCaptions, + this.config.autoCaptionSettings, + this.config.width, + this.config.height, + timeMs, + ); + } } private updateLayout(): void { @@ -1220,6 +1254,7 @@ export class FrameRenderer { this.videoSprite.destroy({ texture: false, textureSource: false }); videoTexture?.destroy(true); this.videoSprite = null; + this.videoTextureSource = null; } this.backgroundSprite = null; if (this.app) { diff --git a/src/lib/exporter/gifExporter.ts b/src/lib/exporter/gifExporter.ts index 8b0db134..08dffff3 100644 --- a/src/lib/exporter/gifExporter.ts +++ b/src/lib/exporter/gifExporter.ts @@ -9,13 +9,17 @@ import type { import { StreamingVideoDecoder } from "./streamingDecoder"; import { FrameRenderer } from "./frameRenderer"; import type { + AutoCaptionSettings, ZoomRegion, CropRegion, TrimRegion, AnnotationRegion, + CaptionCue, SpeedRegion, + CursorStyle, CursorTelemetryPoint, WebcamOverlaySettings, + ZoomTransitionEasing, } from "@/components/video-editor/types"; const GIF_WORKER_URL = new URL( @@ -39,6 +43,14 @@ interface GifExporterConfig { backgroundBlur: number; zoomMotionBlur?: number; connectZooms?: boolean; + zoomInDurationMs?: number; + zoomInOverlapMs?: number; + zoomOutDurationMs?: number; + connectedZoomGapMs?: number; + connectedZoomDurationMs?: number; + zoomInEasing?: ZoomTransitionEasing; + zoomOutEasing?: ZoomTransitionEasing; + connectedZoomEasing?: ZoomTransitionEasing; borderRadius?: number; padding?: number; videoPadding?: number; @@ -46,8 +58,11 @@ interface GifExporterConfig { webcam?: WebcamOverlaySettings; webcamUrl?: string | null; annotationRegions?: AnnotationRegion[]; + autoCaptions?: CaptionCue[]; + autoCaptionSettings?: AutoCaptionSettings; cursorTelemetry?: CursorTelemetryPoint[]; showCursor?: boolean; + cursorStyle?: CursorStyle; cursorSize?: number; cursorSmoothing?: number; cursorMotionBlur?: number; @@ -126,6 +141,14 @@ export class GifExporter { backgroundBlur: this.config.backgroundBlur, zoomMotionBlur: this.config.zoomMotionBlur, connectZooms: this.config.connectZooms, + zoomInDurationMs: this.config.zoomInDurationMs, + zoomInOverlapMs: this.config.zoomInOverlapMs, + zoomOutDurationMs: this.config.zoomOutDurationMs, + connectedZoomGapMs: this.config.connectedZoomGapMs, + connectedZoomDurationMs: this.config.connectedZoomDurationMs, + zoomInEasing: this.config.zoomInEasing, + zoomOutEasing: this.config.zoomOutEasing, + connectedZoomEasing: this.config.connectedZoomEasing, borderRadius: this.config.borderRadius, padding: this.config.padding, cropRegion: this.config.cropRegion, @@ -134,11 +157,14 @@ export class GifExporter { videoWidth: videoInfo.width, videoHeight: videoInfo.height, annotationRegions: this.config.annotationRegions, + autoCaptions: this.config.autoCaptions, + autoCaptionSettings: this.config.autoCaptionSettings, speedRegions: this.config.speedRegions, previewWidth: this.config.previewWidth, previewHeight: this.config.previewHeight, cursorTelemetry: this.config.cursorTelemetry, showCursor: this.config.showCursor, + cursorStyle: this.config.cursorStyle, cursorSize: this.config.cursorSize, cursorSmoothing: this.config.cursorSmoothing, cursorMotionBlur: this.config.cursorMotionBlur, diff --git a/src/lib/exporter/muxer.ts b/src/lib/exporter/muxer.ts index 5934af04..a052fec0 100644 --- a/src/lib/exporter/muxer.ts +++ b/src/lib/exporter/muxer.ts @@ -86,5 +86,12 @@ export class VideoMuxer { return new Blob([buffer], { type: 'video/mp4' }); } + + destroy(): void { + this.output = null; + this.videoSource = null; + this.audioSource = null; + this.target = null; + } } diff --git a/src/lib/exporter/types.ts b/src/lib/exporter/types.ts index 812d850d..a92d28db 100644 --- a/src/lib/exporter/types.ts +++ b/src/lib/exporter/types.ts @@ -11,7 +11,7 @@ export interface ExportProgress { totalFrames: number; percentage: number; estimatedTimeRemaining: number; // in seconds - phase?: 'extracting' | 'finalizing'; // Phase of export + phase?: 'extracting' | 'finalizing' | 'saving'; // Phase of export renderProgress?: number; // 0-100, progress of GIF rendering phase } diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index 12096587..435ac483 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -1,11 +1,15 @@ import type { + AutoCaptionSettings, AnnotationRegion, AudioRegion, + CaptionCue, CropRegion, + CursorStyle, CursorTelemetryPoint, SpeedRegion, TrimRegion, WebcamOverlaySettings, + ZoomTransitionEasing, ZoomRegion, } from "@/components/video-editor/types"; import { AudioProcessor } from "./audioEncoder"; @@ -25,6 +29,14 @@ interface VideoExporterConfig extends ExportConfig { backgroundBlur: number; zoomMotionBlur?: number; connectZooms?: boolean; + zoomInDurationMs?: number; + zoomInOverlapMs?: number; + zoomOutDurationMs?: number; + connectedZoomGapMs?: number; + connectedZoomDurationMs?: number; + zoomInEasing?: ZoomTransitionEasing; + zoomOutEasing?: ZoomTransitionEasing; + connectedZoomEasing?: ZoomTransitionEasing; borderRadius?: number; padding?: number; videoPadding?: number; @@ -32,8 +44,11 @@ interface VideoExporterConfig extends ExportConfig { webcam?: WebcamOverlaySettings; webcamUrl?: string | null; annotationRegions?: AnnotationRegion[]; + autoCaptions?: CaptionCue[]; + autoCaptionSettings?: AutoCaptionSettings; cursorTelemetry?: CursorTelemetryPoint[]; showCursor?: boolean; + cursorStyle?: CursorStyle; cursorSize?: number; cursorSmoothing?: number; cursorMotionBlur?: number; @@ -87,6 +102,14 @@ export class VideoExporter { backgroundBlur: this.config.backgroundBlur, zoomMotionBlur: this.config.zoomMotionBlur, connectZooms: this.config.connectZooms, + zoomInDurationMs: this.config.zoomInDurationMs, + zoomInOverlapMs: this.config.zoomInOverlapMs, + zoomOutDurationMs: this.config.zoomOutDurationMs, + connectedZoomGapMs: this.config.connectedZoomGapMs, + connectedZoomDurationMs: this.config.connectedZoomDurationMs, + zoomInEasing: this.config.zoomInEasing, + zoomOutEasing: this.config.zoomOutEasing, + connectedZoomEasing: this.config.connectedZoomEasing, borderRadius: this.config.borderRadius, padding: this.config.padding, cropRegion: this.config.cropRegion, @@ -95,11 +118,14 @@ export class VideoExporter { videoWidth: videoInfo.width, videoHeight: videoInfo.height, annotationRegions: this.config.annotationRegions, + autoCaptions: this.config.autoCaptions, + autoCaptionSettings: this.config.autoCaptionSettings, speedRegions: this.config.speedRegions, previewWidth: this.config.previewWidth, previewHeight: this.config.previewHeight, cursorTelemetry: this.config.cursorTelemetry, showCursor: this.config.showCursor, + cursorStyle: this.config.cursorStyle, cursorSize: this.config.cursorSize, cursorSmoothing: this.config.cursorSmoothing, cursorMotionBlur: this.config.cursorMotionBlur, @@ -440,6 +466,14 @@ export class VideoExporter { this.renderer = null; } + if (this.muxer) { + try { + this.muxer.destroy(); + } catch (e) { + console.warn("Error destroying muxer:", e); + } + } + this.muxer = null; this.audioProcessor = null; this.encodeQueue = 0;