diff --git a/src/lib/exporter/modernFrameRenderer.ts b/src/lib/exporter/modernFrameRenderer.ts new file mode 100644 index 00000000..08569fbb --- /dev/null +++ b/src/lib/exporter/modernFrameRenderer.ts @@ -0,0 +1,2486 @@ +import { Application, BlurFilter, Container, Graphics, Sprite, Texture } from "pixi.js"; +import { MotionBlurFilter } from "pixi-filters/motion-blur"; +import { buildActiveCaptionLayout } from "@/components/video-editor/captionLayout"; +import { + CAPTION_FONT_WEIGHT, + CAPTION_LINE_HEIGHT, + getCaptionPadding, + getCaptionScaledFontSize, + getCaptionScaledRadius, + getCaptionTextMaxWidth, + getCaptionWordVisualState, +} from "@/components/video-editor/captionStyle"; +import type { + AnnotationRegion, + AutoCaptionSettings, + CaptionCue, + CropRegion, + CursorStyle, + CursorTelemetryPoint, + SpeedRegion, + WebcamOverlaySettings, + ZoomRegion, + ZoomTransitionEasing, +} from "@/components/video-editor/types"; +import { getDefaultCaptionFontFamily, ZOOM_DEPTH_SCALES } from "@/components/video-editor/types"; +import { + DEFAULT_FOCUS, + ZOOM_SCALE_DEADZONE, + ZOOM_TRANSLATION_DEADZONE_PX, +} from "@/components/video-editor/videoPlayback/constants"; +import { + DEFAULT_CURSOR_CONFIG, + PixiCursorOverlay, + preloadCursorAssets, +} from "@/components/video-editor/videoPlayback/cursorRenderer"; +import { findDominantRegion } from "@/components/video-editor/videoPlayback/zoomRegionUtils"; +import { + applyZoomTransform, + computeFocusFromTransform, + computeZoomTransform, + createMotionBlurState, + type MotionBlurState, +} from "@/components/video-editor/videoPlayback/zoomTransform"; +import { + getWebcamOverlayPosition, + getWebcamOverlaySizePx, +} from "@/components/video-editor/webcamOverlay"; +import { getAssetPath, getRenderableAssetUrl } from "@/lib/assetPath"; +import { drawSquircleOnCanvas, drawSquircleOnGraphics } from "@/lib/geometry/squircle"; +import { clampMediaTimeToDuration } from "@/lib/mediaTiming"; +import { isVideoWallpaperSource } from "@/lib/wallpapers"; +import { + type AnnotationRenderAssets, + preloadAnnotationAssets, + renderAnnotationToCanvas, +} from "./annotationRenderer"; +import { ForwardFrameSource } from "./forwardFrameSource"; +import { resolveMediaElementSource } from "./localMediaSource"; +import { + getShadowFilterPadding, + VIDEO_SHADOW_LAYER_PROFILES, + WEBCAM_SHADOW_LAYER_PROFILES, +} from "./shadowProfile"; +import type { ExportRenderBackend } from "./types"; + +interface FrameRenderConfig { + width: number; + height: number; + preferredRenderBackend?: ExportRenderBackend; + nativeReadbackMode?: "canvas" | "pixels"; + wallpaper: string; + zoomRegions: ZoomRegion[]; + showShadow: boolean; + shadowIntensity: number; + backgroundBlur: number; + zoomMotionBlur?: number; + connectZooms?: boolean; + zoomInDurationMs?: number; + zoomInOverlapMs?: number; + zoomOutDurationMs?: number; + connectedZoomGapMs?: number; + connectedZoomDurationMs?: number; + zoomInEasing?: ZoomTransitionEasing; + zoomOutEasing?: ZoomTransitionEasing; + connectedZoomEasing?: ZoomTransitionEasing; + borderRadius?: number; + padding?: number; + cropRegion: CropRegion; + webcam?: WebcamOverlaySettings; + webcamUrl?: string | null; + videoWidth: number; + videoHeight: number; + annotationRegions?: AnnotationRegion[]; + autoCaptions?: CaptionCue[]; + autoCaptionSettings?: AutoCaptionSettings; + speedRegions?: SpeedRegion[]; + previewWidth?: number; + previewHeight?: number; + cursorTelemetry?: CursorTelemetryPoint[]; + showCursor?: boolean; + cursorStyle?: CursorStyle; + cursorSize?: number; + cursorSmoothing?: number; + cursorMotionBlur?: number; + cursorClickBounce?: number; + cursorClickBounceDuration?: number; + cursorSway?: number; +} + +interface AnimationState { + scale: number; + appliedScale: number; + focusX: number; + focusY: number; + progress: number; + x: number; + y: number; +} + +interface LayoutCache { + stageSize: { width: number; height: number }; + videoSize: { width: number; height: number }; + baseScale: number; + baseOffset: { x: number; y: number }; + maskRect: { + x: number; + y: number; + width: number; + height: number; + sourceCrop: CropRegion; + }; +} + +interface MutableVideoTextureSource { + resource: CanvasImageSource | VideoFrame; + update: () => void; +} + +interface ShadowLayer { + container: Container; + sprite: Sprite | null; + canvas: HTMLCanvasElement | null; + context: CanvasRenderingContext2D | null; + textureSource: MutableVideoTextureSource | null; + offsetScale: number; + alphaScale: number; + blurScale: number; +} + +interface WebcamRenderSource { + source: CanvasImageSource | VideoFrame; + width: number; + height: number; + mode: "live" | "cached"; +} + +interface WebcamLayoutCache { + sourceWidth: number; + sourceHeight: number; + size: number; + positionX: number; + positionY: number; + radius: number; + shadowStrength: number; + mirror: boolean; +} + +interface AnnotationSpriteEntry { + annotation: AnnotationRegion; + sprite: Sprite; + texture: Texture; +} + +type ResolvedCaptionLayout = NonNullable>; + +interface CaptionRenderState { + key: string; + layout: ResolvedCaptionLayout; + fontFamily: string; + fontSize: number; + lineHeight: number; + boxWidth: number; + boxHeight: number; + centerX: number; + centerY: number; +} + +function createAnimationState(): AnimationState { + return { + scale: 1, + appliedScale: 1, + focusX: DEFAULT_FOCUS.cx, + focusY: DEFAULT_FOCUS.cy, + progress: 0, + x: 0, + y: 0, + }; +} + +function configureHighQuality2DContext( + context: CanvasRenderingContext2D | null, +): CanvasRenderingContext2D | null { + if (!context) { + return null; + } + + context.imageSmoothingEnabled = true; + context.imageSmoothingQuality = "high"; + + return context; +} + +function drawSourceCoverToCanvas( + ctx: CanvasRenderingContext2D, + source: CanvasImageSource, + sourceWidth: number, + sourceHeight: number, + targetWidth: number, + targetHeight: number, +): void { + const safeSourceWidth = Math.max(1, sourceWidth); + const safeSourceHeight = Math.max(1, sourceHeight); + const sourceAspect = safeSourceWidth / safeSourceHeight; + const targetAspect = targetWidth / targetHeight; + + let drawWidth = targetWidth; + let drawHeight = targetHeight; + let drawX = 0; + let drawY = 0; + + if (sourceAspect > targetAspect) { + drawHeight = targetHeight; + drawWidth = drawHeight * sourceAspect; + drawX = (targetWidth - drawWidth) / 2; + } else { + drawWidth = targetWidth; + drawHeight = drawWidth / sourceAspect; + drawY = (targetHeight - drawHeight) / 2; + } + + ctx.clearRect(0, 0, targetWidth, targetHeight); + ctx.drawImage(source, drawX, drawY, drawWidth, drawHeight); +} + +function applyCoverLayoutToSprite( + sprite: Sprite, + sourceWidth: number, + sourceHeight: number, + targetWidth: number, + targetHeight: number, + centerX: number, + centerY: number, + mirror = false, +): void { + const safeSourceWidth = Math.max(1, sourceWidth); + const safeSourceHeight = Math.max(1, sourceHeight); + const coverScale = Math.max(targetWidth / safeSourceWidth, targetHeight / safeSourceHeight); + + sprite.anchor.set(0.5); + sprite.position.set(centerX, centerY); + sprite.scale.set(coverScale * (mirror ? -1 : 1), coverScale); +} + +function clampUnitInterval(value: number): number { + return Math.min(1, Math.max(0, value)); +} + +function areNearlyEqual(first: number, second: number, epsilon = 0.01): boolean { + return Math.abs(first - second) <= epsilon; +} + +const VIDEO_FRAME_STARTUP_STAGING_WINDOW_SEC = 2.25; + +// Renders video frames with all effects directly into a GPU-backed Pixi scene for export. +export class FrameRenderer { + private app: Application | null = null; + private rendererBackend: ExportRenderBackend = "webgl"; + private backgroundContainer: Container | null = null; + private cameraContainer: Container | null = null; + private videoEffectsContainer: Container | null = null; + private videoContainer: Container | null = null; + private cursorContainer: Container | null = null; + private overlayContainer: Container | null = null; + private annotationContainer: Container | null = null; + private captionContainer: Container | null = null; + private webcamRootContainer: Container | null = null; + private webcamContainer: Container | null = null; + private videoSprite: Sprite | null = null; + private videoTextureSource: MutableVideoTextureSource | null = null; + private backgroundSprite: Sprite | null = null; + private backgroundTextureSource: MutableVideoTextureSource | null = null; + private videoMaskGraphics: Graphics | null = null; + private webcamMaskGraphics: Graphics | null = null; + private blurFilter: BlurFilter | null = null; + private motionBlurFilter: MotionBlurFilter | null = null; + private backgroundBlurFilter: BlurFilter | null = null; + private annotationAssets: AnnotationRenderAssets | null = null; + private annotationScaleFactor = 1; + private annotationSprites: AnnotationSpriteEntry[] = []; + private backgroundForwardFrameSource: ForwardFrameSource | null = null; + private backgroundDecodedFrame: VideoFrame | null = null; + private backgroundVideoElement: HTMLVideoElement | null = null; + private videoShadowLayers: ShadowLayer[] = []; + private webcamShadowLayers: ShadowLayer[] = []; + private webcamSprite: Sprite | null = null; + private webcamTextureSource: MutableVideoTextureSource | null = null; + private webcamForwardFrameSource: ForwardFrameSource | null = null; + private webcamDecodedFrame: VideoFrame | null = null; + private webcamVideoElement: HTMLVideoElement | null = null; + private webcamSeekPromise: Promise | null = null; + private webcamFrameCacheCanvas: HTMLCanvasElement | null = null; + private webcamFrameCacheCtx: CanvasRenderingContext2D | null = null; + private sceneVideoFrameStagingCanvas: HTMLCanvasElement | null = null; + private sceneVideoFrameStagingCtx: CanvasRenderingContext2D | null = null; + private webcamVideoFrameStagingCanvas: HTMLCanvasElement | null = null; + private webcamVideoFrameStagingCtx: CanvasRenderingContext2D | null = null; + private captionMeasureCanvas: HTMLCanvasElement | null = null; + private captionMeasureCtx: CanvasRenderingContext2D | null = null; + private captionCanvas: HTMLCanvasElement | null = null; + private captionCtx: CanvasRenderingContext2D | null = null; + private captionSprite: Sprite | null = null; + private captionTextureSource: MutableVideoTextureSource | null = null; + private captionRenderKey: string | null = null; + private config: FrameRenderConfig; + private animationState: AnimationState; + private motionBlurState: MotionBlurState; + private layoutCache: LayoutCache | null = null; + private currentVideoTime = 0; + private lastMotionVector = { x: 0, y: 0 }; + private cursorOverlay: PixiCursorOverlay | null = null; + private lastSyncedWebcamTime: number | null = null; + private lastWebcamCacheRefreshTime: number | null = null; + private webcamRenderMode: "hidden" | "live" | "cached" = "hidden"; + private webcamLayoutCache: WebcamLayoutCache | null = null; + private videoTextureUsesStartupStaging = false; + private webcamTextureUsesStartupStaging = false; + private nativePixelReadbackWarningShown = false; + private nativeReadbackBuffer: Uint8Array | null = null; + private cleanupWebcamSource: (() => void) | null = null; + + constructor(config: FrameRenderConfig) { + this.config = config; + this.animationState = createAnimationState(); + this.motionBlurState = createMotionBlurState(); + } + + private shouldUseZoomMotionBlur(): boolean { + return (this.config.zoomMotionBlur ?? 0) > 0; + } + + private updateVideoEffectsFilterState(): void { + if (!this.videoEffectsContainer) { + return; + } + + const activeFilters = this.shouldUseZoomMotionBlur() && this.motionBlurFilter + ? [this.motionBlurFilter] + : null; + this.videoEffectsContainer.filters = activeFilters; + } + + async initialize(): Promise { + let cursorOverlayEnabled = true; + try { + await preloadCursorAssets(); + } catch (error) { + cursorOverlayEnabled = false; + console.warn( + "[FrameRenderer] Native cursor assets are unavailable; continuing export without cursor overlay.", + error, + ); + } + + const canvas = document.createElement("canvas"); + canvas.width = this.config.width; + canvas.height = this.config.height; + + try { + const exportCanvas = canvas as HTMLCanvasElement & { colorSpace?: string }; + if ("colorSpace" in exportCanvas) { + exportCanvas.colorSpace = "srgb"; + } + } catch (error) { + console.warn("[FrameRenderer] colorSpace not supported on this platform:", error); + } + + const application = await this.createPixiApplication(canvas); + this.app = application.app; + this.rendererBackend = application.backend; + + this.backgroundContainer = new Container(); + this.cameraContainer = new Container(); + this.videoEffectsContainer = new Container(); + this.videoContainer = new Container(); + this.cursorContainer = new Container(); + this.overlayContainer = new Container(); + this.annotationContainer = new Container(); + this.captionContainer = new Container(); + this.webcamRootContainer = new Container(); + this.webcamContainer = new Container(); + + this.app.stage.addChild(this.backgroundContainer); + this.app.stage.addChild(this.cameraContainer); + this.app.stage.addChild(this.overlayContainer); + + this.videoShadowLayers = this.createShadowLayers( + this.cameraContainer, + VIDEO_SHADOW_LAYER_PROFILES, + ); + + this.cameraContainer.addChild(this.videoEffectsContainer); + this.cameraContainer.addChild(this.cursorContainer); + this.videoEffectsContainer.addChild(this.videoContainer); + + this.webcamShadowLayers = this.createShadowLayers( + this.webcamRootContainer, + WEBCAM_SHADOW_LAYER_PROFILES, + ); + this.webcamRootContainer.addChild(this.webcamContainer); + this.webcamRootContainer.visible = false; + + this.overlayContainer.addChild(this.webcamRootContainer); + this.overlayContainer.addChild(this.annotationContainer); + this.overlayContainer.addChild(this.captionContainer); + + this.videoMaskGraphics = new Graphics(); + this.videoContainer.addChild(this.videoMaskGraphics); + this.videoContainer.mask = this.videoMaskGraphics; + + this.webcamMaskGraphics = new Graphics(); + this.webcamContainer.addChild(this.webcamMaskGraphics); + this.webcamContainer.mask = this.webcamMaskGraphics; + + if (cursorOverlayEnabled) { + this.cursorOverlay = new PixiCursorOverlay({ + dotRadius: DEFAULT_CURSOR_CONFIG.dotRadius * (this.config.cursorSize ?? 1.4), + style: this.config.cursorStyle ?? "tahoe", + smoothingFactor: this.config.cursorSmoothing ?? DEFAULT_CURSOR_CONFIG.smoothingFactor, + motionBlur: this.config.cursorMotionBlur ?? 0, + clickBounce: this.config.cursorClickBounce ?? DEFAULT_CURSOR_CONFIG.clickBounce, + clickBounceDuration: + this.config.cursorClickBounceDuration ?? DEFAULT_CURSOR_CONFIG.clickBounceDuration, + sway: this.config.cursorSway ?? DEFAULT_CURSOR_CONFIG.sway, + }); + this.cursorContainer.addChild(this.cursorOverlay.container); + } + + await this.setupBackground(); + await this.setupWebcamSource(); + + this.annotationScaleFactor = this.calculateAnnotationScaleFactor(); + this.annotationAssets = await preloadAnnotationAssets(this.config.annotationRegions ?? []); + await this.setupAnnotationLayer(); + this.setupCaptionResources(); + + if (this.shouldUseZoomMotionBlur()) { + this.motionBlurFilter = new MotionBlurFilter([0, 0], 5, 0); + } + this.updateVideoEffectsFilterState(); + + console.log(`[FrameRenderer] Export renderer backend: ${this.rendererBackend}`); + } + + private async createPixiApplication( + canvas: HTMLCanvasElement, + ): Promise<{ app: Application; backend: ExportRenderBackend }> { + const baseOptions = { + canvas, + width: this.config.width, + height: this.config.height, + backgroundAlpha: 0, + antialias: true, + resolution: 1, + autoDensity: true, + autoStart: false, + sharedTicker: false, + powerPreference: "high-performance" as const, + }; + + const preferredRenderBackend = this.config.preferredRenderBackend; + const backendOrder: ExportRenderBackend[] = + preferredRenderBackend === "webgl" + ? ["webgl", "webgpu"] + : preferredRenderBackend === "webgpu" + ? ["webgpu", "webgl"] + : typeof navigator !== "undefined" && "gpu" in navigator + ? ["webgpu", "webgl"] + : ["webgl"]; + let lastError: unknown = null; + + for (const backend of backendOrder) { + if (backend === "webgpu") { + if (!(typeof navigator !== "undefined" && "gpu" in navigator)) { + continue; + } + + const webgpuApp = new Application(); + try { + await webgpuApp.init({ + ...baseOptions, + preference: "webgpu", + }); + return { app: webgpuApp, backend: "webgpu" }; + } catch (error) { + lastError = error; + console.warn( + "[FrameRenderer] WebGPU export renderer unavailable; trying next backend:", + error, + ); + webgpuApp.destroy(true); + } + continue; + } + + const webglApp = new Application(); + try { + await webglApp.init({ + ...baseOptions, + preference: "webgl", + }); + return { app: webglApp, backend: "webgl" }; + } catch (error) { + lastError = error; + console.warn( + "[FrameRenderer] WebGL export renderer unavailable; trying next backend:", + error, + ); + webglApp.destroy(true); + } + } + + throw lastError instanceof Error + ? lastError + : new Error("No supported Pixi export renderer was available"); + } + + private createShadowLayers( + parent: Container, + configs: ReadonlyArray<{ offsetScale: number; alphaScale: number; blurScale: number }>, + ): ShadowLayer[] { + return configs.map((config) => { + const container = new Container(); + container.visible = false; + parent.addChild(container); + + return { + container, + sprite: null, + canvas: null, + context: null, + textureSource: null, + ...config, + }; + }); + } + + private ensureShadowLayerCanvas(layer: ShadowLayer, width: number, height: number): void { + const targetWidth = Math.max(1, Math.ceil(width)); + const targetHeight = Math.max(1, Math.ceil(height)); + + if ( + layer.canvas && + layer.canvas.width === targetWidth && + layer.canvas.height === targetHeight && + layer.context && + layer.sprite + ) { + return; + } + + layer.canvas = document.createElement("canvas"); + layer.canvas.width = targetWidth; + layer.canvas.height = targetHeight; + layer.context = configureHighQuality2DContext(layer.canvas.getContext("2d")); + + if (!layer.context) { + throw new Error("Failed to create shadow export canvas"); + } + + const nextTexture = Texture.from(layer.canvas); + if (layer.sprite) { + const previousTexture = layer.sprite.texture; + layer.sprite.texture = nextTexture; + layer.textureSource = nextTexture.source as unknown as MutableVideoTextureSource; + previousTexture.destroy(true); + } else { + layer.sprite = new Sprite(nextTexture); + layer.container.addChild(layer.sprite); + layer.textureSource = nextTexture.source as unknown as MutableVideoTextureSource; + } + } + + private rasterizeShadowLayer( + layer: ShadowLayer, + options: { + x: number; + y: number; + width: number; + height: number; + radius: number; + offsetY: number; + alpha: number; + blur: number; + }, + ): void { + if (options.alpha <= 0 || options.width <= 0 || options.height <= 0) { + layer.container.visible = false; + return; + } + + const padding = getShadowFilterPadding(options.blur, options.offsetY); + this.ensureShadowLayerCanvas( + layer, + options.width + padding * 2, + options.height + padding * 2, + ); + + if (!layer.context || !layer.canvas || !layer.sprite) { + layer.container.visible = false; + return; + } + + layer.context.clearRect(0, 0, layer.canvas.width, layer.canvas.height); + layer.context.save(); + layer.context.filter = options.blur > 0 ? `blur(${options.blur}px)` : "none"; + layer.context.fillStyle = `rgba(0, 0, 0, ${options.alpha})`; + drawSquircleOnCanvas(layer.context, { + x: padding, + y: padding + options.offsetY, + width: options.width, + height: options.height, + radius: options.radius, + }); + layer.context.fill(); + layer.context.restore(); + + layer.sprite.position.set(options.x - padding, options.y - padding); + layer.textureSource?.update(); + layer.container.alpha = 1; + layer.container.visible = true; + } + + private shouldUseStartupVideoFrameStaging(): boolean { + return ( + this.rendererBackend === "webgpu" && + this.currentVideoTime < VIDEO_FRAME_STARTUP_STAGING_WINDOW_SEC + ); + } + + private ensureVideoFrameStagingCanvas( + kind: "scene" | "webcam", + width: number, + height: number, + ): { canvas: HTMLCanvasElement; context: CanvasRenderingContext2D } | null { + const targetWidth = Math.max(1, Math.ceil(width)); + const targetHeight = Math.max(1, Math.ceil(height)); + const currentCanvas = + kind === "scene" ? this.sceneVideoFrameStagingCanvas : this.webcamVideoFrameStagingCanvas; + const currentContext = + kind === "scene" ? this.sceneVideoFrameStagingCtx : this.webcamVideoFrameStagingCtx; + + if ( + currentCanvas && + currentCanvas.width === targetWidth && + currentCanvas.height === targetHeight && + currentContext + ) { + return { canvas: currentCanvas, context: currentContext }; + } + + const canvas = document.createElement("canvas"); + canvas.width = targetWidth; + canvas.height = targetHeight; + const context = configureHighQuality2DContext(canvas.getContext("2d")); + if (!context) { + return null; + } + + if (kind === "scene") { + this.sceneVideoFrameStagingCanvas = canvas; + this.sceneVideoFrameStagingCtx = context; + } else { + this.webcamVideoFrameStagingCanvas = canvas; + this.webcamVideoFrameStagingCtx = context; + } + + return { canvas, context }; + } + + private stageVideoFrameForTexture( + frame: VideoFrame, + kind: "scene" | "webcam", + fallbackWidth: number, + fallbackHeight: number, + ): CanvasImageSource | VideoFrame { + if (!this.shouldUseStartupVideoFrameStaging()) { + return frame; + } + + const width = Math.max(1, frame.displayWidth || fallbackWidth); + const height = Math.max(1, frame.displayHeight || fallbackHeight); + const staging = this.ensureVideoFrameStagingCanvas(kind, width, height); + if (!staging) { + return frame; + } + + staging.context.clearRect(0, 0, staging.canvas.width, staging.canvas.height); + staging.context.drawImage(frame, 0, 0, staging.canvas.width, staging.canvas.height); + return staging.canvas; + } + + private replaceSpriteTexture( + sprite: Sprite, + source: CanvasImageSource | VideoFrame, + ): MutableVideoTextureSource { + const nextTexture = this.createTextureFromSource(source); + const previousTexture = sprite.texture; + sprite.texture = nextTexture; + previousTexture.destroy(true); + return nextTexture.source as unknown as MutableVideoTextureSource; + } + + private createTextureFromSource(source: CanvasImageSource | VideoFrame): Texture { + if (typeof VideoFrame !== "undefined" && source instanceof VideoFrame) { + return Texture.from(source as unknown as ImageBitmap); + } + + return Texture.from( + source as HTMLCanvasElement | HTMLVideoElement | HTMLImageElement | ImageBitmap, + ); + } + + private getWebcamSourceDimensions(source: HTMLVideoElement | VideoFrame): { + width: number; + height: number; + } { + if ("displayWidth" in source && "displayHeight" in source) { + return { + width: source.displayWidth, + height: source.displayHeight, + }; + } + + return { + width: source.videoWidth, + height: source.videoHeight, + }; + } + + private resetBackgroundLayer(): void { + this.backgroundForwardFrameSource?.cancel(); + void this.backgroundForwardFrameSource?.destroy(); + this.backgroundForwardFrameSource = null; + this.closeBackgroundDecodedFrame(); + + if (this.backgroundVideoElement) { + this.backgroundVideoElement.pause(); + this.backgroundVideoElement.src = ""; + this.backgroundVideoElement.load(); + this.backgroundVideoElement = null; + } + + const backgroundTexture = this.backgroundSprite?.texture ?? null; + this.backgroundSprite?.destroy({ texture: false, textureSource: false }); + backgroundTexture?.destroy(true); + + this.backgroundSprite = null; + this.backgroundTextureSource = null; + this.backgroundBlurFilter = null; + this.backgroundContainer?.removeChildren(); + } + + private async setupBackground(): Promise { + this.resetBackgroundLayer(); + + const wallpaper = await this.resolveWallpaperForExport(this.config.wallpaper); + + try { + if (isVideoWallpaperSource(wallpaper)) { + let videoSrc = wallpaper; + if (wallpaper.startsWith("/") && !wallpaper.startsWith("//")) { + videoSrc = await getAssetPath(wallpaper.replace(/^\//, "")); + } + + try { + const frameSource = new ForwardFrameSource(); + await frameSource.initialize(videoSrc); + this.backgroundForwardFrameSource = frameSource; + return; + } catch (error) { + console.warn( + "[FrameRenderer] Decoder-backed wallpaper source unavailable during export; falling back to media element sync:", + error, + ); + } + + const video = document.createElement("video"); + video.muted = true; + video.loop = true; + video.playsInline = true; + video.preload = "auto"; + video.src = videoSrc; + + await new Promise((resolve, reject) => { + video.onloadeddata = () => resolve(); + video.onerror = () => reject(new Error(`Failed to load video wallpaper: ${wallpaper}`)); + }); + + this.backgroundVideoElement = video; + this.ensureBackgroundSprite(video, video.videoWidth, video.videoHeight); + return; + } + + const bgCanvas = document.createElement("canvas"); + bgCanvas.width = this.config.width; + bgCanvas.height = this.config.height; + const bgCtx = configureHighQuality2DContext(bgCanvas.getContext("2d")); + + if (!bgCtx) { + throw new Error("Failed to get 2D context for background canvas"); + } + + if ( + wallpaper.startsWith("file://") || + wallpaper.startsWith("data:") || + wallpaper.startsWith("/") || + wallpaper.startsWith("http") + ) { + const img = new Image(); + const imageUrl = await this.resolveWallpaperImageUrl(wallpaper); + if ( + imageUrl.startsWith("http") && + window.location.origin && + !imageUrl.startsWith(window.location.origin) + ) { + img.crossOrigin = "anonymous"; + } + + await new Promise((resolve, reject) => { + img.onload = () => resolve(); + img.onerror = (err) => { + console.error("[FrameRenderer] Failed to load background image:", imageUrl, err); + reject(new Error(`Failed to load background image: ${imageUrl}`)); + }; + img.src = imageUrl; + }); + + drawSourceCoverToCanvas( + bgCtx, + img, + img.width, + img.height, + this.config.width, + this.config.height, + ); + } else if (wallpaper.startsWith("#")) { + bgCtx.fillStyle = wallpaper; + bgCtx.fillRect(0, 0, this.config.width, this.config.height); + } else if ( + wallpaper.startsWith("linear-gradient") || + wallpaper.startsWith("radial-gradient") + ) { + const gradientMatch = wallpaper.match(/(linear|radial)-gradient\((.+)\)/); + if (!gradientMatch) { + bgCtx.fillStyle = "#000000"; + bgCtx.fillRect(0, 0, this.config.width, this.config.height); + } else { + const [, type, params] = gradientMatch; + const parts = params.split(",").map((value) => value.trim()); + const gradient = + type === "linear" + ? bgCtx.createLinearGradient(0, 0, 0, this.config.height) + : bgCtx.createRadialGradient( + this.config.width / 2, + this.config.height / 2, + 0, + this.config.width / 2, + this.config.height / 2, + Math.max(this.config.width, this.config.height) / 2, + ); + + parts.forEach((part, index) => { + if (type === "linear" && (part.startsWith("to ") || part.includes("deg"))) { + return; + } + + const colorMatch = part.match(/^(#[0-9a-fA-F]{3,8}|rgba?\([^)]+\)|[a-z]+)/); + if (!colorMatch) { + return; + } + + const position = index / Math.max(parts.length - 1, 1); + gradient.addColorStop(position, colorMatch[1]); + }); + + bgCtx.fillStyle = gradient; + bgCtx.fillRect(0, 0, this.config.width, this.config.height); + } + } else { + bgCtx.fillStyle = wallpaper; + bgCtx.fillRect(0, 0, this.config.width, this.config.height); + } + + const blurredCanvas = + this.config.backgroundBlur > 0 ? this.createPreblurredBackgroundCanvas(bgCanvas) : null; + const backgroundSource = blurredCanvas ?? bgCanvas; + const backgroundTexture = Texture.from(backgroundSource); + this.backgroundSprite = new Sprite(backgroundTexture); + this.backgroundContainer?.addChild(this.backgroundSprite); + applyCoverLayoutToSprite( + this.backgroundSprite, + backgroundSource.width, + backgroundSource.height, + this.config.width, + this.config.height, + this.config.width / 2, + this.config.height / 2, + ); + } catch (error) { + console.error("[FrameRenderer] Error setting up background, using fallback:", error); + const fallback = document.createElement("canvas"); + fallback.width = this.config.width; + fallback.height = this.config.height; + const ctx = configureHighQuality2DContext(fallback.getContext("2d")); + if (!ctx) { + throw new Error("Failed to create fallback background context"); + } + ctx.fillStyle = "#000000"; + ctx.fillRect(0, 0, fallback.width, fallback.height); + this.backgroundSprite = new Sprite(Texture.from(fallback)); + this.backgroundContainer?.addChild(this.backgroundSprite); + applyCoverLayoutToSprite( + this.backgroundSprite, + fallback.width, + fallback.height, + this.config.width, + this.config.height, + this.config.width / 2, + this.config.height / 2, + ); + } + } + + private ensureBackgroundSprite( + source: CanvasImageSource | VideoFrame, + sourceWidth: number, + sourceHeight: number, + ): void { + if (!this.backgroundContainer) { + return; + } + + if (!this.backgroundSprite) { + const texture = this.createTextureFromSource(source); + this.backgroundSprite = new Sprite(texture); + this.backgroundTextureSource = texture.source as unknown as MutableVideoTextureSource; + this.backgroundContainer.addChild(this.backgroundSprite); + + if (this.config.backgroundBlur > 0) { + this.backgroundBlurFilter = new BlurFilter(); + this.backgroundBlurFilter.blur = this.config.backgroundBlur * 3; + this.backgroundBlurFilter.quality = 4; + this.backgroundBlurFilter.resolution = this.app?.renderer.resolution ?? 1; + this.backgroundSprite.filters = [this.backgroundBlurFilter]; + } + } else if (this.backgroundTextureSource) { + this.backgroundTextureSource.resource = source; + this.backgroundTextureSource.update(); + } + + applyCoverLayoutToSprite( + this.backgroundSprite, + sourceWidth, + sourceHeight, + this.config.width, + this.config.height, + this.config.width / 2, + this.config.height / 2, + ); + } + + private createPreblurredBackgroundCanvas( + sourceCanvas: HTMLCanvasElement, + ): HTMLCanvasElement | null { + const blurredCanvas = document.createElement("canvas"); + blurredCanvas.width = sourceCanvas.width; + blurredCanvas.height = sourceCanvas.height; + const blurredCtx = configureHighQuality2DContext(blurredCanvas.getContext("2d")); + if (!blurredCtx) { + return null; + } + + blurredCtx.save(); + blurredCtx.filter = `blur(${this.config.backgroundBlur * 3}px)`; + blurredCtx.drawImage(sourceCanvas, 0, 0, blurredCanvas.width, blurredCanvas.height); + blurredCtx.restore(); + + return blurredCanvas; + } + + private closeBackgroundDecodedFrame(): void { + if (!this.backgroundDecodedFrame) { + return; + } + + this.backgroundDecodedFrame.close(); + this.backgroundDecodedFrame = null; + } + + private calculateAnnotationScaleFactor(): number { + const previewWidth = this.config.previewWidth || 1920; + const previewHeight = this.config.previewHeight || 1080; + const scaleX = this.config.width / previewWidth; + const scaleY = this.config.height / previewHeight; + return (scaleX + scaleY) / 2; + } + + private async setupAnnotationLayer(): Promise { + if (!this.annotationContainer) { + return; + } + + for (const entry of this.annotationSprites) { + entry.sprite.destroy({ texture: false, textureSource: false }); + entry.texture.destroy(true); + } + this.annotationSprites = []; + this.annotationContainer.removeChildren(); + + const annotations = [...(this.config.annotationRegions ?? [])].sort( + (first, second) => first.zIndex - second.zIndex, + ); + + for (const annotation of annotations) { + const x = (annotation.position.x / 100) * this.config.width; + const y = (annotation.position.y / 100) * this.config.height; + const width = (annotation.size.width / 100) * this.config.width; + const height = (annotation.size.height / 100) * this.config.height; + + if (width <= 0 || height <= 0) { + continue; + } + + const canvas = await renderAnnotationToCanvas( + annotation, + width, + height, + this.annotationScaleFactor, + this.annotationAssets ?? undefined, + ); + if (!canvas) { + continue; + } + + const texture = Texture.from(canvas); + const sprite = new Sprite(texture); + sprite.position.set(x, y); + sprite.visible = false; + this.annotationContainer.addChild(sprite); + this.annotationSprites.push({ annotation, sprite, texture }); + } + } + + private updateAnnotationLayer(currentTimeMs: number): void { + for (const entry of this.annotationSprites) { + entry.sprite.visible = + currentTimeMs >= entry.annotation.startMs && currentTimeMs <= entry.annotation.endMs; + } + } + + private setupCaptionResources(): void { + if (!this.config.autoCaptions?.length || !this.config.autoCaptionSettings) { + return; + } + + this.captionMeasureCanvas = document.createElement("canvas"); + this.captionMeasureCanvas.width = 1; + this.captionMeasureCanvas.height = 1; + this.captionMeasureCtx = configureHighQuality2DContext( + this.captionMeasureCanvas.getContext("2d"), + ); + } + + private buildCaptionRenderState(timeMs: number): CaptionRenderState | null { + const settings = this.config.autoCaptionSettings; + const cues = this.config.autoCaptions; + const measureCtx = this.captionMeasureCtx; + + if (!settings || !cues?.length || !measureCtx) { + return null; + } + + const fontFamily = settings.fontFamily || getDefaultCaptionFontFamily(); + const fontSize = getCaptionScaledFontSize( + settings.fontSize, + this.config.width, + settings.maxWidth, + ); + measureCtx.font = `${CAPTION_FONT_WEIGHT} ${fontSize}px ${fontFamily}`; + + const layout = buildActiveCaptionLayout({ + cues, + timeMs, + settings, + maxWidthPx: getCaptionTextMaxWidth(this.config.width, settings.maxWidth, fontSize), + measureText: (text) => measureCtx.measureText(text).width, + }); + if (!layout) { + return null; + } + + const padding = getCaptionPadding(fontSize); + const lineHeight = fontSize * CAPTION_LINE_HEIGHT; + const textBlockHeight = layout.visibleLines.length * lineHeight; + const boxHeight = textBlockHeight + padding.y * 2; + const maxMeasuredWidth = layout.visibleLines.reduce( + (largest, line) => Math.max(largest, line.width), + 0, + ); + const boxWidth = Math.min( + this.config.width * (settings.maxWidth / 100) + padding.x * 2, + maxMeasuredWidth + padding.x * 2, + ); + const centerX = this.config.width / 2; + const centerY = + this.config.height - (this.config.height * settings.bottomOffset) / 100 - boxHeight / 2; + + return { + key: `${layout.blockKey}:${layout.visiblePageIndex}:${layout.activeWordIndex}`, + layout, + fontFamily, + fontSize, + lineHeight, + boxWidth, + boxHeight, + centerX, + centerY, + }; + } + + private ensureCaptionCanvas(width: number, height: number): void { + const targetWidth = Math.max(1, Math.ceil(width)); + const targetHeight = Math.max(1, Math.ceil(height)); + + if ( + this.captionCanvas && + this.captionCanvas.width === targetWidth && + this.captionCanvas.height === targetHeight && + this.captionCtx && + this.captionSprite + ) { + return; + } + + this.captionCanvas = document.createElement("canvas"); + this.captionCanvas.width = targetWidth; + this.captionCanvas.height = targetHeight; + this.captionCtx = configureHighQuality2DContext(this.captionCanvas.getContext("2d")); + + if (!this.captionCtx) { + throw new Error("Failed to create caption export canvas"); + } + + const nextTexture = Texture.from(this.captionCanvas); + if (this.captionSprite) { + const previousTexture = this.captionSprite.texture; + this.captionSprite.texture = nextTexture; + this.captionTextureSource = nextTexture.source as unknown as MutableVideoTextureSource; + previousTexture.destroy(true); + } else { + this.captionSprite = new Sprite(nextTexture); + this.captionSprite.anchor.set(0.5); + this.captionContainer?.addChild(this.captionSprite); + this.captionTextureSource = nextTexture.source as unknown as MutableVideoTextureSource; + } + } + + private rasterizeCaptionSprite(state: CaptionRenderState): void { + this.ensureCaptionCanvas(state.boxWidth, state.boxHeight); + + if (!this.captionCtx || !this.captionCanvas || !this.captionSprite) { + return; + } + + const ctx = this.captionCtx; + const settings = this.config.autoCaptionSettings; + if (!settings) { + return; + } + + ctx.clearRect(0, 0, this.captionCanvas.width, this.captionCanvas.height); + ctx.font = `${CAPTION_FONT_WEIGHT} ${state.fontSize}px ${state.fontFamily}`; + ctx.fillStyle = `rgba(0, 0, 0, ${settings.backgroundOpacity})`; + drawSquircleOnCanvas(ctx, { + x: 0, + y: 0, + width: state.boxWidth, + height: state.boxHeight, + radius: getCaptionScaledRadius(settings.boxRadius, state.fontSize), + }); + ctx.fill(); + + const padding = getCaptionPadding(state.fontSize); + ctx.textAlign = "left"; + ctx.textBaseline = "middle"; + + state.layout.visibleLines.forEach((line, lineIndex) => { + let cursorX = (state.boxWidth - line.width) / 2; + const lineY = padding.y + state.lineHeight * lineIndex + state.lineHeight / 2; + + line.words.forEach((word) => { + const segmentText = `${word.leadingSpace ? " " : ""}${word.text}`; + const segmentWidth = ctx.measureText(segmentText).width; + const visualState = getCaptionWordVisualState(state.layout.hasWordTimings, word.state); + + ctx.save(); + ctx.translate(cursorX, lineY); + ctx.fillStyle = visualState.isInactive ? settings.inactiveTextColor : settings.textColor; + ctx.globalAlpha = visualState.opacity; + ctx.fillText(segmentText, 0, 0); + ctx.restore(); + + cursorX += segmentWidth; + }); + }); + + this.captionTextureSource?.update(); + this.captionRenderKey = state.key; + } + + private updateCaptionLayer(timeMs: number): void { + const state = this.buildCaptionRenderState(timeMs); + if (!state || !this.captionContainer) { + if (this.captionSprite) { + this.captionSprite.visible = false; + } + if (this.captionContainer) { + this.captionContainer.visible = false; + } + this.captionRenderKey = null; + return; + } + + const needsReraster = + !this.captionSprite || + !this.captionCanvas || + this.captionCanvas.width !== Math.max(1, Math.ceil(state.boxWidth)) || + this.captionCanvas.height !== Math.max(1, Math.ceil(state.boxHeight)) || + this.captionRenderKey !== state.key; + + if (needsReraster) { + this.rasterizeCaptionSprite(state); + } + + if (!this.captionSprite) { + return; + } + + this.captionContainer.visible = true; + this.captionSprite.visible = true; + this.captionSprite.position.set(state.centerX, state.centerY + state.layout.translateY); + this.captionSprite.scale.set(state.layout.scale); + this.captionSprite.alpha = state.layout.opacity; + } + + private async syncBackgroundFrame(timeSeconds: number): Promise { + if (this.backgroundForwardFrameSource) { + const decodedFrame = await this.backgroundForwardFrameSource.getFrameAtTime( + Math.max(0, timeSeconds), + ); + this.closeBackgroundDecodedFrame(); + this.backgroundDecodedFrame = decodedFrame; + if (decodedFrame) { + this.ensureBackgroundSprite( + decodedFrame, + decodedFrame.displayWidth, + decodedFrame.displayHeight, + ); + } + return; + } + + const video = this.backgroundVideoElement; + if (!video) { + return; + } + + if (video.duration && Number.isFinite(video.duration)) { + const targetTime = timeSeconds % video.duration; + if (Math.abs(video.currentTime - targetTime) > 0.008) { + video.currentTime = targetTime; + await new Promise((resolve) => { + const onSeeked = () => { + video.removeEventListener("seeked", onSeeked); + resolve(); + }; + video.addEventListener("seeked", onSeeked); + }); + } + } + + this.ensureBackgroundSprite(video, video.videoWidth, video.videoHeight); + } + + private async resolveWallpaperImageUrl(wallpaper: string): Promise { + if ( + wallpaper.startsWith("file://") || + wallpaper.startsWith("data:") || + wallpaper.startsWith("http") + ) { + return wallpaper; + } + + const resolved = await getAssetPath(wallpaper.replace(/^\/+/, "")); + if (resolved.startsWith("/") && window.location.protocol.startsWith("http")) { + return `${window.location.origin}${resolved}`; + } + + return resolved; + } + + private async resolveWallpaperForExport(wallpaper: string): Promise { + if (!wallpaper) { + return wallpaper; + } + + if ( + wallpaper.startsWith("#") || + wallpaper.startsWith("linear-gradient") || + wallpaper.startsWith("radial-gradient") + ) { + return wallpaper; + } + + const looksLikeAbsoluteFilePath = + wallpaper.startsWith("/") && + !wallpaper.startsWith("//") && + !wallpaper.startsWith("/wallpapers/") && + !wallpaper.startsWith("/app-icons/"); + + const wallpaperAsset = looksLikeAbsoluteFilePath ? `file://${encodeURI(wallpaper)}` : wallpaper; + return getRenderableAssetUrl(wallpaperAsset); + } + + private async setupWebcamSource(): Promise { + const webcamUrl = this.config.webcamUrl; + if (!this.config.webcam?.enabled || !webcamUrl) { + this.webcamForwardFrameSource?.cancel(); + void this.webcamForwardFrameSource?.destroy(); + this.webcamForwardFrameSource = null; + this.closeWebcamDecodedFrame(); + this.cleanupWebcamSource?.(); + this.cleanupWebcamSource = null; + this.webcamVideoElement = null; + this.webcamFrameCacheCanvas = null; + this.webcamFrameCacheCtx = null; + this.lastSyncedWebcamTime = null; + this.lastWebcamCacheRefreshTime = null; + this.webcamLayoutCache = null; + this.webcamRenderMode = "hidden"; + return; + } + + this.webcamForwardFrameSource?.cancel(); + void this.webcamForwardFrameSource?.destroy(); + this.webcamForwardFrameSource = null; + this.closeWebcamDecodedFrame(); + this.cleanupWebcamSource?.(); + this.cleanupWebcamSource = null; + this.webcamFrameCacheCanvas = null; + this.webcamFrameCacheCtx = null; + this.lastWebcamCacheRefreshTime = null; + this.webcamLayoutCache = null; + this.webcamRenderMode = "hidden"; + + try { + const frameSource = new ForwardFrameSource(); + await frameSource.initialize(webcamUrl); + this.webcamForwardFrameSource = frameSource; + this.webcamVideoElement = null; + this.webcamSeekPromise = null; + this.lastSyncedWebcamTime = null; + this.lastWebcamCacheRefreshTime = null; + return; + } catch (error) { + console.warn( + "[FrameRenderer] Decoder-backed webcam source unavailable during export; falling back to media element sync:", + error, + ); + } + + const webcamSource = await resolveMediaElementSource(webcamUrl); + this.cleanupWebcamSource = webcamSource.revoke; + + const video = document.createElement("video"); + video.src = webcamSource.src; + video.muted = true; + video.preload = "auto"; + video.playsInline = true; + video.load(); + + await new Promise((resolve, reject) => { + const onReady = () => { + if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) { + return; + } + cleanup(); + resolve(); + }; + const onError = () => { + cleanup(); + reject(new Error("Failed to load webcam source for export")); + }; + const cleanup = () => { + video.removeEventListener("loadeddata", onReady); + video.removeEventListener("canplay", onReady); + video.removeEventListener("canplaythrough", onReady); + video.removeEventListener("error", onError); + }; + + if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) { + resolve(); + return; + } + + video.addEventListener("loadeddata", onReady, { once: true }); + video.addEventListener("canplay", onReady, { once: true }); + video.addEventListener("canplaythrough", onReady, { once: true }); + video.addEventListener("error", onError, { once: true }); + }).catch((error) => { + console.warn("[FrameRenderer] Webcam overlay unavailable during export:", error); + this.webcamVideoElement = null; + }); + + if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) { + this.webcamVideoElement = video; + return; + } + + this.webcamVideoElement = null; + this.lastSyncedWebcamTime = null; + } + + private ensureWebcamSprite( + source: CanvasImageSource | VideoFrame, + sourceWidth: number, + sourceHeight: number, + ): void { + if (!this.webcamContainer) { + return; + } + + const resolvedSource = + typeof VideoFrame !== "undefined" && source instanceof VideoFrame + ? this.stageVideoFrameForTexture(source, "webcam", sourceWidth, sourceHeight) + : source; + const usesStartupStaging = resolvedSource !== source; + + if (!this.webcamSprite) { + const texture = this.createTextureFromSource(resolvedSource); + this.webcamSprite = new Sprite(texture); + this.webcamTextureSource = texture.source as unknown as MutableVideoTextureSource; + this.webcamContainer.addChildAt(this.webcamSprite, 0); + this.webcamTextureUsesStartupStaging = usesStartupStaging; + } else if (this.webcamTextureUsesStartupStaging !== usesStartupStaging) { + this.webcamTextureSource = this.replaceSpriteTexture(this.webcamSprite, resolvedSource); + this.webcamTextureUsesStartupStaging = usesStartupStaging; + } else if (this.webcamTextureSource) { + this.webcamTextureSource.resource = resolvedSource; + this.webcamTextureSource.update(); + } + + if (this.webcamRootContainer) { + this.webcamRootContainer.visible = sourceWidth > 0 && sourceHeight > 0; + } + } + + private setWebcamRenderMode(nextMode: "hidden" | "live" | "cached"): void { + if (this.webcamRenderMode === nextMode) { + return; + } + + if (nextMode === "cached") { + console.log("[FrameRenderer] Webcam export source fell back to the last synced frame"); + } else if (this.webcamRenderMode === "cached" && nextMode === "live") { + console.log("[FrameRenderer] Webcam export source resynchronized"); + } + + this.webcamRenderMode = nextMode; + } + + private shouldRefreshWebcamFrameCache(width: number, height: number): boolean { + const targetWidth = Math.max(1, Math.ceil(width)); + const targetHeight = Math.max(1, Math.ceil(height)); + + if ( + !this.webcamFrameCacheCanvas || + this.webcamFrameCacheCanvas.width !== targetWidth || + this.webcamFrameCacheCanvas.height !== targetHeight + ) { + return true; + } + + if (this.lastWebcamCacheRefreshTime === null) { + return true; + } + + return Math.abs(this.currentVideoTime - this.lastWebcamCacheRefreshTime) >= 0.25; + } + + private ensureWebcamFrameCache(width: number, height: number): boolean { + const targetWidth = Math.max(1, Math.ceil(width)); + const targetHeight = Math.max(1, Math.ceil(height)); + + if ( + this.webcamFrameCacheCanvas && + this.webcamFrameCacheCanvas.width === targetWidth && + this.webcamFrameCacheCanvas.height === targetHeight && + this.webcamFrameCacheCtx + ) { + return true; + } + + this.webcamFrameCacheCanvas = document.createElement("canvas"); + this.webcamFrameCacheCanvas.width = targetWidth; + this.webcamFrameCacheCanvas.height = targetHeight; + this.webcamFrameCacheCtx = configureHighQuality2DContext( + this.webcamFrameCacheCanvas.getContext("2d"), + ); + + return !!this.webcamFrameCacheCtx; + } + + private refreshWebcamFrameCache( + source: CanvasImageSource | VideoFrame, + width: number, + height: number, + ): boolean { + if (!this.ensureWebcamFrameCache(width, height)) { + return false; + } + + if (!this.webcamFrameCacheCanvas || !this.webcamFrameCacheCtx) { + return false; + } + + this.webcamFrameCacheCtx.clearRect( + 0, + 0, + this.webcamFrameCacheCanvas.width, + this.webcamFrameCacheCanvas.height, + ); + this.webcamFrameCacheCtx.drawImage( + source, + 0, + 0, + this.webcamFrameCacheCanvas.width, + this.webcamFrameCacheCanvas.height, + ); + this.lastWebcamCacheRefreshTime = this.currentVideoTime; + return true; + } + + private getCachedWebcamRenderSource(): WebcamRenderSource | null { + if ( + !this.webcamFrameCacheCanvas || + this.webcamFrameCacheCanvas.width <= 0 || + this.webcamFrameCacheCanvas.height <= 0 + ) { + return null; + } + + return { + source: this.webcamFrameCacheCanvas, + width: this.webcamFrameCacheCanvas.width, + height: this.webcamFrameCacheCanvas.height, + mode: "cached", + }; + } + + private resolveRenderableWebcamSource( + liveSource: CanvasImageSource | VideoFrame | null, + liveSourceWidth: number, + liveSourceHeight: number, + canUseLiveSource: boolean, + ): WebcamRenderSource | null { + if (canUseLiveSource && liveSource && liveSourceWidth > 0 && liveSourceHeight > 0) { + if (this.shouldRefreshWebcamFrameCache(liveSourceWidth, liveSourceHeight)) { + this.refreshWebcamFrameCache(liveSource, liveSourceWidth, liveSourceHeight); + } + this.setWebcamRenderMode("live"); + return { + source: liveSource, + width: liveSourceWidth, + height: liveSourceHeight, + mode: "live", + }; + } + + const cachedSource = this.getCachedWebcamRenderSource(); + if (cachedSource) { + this.setWebcamRenderMode("cached"); + return cachedSource; + } + + if (canUseLiveSource && liveSource && liveSourceWidth > 0 && liveSourceHeight > 0) { + this.setWebcamRenderMode("live"); + return { + source: liveSource, + width: liveSourceWidth, + height: liveSourceHeight, + mode: "live", + }; + } + + this.setWebcamRenderMode("hidden"); + return null; + } + + private hasMatchingWebcamLayout(nextLayout: WebcamLayoutCache): boolean { + const previousLayout = this.webcamLayoutCache; + if (!previousLayout) { + return false; + } + + return ( + previousLayout.mirror === nextLayout.mirror && + areNearlyEqual(previousLayout.sourceWidth, nextLayout.sourceWidth) && + areNearlyEqual(previousLayout.sourceHeight, nextLayout.sourceHeight) && + areNearlyEqual(previousLayout.size, nextLayout.size) && + areNearlyEqual(previousLayout.positionX, nextLayout.positionX) && + areNearlyEqual(previousLayout.positionY, nextLayout.positionY) && + areNearlyEqual(previousLayout.radius, nextLayout.radius) && + areNearlyEqual(previousLayout.shadowStrength, nextLayout.shadowStrength) + ); + } + + private applyWebcamLayout(nextLayout: WebcamLayoutCache): void { + if (!this.webcamRootContainer || !this.webcamSprite || !this.webcamMaskGraphics) { + return; + } + + this.webcamRootContainer.position.set(nextLayout.positionX, nextLayout.positionY); + + applyCoverLayoutToSprite( + this.webcamSprite, + nextLayout.sourceWidth, + nextLayout.sourceHeight, + nextLayout.size, + nextLayout.size, + nextLayout.size / 2, + nextLayout.size / 2, + nextLayout.mirror, + ); + + this.webcamMaskGraphics.clear(); + drawSquircleOnGraphics(this.webcamMaskGraphics, { + x: 0, + y: 0, + width: nextLayout.size, + height: nextLayout.size, + radius: nextLayout.radius, + }); + this.webcamMaskGraphics.fill({ color: 0xffffff }); + + for (const layer of this.webcamShadowLayers) { + if (nextLayout.shadowStrength <= 0) { + layer.container.visible = false; + continue; + } + + const offsetY = nextLayout.size * layer.offsetScale * nextLayout.shadowStrength; + this.rasterizeShadowLayer(layer, { + x: 0, + y: 0, + width: nextLayout.size, + height: nextLayout.size, + radius: nextLayout.radius, + offsetY, + alpha: layer.alphaScale * nextLayout.shadowStrength, + blur: Math.max(0, nextLayout.size * layer.blurScale * nextLayout.shadowStrength), + }); + } + + this.webcamLayoutCache = { ...nextLayout }; + } + + private async syncWebcamFrame(targetTime: number): Promise { + const webcamTimeOffsetSec = (this.config.webcam?.timeOffsetMs ?? 0) / 1000; + const webcamTargetTime = Math.max(0, targetTime + webcamTimeOffsetSec); + + if (this.webcamForwardFrameSource) { + const clampedTime = clampMediaTimeToDuration(webcamTargetTime, null); + const decodedFrame = await this.webcamForwardFrameSource.getFrameAtTime(clampedTime); + this.closeWebcamDecodedFrame(); + this.webcamDecodedFrame = decodedFrame; + if (decodedFrame) { + this.lastSyncedWebcamTime = clampedTime; + } + return; + } + + const webcamVideo = this.webcamVideoElement; + if (!webcamVideo) { + return; + } + + const clampedTime = clampMediaTimeToDuration( + webcamTargetTime, + Number.isFinite(webcamVideo.duration) ? webcamVideo.duration : null, + ); + + if (Math.abs(webcamVideo.currentTime - clampedTime) <= 0.008) { + this.lastSyncedWebcamTime = clampedTime; + return; + } + + if (this.webcamSeekPromise) { + await this.webcamSeekPromise; + } + + this.webcamSeekPromise = new Promise((resolve) => { + let settled = false; + let fallbackTimeout: number | null = null; + let animationFrameRequestId: number | null = null; + let videoFrameRequestId: number | null = null; + + const waitForPresentedFrame = () => { + const requestVideoFrameCallback = ( + webcamVideo as HTMLVideoElement & { + requestVideoFrameCallback?: ( + callback: (now: DOMHighResTimeStamp, metadata: VideoFrameCallbackMetadata) => void, + ) => number; + cancelVideoFrameCallback?: (handle: number) => void; + } + ).requestVideoFrameCallback; + + const scheduleAnimationFrameFinish = () => { + animationFrameRequestId = requestAnimationFrame(() => { + animationFrameRequestId = null; + finish(); + }); + }; + + scheduleAnimationFrameFinish(); + + if (typeof requestVideoFrameCallback === "function") { + videoFrameRequestId = requestVideoFrameCallback.call(webcamVideo, () => { + videoFrameRequestId = null; + finish(); + }); + } + }; + + const finish = () => { + if (settled) { + return; + } + settled = true; + if (Math.abs(webcamVideo.currentTime - clampedTime) <= 0.02) { + this.lastSyncedWebcamTime = clampedTime; + } + cleanup(); + resolve(); + }; + + const handleMediaReady = () => { + if ( + !webcamVideo.seeking && + Math.abs(webcamVideo.currentTime - clampedTime) <= 0.01 && + webcamVideo.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA + ) { + waitForPresentedFrame(); + } + }; + + const cleanup = () => { + webcamVideo.removeEventListener("seeked", waitForPresentedFrame); + webcamVideo.removeEventListener("loadeddata", handleMediaReady); + webcamVideo.removeEventListener("canplay", handleMediaReady); + webcamVideo.removeEventListener("error", finish); + if (animationFrameRequestId !== null) { + cancelAnimationFrame(animationFrameRequestId); + animationFrameRequestId = null; + } + if ( + videoFrameRequestId !== null && + typeof ( + webcamVideo as HTMLVideoElement & { + cancelVideoFrameCallback?: (handle: number) => void; + } + ).cancelVideoFrameCallback === "function" + ) { + ( + webcamVideo as HTMLVideoElement & { + cancelVideoFrameCallback: (handle: number) => void; + } + ).cancelVideoFrameCallback(videoFrameRequestId); + videoFrameRequestId = null; + } + if (fallbackTimeout !== null) { + window.clearTimeout(fallbackTimeout); + } + }; + + webcamVideo.addEventListener("seeked", waitForPresentedFrame, { once: true }); + webcamVideo.addEventListener("loadeddata", handleMediaReady, { once: true }); + webcamVideo.addEventListener("canplay", handleMediaReady, { once: true }); + webcamVideo.addEventListener("error", finish, { once: true }); + + fallbackTimeout = window.setTimeout(() => { + finish(); + }, 50); + + try { + webcamVideo.currentTime = clampedTime; + } catch { + finish(); + return; + } + + if ( + !webcamVideo.seeking && + Math.abs(webcamVideo.currentTime - clampedTime) <= 0.001 && + webcamVideo.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA + ) { + waitForPresentedFrame(); + } + }); + + try { + await this.webcamSeekPromise; + } finally { + this.webcamSeekPromise = null; + } + } + + private updateWebcamOverlay(): void { + const webcam = this.config.webcam; + if (!webcam?.enabled || !this.webcamRootContainer || !this.webcamMaskGraphics) { + if (this.webcamRootContainer) { + this.webcamRootContainer.visible = false; + } + this.webcamLayoutCache = null; + this.setWebcamRenderMode("hidden"); + return; + } + + const webcamSource = this.webcamDecodedFrame ?? this.webcamVideoElement; + const liveSourceDimensions = webcamSource + ? this.getWebcamSourceDimensions(webcamSource) + : { width: 0, height: 0 }; + const activeWebcamVideoElement = + webcamSource === this.webcamVideoElement ? this.webcamVideoElement : null; + const webcamTimeDrift = + this.lastSyncedWebcamTime === null + ? 0 + : Math.abs(this.lastSyncedWebcamTime - this.currentVideoTime); + const canUseLiveSource = + !!webcamSource && + liveSourceDimensions.width > 0 && + liveSourceDimensions.height > 0 && + webcamTimeDrift <= 0.08 && + (!activeWebcamVideoElement || + (activeWebcamVideoElement.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA && + !activeWebcamVideoElement.seeking)); + const renderableWebcamSource = this.resolveRenderableWebcamSource( + webcamSource, + liveSourceDimensions.width, + liveSourceDimensions.height, + canUseLiveSource, + ); + + if (!renderableWebcamSource) { + this.webcamRootContainer.visible = false; + this.webcamLayoutCache = null; + return; + } + + this.ensureWebcamSprite( + renderableWebcamSource.source, + renderableWebcamSource.width, + renderableWebcamSource.height, + ); + if (!this.webcamSprite) { + this.webcamRootContainer.visible = false; + return; + } + + const margin = webcam.margin ?? 24; + const size = getWebcamOverlaySizePx({ + containerWidth: this.config.width, + containerHeight: this.config.height, + sizePercent: webcam.size ?? 50, + margin, + zoomScale: this.animationState.appliedScale || 1, + reactToZoom: webcam.reactToZoom ?? true, + }); + const position = getWebcamOverlayPosition({ + containerWidth: this.config.width, + containerHeight: this.config.height, + size, + margin, + positionPreset: webcam.positionPreset ?? webcam.corner, + positionX: webcam.positionX ?? 1, + positionY: webcam.positionY ?? 1, + legacyCorner: webcam.corner, + }); + const radius = Math.max(0, webcam.cornerRadius ?? 18); + const shadowStrength = clampUnitInterval(webcam.shadow ?? 0); + + this.webcamRootContainer.visible = true; + + const nextLayout: WebcamLayoutCache = { + sourceWidth: renderableWebcamSource.width, + sourceHeight: renderableWebcamSource.height, + size, + positionX: position.x, + positionY: position.y, + radius, + shadowStrength, + mirror: webcam.mirror, + }; + + if (!this.hasMatchingWebcamLayout(nextLayout)) { + this.applyWebcamLayout(nextLayout); + } + } + + async renderFrame(videoFrame: VideoFrame, timestamp: number): Promise { + if (!this.app || !this.videoContainer || !this.cameraContainer || !this.videoMaskGraphics) { + throw new Error("Renderer not initialized"); + } + + this.currentVideoTime = timestamp / 1_000_000; + + if (this.webcamForwardFrameSource || this.webcamVideoElement) { + await this.syncWebcamFrame(Math.max(0, this.currentVideoTime)); + } + + if (this.backgroundForwardFrameSource || this.backgroundVideoElement) { + await this.syncBackgroundFrame(this.currentVideoTime); + } + + const resolvedVideoSource = this.stageVideoFrameForTexture( + videoFrame, + "scene", + this.config.videoWidth, + this.config.videoHeight, + ); + const usesStartupStaging = resolvedVideoSource !== videoFrame; + + if (!this.videoSprite) { + const texture = this.createTextureFromSource(resolvedVideoSource); + this.videoSprite = new Sprite(texture); + this.videoTextureSource = texture.source as unknown as MutableVideoTextureSource; + this.videoContainer.addChildAt(this.videoSprite, 0); + this.videoTextureUsesStartupStaging = usesStartupStaging; + } else if (this.videoTextureUsesStartupStaging !== usesStartupStaging) { + this.videoTextureSource = this.replaceSpriteTexture(this.videoSprite, resolvedVideoSource); + this.videoTextureUsesStartupStaging = usesStartupStaging; + } else if (this.videoTextureSource) { + this.videoTextureSource.resource = resolvedVideoSource; + this.videoTextureSource.update(); + } + + if (!this.layoutCache) { + this.updateLayout(); + } + const layoutCache = this.layoutCache; + if (!layoutCache) { + throw new Error("Renderer layout cache is unavailable"); + } + + const timeMs = this.currentVideoTime * 1000; + + if (this.cursorOverlay) { + this.cursorOverlay.update( + this.config.cursorTelemetry ?? [], + timeMs, + layoutCache.maskRect, + this.config.showCursor ?? true, + false, + ); + } + + let maxMotionIntensity = 0; + const motionIntensity = this.updateAnimationState(timeMs); + maxMotionIntensity = Math.max(maxMotionIntensity, motionIntensity); + + applyZoomTransform({ + cameraContainer: this.cameraContainer, + blurFilter: this.blurFilter, + motionBlurFilter: this.motionBlurFilter, + stageSize: layoutCache.stageSize, + baseMask: layoutCache.maskRect, + zoomScale: this.animationState.scale, + zoomProgress: this.animationState.progress, + focusX: this.animationState.focusX, + focusY: this.animationState.focusY, + motionIntensity: maxMotionIntensity, + motionVector: this.lastMotionVector, + isPlaying: true, + motionBlurAmount: this.config.zoomMotionBlur ?? 0, + transformOverride: { + scale: this.animationState.appliedScale, + x: this.animationState.x, + y: this.animationState.y, + }, + motionBlurState: this.motionBlurState, + frameTimeMs: timeMs, + }); + + this.updateAnnotationLayer(timeMs); + this.updateCaptionLayer(timeMs); + this.updateWebcamOverlay(); + this.app.render(); + } + + private updateLayout(): void { + if (!this.videoSprite || !this.videoMaskGraphics || !this.videoContainer) { + return; + } + + const { width, height } = this.config; + const { cropRegion, borderRadius = 0, padding = 0 } = this.config; + const videoWidth = this.config.videoWidth; + const videoHeight = this.config.videoHeight; + + const cropStartX = cropRegion.x; + const cropStartY = cropRegion.y; + const cropEndX = cropRegion.x + cropRegion.width; + const cropEndY = cropRegion.y + cropRegion.height; + + const croppedVideoWidth = videoWidth * (cropEndX - cropStartX); + const croppedVideoHeight = videoHeight * (cropEndY - cropStartY); + + const paddingScale = 1.0 - (padding / 100) * 0.4; + const viewportWidth = width * paddingScale; + const viewportHeight = height * paddingScale; + const scale = Math.min(viewportWidth / croppedVideoWidth, viewportHeight / croppedVideoHeight); + + this.videoSprite.scale.set(scale); + + const fullVideoDisplayWidth = videoWidth * scale; + const fullVideoDisplayHeight = videoHeight * scale; + const croppedDisplayWidth = croppedVideoWidth * scale; + const croppedDisplayHeight = croppedVideoHeight * scale; + const centerOffsetX = (width - croppedDisplayWidth) / 2; + const centerOffsetY = (height - croppedDisplayHeight) / 2; + + const spriteX = centerOffsetX - cropRegion.x * fullVideoDisplayWidth; + const spriteY = centerOffsetY - cropRegion.y * fullVideoDisplayHeight; + this.videoSprite.position.set(spriteX, spriteY); + + const previewWidth = this.config.previewWidth || 1920; + const previewHeight = this.config.previewHeight || 1080; + const canvasScaleFactor = Math.min(width / previewWidth, height / previewHeight); + const scaledBorderRadius = borderRadius * canvasScaleFactor; + + this.videoMaskGraphics.clear(); + drawSquircleOnGraphics(this.videoMaskGraphics, { + x: centerOffsetX, + y: centerOffsetY, + width: croppedDisplayWidth, + height: croppedDisplayHeight, + radius: scaledBorderRadius, + }); + this.videoMaskGraphics.fill({ color: 0xffffff }); + + this.updateVideoShadowLayout({ + maskX: centerOffsetX, + maskY: centerOffsetY, + maskWidth: croppedDisplayWidth, + maskHeight: croppedDisplayHeight, + maskRadius: scaledBorderRadius, + }); + + this.layoutCache = { + stageSize: { width, height }, + videoSize: { width: croppedVideoWidth, height: croppedVideoHeight }, + baseScale: scale, + baseOffset: { x: spriteX, y: spriteY }, + maskRect: { + x: centerOffsetX, + y: centerOffsetY, + width: croppedDisplayWidth, + height: croppedDisplayHeight, + sourceCrop: cropRegion, + }, + }; + } + + private updateVideoShadowLayout(layout: { + maskX: number; + maskY: number; + maskWidth: number; + maskHeight: number; + maskRadius: number; + }): void { + const shadowStrength = clampUnitInterval(this.config.shadowIntensity); + for (const layer of this.videoShadowLayers) { + if (!this.config.showShadow || shadowStrength <= 0) { + layer.container.visible = false; + continue; + } + + const offsetY = layer.offsetScale * shadowStrength; + this.rasterizeShadowLayer(layer, { + x: layout.maskX, + y: layout.maskY, + width: layout.maskWidth, + height: layout.maskHeight, + radius: layout.maskRadius, + offsetY, + alpha: layer.alphaScale * shadowStrength, + blur: Math.max(0, layer.blurScale * shadowStrength), + }); + } + } + + private updateAnimationState(timeMs: number): number { + if (!this.cameraContainer || !this.layoutCache) { + return 0; + } + + const { region, strength, blendedScale, transition } = findDominantRegion( + this.config.zoomRegions, + timeMs, + { + connectZooms: this.config.connectZooms, + }, + ); + + let targetScaleFactor = 1; + let targetFocus = { ...DEFAULT_FOCUS }; + let targetProgress = 0; + + if (region && strength > 0) { + targetScaleFactor = blendedScale ?? ZOOM_DEPTH_SCALES[region.depth]; + targetFocus = region.focus; + targetProgress = strength; + + if (transition) { + const startTransform = computeZoomTransform({ + stageSize: this.layoutCache.stageSize, + baseMask: this.layoutCache.maskRect, + zoomScale: transition.startScale, + zoomProgress: 1, + focusX: transition.startFocus.cx, + focusY: transition.startFocus.cy, + }); + const endTransform = computeZoomTransform({ + stageSize: this.layoutCache.stageSize, + baseMask: this.layoutCache.maskRect, + zoomScale: transition.endScale, + zoomProgress: 1, + focusX: transition.endFocus.cx, + focusY: transition.endFocus.cy, + }); + + const interpolatedTransform = { + scale: + startTransform.scale + + (endTransform.scale - startTransform.scale) * transition.progress, + x: startTransform.x + (endTransform.x - startTransform.x) * transition.progress, + y: startTransform.y + (endTransform.y - startTransform.y) * transition.progress, + }; + + targetScaleFactor = interpolatedTransform.scale; + targetFocus = computeFocusFromTransform({ + stageSize: this.layoutCache.stageSize, + baseMask: this.layoutCache.maskRect, + zoomScale: interpolatedTransform.scale, + x: interpolatedTransform.x, + y: interpolatedTransform.y, + }); + targetProgress = 1; + } + } + + const state = this.animationState; + const previousScale = state.appliedScale; + const previousX = state.x; + const previousY = state.y; + + state.scale = targetScaleFactor; + state.focusX = targetFocus.cx; + state.focusY = targetFocus.cy; + state.progress = targetProgress; + + const projectedTransform = computeZoomTransform({ + stageSize: this.layoutCache.stageSize, + baseMask: this.layoutCache.maskRect, + zoomScale: state.scale, + zoomProgress: state.progress, + focusX: state.focusX, + focusY: state.focusY, + }); + + state.appliedScale = + Math.abs(projectedTransform.scale - previousScale) < ZOOM_SCALE_DEADZONE + ? projectedTransform.scale + : projectedTransform.scale; + state.x = + Math.abs(projectedTransform.x - previousX) < ZOOM_TRANSLATION_DEADZONE_PX + ? projectedTransform.x + : projectedTransform.x; + state.y = + Math.abs(projectedTransform.y - previousY) < ZOOM_TRANSLATION_DEADZONE_PX + ? projectedTransform.y + : projectedTransform.y; + + this.lastMotionVector = { + x: state.x - previousX, + y: state.y - previousY, + }; + + return Math.max( + Math.abs(state.appliedScale - previousScale), + Math.abs(state.x - previousX) / Math.max(1, this.layoutCache.stageSize.width), + Math.abs(state.y - previousY) / Math.max(1, this.layoutCache.stageSize.height), + ); + } + + private closeWebcamDecodedFrame(): void { + if (!this.webcamDecodedFrame) { + return; + } + + this.webcamDecodedFrame.close(); + this.webcamDecodedFrame = null; + } + + private getNativeReadbackBuffer(byteLength: number): Uint8Array { + if (!this.nativeReadbackBuffer || this.nativeReadbackBuffer.byteLength !== byteLength) { + this.nativeReadbackBuffer = new Uint8Array(byteLength); + } + + return this.nativeReadbackBuffer; + } + + private capturePixelsFromWebGL(): Uint8Array | null { + if (!this.app) { + throw new Error("Renderer not initialized"); + } + + const renderer = this.app.renderer as typeof this.app.renderer & { + gl?: WebGLRenderingContext | WebGL2RenderingContext; + }; + const gl = renderer.gl; + if (!gl) { + return null; + } + + const width = this.config.width; + const height = this.config.height; + const pixelByteLength = width * height * 4; + const pixelBuffer = this.getNativeReadbackBuffer(pixelByteLength); + while (gl.getError() !== gl.NO_ERROR) { + // Clear stale GL errors from earlier work before checking this readback attempt. + } + + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + gl.pixelStorei(gl.PACK_ALIGNMENT, 1); + gl.readPixels(0, 0, width, height, gl.RGBA, gl.UNSIGNED_BYTE, pixelBuffer); + + let glError = gl.getError(); + if (glError !== gl.NO_ERROR) { + // ANGLE/Metal may reject RGBA+UNSIGNED_BYTE on the default framebuffer. + // Blit to an FBO with explicit RGBA8 attachment where this is guaranteed. + const gl2 = gl as WebGL2RenderingContext; + if (typeof gl2.blitFramebuffer !== "function") { + throw new Error(`[FrameRenderer] WebGL readPixels failed with error code ${glError}`); + } + + while (gl2.getError() !== gl2.NO_ERROR) { + // drain + } + + const readFb = gl2.createFramebuffer(); + const rb = gl2.createRenderbuffer(); + try { + gl2.bindRenderbuffer(gl2.RENDERBUFFER, rb); + gl2.renderbufferStorage(gl2.RENDERBUFFER, gl2.RGBA8, width, height); + gl2.bindFramebuffer(gl2.DRAW_FRAMEBUFFER, readFb); + gl2.framebufferRenderbuffer( + gl2.DRAW_FRAMEBUFFER, + gl2.COLOR_ATTACHMENT0, + gl2.RENDERBUFFER, + rb, + ); + gl2.bindFramebuffer(gl2.READ_FRAMEBUFFER, null); + gl2.blitFramebuffer( + 0, 0, width, height, + 0, 0, width, height, + gl2.COLOR_BUFFER_BIT, + gl2.NEAREST, + ); + gl2.bindFramebuffer(gl2.FRAMEBUFFER, readFb); + gl2.readPixels(0, 0, width, height, gl2.RGBA, gl2.UNSIGNED_BYTE, pixelBuffer); + glError = gl2.getError(); + if (glError !== gl2.NO_ERROR) { + throw new Error( + `[FrameRenderer] WebGL FBO readPixels failed with error code ${glError}`, + ); + } + } finally { + gl2.bindFramebuffer(gl2.FRAMEBUFFER, null); + gl2.deleteFramebuffer(readFb); + gl2.deleteRenderbuffer(rb); + } + } + + return pixelBuffer; + } + + capturePixelsForNativeExport(): Uint8Array | null { + if (!this.app) { + throw new Error("Renderer not initialized"); + } + + if (this.config.nativeReadbackMode !== "pixels") { + return null; + } + + try { + return this.capturePixelsFromWebGL(); + } catch (error) { + if (!this.nativePixelReadbackWarningShown) { + this.nativePixelReadbackWarningShown = true; + console.warn( + "[FrameRenderer] Direct WebGL readback unavailable for native export; falling back to canvas capture.", + error, + ); + } + + return null; + } + } + + getCanvas(): HTMLCanvasElement { + if (!this.app) { + throw new Error("Renderer not initialized"); + } + + return this.app.canvas as HTMLCanvasElement; + } + + getRendererBackend(): ExportRenderBackend { + return this.rendererBackend; + } + + destroy(): void { + const texturesToDestroy = new Set(); + if (this.videoSprite?.texture) { + texturesToDestroy.add(this.videoSprite.texture); + } + if (this.backgroundSprite?.texture) { + texturesToDestroy.add(this.backgroundSprite.texture); + } + if (this.webcamSprite?.texture) { + texturesToDestroy.add(this.webcamSprite.texture); + } + if (this.captionSprite?.texture) { + texturesToDestroy.add(this.captionSprite.texture); + } + for (const layer of this.videoShadowLayers) { + if (layer.sprite?.texture) { + texturesToDestroy.add(layer.sprite.texture); + } + } + for (const layer of this.webcamShadowLayers) { + if (layer.sprite?.texture) { + texturesToDestroy.add(layer.sprite.texture); + } + } + for (const entry of this.annotationSprites) { + texturesToDestroy.add(entry.texture); + } + + if (this.cursorOverlay) { + this.cursorOverlay.destroy(); + this.cursorOverlay = null; + } + + if (this.videoEffectsContainer) { + this.videoEffectsContainer.filters = null; + } + if (this.backgroundSprite) { + this.backgroundSprite.filters = null; + } + this.blurFilter?.destroy(); + this.motionBlurFilter?.destroy(); + this.backgroundBlurFilter?.destroy(); + + this.app?.destroy(true, { + children: true, + texture: false, + textureSource: false, + }); + + for (const texture of texturesToDestroy) { + try { + texture.destroy(true); + } catch (error) { + console.warn("[FrameRenderer] Failed to destroy texture during cleanup:", error); + } + } + + this.app = null; + this.backgroundContainer = null; + this.cameraContainer = null; + this.videoEffectsContainer = null; + this.videoContainer = null; + this.cursorContainer = null; + this.overlayContainer = null; + this.annotationContainer = null; + this.captionContainer = null; + this.webcamRootContainer = null; + this.webcamContainer = null; + this.videoSprite = null; + this.videoTextureSource = null; + this.backgroundSprite = null; + this.backgroundTextureSource = null; + this.videoMaskGraphics = null; + this.webcamMaskGraphics = null; + this.blurFilter = null; + this.motionBlurFilter = null; + this.backgroundBlurFilter = null; + this.annotationAssets = null; + this.annotationSprites = []; + this.videoShadowLayers = []; + this.webcamShadowLayers = []; + this.webcamSprite = null; + this.webcamTextureSource = null; + + this.closeBackgroundDecodedFrame(); + this.backgroundForwardFrameSource?.cancel(); + void this.backgroundForwardFrameSource?.destroy(); + this.backgroundForwardFrameSource = null; + if (this.backgroundVideoElement) { + this.backgroundVideoElement.pause(); + this.backgroundVideoElement.src = ""; + this.backgroundVideoElement.load(); + this.backgroundVideoElement = null; + } + + this.webcamForwardFrameSource?.cancel(); + void this.webcamForwardFrameSource?.destroy(); + this.webcamForwardFrameSource = null; + this.closeWebcamDecodedFrame(); + if (this.webcamVideoElement) { + this.webcamVideoElement.pause(); + this.webcamVideoElement.src = ""; + this.webcamVideoElement.load(); + this.webcamVideoElement = null; + } + this.cleanupWebcamSource?.(); + this.cleanupWebcamSource = null; + this.webcamFrameCacheCanvas = null; + this.webcamFrameCacheCtx = null; + this.sceneVideoFrameStagingCanvas = null; + this.sceneVideoFrameStagingCtx = null; + this.webcamVideoFrameStagingCanvas = null; + this.webcamVideoFrameStagingCtx = null; + this.videoTextureUsesStartupStaging = false; + this.webcamTextureUsesStartupStaging = false; + + this.captionCanvas = null; + this.captionCtx = null; + this.captionMeasureCanvas = null; + this.captionMeasureCtx = null; + this.captionSprite = null; + this.captionTextureSource = null; + this.captionRenderKey = null; + this.nativeReadbackBuffer = null; + + this.annotationScaleFactor = 1; + this.lastSyncedWebcamTime = null; + this.lastWebcamCacheRefreshTime = null; + this.webcamRenderMode = "hidden"; + this.webcamLayoutCache = null; + this.layoutCache = null; + } +}