feat(export): add extension render hooks to export pipeline

- Integrate extension render hooks into frameRenderer and modernFrameRenderer
- Add modernFrameRenderer for compositing extensions during export
- Wire extension audio processing in audioEncoder
- Support extension hooks in gif and video export paths
This commit is contained in:
webadderall
2026-04-12 01:38:36 +10:00
parent bcf67bff1d
commit 5fc78fd90b
6 changed files with 543 additions and 15 deletions
+30 -8
View File
@@ -562,7 +562,7 @@ export class AudioProcessor {
}
const { recorder, recordedBlobPromise } = this.startAudioRecording(destinationNode.stream)
let rafId: number | null = null
let tickTimerId: ReturnType<typeof setTimeout> | null = null
try {
if (audioContext.state === 'suspended') {
@@ -577,9 +577,9 @@ export class AudioProcessor {
await new Promise<void>((resolve, reject) => {
const cleanup = () => {
if (rafId !== null) {
cancelAnimationFrame(rafId)
rafId = null
if (tickTimerId !== null) {
clearTimeout(tickTimerId)
tickTimerId = null
}
timelineMedia.removeEventListener('error', onError)
timelineMedia.removeEventListener('ended', onEnded)
@@ -701,7 +701,7 @@ export class AudioProcessor {
}
if (!timelineMedia.paused && !timelineMedia.ended) {
rafId = requestAnimationFrame(tick)
tickTimerId = setTimeout(tick, 16)
} else {
cleanup()
resolve()
@@ -710,11 +710,11 @@ export class AudioProcessor {
timelineMedia.addEventListener('error', onError, { once: true })
timelineMedia.addEventListener('ended', onEnded, { once: true })
rafId = requestAnimationFrame(tick)
tickTimerId = setTimeout(tick, 16)
})
} finally {
if (rafId !== null) {
cancelAnimationFrame(rafId)
if (tickTimerId !== null) {
clearTimeout(tickTimerId)
}
timelineMedia.pause()
timelineAudioSourceNode?.disconnect()
@@ -856,6 +856,8 @@ export class AudioProcessor {
}
return new Promise<void>((resolve, reject) => {
let timeoutId: ReturnType<typeof setTimeout> | null = null
const onLoaded = () => {
cleanup()
resolve()
@@ -864,11 +866,20 @@ export class AudioProcessor {
cleanup()
reject(new Error('Failed to load media metadata for speed-adjusted audio'))
}
const onTimeout = () => {
cleanup()
reject(new Error('Timed out waiting for media metadata (30s)'))
}
const cleanup = () => {
if (timeoutId !== null) {
clearTimeout(timeoutId)
timeoutId = null
}
media.removeEventListener('loadedmetadata', onLoaded)
media.removeEventListener('error', onError)
}
timeoutId = setTimeout(onTimeout, 30_000)
media.addEventListener('loadedmetadata', onLoaded)
media.addEventListener('error', onError, { once: true })
})
@@ -880,6 +891,8 @@ export class AudioProcessor {
}
return new Promise<void>((resolve, reject) => {
let timeoutId: ReturnType<typeof setTimeout> | null = null
const onSeeked = () => {
cleanup()
resolve()
@@ -888,11 +901,20 @@ export class AudioProcessor {
cleanup()
reject(new Error('Failed to seek media for speed-adjusted audio'))
}
const onTimeout = () => {
cleanup()
reject(new Error('Timed out waiting for media seek (30s)'))
}
const cleanup = () => {
if (timeoutId !== null) {
clearTimeout(timeoutId)
timeoutId = null
}
media.removeEventListener('seeked', onSeeked)
media.removeEventListener('error', onError)
}
timeoutId = setTimeout(onTimeout, 30_000)
media.addEventListener('seeked', onSeeked, { once: true })
media.addEventListener('error', onError, { once: true })
media.currentTime = targetSec
+277 -4
View File
@@ -23,6 +23,7 @@ import type {
import { ZOOM_DEPTH_SCALES, BASE_PREVIEW_WIDTH, BASE_PREVIEW_HEIGHT } from "@/components/video-editor/types";
import { getAssetPath, getRenderableAssetUrl } from "@/lib/assetPath";
import { extensionHost } from "@/lib/extensions";
import { findDominantRegion } from "@/components/video-editor/videoPlayback/zoomRegionUtils";
import {
type CursorFollowCameraState,
@@ -49,6 +50,16 @@ import {
} from "@/components/video-editor/videoPlayback/motionSmoothing";
import { renderAnnotations } from "./annotationRenderer";
import { renderCaptions } from "./captionRenderer";
import {
executeExtensionRenderHooks,
executeExtensionCursorEffects,
notifyCursorInteraction,
} from "@/lib/extensions/renderHooks";
import {
mapCursorToCanvasNormalized,
mapSmoothedCursorToCanvasNormalized,
} from "@/lib/extensions/cursorCoordinates";
import { applyCanvasSceneTransform } from "@/lib/extensions/sceneTransform";
import {
PixiCursorOverlay,
DEFAULT_CURSOR_CONFIG,
@@ -104,6 +115,7 @@ interface FrameRenderConfig {
cursorClickBounce?: number;
cursorClickBounceDuration?: number;
cursorSway?: number;
frame?: string | null;
}
interface AnimationState {
@@ -183,6 +195,9 @@ export class FrameRenderer {
private webcamBubbleCtx: CanvasRenderingContext2D | null = null;
private lastSyncedWebcamTime: number | null = null;
private cleanupWebcamSource: (() => void) | null = null;
private frameImage: HTMLImageElement | null = null;
private frameInsets: { top: number; right: number; bottom: number; left: number } | null = null;
private frameDraw: ((ctx: CanvasRenderingContext2D, w: number, h: number) => void) | null = null;
constructor(config: FrameRenderConfig) {
this.config = config;
@@ -271,6 +286,7 @@ export class FrameRenderer {
// Setup background (render separately, not in PixiJS)
await this.setupBackground();
await this.setupWebcamSource();
await this.setupFrame();
// Setup blur filter for video container
this.blurFilter = new BlurFilter();
@@ -826,6 +842,37 @@ export class FrameRenderer {
}
}
private async setupFrame(): Promise<void> {
const frameId = this.config.frame;
if (!frameId) return;
const { extensionHost } = await import("@/lib/extensions/extensionHost");
const frames = extensionHost.getFrames();
const frame = frames.find((f) => f.id === frameId);
if (!frame) {
console.warn(`[FrameRenderer] Device frame "${frameId}" not found`);
return;
}
this.frameInsets = frame.screenInsets;
if (frame.draw) {
// Prefer draw function — renders at export resolution, no bitmap scaling
this.frameDraw = frame.draw;
return;
}
const img = new Image();
img.crossOrigin = "anonymous";
await new Promise<void>((resolve, reject) => {
img.onload = () => resolve();
img.onerror = () => reject(new Error(`Failed to load device frame image: ${frameId}`));
img.src = frame.filePath;
});
this.frameImage = img;
}
async renderFrame(
videoFrame: VideoFrame,
timestamp: number,
@@ -881,6 +928,25 @@ export class FrameRenderer {
);
}
const smoothedCursor = mapSmoothedCursorToCanvasNormalized(
this.cursorOverlay?.getSmoothedCursorSnapshot() ?? null,
{
maskRect: this.layoutCache.maskRect,
canvasWidth: this.config.width,
canvasHeight: this.config.height,
},
);
extensionHost.setSmoothedCursor(
smoothedCursor
? {
timeMs,
cx: smoothedCursor.cx,
cy: smoothedCursor.cy,
trail: smoothedCursor.trail,
}
: null,
);
const TICKS_PER_FRAME = 1;
let maxMotionIntensity = 0;
@@ -919,6 +985,9 @@ export class FrameRenderer {
// Composite with shadows to final output canvas
this.compositeWithShadows();
// Draw device frame overlay on top of video content
this.drawFrame();
// Render annotations on top if present
if (
this.config.annotationRegions &&
@@ -956,6 +1025,140 @@ export class FrameRenderer {
timeMs,
);
}
// Extension render hooks — run after all built-in rendering
if (this.compositeCtx) {
const maskRect = this.layoutCache?.maskRect;
const hookParams = {
width: this.config.width,
height: this.config.height,
timeMs,
durationMs: 0,
cursor: smoothedCursor
? {
cx: smoothedCursor.cx,
cy: smoothedCursor.cy,
interactionType: this.getCursorPosition(cursorTimeMs)?.interactionType,
}
: this.getCursorPosition(cursorTimeMs),
smoothedCursor,
videoLayout: maskRect ? {
maskRect: { x: maskRect.x, y: maskRect.y, width: maskRect.width, height: maskRect.height },
borderRadius: this.config.borderRadius ?? 0,
padding: this.config.padding ?? 0,
} : undefined,
zoom: {
scale: this.animationState.scale,
focusX: this.animationState.focusX,
focusY: this.animationState.focusY,
progress: this.animationState.progress,
},
shadow: {
enabled: this.config.showShadow,
intensity: this.config.shadowIntensity,
},
sceneTransform: {
scale: this.animationState.appliedScale,
x: this.animationState.x,
y: this.animationState.y,
},
};
this.compositeCtx.save();
applyCanvasSceneTransform(this.compositeCtx, {
scale: this.animationState.appliedScale,
x: this.animationState.x,
y: this.animationState.y,
});
executeExtensionRenderHooks('post-video', this.compositeCtx, hookParams);
executeExtensionRenderHooks('post-zoom', this.compositeCtx, hookParams);
executeExtensionRenderHooks('post-cursor', this.compositeCtx, hookParams);
// Cursor click effects
this.emitCursorInteractions(cursorTimeMs);
executeExtensionCursorEffects(
this.compositeCtx,
timeMs,
this.config.width,
this.config.height,
{
zoom: hookParams.zoom,
sceneTransform: hookParams.sceneTransform,
videoLayout: hookParams.videoLayout,
},
);
this.compositeCtx.restore();
executeExtensionRenderHooks('post-webcam', this.compositeCtx, hookParams);
executeExtensionRenderHooks('post-annotations', this.compositeCtx, hookParams);
executeExtensionRenderHooks('final', this.compositeCtx, hookParams);
}
}
/**
* Get the cursor position (normalized 0-1) at the given time.
*/
private getCursorPosition(timeMs: number): { cx: number; cy: number; interactionType?: string } | null {
const telemetry = this.config.cursorTelemetry;
if (!telemetry || telemetry.length === 0) return null;
// Find the closest telemetry point
let closest = telemetry[0];
let minDist = Math.abs(telemetry[0].timeMs - timeMs);
for (let i = 1; i < telemetry.length; i++) {
const dist = Math.abs(telemetry[i].timeMs - timeMs);
if (dist < minDist) {
minDist = dist;
closest = telemetry[i];
}
if (telemetry[i].timeMs > timeMs) break;
}
return mapCursorToCanvasNormalized(
{ cx: closest.cx, cy: closest.cy, interactionType: closest.interactionType },
{
maskRect: this.layoutCache?.maskRect,
canvasWidth: this.config.width,
canvasHeight: this.config.height,
},
);
}
/**
* Emit cursor interaction events for extensions based on telemetry clicks.
*/
private lastEmittedClickTimeMs = -1;
private emitCursorInteractions(timeMs: number): void {
const telemetry = this.config.cursorTelemetry;
if (!telemetry || telemetry.length === 0) return;
// Find click events near this time
for (const point of telemetry) {
if (point.timeMs > timeMs) break;
if (point.timeMs < timeMs - 100) continue;
if (!point.interactionType || point.interactionType === 'move') continue;
if (point.timeMs === this.lastEmittedClickTimeMs) continue;
const mappedCursor = mapCursorToCanvasNormalized(
{ cx: point.cx, cy: point.cy, interactionType: point.interactionType },
{
maskRect: this.layoutCache?.maskRect,
canvasWidth: this.config.width,
canvasHeight: this.config.height,
},
);
if (!mappedCursor) continue;
this.lastEmittedClickTimeMs = point.timeMs;
notifyCursorInteraction(
point.timeMs,
mappedCursor.cx,
mappedCursor.cy,
point.interactionType,
);
}
}
private updateLayout(): void {
@@ -984,9 +1187,17 @@ export class FrameRenderer {
const paddingScale = 1.0 - (padding / 100) * 0.4;
const viewportWidth = width * paddingScale;
const viewportHeight = height * paddingScale;
// When a device frame is active, scale to fit the ENTIRE frame (video + bezels)
const insets = this.frameInsets;
const screenFracW = insets ? (1 - insets.left - insets.right) : 1;
const screenFracH = insets ? (1 - insets.top - insets.bottom) : 1;
const fullFrameVideoW = croppedVideoWidth / screenFracW;
const fullFrameVideoH = croppedVideoHeight / screenFracH;
const scale = Math.min(
viewportWidth / croppedVideoWidth,
viewportHeight / croppedVideoHeight,
viewportWidth / fullFrameVideoW,
viewportHeight / fullFrameVideoH,
);
this.videoSprite.scale.set(scale);
@@ -995,8 +1206,18 @@ export class FrameRenderer {
const fullVideoDisplayHeight = videoHeight * scale;
const croppedDisplayWidth = croppedVideoWidth * scale;
const croppedDisplayHeight = croppedVideoHeight * scale;
const centerOffsetX = (width - croppedDisplayWidth) / 2;
const centerOffsetY = (height - croppedDisplayHeight) / 2;
// Center the full frame (video + bezels) in the output canvas
const fullFrameDisplayW = fullFrameVideoW * scale;
const fullFrameDisplayH = fullFrameVideoH * scale;
const frameCenterX = (width - fullFrameDisplayW) / 2;
const frameCenterY = (height - fullFrameDisplayH) / 2;
const centerOffsetX = insets
? frameCenterX + insets.left * fullFrameDisplayW
: (width - croppedDisplayWidth) / 2;
const centerOffsetY = insets
? frameCenterY + insets.top * fullFrameDisplayH
: (height - croppedDisplayHeight) / 2;
const spriteX = centerOffsetX - cropRegion.x * fullVideoDisplayWidth;
const spriteY = centerOffsetY - cropRegion.y * fullVideoDisplayHeight;
@@ -1236,6 +1457,55 @@ export class FrameRenderer {
this.drawWebcamOverlay(ctx, w, h);
}
private drawFrame(): void {
if ((!this.frameImage && !this.frameDraw) || !this.compositeCtx || !this.layoutCache) return;
const ctx = this.compositeCtx;
const maskRect = this.layoutCache.maskRect;
const insets = this.frameInsets;
if (!insets) {
// No insets: draw frame spanning entire mask area
if (this.frameDraw) {
const c = document.createElement('canvas');
c.width = Math.round(maskRect.width);
c.height = Math.round(maskRect.height);
const dCtx = c.getContext('2d');
if (dCtx) this.frameDraw(dCtx, c.width, c.height);
ctx.drawImage(c, maskRect.x, maskRect.y, maskRect.width, maskRect.height);
} else {
ctx.drawImage(
this.frameImage!,
maskRect.x,
maskRect.y,
maskRect.width,
maskRect.height,
);
}
return;
}
// Calculate frame dimensions from insets
const screenW = maskRect.width;
const screenH = maskRect.height;
const frameW = screenW / (1 - insets.left - insets.right);
const frameH = screenH / (1 - insets.top - insets.bottom);
const frameX = maskRect.x - insets.left * frameW;
const frameY = maskRect.y - insets.top * frameH;
if (this.frameDraw) {
// Draw at the exact export resolution — no bitmap scaling
const c = document.createElement('canvas');
c.width = Math.round(frameW);
c.height = Math.round(frameH);
const dCtx = c.getContext('2d');
if (dCtx) this.frameDraw(dCtx, c.width, c.height);
ctx.drawImage(c, frameX, frameY, frameW, frameH);
} else {
ctx.drawImage(this.frameImage!, frameX, frameY, frameW, frameH);
}
}
private drawWebcamOverlay(
ctx: CanvasRenderingContext2D,
width: number,
@@ -1469,5 +1739,8 @@ export class FrameRenderer {
this.webcamBubbleCanvas = null;
this.webcamBubbleCtx = null;
this.lastSyncedWebcamTime = null;
this.frameImage = null;
this.frameInsets = null;
this.frameDraw = null;
}
}
+2
View File
@@ -73,6 +73,7 @@ interface GifExporterConfig {
cursorClickBounce?: number;
cursorClickBounceDuration?: number;
cursorSway?: number;
frame?: string | null;
previewWidth?: number;
previewHeight?: number;
maxDecodeQueue?: number;
@@ -190,6 +191,7 @@ export class GifExporter {
cursorClickBounce: this.config.cursorClickBounce,
cursorClickBounceDuration: this.config.cursorClickBounceDuration,
cursorSway: this.config.cursorSway,
frame: this.config.frame,
});
await this.renderer.initialize();
+218 -1
View File
@@ -57,8 +57,19 @@ import {
getWebcamOverlaySizePx,
} from "@/components/video-editor/webcamOverlay";
import { getAssetPath, getRenderableAssetUrl } from "@/lib/assetPath";
import { extensionHost } from "@/lib/extensions";
import {
mapCursorToCanvasNormalized,
mapSmoothedCursorToCanvasNormalized,
} from "@/lib/extensions/cursorCoordinates";
import { applyCanvasSceneTransform } from "@/lib/extensions/sceneTransform";
import { drawSquircleOnCanvas, drawSquircleOnGraphics } from "@/lib/geometry/squircle";
import { clampMediaTimeToDuration } from "@/lib/mediaTiming";
import {
executeExtensionCursorEffects,
executeExtensionRenderHooks,
notifyCursorInteraction,
} from "@/lib/extensions/renderHooks";
import { isVideoWallpaperSource } from "@/lib/wallpapers";
import {
type AnnotationRenderAssets,
@@ -119,6 +130,7 @@ interface FrameRenderConfig {
cursorSway?: number;
zoomSmoothness?: number;
zoomClassicMode?: boolean;
frame?: string | null;
}
interface AnimationState {
@@ -362,6 +374,9 @@ export class FrameRenderer {
private webcamTextureUsesStartupStaging = false;
private nativePixelReadbackWarningShown = false;
private nativeReadbackBuffer: Uint8Array | null = null;
private compositeCanvas: HTMLCanvasElement | null = null;
private compositeCtx: CanvasRenderingContext2D | null = null;
private lastEmittedClickTimeMs = -1;
private cleanupWebcamSource: (() => void) | null = null;
constructor(config: FrameRenderConfig) {
@@ -483,6 +498,18 @@ export class FrameRenderer {
await this.setupAnnotationLayer();
this.setupCaptionResources();
this.compositeCanvas = document.createElement("canvas");
this.compositeCanvas.width = this.config.width;
this.compositeCanvas.height = this.config.height;
this.compositeCtx = configureHighQuality2DContext(
this.compositeCanvas.getContext("2d", {
willReadFrequently: false,
}),
);
if (!this.compositeCtx) {
throw new Error("Failed to get 2D context for composite canvas");
}
if (this.shouldUseZoomMotionBlur()) {
this.motionBlurFilter = new MotionBlurFilter([0, 0], 5, 0);
}
@@ -2165,6 +2192,192 @@ export class FrameRenderer {
this.outputCanvasOverride = null;
this.app.render();
this.compositeExtensions(timeMs, cursorTimeMs);
}
private shouldCompositeExtensionFrame(): boolean {
return (
extensionHost.hasCursorEffects() ||
extensionHost.hasRenderHooks("post-zoom") ||
extensionHost.hasRenderHooks("post-cursor") ||
extensionHost.hasRenderHooks("post-annotations") ||
extensionHost.hasRenderHooks("final")
);
}
private compositeExtensions(timeMs: number, cursorTimeMs: number): void {
if (!this.app || !this.compositeCtx || !this.compositeCanvas) {
return;
}
if (!this.shouldCompositeExtensionFrame()) {
return;
}
this.compositeCtx.clearRect(0, 0, this.config.width, this.config.height);
this.compositeCtx.drawImage(this.app.canvas as HTMLCanvasElement, 0, 0);
const maskRect = this.layoutCache?.maskRect;
const smoothedCursor = mapSmoothedCursorToCanvasNormalized(
this.cursorOverlay?.getSmoothedCursorSnapshot() ?? null,
{
maskRect,
canvasWidth: this.config.width,
canvasHeight: this.config.height,
},
);
extensionHost.setSmoothedCursor(
smoothedCursor
? {
timeMs,
cx: smoothedCursor.cx,
cy: smoothedCursor.cy,
trail: smoothedCursor.trail,
}
: null,
);
const rawCursor = this.getCursorPosition(cursorTimeMs);
const hookParams = {
width: this.config.width,
height: this.config.height,
timeMs,
durationMs: 0,
cursor: smoothedCursor
? {
cx: smoothedCursor.cx,
cy: smoothedCursor.cy,
interactionType: rawCursor?.interactionType,
}
: rawCursor,
smoothedCursor,
videoLayout: maskRect
? {
maskRect: {
x: maskRect.x,
y: maskRect.y,
width: maskRect.width,
height: maskRect.height,
},
borderRadius: this.config.borderRadius ?? 0,
padding: this.config.padding ?? 0,
}
: undefined,
zoom: {
scale: this.animationState.scale,
focusX: this.animationState.focusX,
focusY: this.animationState.focusY,
progress: this.animationState.progress,
},
shadow: {
enabled: this.config.showShadow,
intensity: this.config.shadowIntensity,
},
sceneTransform: {
scale: this.animationState.appliedScale,
x: this.animationState.x,
y: this.animationState.y,
},
};
this.compositeCtx.save();
applyCanvasSceneTransform(this.compositeCtx, {
scale: this.animationState.appliedScale,
x: this.animationState.x,
y: this.animationState.y,
});
executeExtensionRenderHooks("post-video", this.compositeCtx, hookParams);
executeExtensionRenderHooks("post-zoom", this.compositeCtx, hookParams);
executeExtensionRenderHooks("post-cursor", this.compositeCtx, hookParams);
this.emitCursorInteractions(cursorTimeMs);
executeExtensionCursorEffects(
this.compositeCtx,
timeMs,
this.config.width,
this.config.height,
{
zoom: hookParams.zoom,
sceneTransform: hookParams.sceneTransform,
videoLayout: hookParams.videoLayout,
},
);
this.compositeCtx.restore();
executeExtensionRenderHooks("post-webcam", this.compositeCtx, hookParams);
executeExtensionRenderHooks("post-annotations", this.compositeCtx, hookParams);
executeExtensionRenderHooks("final", this.compositeCtx, hookParams);
}
private getCursorPosition(
timeMs: number,
): { cx: number; cy: number; interactionType?: string } | null {
const telemetry = this.config.cursorTelemetry;
if (!telemetry || telemetry.length === 0) {
return null;
}
let closest = telemetry[0];
let minDist = Math.abs(telemetry[0].timeMs - timeMs);
for (let i = 1; i < telemetry.length; i++) {
const dist = Math.abs(telemetry[i].timeMs - timeMs);
if (dist < minDist) {
minDist = dist;
closest = telemetry[i];
}
if (telemetry[i].timeMs > timeMs) {
break;
}
}
return mapCursorToCanvasNormalized(
{ cx: closest.cx, cy: closest.cy, interactionType: closest.interactionType },
{
maskRect: this.layoutCache?.maskRect,
canvasWidth: this.config.width,
canvasHeight: this.config.height,
},
);
}
private emitCursorInteractions(timeMs: number): void {
const telemetry = this.config.cursorTelemetry;
if (!telemetry || telemetry.length === 0) {
return;
}
for (const point of telemetry) {
if (point.timeMs > timeMs) {
break;
}
if (point.timeMs < timeMs - 100) {
continue;
}
if (!point.interactionType || point.interactionType === "move") {
continue;
}
if (point.timeMs === this.lastEmittedClickTimeMs) {
continue;
}
const mappedCursor = mapCursorToCanvasNormalized(
{ cx: point.cx, cy: point.cy, interactionType: point.interactionType },
{
maskRect: this.layoutCache?.maskRect,
canvasWidth: this.config.width,
canvasHeight: this.config.height,
},
);
if (!mappedCursor) {
continue;
}
this.lastEmittedClickTimeMs = point.timeMs;
notifyCursorInteraction(
point.timeMs,
mappedCursor.cx,
mappedCursor.cy,
point.interactionType,
);
}
}
private updateLayout(): void {
@@ -2495,7 +2708,7 @@ export class FrameRenderer {
throw new Error("Renderer not initialized");
}
if (this.outputCanvasOverride) {
if (this.outputCanvasOverride || this.shouldCompositeExtensionFrame()) {
return null;
}
@@ -2523,6 +2736,10 @@ export class FrameRenderer {
throw new Error("Renderer not initialized");
}
if (this.shouldCompositeExtensionFrame() && this.compositeCanvas) {
return this.compositeCanvas;
}
return this.outputCanvasOverride ?? (this.app.canvas as HTMLCanvasElement);
}
+14 -2
View File
@@ -25,6 +25,7 @@ import {
} from "./mp4Support";
import { VideoMuxer } from "./muxer";
import { captureCanvasFrameForNativeExport } from "./nativeFrameCapture";
import { extensionHost } from "@/lib/extensions";
import { type DecodedVideoInfo, StreamingVideoDecoder } from "./streamingDecoder";
import type {
ExportConfig,
@@ -81,6 +82,7 @@ interface VideoExporterConfig extends ExportConfig {
cursorSway?: number;
zoomSmoothness?: number;
zoomClassicMode?: boolean;
frame?: string | null;
audioRegions?: AudioRegion[];
sourceAudioFallbackPaths?: string[];
previewWidth?: number;
@@ -341,6 +343,7 @@ export class ModernVideoExporter {
cursorSway: this.config.cursorSway,
zoomSmoothness: this.config.zoomSmoothness,
zoomClassicMode: this.config.zoomClassicMode,
frame: this.config.frame,
});
await this.renderer.initialize();
this.rendererInitTimeMs = this.getNowMs() - stageStartedAt;
@@ -400,6 +403,7 @@ export class ModernVideoExporter {
frameIndex++;
this.processedFrameCount = frameIndex;
this.reportProgress(frameIndex, totalFrames, "extracting");
extensionHost.emitEvent({ type: 'export:frame', data: { frameIndex, totalFrames } });
},
);
this.decodeLoopTimeMs = this.getNowMs() - decodeLoopStartedAt;
@@ -455,6 +459,9 @@ export class ModernVideoExporter {
(this.config.sourceAudioFallbackPaths ?? []).length > 0
) {
this.audioProcessor = new AudioProcessor();
this.audioProcessor.setOnProgress((progress) => {
this.reportFinalizingProgress(totalFrames, 99, progress);
});
this.reportFinalizingProgress(totalFrames, 99);
await this.awaitWithFinalizationTimeout(
this.audioProcessor.process(
@@ -857,6 +864,9 @@ export class ModernVideoExporter {
if (audioPlan.audioMode === "edited-track") {
this.audioProcessor = new AudioProcessor();
this.audioProcessor.setOnProgress((progress) => {
this.reportFinalizingProgress(this.processedFrameCount, 99, progress);
});
const audioBlob = await this.awaitWithFinalizationTimeout(
this.audioProcessor.renderEditedAudioTrack(
this.config.videoUrl,
@@ -968,8 +978,8 @@ export class ModernVideoExporter {
}
}
private reportFinalizingProgress(totalFrames: number, renderProgress: number) {
this.reportProgress(totalFrames, totalFrames, "finalizing", renderProgress);
private reportFinalizingProgress(totalFrames: number, renderProgress: number, audioProgress?: number) {
this.reportProgress(totalFrames, totalFrames, "finalizing", renderProgress, audioProgress);
}
private reportProgress(
@@ -977,6 +987,7 @@ export class ModernVideoExporter {
totalFrames: number,
phase: ExportProgress["phase"] = "extracting",
renderProgress?: number,
audioProgress?: number,
) {
const nowMs = this.getNowMs();
const elapsedSeconds = Math.max((nowMs - this.exportStartTimeMs) / 1000, 0.001);
@@ -1047,6 +1058,7 @@ export class ModernVideoExporter {
encoderName: this.encoderName ?? undefined,
phase,
renderProgress: safeRenderProgress,
audioProgress,
});
}
}
+2
View File
@@ -62,6 +62,7 @@ interface VideoExporterConfig extends ExportConfig {
cursorClickBounceDuration?: number;
cursorSway?: number;
zoomSmoothness?: number;
frame?: string | null;
audioRegions?: AudioRegion[];
sourceAudioFallbackPaths?: string[];
previewWidth?: number;
@@ -177,6 +178,7 @@ export class VideoExporter {
cursorClickBounceDuration: this.config.cursorClickBounceDuration,
cursorSway: this.config.cursorSway,
zoomSmoothness: this.config.zoomSmoothness,
frame: this.config.frame,
});
await this.renderer.initialize();