mirror of
https://github.com/webadderallorg/Recordly.git
synced 2026-09-27 16:25:35 +00:00
feat(export): add extension render hooks to export pipeline
- Integrate extension render hooks into frameRenderer and modernFrameRenderer - Add modernFrameRenderer for compositing extensions during export - Wire extension audio processing in audioEncoder - Support extension hooks in gif and video export paths
This commit is contained in:
@@ -562,7 +562,7 @@ export class AudioProcessor {
|
||||
}
|
||||
|
||||
const { recorder, recordedBlobPromise } = this.startAudioRecording(destinationNode.stream)
|
||||
let rafId: number | null = null
|
||||
let tickTimerId: ReturnType<typeof setTimeout> | null = null
|
||||
|
||||
try {
|
||||
if (audioContext.state === 'suspended') {
|
||||
@@ -577,9 +577,9 @@ export class AudioProcessor {
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const cleanup = () => {
|
||||
if (rafId !== null) {
|
||||
cancelAnimationFrame(rafId)
|
||||
rafId = null
|
||||
if (tickTimerId !== null) {
|
||||
clearTimeout(tickTimerId)
|
||||
tickTimerId = null
|
||||
}
|
||||
timelineMedia.removeEventListener('error', onError)
|
||||
timelineMedia.removeEventListener('ended', onEnded)
|
||||
@@ -701,7 +701,7 @@ export class AudioProcessor {
|
||||
}
|
||||
|
||||
if (!timelineMedia.paused && !timelineMedia.ended) {
|
||||
rafId = requestAnimationFrame(tick)
|
||||
tickTimerId = setTimeout(tick, 16)
|
||||
} else {
|
||||
cleanup()
|
||||
resolve()
|
||||
@@ -710,11 +710,11 @@ export class AudioProcessor {
|
||||
|
||||
timelineMedia.addEventListener('error', onError, { once: true })
|
||||
timelineMedia.addEventListener('ended', onEnded, { once: true })
|
||||
rafId = requestAnimationFrame(tick)
|
||||
tickTimerId = setTimeout(tick, 16)
|
||||
})
|
||||
} finally {
|
||||
if (rafId !== null) {
|
||||
cancelAnimationFrame(rafId)
|
||||
if (tickTimerId !== null) {
|
||||
clearTimeout(tickTimerId)
|
||||
}
|
||||
timelineMedia.pause()
|
||||
timelineAudioSourceNode?.disconnect()
|
||||
@@ -856,6 +856,8 @@ export class AudioProcessor {
|
||||
}
|
||||
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
let timeoutId: ReturnType<typeof setTimeout> | null = null
|
||||
|
||||
const onLoaded = () => {
|
||||
cleanup()
|
||||
resolve()
|
||||
@@ -864,11 +866,20 @@ export class AudioProcessor {
|
||||
cleanup()
|
||||
reject(new Error('Failed to load media metadata for speed-adjusted audio'))
|
||||
}
|
||||
const onTimeout = () => {
|
||||
cleanup()
|
||||
reject(new Error('Timed out waiting for media metadata (30s)'))
|
||||
}
|
||||
const cleanup = () => {
|
||||
if (timeoutId !== null) {
|
||||
clearTimeout(timeoutId)
|
||||
timeoutId = null
|
||||
}
|
||||
media.removeEventListener('loadedmetadata', onLoaded)
|
||||
media.removeEventListener('error', onError)
|
||||
}
|
||||
|
||||
timeoutId = setTimeout(onTimeout, 30_000)
|
||||
media.addEventListener('loadedmetadata', onLoaded)
|
||||
media.addEventListener('error', onError, { once: true })
|
||||
})
|
||||
@@ -880,6 +891,8 @@ export class AudioProcessor {
|
||||
}
|
||||
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
let timeoutId: ReturnType<typeof setTimeout> | null = null
|
||||
|
||||
const onSeeked = () => {
|
||||
cleanup()
|
||||
resolve()
|
||||
@@ -888,11 +901,20 @@ export class AudioProcessor {
|
||||
cleanup()
|
||||
reject(new Error('Failed to seek media for speed-adjusted audio'))
|
||||
}
|
||||
const onTimeout = () => {
|
||||
cleanup()
|
||||
reject(new Error('Timed out waiting for media seek (30s)'))
|
||||
}
|
||||
const cleanup = () => {
|
||||
if (timeoutId !== null) {
|
||||
clearTimeout(timeoutId)
|
||||
timeoutId = null
|
||||
}
|
||||
media.removeEventListener('seeked', onSeeked)
|
||||
media.removeEventListener('error', onError)
|
||||
}
|
||||
|
||||
timeoutId = setTimeout(onTimeout, 30_000)
|
||||
media.addEventListener('seeked', onSeeked, { once: true })
|
||||
media.addEventListener('error', onError, { once: true })
|
||||
media.currentTime = targetSec
|
||||
|
||||
@@ -23,6 +23,7 @@ import type {
|
||||
import { ZOOM_DEPTH_SCALES, BASE_PREVIEW_WIDTH, BASE_PREVIEW_HEIGHT } from "@/components/video-editor/types";
|
||||
|
||||
import { getAssetPath, getRenderableAssetUrl } from "@/lib/assetPath";
|
||||
import { extensionHost } from "@/lib/extensions";
|
||||
import { findDominantRegion } from "@/components/video-editor/videoPlayback/zoomRegionUtils";
|
||||
import {
|
||||
type CursorFollowCameraState,
|
||||
@@ -49,6 +50,16 @@ import {
|
||||
} from "@/components/video-editor/videoPlayback/motionSmoothing";
|
||||
import { renderAnnotations } from "./annotationRenderer";
|
||||
import { renderCaptions } from "./captionRenderer";
|
||||
import {
|
||||
executeExtensionRenderHooks,
|
||||
executeExtensionCursorEffects,
|
||||
notifyCursorInteraction,
|
||||
} from "@/lib/extensions/renderHooks";
|
||||
import {
|
||||
mapCursorToCanvasNormalized,
|
||||
mapSmoothedCursorToCanvasNormalized,
|
||||
} from "@/lib/extensions/cursorCoordinates";
|
||||
import { applyCanvasSceneTransform } from "@/lib/extensions/sceneTransform";
|
||||
import {
|
||||
PixiCursorOverlay,
|
||||
DEFAULT_CURSOR_CONFIG,
|
||||
@@ -104,6 +115,7 @@ interface FrameRenderConfig {
|
||||
cursorClickBounce?: number;
|
||||
cursorClickBounceDuration?: number;
|
||||
cursorSway?: number;
|
||||
frame?: string | null;
|
||||
}
|
||||
|
||||
interface AnimationState {
|
||||
@@ -183,6 +195,9 @@ export class FrameRenderer {
|
||||
private webcamBubbleCtx: CanvasRenderingContext2D | null = null;
|
||||
private lastSyncedWebcamTime: number | null = null;
|
||||
private cleanupWebcamSource: (() => void) | null = null;
|
||||
private frameImage: HTMLImageElement | null = null;
|
||||
private frameInsets: { top: number; right: number; bottom: number; left: number } | null = null;
|
||||
private frameDraw: ((ctx: CanvasRenderingContext2D, w: number, h: number) => void) | null = null;
|
||||
|
||||
constructor(config: FrameRenderConfig) {
|
||||
this.config = config;
|
||||
@@ -271,6 +286,7 @@ export class FrameRenderer {
|
||||
// Setup background (render separately, not in PixiJS)
|
||||
await this.setupBackground();
|
||||
await this.setupWebcamSource();
|
||||
await this.setupFrame();
|
||||
|
||||
// Setup blur filter for video container
|
||||
this.blurFilter = new BlurFilter();
|
||||
@@ -826,6 +842,37 @@ export class FrameRenderer {
|
||||
}
|
||||
}
|
||||
|
||||
private async setupFrame(): Promise<void> {
|
||||
const frameId = this.config.frame;
|
||||
if (!frameId) return;
|
||||
|
||||
const { extensionHost } = await import("@/lib/extensions/extensionHost");
|
||||
const frames = extensionHost.getFrames();
|
||||
const frame = frames.find((f) => f.id === frameId);
|
||||
if (!frame) {
|
||||
console.warn(`[FrameRenderer] Device frame "${frameId}" not found`);
|
||||
return;
|
||||
}
|
||||
|
||||
this.frameInsets = frame.screenInsets;
|
||||
|
||||
if (frame.draw) {
|
||||
// Prefer draw function — renders at export resolution, no bitmap scaling
|
||||
this.frameDraw = frame.draw;
|
||||
return;
|
||||
}
|
||||
|
||||
const img = new Image();
|
||||
img.crossOrigin = "anonymous";
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
img.onload = () => resolve();
|
||||
img.onerror = () => reject(new Error(`Failed to load device frame image: ${frameId}`));
|
||||
img.src = frame.filePath;
|
||||
});
|
||||
|
||||
this.frameImage = img;
|
||||
}
|
||||
|
||||
async renderFrame(
|
||||
videoFrame: VideoFrame,
|
||||
timestamp: number,
|
||||
@@ -881,6 +928,25 @@ export class FrameRenderer {
|
||||
);
|
||||
}
|
||||
|
||||
const smoothedCursor = mapSmoothedCursorToCanvasNormalized(
|
||||
this.cursorOverlay?.getSmoothedCursorSnapshot() ?? null,
|
||||
{
|
||||
maskRect: this.layoutCache.maskRect,
|
||||
canvasWidth: this.config.width,
|
||||
canvasHeight: this.config.height,
|
||||
},
|
||||
);
|
||||
extensionHost.setSmoothedCursor(
|
||||
smoothedCursor
|
||||
? {
|
||||
timeMs,
|
||||
cx: smoothedCursor.cx,
|
||||
cy: smoothedCursor.cy,
|
||||
trail: smoothedCursor.trail,
|
||||
}
|
||||
: null,
|
||||
);
|
||||
|
||||
const TICKS_PER_FRAME = 1;
|
||||
|
||||
let maxMotionIntensity = 0;
|
||||
@@ -919,6 +985,9 @@ export class FrameRenderer {
|
||||
// Composite with shadows to final output canvas
|
||||
this.compositeWithShadows();
|
||||
|
||||
// Draw device frame overlay on top of video content
|
||||
this.drawFrame();
|
||||
|
||||
// Render annotations on top if present
|
||||
if (
|
||||
this.config.annotationRegions &&
|
||||
@@ -956,6 +1025,140 @@ export class FrameRenderer {
|
||||
timeMs,
|
||||
);
|
||||
}
|
||||
|
||||
// Extension render hooks — run after all built-in rendering
|
||||
if (this.compositeCtx) {
|
||||
const maskRect = this.layoutCache?.maskRect;
|
||||
const hookParams = {
|
||||
width: this.config.width,
|
||||
height: this.config.height,
|
||||
timeMs,
|
||||
durationMs: 0,
|
||||
cursor: smoothedCursor
|
||||
? {
|
||||
cx: smoothedCursor.cx,
|
||||
cy: smoothedCursor.cy,
|
||||
interactionType: this.getCursorPosition(cursorTimeMs)?.interactionType,
|
||||
}
|
||||
: this.getCursorPosition(cursorTimeMs),
|
||||
smoothedCursor,
|
||||
videoLayout: maskRect ? {
|
||||
maskRect: { x: maskRect.x, y: maskRect.y, width: maskRect.width, height: maskRect.height },
|
||||
borderRadius: this.config.borderRadius ?? 0,
|
||||
padding: this.config.padding ?? 0,
|
||||
} : undefined,
|
||||
zoom: {
|
||||
scale: this.animationState.scale,
|
||||
focusX: this.animationState.focusX,
|
||||
focusY: this.animationState.focusY,
|
||||
progress: this.animationState.progress,
|
||||
},
|
||||
shadow: {
|
||||
enabled: this.config.showShadow,
|
||||
intensity: this.config.shadowIntensity,
|
||||
},
|
||||
sceneTransform: {
|
||||
scale: this.animationState.appliedScale,
|
||||
x: this.animationState.x,
|
||||
y: this.animationState.y,
|
||||
},
|
||||
};
|
||||
|
||||
this.compositeCtx.save();
|
||||
applyCanvasSceneTransform(this.compositeCtx, {
|
||||
scale: this.animationState.appliedScale,
|
||||
x: this.animationState.x,
|
||||
y: this.animationState.y,
|
||||
});
|
||||
executeExtensionRenderHooks('post-video', this.compositeCtx, hookParams);
|
||||
executeExtensionRenderHooks('post-zoom', this.compositeCtx, hookParams);
|
||||
executeExtensionRenderHooks('post-cursor', this.compositeCtx, hookParams);
|
||||
|
||||
// Cursor click effects
|
||||
this.emitCursorInteractions(cursorTimeMs);
|
||||
executeExtensionCursorEffects(
|
||||
this.compositeCtx,
|
||||
timeMs,
|
||||
this.config.width,
|
||||
this.config.height,
|
||||
{
|
||||
zoom: hookParams.zoom,
|
||||
sceneTransform: hookParams.sceneTransform,
|
||||
videoLayout: hookParams.videoLayout,
|
||||
},
|
||||
);
|
||||
this.compositeCtx.restore();
|
||||
|
||||
executeExtensionRenderHooks('post-webcam', this.compositeCtx, hookParams);
|
||||
executeExtensionRenderHooks('post-annotations', this.compositeCtx, hookParams);
|
||||
|
||||
executeExtensionRenderHooks('final', this.compositeCtx, hookParams);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the cursor position (normalized 0-1) at the given time.
|
||||
*/
|
||||
private getCursorPosition(timeMs: number): { cx: number; cy: number; interactionType?: string } | null {
|
||||
const telemetry = this.config.cursorTelemetry;
|
||||
if (!telemetry || telemetry.length === 0) return null;
|
||||
|
||||
// Find the closest telemetry point
|
||||
let closest = telemetry[0];
|
||||
let minDist = Math.abs(telemetry[0].timeMs - timeMs);
|
||||
for (let i = 1; i < telemetry.length; i++) {
|
||||
const dist = Math.abs(telemetry[i].timeMs - timeMs);
|
||||
if (dist < minDist) {
|
||||
minDist = dist;
|
||||
closest = telemetry[i];
|
||||
}
|
||||
if (telemetry[i].timeMs > timeMs) break;
|
||||
}
|
||||
|
||||
return mapCursorToCanvasNormalized(
|
||||
{ cx: closest.cx, cy: closest.cy, interactionType: closest.interactionType },
|
||||
{
|
||||
maskRect: this.layoutCache?.maskRect,
|
||||
canvasWidth: this.config.width,
|
||||
canvasHeight: this.config.height,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Emit cursor interaction events for extensions based on telemetry clicks.
|
||||
*/
|
||||
private lastEmittedClickTimeMs = -1;
|
||||
|
||||
private emitCursorInteractions(timeMs: number): void {
|
||||
const telemetry = this.config.cursorTelemetry;
|
||||
if (!telemetry || telemetry.length === 0) return;
|
||||
|
||||
// Find click events near this time
|
||||
for (const point of telemetry) {
|
||||
if (point.timeMs > timeMs) break;
|
||||
if (point.timeMs < timeMs - 100) continue;
|
||||
if (!point.interactionType || point.interactionType === 'move') continue;
|
||||
if (point.timeMs === this.lastEmittedClickTimeMs) continue;
|
||||
|
||||
const mappedCursor = mapCursorToCanvasNormalized(
|
||||
{ cx: point.cx, cy: point.cy, interactionType: point.interactionType },
|
||||
{
|
||||
maskRect: this.layoutCache?.maskRect,
|
||||
canvasWidth: this.config.width,
|
||||
canvasHeight: this.config.height,
|
||||
},
|
||||
);
|
||||
if (!mappedCursor) continue;
|
||||
|
||||
this.lastEmittedClickTimeMs = point.timeMs;
|
||||
notifyCursorInteraction(
|
||||
point.timeMs,
|
||||
mappedCursor.cx,
|
||||
mappedCursor.cy,
|
||||
point.interactionType,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private updateLayout(): void {
|
||||
@@ -984,9 +1187,17 @@ export class FrameRenderer {
|
||||
const paddingScale = 1.0 - (padding / 100) * 0.4;
|
||||
const viewportWidth = width * paddingScale;
|
||||
const viewportHeight = height * paddingScale;
|
||||
|
||||
// When a device frame is active, scale to fit the ENTIRE frame (video + bezels)
|
||||
const insets = this.frameInsets;
|
||||
const screenFracW = insets ? (1 - insets.left - insets.right) : 1;
|
||||
const screenFracH = insets ? (1 - insets.top - insets.bottom) : 1;
|
||||
const fullFrameVideoW = croppedVideoWidth / screenFracW;
|
||||
const fullFrameVideoH = croppedVideoHeight / screenFracH;
|
||||
|
||||
const scale = Math.min(
|
||||
viewportWidth / croppedVideoWidth,
|
||||
viewportHeight / croppedVideoHeight,
|
||||
viewportWidth / fullFrameVideoW,
|
||||
viewportHeight / fullFrameVideoH,
|
||||
);
|
||||
|
||||
this.videoSprite.scale.set(scale);
|
||||
@@ -995,8 +1206,18 @@ export class FrameRenderer {
|
||||
const fullVideoDisplayHeight = videoHeight * scale;
|
||||
const croppedDisplayWidth = croppedVideoWidth * scale;
|
||||
const croppedDisplayHeight = croppedVideoHeight * scale;
|
||||
const centerOffsetX = (width - croppedDisplayWidth) / 2;
|
||||
const centerOffsetY = (height - croppedDisplayHeight) / 2;
|
||||
|
||||
// Center the full frame (video + bezels) in the output canvas
|
||||
const fullFrameDisplayW = fullFrameVideoW * scale;
|
||||
const fullFrameDisplayH = fullFrameVideoH * scale;
|
||||
const frameCenterX = (width - fullFrameDisplayW) / 2;
|
||||
const frameCenterY = (height - fullFrameDisplayH) / 2;
|
||||
const centerOffsetX = insets
|
||||
? frameCenterX + insets.left * fullFrameDisplayW
|
||||
: (width - croppedDisplayWidth) / 2;
|
||||
const centerOffsetY = insets
|
||||
? frameCenterY + insets.top * fullFrameDisplayH
|
||||
: (height - croppedDisplayHeight) / 2;
|
||||
|
||||
const spriteX = centerOffsetX - cropRegion.x * fullVideoDisplayWidth;
|
||||
const spriteY = centerOffsetY - cropRegion.y * fullVideoDisplayHeight;
|
||||
@@ -1236,6 +1457,55 @@ export class FrameRenderer {
|
||||
this.drawWebcamOverlay(ctx, w, h);
|
||||
}
|
||||
|
||||
private drawFrame(): void {
|
||||
if ((!this.frameImage && !this.frameDraw) || !this.compositeCtx || !this.layoutCache) return;
|
||||
|
||||
const ctx = this.compositeCtx;
|
||||
const maskRect = this.layoutCache.maskRect;
|
||||
const insets = this.frameInsets;
|
||||
|
||||
if (!insets) {
|
||||
// No insets: draw frame spanning entire mask area
|
||||
if (this.frameDraw) {
|
||||
const c = document.createElement('canvas');
|
||||
c.width = Math.round(maskRect.width);
|
||||
c.height = Math.round(maskRect.height);
|
||||
const dCtx = c.getContext('2d');
|
||||
if (dCtx) this.frameDraw(dCtx, c.width, c.height);
|
||||
ctx.drawImage(c, maskRect.x, maskRect.y, maskRect.width, maskRect.height);
|
||||
} else {
|
||||
ctx.drawImage(
|
||||
this.frameImage!,
|
||||
maskRect.x,
|
||||
maskRect.y,
|
||||
maskRect.width,
|
||||
maskRect.height,
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Calculate frame dimensions from insets
|
||||
const screenW = maskRect.width;
|
||||
const screenH = maskRect.height;
|
||||
const frameW = screenW / (1 - insets.left - insets.right);
|
||||
const frameH = screenH / (1 - insets.top - insets.bottom);
|
||||
const frameX = maskRect.x - insets.left * frameW;
|
||||
const frameY = maskRect.y - insets.top * frameH;
|
||||
|
||||
if (this.frameDraw) {
|
||||
// Draw at the exact export resolution — no bitmap scaling
|
||||
const c = document.createElement('canvas');
|
||||
c.width = Math.round(frameW);
|
||||
c.height = Math.round(frameH);
|
||||
const dCtx = c.getContext('2d');
|
||||
if (dCtx) this.frameDraw(dCtx, c.width, c.height);
|
||||
ctx.drawImage(c, frameX, frameY, frameW, frameH);
|
||||
} else {
|
||||
ctx.drawImage(this.frameImage!, frameX, frameY, frameW, frameH);
|
||||
}
|
||||
}
|
||||
|
||||
private drawWebcamOverlay(
|
||||
ctx: CanvasRenderingContext2D,
|
||||
width: number,
|
||||
@@ -1469,5 +1739,8 @@ export class FrameRenderer {
|
||||
this.webcamBubbleCanvas = null;
|
||||
this.webcamBubbleCtx = null;
|
||||
this.lastSyncedWebcamTime = null;
|
||||
this.frameImage = null;
|
||||
this.frameInsets = null;
|
||||
this.frameDraw = null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,6 +73,7 @@ interface GifExporterConfig {
|
||||
cursorClickBounce?: number;
|
||||
cursorClickBounceDuration?: number;
|
||||
cursorSway?: number;
|
||||
frame?: string | null;
|
||||
previewWidth?: number;
|
||||
previewHeight?: number;
|
||||
maxDecodeQueue?: number;
|
||||
@@ -190,6 +191,7 @@ export class GifExporter {
|
||||
cursorClickBounce: this.config.cursorClickBounce,
|
||||
cursorClickBounceDuration: this.config.cursorClickBounceDuration,
|
||||
cursorSway: this.config.cursorSway,
|
||||
frame: this.config.frame,
|
||||
});
|
||||
await this.renderer.initialize();
|
||||
|
||||
|
||||
@@ -57,8 +57,19 @@ import {
|
||||
getWebcamOverlaySizePx,
|
||||
} from "@/components/video-editor/webcamOverlay";
|
||||
import { getAssetPath, getRenderableAssetUrl } from "@/lib/assetPath";
|
||||
import { extensionHost } from "@/lib/extensions";
|
||||
import {
|
||||
mapCursorToCanvasNormalized,
|
||||
mapSmoothedCursorToCanvasNormalized,
|
||||
} from "@/lib/extensions/cursorCoordinates";
|
||||
import { applyCanvasSceneTransform } from "@/lib/extensions/sceneTransform";
|
||||
import { drawSquircleOnCanvas, drawSquircleOnGraphics } from "@/lib/geometry/squircle";
|
||||
import { clampMediaTimeToDuration } from "@/lib/mediaTiming";
|
||||
import {
|
||||
executeExtensionCursorEffects,
|
||||
executeExtensionRenderHooks,
|
||||
notifyCursorInteraction,
|
||||
} from "@/lib/extensions/renderHooks";
|
||||
import { isVideoWallpaperSource } from "@/lib/wallpapers";
|
||||
import {
|
||||
type AnnotationRenderAssets,
|
||||
@@ -119,6 +130,7 @@ interface FrameRenderConfig {
|
||||
cursorSway?: number;
|
||||
zoomSmoothness?: number;
|
||||
zoomClassicMode?: boolean;
|
||||
frame?: string | null;
|
||||
}
|
||||
|
||||
interface AnimationState {
|
||||
@@ -362,6 +374,9 @@ export class FrameRenderer {
|
||||
private webcamTextureUsesStartupStaging = false;
|
||||
private nativePixelReadbackWarningShown = false;
|
||||
private nativeReadbackBuffer: Uint8Array | null = null;
|
||||
private compositeCanvas: HTMLCanvasElement | null = null;
|
||||
private compositeCtx: CanvasRenderingContext2D | null = null;
|
||||
private lastEmittedClickTimeMs = -1;
|
||||
private cleanupWebcamSource: (() => void) | null = null;
|
||||
|
||||
constructor(config: FrameRenderConfig) {
|
||||
@@ -483,6 +498,18 @@ export class FrameRenderer {
|
||||
await this.setupAnnotationLayer();
|
||||
this.setupCaptionResources();
|
||||
|
||||
this.compositeCanvas = document.createElement("canvas");
|
||||
this.compositeCanvas.width = this.config.width;
|
||||
this.compositeCanvas.height = this.config.height;
|
||||
this.compositeCtx = configureHighQuality2DContext(
|
||||
this.compositeCanvas.getContext("2d", {
|
||||
willReadFrequently: false,
|
||||
}),
|
||||
);
|
||||
if (!this.compositeCtx) {
|
||||
throw new Error("Failed to get 2D context for composite canvas");
|
||||
}
|
||||
|
||||
if (this.shouldUseZoomMotionBlur()) {
|
||||
this.motionBlurFilter = new MotionBlurFilter([0, 0], 5, 0);
|
||||
}
|
||||
@@ -2165,6 +2192,192 @@ export class FrameRenderer {
|
||||
|
||||
this.outputCanvasOverride = null;
|
||||
this.app.render();
|
||||
this.compositeExtensions(timeMs, cursorTimeMs);
|
||||
}
|
||||
|
||||
private shouldCompositeExtensionFrame(): boolean {
|
||||
return (
|
||||
extensionHost.hasCursorEffects() ||
|
||||
extensionHost.hasRenderHooks("post-zoom") ||
|
||||
extensionHost.hasRenderHooks("post-cursor") ||
|
||||
extensionHost.hasRenderHooks("post-annotations") ||
|
||||
extensionHost.hasRenderHooks("final")
|
||||
);
|
||||
}
|
||||
|
||||
private compositeExtensions(timeMs: number, cursorTimeMs: number): void {
|
||||
if (!this.app || !this.compositeCtx || !this.compositeCanvas) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!this.shouldCompositeExtensionFrame()) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.compositeCtx.clearRect(0, 0, this.config.width, this.config.height);
|
||||
this.compositeCtx.drawImage(this.app.canvas as HTMLCanvasElement, 0, 0);
|
||||
|
||||
const maskRect = this.layoutCache?.maskRect;
|
||||
const smoothedCursor = mapSmoothedCursorToCanvasNormalized(
|
||||
this.cursorOverlay?.getSmoothedCursorSnapshot() ?? null,
|
||||
{
|
||||
maskRect,
|
||||
canvasWidth: this.config.width,
|
||||
canvasHeight: this.config.height,
|
||||
},
|
||||
);
|
||||
extensionHost.setSmoothedCursor(
|
||||
smoothedCursor
|
||||
? {
|
||||
timeMs,
|
||||
cx: smoothedCursor.cx,
|
||||
cy: smoothedCursor.cy,
|
||||
trail: smoothedCursor.trail,
|
||||
}
|
||||
: null,
|
||||
);
|
||||
const rawCursor = this.getCursorPosition(cursorTimeMs);
|
||||
const hookParams = {
|
||||
width: this.config.width,
|
||||
height: this.config.height,
|
||||
timeMs,
|
||||
durationMs: 0,
|
||||
cursor: smoothedCursor
|
||||
? {
|
||||
cx: smoothedCursor.cx,
|
||||
cy: smoothedCursor.cy,
|
||||
interactionType: rawCursor?.interactionType,
|
||||
}
|
||||
: rawCursor,
|
||||
smoothedCursor,
|
||||
videoLayout: maskRect
|
||||
? {
|
||||
maskRect: {
|
||||
x: maskRect.x,
|
||||
y: maskRect.y,
|
||||
width: maskRect.width,
|
||||
height: maskRect.height,
|
||||
},
|
||||
borderRadius: this.config.borderRadius ?? 0,
|
||||
padding: this.config.padding ?? 0,
|
||||
}
|
||||
: undefined,
|
||||
zoom: {
|
||||
scale: this.animationState.scale,
|
||||
focusX: this.animationState.focusX,
|
||||
focusY: this.animationState.focusY,
|
||||
progress: this.animationState.progress,
|
||||
},
|
||||
shadow: {
|
||||
enabled: this.config.showShadow,
|
||||
intensity: this.config.shadowIntensity,
|
||||
},
|
||||
sceneTransform: {
|
||||
scale: this.animationState.appliedScale,
|
||||
x: this.animationState.x,
|
||||
y: this.animationState.y,
|
||||
},
|
||||
};
|
||||
|
||||
this.compositeCtx.save();
|
||||
applyCanvasSceneTransform(this.compositeCtx, {
|
||||
scale: this.animationState.appliedScale,
|
||||
x: this.animationState.x,
|
||||
y: this.animationState.y,
|
||||
});
|
||||
executeExtensionRenderHooks("post-video", this.compositeCtx, hookParams);
|
||||
executeExtensionRenderHooks("post-zoom", this.compositeCtx, hookParams);
|
||||
executeExtensionRenderHooks("post-cursor", this.compositeCtx, hookParams);
|
||||
this.emitCursorInteractions(cursorTimeMs);
|
||||
executeExtensionCursorEffects(
|
||||
this.compositeCtx,
|
||||
timeMs,
|
||||
this.config.width,
|
||||
this.config.height,
|
||||
{
|
||||
zoom: hookParams.zoom,
|
||||
sceneTransform: hookParams.sceneTransform,
|
||||
videoLayout: hookParams.videoLayout,
|
||||
},
|
||||
);
|
||||
this.compositeCtx.restore();
|
||||
|
||||
executeExtensionRenderHooks("post-webcam", this.compositeCtx, hookParams);
|
||||
executeExtensionRenderHooks("post-annotations", this.compositeCtx, hookParams);
|
||||
executeExtensionRenderHooks("final", this.compositeCtx, hookParams);
|
||||
}
|
||||
|
||||
private getCursorPosition(
|
||||
timeMs: number,
|
||||
): { cx: number; cy: number; interactionType?: string } | null {
|
||||
const telemetry = this.config.cursorTelemetry;
|
||||
if (!telemetry || telemetry.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let closest = telemetry[0];
|
||||
let minDist = Math.abs(telemetry[0].timeMs - timeMs);
|
||||
for (let i = 1; i < telemetry.length; i++) {
|
||||
const dist = Math.abs(telemetry[i].timeMs - timeMs);
|
||||
if (dist < minDist) {
|
||||
minDist = dist;
|
||||
closest = telemetry[i];
|
||||
}
|
||||
if (telemetry[i].timeMs > timeMs) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return mapCursorToCanvasNormalized(
|
||||
{ cx: closest.cx, cy: closest.cy, interactionType: closest.interactionType },
|
||||
{
|
||||
maskRect: this.layoutCache?.maskRect,
|
||||
canvasWidth: this.config.width,
|
||||
canvasHeight: this.config.height,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private emitCursorInteractions(timeMs: number): void {
|
||||
const telemetry = this.config.cursorTelemetry;
|
||||
if (!telemetry || telemetry.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const point of telemetry) {
|
||||
if (point.timeMs > timeMs) {
|
||||
break;
|
||||
}
|
||||
if (point.timeMs < timeMs - 100) {
|
||||
continue;
|
||||
}
|
||||
if (!point.interactionType || point.interactionType === "move") {
|
||||
continue;
|
||||
}
|
||||
if (point.timeMs === this.lastEmittedClickTimeMs) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const mappedCursor = mapCursorToCanvasNormalized(
|
||||
{ cx: point.cx, cy: point.cy, interactionType: point.interactionType },
|
||||
{
|
||||
maskRect: this.layoutCache?.maskRect,
|
||||
canvasWidth: this.config.width,
|
||||
canvasHeight: this.config.height,
|
||||
},
|
||||
);
|
||||
if (!mappedCursor) {
|
||||
continue;
|
||||
}
|
||||
|
||||
this.lastEmittedClickTimeMs = point.timeMs;
|
||||
notifyCursorInteraction(
|
||||
point.timeMs,
|
||||
mappedCursor.cx,
|
||||
mappedCursor.cy,
|
||||
point.interactionType,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private updateLayout(): void {
|
||||
@@ -2495,7 +2708,7 @@ export class FrameRenderer {
|
||||
throw new Error("Renderer not initialized");
|
||||
}
|
||||
|
||||
if (this.outputCanvasOverride) {
|
||||
if (this.outputCanvasOverride || this.shouldCompositeExtensionFrame()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -2523,6 +2736,10 @@ export class FrameRenderer {
|
||||
throw new Error("Renderer not initialized");
|
||||
}
|
||||
|
||||
if (this.shouldCompositeExtensionFrame() && this.compositeCanvas) {
|
||||
return this.compositeCanvas;
|
||||
}
|
||||
|
||||
return this.outputCanvasOverride ?? (this.app.canvas as HTMLCanvasElement);
|
||||
}
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
} from "./mp4Support";
|
||||
import { VideoMuxer } from "./muxer";
|
||||
import { captureCanvasFrameForNativeExport } from "./nativeFrameCapture";
|
||||
import { extensionHost } from "@/lib/extensions";
|
||||
import { type DecodedVideoInfo, StreamingVideoDecoder } from "./streamingDecoder";
|
||||
import type {
|
||||
ExportConfig,
|
||||
@@ -81,6 +82,7 @@ interface VideoExporterConfig extends ExportConfig {
|
||||
cursorSway?: number;
|
||||
zoomSmoothness?: number;
|
||||
zoomClassicMode?: boolean;
|
||||
frame?: string | null;
|
||||
audioRegions?: AudioRegion[];
|
||||
sourceAudioFallbackPaths?: string[];
|
||||
previewWidth?: number;
|
||||
@@ -341,6 +343,7 @@ export class ModernVideoExporter {
|
||||
cursorSway: this.config.cursorSway,
|
||||
zoomSmoothness: this.config.zoomSmoothness,
|
||||
zoomClassicMode: this.config.zoomClassicMode,
|
||||
frame: this.config.frame,
|
||||
});
|
||||
await this.renderer.initialize();
|
||||
this.rendererInitTimeMs = this.getNowMs() - stageStartedAt;
|
||||
@@ -400,6 +403,7 @@ export class ModernVideoExporter {
|
||||
frameIndex++;
|
||||
this.processedFrameCount = frameIndex;
|
||||
this.reportProgress(frameIndex, totalFrames, "extracting");
|
||||
extensionHost.emitEvent({ type: 'export:frame', data: { frameIndex, totalFrames } });
|
||||
},
|
||||
);
|
||||
this.decodeLoopTimeMs = this.getNowMs() - decodeLoopStartedAt;
|
||||
@@ -455,6 +459,9 @@ export class ModernVideoExporter {
|
||||
(this.config.sourceAudioFallbackPaths ?? []).length > 0
|
||||
) {
|
||||
this.audioProcessor = new AudioProcessor();
|
||||
this.audioProcessor.setOnProgress((progress) => {
|
||||
this.reportFinalizingProgress(totalFrames, 99, progress);
|
||||
});
|
||||
this.reportFinalizingProgress(totalFrames, 99);
|
||||
await this.awaitWithFinalizationTimeout(
|
||||
this.audioProcessor.process(
|
||||
@@ -857,6 +864,9 @@ export class ModernVideoExporter {
|
||||
|
||||
if (audioPlan.audioMode === "edited-track") {
|
||||
this.audioProcessor = new AudioProcessor();
|
||||
this.audioProcessor.setOnProgress((progress) => {
|
||||
this.reportFinalizingProgress(this.processedFrameCount, 99, progress);
|
||||
});
|
||||
const audioBlob = await this.awaitWithFinalizationTimeout(
|
||||
this.audioProcessor.renderEditedAudioTrack(
|
||||
this.config.videoUrl,
|
||||
@@ -968,8 +978,8 @@ export class ModernVideoExporter {
|
||||
}
|
||||
}
|
||||
|
||||
private reportFinalizingProgress(totalFrames: number, renderProgress: number) {
|
||||
this.reportProgress(totalFrames, totalFrames, "finalizing", renderProgress);
|
||||
private reportFinalizingProgress(totalFrames: number, renderProgress: number, audioProgress?: number) {
|
||||
this.reportProgress(totalFrames, totalFrames, "finalizing", renderProgress, audioProgress);
|
||||
}
|
||||
|
||||
private reportProgress(
|
||||
@@ -977,6 +987,7 @@ export class ModernVideoExporter {
|
||||
totalFrames: number,
|
||||
phase: ExportProgress["phase"] = "extracting",
|
||||
renderProgress?: number,
|
||||
audioProgress?: number,
|
||||
) {
|
||||
const nowMs = this.getNowMs();
|
||||
const elapsedSeconds = Math.max((nowMs - this.exportStartTimeMs) / 1000, 0.001);
|
||||
@@ -1047,6 +1058,7 @@ export class ModernVideoExporter {
|
||||
encoderName: this.encoderName ?? undefined,
|
||||
phase,
|
||||
renderProgress: safeRenderProgress,
|
||||
audioProgress,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -62,6 +62,7 @@ interface VideoExporterConfig extends ExportConfig {
|
||||
cursorClickBounceDuration?: number;
|
||||
cursorSway?: number;
|
||||
zoomSmoothness?: number;
|
||||
frame?: string | null;
|
||||
audioRegions?: AudioRegion[];
|
||||
sourceAudioFallbackPaths?: string[];
|
||||
previewWidth?: number;
|
||||
@@ -177,6 +178,7 @@ export class VideoExporter {
|
||||
cursorClickBounceDuration: this.config.cursorClickBounceDuration,
|
||||
cursorSway: this.config.cursorSway,
|
||||
zoomSmoothness: this.config.zoomSmoothness,
|
||||
frame: this.config.frame,
|
||||
});
|
||||
await this.renderer.initialize();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user