From dce398ee0212c1070616f36e8736a6a2ad6d11af Mon Sep 17 00:00:00 2001 From: webadderall <131426131+webadderall@users.noreply.github.com> Date: Sat, 11 Apr 2026 21:40:37 +1000 Subject: [PATCH] fix(export): render blur annotations in modern pipeline --- src/lib/exporter/annotationRenderer.ts | 4 + src/lib/exporter/modernFrameRenderer.test.ts | 187 +++++++++++++++++++ src/lib/exporter/modernFrameRenderer.ts | 130 ++++++++++++- 3 files changed, 320 insertions(+), 1 deletion(-) create mode 100644 src/lib/exporter/modernFrameRenderer.test.ts diff --git a/src/lib/exporter/annotationRenderer.ts b/src/lib/exporter/annotationRenderer.ts index 8b540219..4407e146 100644 --- a/src/lib/exporter/annotationRenderer.ts +++ b/src/lib/exporter/annotationRenderer.ts @@ -461,6 +461,10 @@ export async function renderAnnotationToCanvas( scaleFactor, ); break; + case "blur": + // Blur annotations must sample already-rendered scene pixels, + // so they cannot be rasterized as standalone sprites. + return null; } return canvas; diff --git a/src/lib/exporter/modernFrameRenderer.test.ts b/src/lib/exporter/modernFrameRenderer.test.ts new file mode 100644 index 00000000..59211093 --- /dev/null +++ b/src/lib/exporter/modernFrameRenderer.test.ts @@ -0,0 +1,187 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { DEFAULT_WEBCAM_OVERLAY } from "@/components/video-editor/types"; + +vi.mock("pixi.js", () => ({ + Application: class {}, + BlurFilter: class {}, + Container: class { + visible = true; + addChild = vi.fn(); + addChildAt = vi.fn(); + removeChildren = vi.fn(); + }, + Graphics: class {}, + Sprite: class { + visible = true; + x = 0; + y = 0; + alpha = 1; + scale = { x: 1, y: 1, set: vi.fn() }; + anchor = { x: 0.5, y: 0.5, set: vi.fn() }; + position = { set: vi.fn() }; + texture: { destroy: ReturnType }; + + constructor(texture = { destroy: vi.fn() }) { + this.texture = texture; + } + }, + Texture: { + from: vi.fn(() => ({ source: { update: vi.fn() }, destroy: vi.fn() })), + }, +})); + +vi.mock("pixi-filters/motion-blur", () => ({ + MotionBlurFilter: class {}, +})); + +vi.mock("@/lib/assetPath", () => ({ + getAssetPath: vi.fn(async (value: string) => value), + getRenderableAssetUrl: vi.fn((value: string) => value), +})); + +vi.mock("@/components/video-editor/videoPlayback/zoomRegionUtils", () => ({ + findDominantRegion: vi.fn(() => ({ + region: null, + strength: 0, + blendedScale: 1, + transition: null, + })), +})); + +vi.mock("@/components/video-editor/videoPlayback/zoomTransform", () => ({ + applyZoomTransform: vi.fn(), + computeFocusFromTransform: vi.fn(() => ({ cx: 0.5, cy: 0.5 })), + computeZoomTransform: vi.fn(() => ({ scale: 1, x: 0, y: 0 })), + createMotionBlurState: vi.fn(() => ({})), +})); + +vi.mock("@/components/video-editor/videoPlayback/cursorRenderer", () => ({ + PixiCursorOverlay: class { + container = {}; + update = vi.fn(); + destroy = vi.fn(); + }, + DEFAULT_CURSOR_CONFIG: { + dotRadius: 28, + smoothingFactor: 0.18, + motionBlur: 0, + clickBounce: 1, + sway: 0, + }, + preloadCursorAssets: vi.fn(async () => undefined), +})); + +vi.mock("./forwardFrameSource", () => ({ + ForwardFrameSource: class {}, +})); + +vi.mock("./localMediaSource", () => ({ + resolveMediaElementSource: vi.fn(async () => null), +})); + +vi.mock("./annotationRenderer", () => ({ + preloadAnnotationAssets: vi.fn(async () => ({ imageCache: new Map() })), + renderAnnotationToCanvas: vi.fn(async () => null), + renderAnnotations: vi.fn(async () => undefined), +})); + +import { renderAnnotations } from "./annotationRenderer"; +import { FrameRenderer } from "./modernFrameRenderer"; + +function createMockContext() { + return { + clearRect: vi.fn(), + drawImage: vi.fn(), + save: vi.fn(), + restore: vi.fn(), + globalAlpha: 1, + imageSmoothingEnabled: true, + imageSmoothingQuality: "high", + } as unknown as CanvasRenderingContext2D; +} + +function createMockCanvas() { + const context = createMockContext(); + return { + width: 0, + height: 0, + getContext: vi.fn(() => context), + context, + }; +} + +function createRenderer() { + return new FrameRenderer({ + width: 1920, + height: 1080, + nativeReadbackMode: "pixels", + wallpaper: "#000000", + zoomRegions: [], + showShadow: false, + shadowIntensity: 0, + backgroundBlur: 0, + cropRegion: { x: 0, y: 0, width: 1, height: 1 }, + webcam: { + ...DEFAULT_WEBCAM_OVERLAY, + enabled: false, + }, + videoWidth: 1920, + videoHeight: 1080, + annotationRegions: [ + { + id: "blur-1", + startMs: 0, + endMs: 1000, + type: "blur", + content: "", + position: { x: 10, y: 10 }, + size: { width: 20, height: 20 }, + style: { + color: "#ffffff", + backgroundColor: "transparent", + fontSize: 24, + fontFamily: "Inter", + fontWeight: "normal", + fontStyle: "normal", + textDecoration: "none", + textAlign: "center", + borderRadius: 0, + }, + zIndex: 1, + blurIntensity: 20, + }, + ], + }); +} + +describe("ModernFrameRenderer blur export path", () => { + beforeEach(() => { + Object.assign(globalThis, { + window: globalThis, + document: { + createElement: vi.fn((tag: string) => { + if (tag !== "canvas") { + throw new Error(`Unexpected element requested in test: ${tag}`); + } + + return createMockCanvas(); + }), + }, + }); + }); + + it("uses a composited canvas and disables pixel readback when blur post-processing is active", async () => { + const renderer = createRenderer() as any; + const sourceCanvas = createMockCanvas(); + + renderer.app = { canvas: sourceCanvas }; + renderer.annotationScaleFactor = 1; + renderer.annotationAssets = { imageCache: new Map() }; + + await renderer.composeBlurAnnotationFrame(500); + + expect(renderAnnotations).toHaveBeenCalledTimes(1); + expect(renderer.getCanvas()).not.toBe(sourceCanvas); + expect(renderer.capturePixelsForNativeExport()).toBeNull(); + }); +}); \ No newline at end of file diff --git a/src/lib/exporter/modernFrameRenderer.ts b/src/lib/exporter/modernFrameRenderer.ts index b2b63117..f0897a5e 100644 --- a/src/lib/exporter/modernFrameRenderer.ts +++ b/src/lib/exporter/modernFrameRenderer.ts @@ -63,6 +63,7 @@ import { isVideoWallpaperSource } from "@/lib/wallpapers"; import { type AnnotationRenderAssets, preloadAnnotationAssets, + renderAnnotations, renderAnnotationToCanvas, } from "./annotationRenderer"; import { ForwardFrameSource } from "./forwardFrameSource"; @@ -184,6 +185,11 @@ interface AnnotationSpriteEntry { texture: Texture; } +interface ExportCompositeCanvasState { + canvas: HTMLCanvasElement; + context: CanvasRenderingContext2D; +} + type ResolvedCaptionLayout = NonNullable>; interface CaptionRenderState { @@ -334,6 +340,8 @@ export class FrameRenderer { private captionSprite: Sprite | null = null; private captionTextureSource: MutableVideoTextureSource | null = null; private captionRenderKey: string | null = null; + private exportCompositeCanvas: ExportCompositeCanvasState | null = null; + private outputCanvasOverride: HTMLCanvasElement | null = null; private config: FrameRenderConfig; private animationState: AnimationState; private motionBlurState: MotionBlurState; @@ -1038,6 +1046,94 @@ export class FrameRenderer { return (scaleX + scaleY) / 2; } + private hasActiveBlurAnnotations(timeMs: number): boolean { + return (this.config.annotationRegions ?? []).some( + (annotation) => + annotation.type === "blur" && + timeMs >= annotation.startMs && + timeMs <= annotation.endMs, + ); + } + + private ensureExportCompositeCanvas(): ExportCompositeCanvasState | null { + const targetWidth = Math.max(1, Math.ceil(this.config.width)); + const targetHeight = Math.max(1, Math.ceil(this.config.height)); + + if ( + this.exportCompositeCanvas && + this.exportCompositeCanvas.canvas.width === targetWidth && + this.exportCompositeCanvas.canvas.height === targetHeight + ) { + return this.exportCompositeCanvas; + } + + const canvas = document.createElement("canvas"); + canvas.width = targetWidth; + canvas.height = targetHeight; + + const context = configureHighQuality2DContext(canvas.getContext("2d")); + if (!context) { + return null; + } + + this.exportCompositeCanvas = { + canvas, + context, + }; + + return this.exportCompositeCanvas; + } + + private drawCaptionOverlay(context: CanvasRenderingContext2D): void { + if ( + !this.captionContainer?.visible || + !this.captionSprite?.visible || + !this.captionCanvas + ) { + return; + } + + const drawWidth = this.captionCanvas.width * this.captionSprite.scale.x; + const drawHeight = this.captionCanvas.height * this.captionSprite.scale.y; + const drawX = this.captionSprite.x - drawWidth * this.captionSprite.anchor.x; + const drawY = this.captionSprite.y - drawHeight * this.captionSprite.anchor.y; + + context.save(); + context.globalAlpha = this.captionSprite.alpha; + context.drawImage(this.captionCanvas, drawX, drawY, drawWidth, drawHeight); + context.restore(); + } + + private async composeBlurAnnotationFrame(timeMs: number): Promise { + if (!this.app) { + this.outputCanvasOverride = null; + return; + } + + const compositeState = this.ensureExportCompositeCanvas(); + if (!compositeState) { + this.outputCanvasOverride = null; + return; + } + + const { canvas, context } = compositeState; + context.clearRect(0, 0, canvas.width, canvas.height); + context.drawImage(this.app.canvas as HTMLCanvasElement, 0, 0); + + await renderAnnotations( + context, + this.config.annotationRegions ?? [], + this.config.width, + this.config.height, + timeMs, + this.annotationScaleFactor, + this.annotationAssets ?? undefined, + ); + + this.drawCaptionOverlay(context); + this.outputCanvasOverride = canvas; + } + private async setupAnnotationLayer(): Promise { if (!this.annotationContainer) { return; @@ -2042,6 +2138,32 @@ export class FrameRenderer { this.updateAnnotationLayer(timeMs); this.updateCaptionLayer(timeMs); this.updateWebcamOverlay(); + + if (this.hasActiveBlurAnnotations(timeMs)) { + const annotationContainerVisible = this.annotationContainer?.visible ?? true; + const captionContainerVisible = this.captionContainer?.visible ?? true; + + if (this.annotationContainer) { + this.annotationContainer.visible = false; + } + if (this.captionContainer) { + this.captionContainer.visible = false; + } + + this.app.render(); + + if (this.annotationContainer) { + this.annotationContainer.visible = annotationContainerVisible; + } + if (this.captionContainer) { + this.captionContainer.visible = captionContainerVisible; + } + + await this.composeBlurAnnotationFrame(timeMs); + return; + } + + this.outputCanvasOverride = null; this.app.render(); } @@ -2373,6 +2495,10 @@ export class FrameRenderer { throw new Error("Renderer not initialized"); } + if (this.outputCanvasOverride) { + return null; + } + if (this.config.nativeReadbackMode !== "pixels") { return null; } @@ -2397,7 +2523,7 @@ export class FrameRenderer { throw new Error("Renderer not initialized"); } - return this.app.canvas as HTMLCanvasElement; + return this.outputCanvasOverride ?? (this.app.canvas as HTMLCanvasElement); } getRendererBackend(): ExportRenderBackend { @@ -2527,6 +2653,8 @@ export class FrameRenderer { this.captionSprite = null; this.captionTextureSource = null; this.captionRenderKey = null; + this.exportCompositeCanvas = null; + this.outputCanvasOverride = null; this.nativeReadbackBuffer = null; this.annotationScaleFactor = 1;