From 427955b04b68259651c30909591c211cfdb1d820 Mon Sep 17 00:00:00 2001 From: wiiiii123 Date: Fri, 8 May 2026 17:25:23 +0700 Subject: [PATCH] fix(export): fallback webcam packet streaming failures --- src/lib/exporter/modernFrameRenderer.test.ts | 195 +++++++++++++++- src/lib/exporter/modernFrameRenderer.ts | 230 ++++++++++++------- 2 files changed, 342 insertions(+), 83 deletions(-) diff --git a/src/lib/exporter/modernFrameRenderer.test.ts b/src/lib/exporter/modernFrameRenderer.test.ts index f8b95fd5..c0c0927e 100644 --- a/src/lib/exporter/modernFrameRenderer.test.ts +++ b/src/lib/exporter/modernFrameRenderer.test.ts @@ -1,7 +1,16 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { DEFAULT_WEBCAM_OVERLAY } from "../../components/video-editor/types"; -const { initializeForwardFrameSourceMock, resolveMediaElementSourceMock } = vi.hoisted(() => ({ +const { + cancelForwardFrameSourceMock, + destroyForwardFrameSourceMock, + getForwardFrameAtTimeMock, + initializeForwardFrameSourceMock, + resolveMediaElementSourceMock, +} = vi.hoisted(() => ({ + cancelForwardFrameSourceMock: vi.fn(), + destroyForwardFrameSourceMock: vi.fn(async () => undefined), + getForwardFrameAtTimeMock: vi.fn(async () => null), initializeForwardFrameSourceMock: vi.fn(async () => undefined), resolveMediaElementSourceMock: vi.fn(async () => ({ src: "blob:background", @@ -82,6 +91,9 @@ vi.mock("@/components/video-editor/videoPlayback/cursorRenderer", () => ({ vi.mock("./forwardFrameSource", () => ({ ForwardFrameSource: class { + cancel = cancelForwardFrameSourceMock; + destroy = destroyForwardFrameSourceMock; + getFrameAtTime = getForwardFrameAtTimeMock; initialize = initializeForwardFrameSourceMock; }, })); @@ -248,6 +260,43 @@ describe("ModernFrameRenderer blur export path", () => { }); describe("ModernFrameRenderer webcam frame cache", () => { + it("uses staging canvas instead of recursing when WebGPU frame retention fails", () => { + const renderer = createRenderer() as any; + const originalVideoFrame = (globalThis as any).VideoFrame; + + (globalThis as any).VideoFrame = class { + constructor() { + throw new Error("retain failed"); + } + }; + + try { + renderer.rendererBackend = "webgpu"; + const frame = { + displayWidth: 320, + displayHeight: 180, + timestamp: 0, + } as VideoFrame; + + const result = renderer.stageVideoFrameForTexture(frame, "webcam", 640, 360); + + expect(result).toBe(renderer.webcamVideoFrameStagingCanvas); + expect(renderer.webcamVideoFrameStagingCtx.drawImage).toHaveBeenCalledWith( + frame, + 0, + 0, + 320, + 180, + ); + } finally { + if (originalVideoFrame === undefined) { + delete (globalThis as any).VideoFrame; + } else { + (globalThis as any).VideoFrame = originalVideoFrame; + } + } + }); + it("keeps the refresh throttle for default crop regions", () => { const renderer = createRenderer() as any; @@ -270,3 +319,147 @@ describe("ModernFrameRenderer webcam frame cache", () => { expect(renderer.shouldRefreshWebcamFrameCache(1280, 720)).toBe(true); }); }); + +describe("ModernFrameRenderer webcam export fallback", () => { + beforeEach(() => { + vi.clearAllMocks(); + initializeForwardFrameSourceMock.mockResolvedValue(undefined); + getForwardFrameAtTimeMock.mockResolvedValue(null); + resolveMediaElementSourceMock.mockResolvedValue({ + src: "blob:webcam", + revoke: vi.fn(), + }); + + Object.assign(globalThis, { + window: { + clearTimeout, + setTimeout, + }, + HTMLMediaElement: { + HAVE_CURRENT_DATA: 2, + }, + cancelAnimationFrame: vi.fn(), + requestAnimationFrame: vi.fn((callback: FrameRequestCallback) => { + callback(0); + return 1; + }), + document: { + createElement: vi.fn((tag: string) => { + if (tag === "video") { + return { + duration: 5, + readyState: 2, + videoWidth: 640, + videoHeight: 360, + muted: true, + loop: true, + playsInline: true, + preload: "auto", + src: "", + currentTime: 0, + seeking: false, + load: vi.fn(), + pause: vi.fn(), + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + }; + } + if (tag !== "canvas") { + throw new Error(`Unexpected element requested in test: ${tag}`); + } + + return createMockCanvas(); + }), + }, + }); + }); + + it("falls back to media-element webcam sync when packet streaming fails after initialize", async () => { + getForwardFrameAtTimeMock.mockRejectedValueOnce( + new Error("readAVPacket pipeline failed: Failed after 3 attempts"), + ); + const renderer = createRenderer() as any; + renderer.config.webcam = { + ...DEFAULT_WEBCAM_OVERLAY, + enabled: true, + }; + renderer.config.webcamUrl = "file:///tmp/webcam.webm"; + + await renderer.setupWebcamSource(); + await expect(renderer.syncWebcamFrame(1)).resolves.toBeUndefined(); + + expect(cancelForwardFrameSourceMock).toHaveBeenCalled(); + expect(destroyForwardFrameSourceMock).toHaveBeenCalled(); + expect(resolveMediaElementSourceMock).toHaveBeenCalledWith("file:///tmp/webcam.webm"); + expect(renderer.webcamForwardFrameSource).toBeNull(); + expect(renderer.webcamVideoElement).toBeTruthy(); + }); + + it("tears down the media-element fallback when readiness times out", async () => { + vi.useFakeTimers(); + const originalCreateElement = (globalThis as any).document.createElement; + const revoke = vi.fn(); + getForwardFrameAtTimeMock.mockRejectedValueOnce( + new Error("readAVPacket pipeline failed: Failed after 3 attempts"), + ); + resolveMediaElementSourceMock.mockResolvedValueOnce({ + src: "blob:webcam-timeout", + revoke, + }); + Object.assign((globalThis as any).window, { + clearTimeout, + setTimeout, + }); + + (globalThis as any).document.createElement = vi.fn((tag: string) => { + if (tag === "video") { + return { + duration: Number.NaN, + readyState: 0, + videoWidth: 0, + videoHeight: 0, + muted: true, + loop: true, + playsInline: true, + preload: "auto", + src: "", + currentTime: 0, + seeking: false, + load: vi.fn(), + pause: vi.fn(), + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + }; + } + if (tag !== "canvas") { + throw new Error(`Unexpected element requested in test: ${tag}`); + } + + return createMockCanvas(); + }); + + try { + const renderer = createRenderer() as any; + renderer.config.webcam = { + ...DEFAULT_WEBCAM_OVERLAY, + enabled: true, + }; + renderer.config.webcamUrl = "file:///tmp/webcam.webm"; + + await renderer.setupWebcamSource(); + const syncPromise = renderer.syncWebcamFrame(1); + + await vi.advanceTimersByTimeAsync(5_001); + await expect(syncPromise).resolves.toBeUndefined(); + + expect(cancelForwardFrameSourceMock).toHaveBeenCalled(); + expect(destroyForwardFrameSourceMock).toHaveBeenCalled(); + expect(revoke).toHaveBeenCalled(); + expect(renderer.webcamForwardFrameSource).toBeNull(); + expect(renderer.webcamVideoElement).toBeNull(); + } finally { + (globalThis as any).document.createElement = originalCreateElement; + vi.useRealTimers(); + } + }); +}); diff --git a/src/lib/exporter/modernFrameRenderer.ts b/src/lib/exporter/modernFrameRenderer.ts index c6830315..02e648aa 100644 --- a/src/lib/exporter/modernFrameRenderer.ts +++ b/src/lib/exporter/modernFrameRenderer.ts @@ -246,10 +246,14 @@ type PixiRendererAttempt = { const CANVAS_RENDERER_NOT_IMPLEMENTED_HINT = "CanvasRenderer is not yet implemented"; const NO_RENDERER_HINT = "no available renderer"; const PIXI_RENDERER_INIT_TIMEOUT_MS = 8_000; +const WEBCAM_MEDIA_ELEMENT_READY_TIMEOUT_MS = 5_000; function isCanvasRenderer(application: Application): boolean { const rendererName = application?.renderer?.constructor?.name?.toLowerCase(); - return Boolean(rendererName && (rendererName.includes("canvasrenderer") || rendererName.includes("canvas"))); + return Boolean( + rendererName && + (rendererName.includes("canvasrenderer") || rendererName.includes("canvas")), + ); } function toErrorMessage(error: unknown): string { @@ -681,7 +685,8 @@ export class FrameRenderer { backend, ); const elapsed = Math.round( - (typeof performance === "undefined" ? Date.now() : performance.now()) - initStarted, + (typeof performance === "undefined" ? Date.now() : performance.now()) - + initStarted, ); if (isCanvasRenderer(app)) { throw new Error( @@ -691,7 +696,8 @@ export class FrameRenderer { return { app, backend }; } catch (error) { const elapsed = Math.round( - (typeof performance === "undefined" ? Date.now() : performance.now()) - initStarted, + (typeof performance === "undefined" ? Date.now() : performance.now()) - + initStarted, ); failures.push({ backend, @@ -894,7 +900,9 @@ export class FrameRenderer { } const cachedTimestamp = - kind === "scene" ? this.retainedSceneBitmapTimestamp : this.retainedBackgroundBitmapTimestamp; + kind === "scene" + ? this.retainedSceneBitmapTimestamp + : this.retainedBackgroundBitmapTimestamp; const cachedBitmap = kind === "scene" ? this.retainedSceneBitmap : this.retainedBackgroundBitmap; if (cachedTimestamp === frame.timestamp && cachedBitmap) { @@ -924,8 +932,8 @@ export class FrameRenderer { private resolveRetainedVideoFrameSource( frame: VideoFrame, kind: "scene" | "background" | "webcam", - _fallbackWidth: number, - _fallbackHeight: number, + fallbackWidth: number, + fallbackHeight: number, ): CanvasImageSource | VideoFrame { if (this.rendererBackend !== "webgpu") { return frame; @@ -948,12 +956,7 @@ export class FrameRenderer { `[ModernFrameRenderer] Failed to retain ${kind} VideoFrame, falling back to staging canvas:`, error, ); - return this.stageVideoFrameForTexture( - frame, - "scene", - this.config.videoWidth, - this.config.videoHeight, - ); + return this.stageVideoFrameOnCanvas(frame, kind, fallbackWidth, fallbackHeight); } } @@ -1010,21 +1013,12 @@ export class FrameRenderer { return { canvas, context }; } - private stageVideoFrameForTexture( + private stageVideoFrameOnCanvas( frame: VideoFrame, kind: "scene" | "background" | "webcam", fallbackWidth: number, fallbackHeight: number, ): CanvasImageSource | VideoFrame { - if (this.rendererBackend === "webgpu") { - return this.resolveRetainedVideoFrameSource( - frame, - kind, - fallbackWidth, - fallbackHeight, - ); - } - const width = Math.max(1, frame.displayWidth || fallbackWidth); const height = Math.max(1, frame.displayHeight || fallbackHeight); const staging = this.ensureVideoFrameStagingCanvas(kind, width, height); @@ -1037,6 +1031,19 @@ export class FrameRenderer { return staging.canvas; } + private stageVideoFrameForTexture( + frame: VideoFrame, + kind: "scene" | "background" | "webcam", + fallbackWidth: number, + fallbackHeight: number, + ): CanvasImageSource | VideoFrame { + if (this.rendererBackend === "webgpu") { + return this.resolveRetainedVideoFrameSource(frame, kind, fallbackWidth, fallbackHeight); + } + + return this.stageVideoFrameOnCanvas(frame, kind, fallbackWidth, fallbackHeight); + } + private replaceSpriteTexture( sprite: Sprite, source: CanvasImageSource | VideoFrame, @@ -2085,6 +2092,99 @@ export class FrameRenderer { return getRenderableAssetUrl(wallpaperAsset); } + private disposeWebcamMediaElement(video: HTMLVideoElement): void { + try { + video.pause(); + video.src = ""; + video.load(); + } catch { + // Ignore media element teardown errors during export fallback. + } + } + + private clearWebcamMediaElement(): void { + if (this.webcamVideoElement) { + this.disposeWebcamMediaElement(this.webcamVideoElement); + } + + this.webcamVideoElement = null; + this.webcamSeekPromise = null; + this.cleanupWebcamSource?.(); + this.cleanupWebcamSource = null; + } + + private async loadWebcamMediaElementSource(webcamUrl: string): Promise { + this.clearWebcamMediaElement(); + + let webcamSource: Awaited>; + try { + webcamSource = await resolveMediaElementSource(webcamUrl); + } catch (error) { + console.warn("[FrameRenderer] Unable to resolve webcam media element source:", error); + return false; + } + this.cleanupWebcamSource = webcamSource.revoke; + + const video = document.createElement("video"); + video.src = webcamSource.src; + video.muted = true; + video.preload = "auto"; + video.playsInline = true; + video.load(); + + const ready = await new Promise((resolve) => { + let readyTimeout: number | null = null; + const onReady = () => { + if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) { + return; + } + cleanup(); + resolve(true); + }; + const onError = () => { + cleanup(); + resolve(false); + }; + const cleanup = () => { + video.removeEventListener("loadeddata", onReady); + video.removeEventListener("canplay", onReady); + video.removeEventListener("canplaythrough", onReady); + video.removeEventListener("error", onError); + if (readyTimeout !== null) { + window.clearTimeout(readyTimeout); + readyTimeout = null; + } + }; + + if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) { + resolve(true); + return; + } + + video.addEventListener("loadeddata", onReady, { once: true }); + video.addEventListener("canplay", onReady, { once: true }); + video.addEventListener("canplaythrough", onReady, { once: true }); + video.addEventListener("error", onError, { once: true }); + readyTimeout = window.setTimeout(() => { + console.warn( + `[FrameRenderer] Webcam media element fallback did not become ready within ${WEBCAM_MEDIA_ELEMENT_READY_TIMEOUT_MS}ms`, + ); + onError(); + }, WEBCAM_MEDIA_ELEMENT_READY_TIMEOUT_MS); + }); + + if (ready && video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) { + this.webcamVideoElement = video; + this.lastSyncedWebcamTime = null; + return true; + } + + console.warn("[FrameRenderer] Webcam overlay unavailable during export"); + this.disposeWebcamMediaElement(video); + this.clearWebcamMediaElement(); + return false; + } + private async setupWebcamSource(): Promise { const webcamUrl = this.config.webcamUrl; if (!this.config.webcam?.enabled || !webcamUrl) { @@ -2092,9 +2192,7 @@ export class FrameRenderer { void this.webcamForwardFrameSource?.destroy(); this.webcamForwardFrameSource = null; this.closeWebcamDecodedFrame(); - this.cleanupWebcamSource?.(); - this.cleanupWebcamSource = null; - this.webcamVideoElement = null; + this.clearWebcamMediaElement(); this.webcamFrameCacheCanvas = null; this.webcamFrameCacheCtx = null; this.lastSyncedWebcamTime = null; @@ -2108,8 +2206,7 @@ export class FrameRenderer { void this.webcamForwardFrameSource?.destroy(); this.webcamForwardFrameSource = null; this.closeWebcamDecodedFrame(); - this.cleanupWebcamSource?.(); - this.cleanupWebcamSource = null; + this.clearWebcamMediaElement(); this.webcamFrameCacheCanvas = null; this.webcamFrameCacheCtx = null; this.lastWebcamCacheRefreshTime = null; @@ -2132,55 +2229,7 @@ export class FrameRenderer { ); } - const webcamSource = await resolveMediaElementSource(webcamUrl); - this.cleanupWebcamSource = webcamSource.revoke; - - const video = document.createElement("video"); - video.src = webcamSource.src; - video.muted = true; - video.preload = "auto"; - video.playsInline = true; - video.load(); - - await new Promise((resolve, reject) => { - const onReady = () => { - if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) { - return; - } - cleanup(); - resolve(); - }; - const onError = () => { - cleanup(); - reject(new Error("Failed to load webcam source for export")); - }; - const cleanup = () => { - video.removeEventListener("loadeddata", onReady); - video.removeEventListener("canplay", onReady); - video.removeEventListener("canplaythrough", onReady); - video.removeEventListener("error", onError); - }; - - if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) { - resolve(); - return; - } - - video.addEventListener("loadeddata", onReady, { once: true }); - video.addEventListener("canplay", onReady, { once: true }); - video.addEventListener("canplaythrough", onReady, { once: true }); - video.addEventListener("error", onError, { once: true }); - }).catch((error) => { - console.warn("[FrameRenderer] Webcam overlay unavailable during export:", error); - this.webcamVideoElement = null; - }); - - if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) { - this.webcamVideoElement = video; - return; - } - - this.webcamVideoElement = null; + await this.loadWebcamMediaElementSource(webcamUrl); this.lastSyncedWebcamTime = null; } @@ -2457,13 +2506,30 @@ export class FrameRenderer { if (this.webcamForwardFrameSource) { const clampedTime = clampMediaTimeToDuration(webcamTargetTime, null); - const decodedFrame = await this.webcamForwardFrameSource.getFrameAtTime(clampedTime); - this.closeWebcamDecodedFrame(); - this.webcamDecodedFrame = decodedFrame; - if (decodedFrame) { - this.lastSyncedWebcamTime = clampedTime; + try { + const decodedFrame = + await this.webcamForwardFrameSource.getFrameAtTime(clampedTime); + this.closeWebcamDecodedFrame(); + this.webcamDecodedFrame = decodedFrame; + if (decodedFrame) { + this.lastSyncedWebcamTime = clampedTime; + } + return; + } catch (error) { + console.warn( + "[FrameRenderer] Decoder-backed webcam source failed during export; falling back to media element sync:", + error, + ); + this.webcamForwardFrameSource.cancel(); + void this.webcamForwardFrameSource.destroy(); + this.webcamForwardFrameSource = null; + this.closeWebcamDecodedFrame(); + this.lastSyncedWebcamTime = null; + const webcamUrl = this.config.webcamUrl; + if (!webcamUrl || !(await this.loadWebcamMediaElementSource(webcamUrl))) { + return; + } } - return; } const webcamVideo = this.webcamVideoElement;