lee/Recordly
1
0
Fork 0
mirror of https://github.com/webadderallorg/Recordly.git synced 2026-09-26 15:55:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(export): fallback webcam packet streaming failures

Browse Source
This commit is contained in:
wiiiii123
2026-05-08 19:42:34 +07:00
parent ae9cc08024
commit 427955b04b
2 changed files with 342 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/exporter/modernFrameRenderer.test.ts
+194 -1
View File
@@ -1,7 +1,16 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { DEFAULT_WEBCAM_OVERLAY } from "../../components/video-editor/types";
const { initializeForwardFrameSourceMock, resolveMediaElementSourceMock } = vi.hoisted(() => ({
const {
cancelForwardFrameSourceMock,
destroyForwardFrameSourceMock,
getForwardFrameAtTimeMock,
initializeForwardFrameSourceMock,
resolveMediaElementSourceMock,
} = vi.hoisted(() => ({
cancelForwardFrameSourceMock: vi.fn(),
destroyForwardFrameSourceMock: vi.fn(async () => undefined),
getForwardFrameAtTimeMock: vi.fn(async () => null),
initializeForwardFrameSourceMock: vi.fn(async () => undefined),
resolveMediaElementSourceMock: vi.fn(async () => ({
src: "blob:background",
@@ -82,6 +91,9 @@ vi.mock("@/components/video-editor/videoPlayback/cursorRenderer", () => ({
vi.mock("./forwardFrameSource", () => ({
ForwardFrameSource: class {
cancel = cancelForwardFrameSourceMock;
destroy = destroyForwardFrameSourceMock;
getFrameAtTime = getForwardFrameAtTimeMock;
initialize = initializeForwardFrameSourceMock;
},
}));
@@ -248,6 +260,43 @@ describe("ModernFrameRenderer blur export path", () => {
});
describe("ModernFrameRenderer webcam frame cache", () => {
it("uses staging canvas instead of recursing when WebGPU frame retention fails", () => {
const renderer = createRenderer() as any;
const originalVideoFrame = (globalThis as any).VideoFrame;
(globalThis as any).VideoFrame = class {
constructor() {
throw new Error("retain failed");
}
};
try {
renderer.rendererBackend = "webgpu";
const frame = {
displayWidth: 320,
displayHeight: 180,
timestamp: 0,
} as VideoFrame;
const result = renderer.stageVideoFrameForTexture(frame, "webcam", 640, 360);
expect(result).toBe(renderer.webcamVideoFrameStagingCanvas);
expect(renderer.webcamVideoFrameStagingCtx.drawImage).toHaveBeenCalledWith(
frame,
0,
0,
320,
180,
);
} finally {
if (originalVideoFrame === undefined) {
delete (globalThis as any).VideoFrame;
} else {
(globalThis as any).VideoFrame = originalVideoFrame;
}
}
});
it("keeps the refresh throttle for default crop regions", () => {
const renderer = createRenderer() as any;
@@ -270,3 +319,147 @@ describe("ModernFrameRenderer webcam frame cache", () => {
expect(renderer.shouldRefreshWebcamFrameCache(1280, 720)).toBe(true);
});
});
describe("ModernFrameRenderer webcam export fallback", () => {
beforeEach(() => {
vi.clearAllMocks();
initializeForwardFrameSourceMock.mockResolvedValue(undefined);
getForwardFrameAtTimeMock.mockResolvedValue(null);
resolveMediaElementSourceMock.mockResolvedValue({
src: "blob:webcam",
revoke: vi.fn(),
});
Object.assign(globalThis, {
window: {
clearTimeout,
setTimeout,
},
HTMLMediaElement: {
HAVE_CURRENT_DATA: 2,
},
cancelAnimationFrame: vi.fn(),
requestAnimationFrame: vi.fn((callback: FrameRequestCallback) => {
callback(0);
return 1;
}),
document: {
createElement: vi.fn((tag: string) => {
if (tag === "video") {
return {
duration: 5,
readyState: 2,
videoWidth: 640,
videoHeight: 360,
muted: true,
loop: true,
playsInline: true,
preload: "auto",
src: "",
currentTime: 0,
seeking: false,
load: vi.fn(),
pause: vi.fn(),
addEventListener: vi.fn(),
removeEventListener: vi.fn(),
};
}
if (tag !== "canvas") {
throw new Error(`Unexpected element requested in test: ${tag}`);
}
return createMockCanvas();
}),
},
});
});
it("falls back to media-element webcam sync when packet streaming fails after initialize", async () => {
getForwardFrameAtTimeMock.mockRejectedValueOnce(
new Error("readAVPacket pipeline failed: Failed after 3 attempts"),
);
const renderer = createRenderer() as any;
renderer.config.webcam = {
...DEFAULT_WEBCAM_OVERLAY,
enabled: true,
};
renderer.config.webcamUrl = "file:///tmp/webcam.webm";
await renderer.setupWebcamSource();
await expect(renderer.syncWebcamFrame(1)).resolves.toBeUndefined();
expect(cancelForwardFrameSourceMock).toHaveBeenCalled();
expect(destroyForwardFrameSourceMock).toHaveBeenCalled();
expect(resolveMediaElementSourceMock).toHaveBeenCalledWith("file:///tmp/webcam.webm");
expect(renderer.webcamForwardFrameSource).toBeNull();
expect(renderer.webcamVideoElement).toBeTruthy();
});
it("tears down the media-element fallback when readiness times out", async () => {
vi.useFakeTimers();
const originalCreateElement = (globalThis as any).document.createElement;
const revoke = vi.fn();
getForwardFrameAtTimeMock.mockRejectedValueOnce(
new Error("readAVPacket pipeline failed: Failed after 3 attempts"),
);
resolveMediaElementSourceMock.mockResolvedValueOnce({
src: "blob:webcam-timeout",
revoke,
});
Object.assign((globalThis as any).window, {
clearTimeout,
setTimeout,
});
(globalThis as any).document.createElement = vi.fn((tag: string) => {
if (tag === "video") {
return {
duration: Number.NaN,
readyState: 0,
videoWidth: 0,
videoHeight: 0,
muted: true,
loop: true,
playsInline: true,
preload: "auto",
src: "",
currentTime: 0,
seeking: false,
load: vi.fn(),
pause: vi.fn(),
addEventListener: vi.fn(),
removeEventListener: vi.fn(),
};
}
if (tag !== "canvas") {
throw new Error(`Unexpected element requested in test: ${tag}`);
}
return createMockCanvas();
});
try {
const renderer = createRenderer() as any;
renderer.config.webcam = {
...DEFAULT_WEBCAM_OVERLAY,
enabled: true,
};
renderer.config.webcamUrl = "file:///tmp/webcam.webm";
await renderer.setupWebcamSource();
const syncPromise = renderer.syncWebcamFrame(1);
await vi.advanceTimersByTimeAsync(5_001);
await expect(syncPromise).resolves.toBeUndefined();
expect(cancelForwardFrameSourceMock).toHaveBeenCalled();
expect(destroyForwardFrameSourceMock).toHaveBeenCalled();
expect(revoke).toHaveBeenCalled();
expect(renderer.webcamForwardFrameSource).toBeNull();
expect(renderer.webcamVideoElement).toBeNull();
} finally {
(globalThis as any).document.createElement = originalCreateElement;
vi.useRealTimers();
}
});
});
src/lib/exporter/modernFrameRenderer.ts
+148 -82
View File
@@ -246,10 +246,14 @@ type PixiRendererAttempt = {
const CANVAS_RENDERER_NOT_IMPLEMENTED_HINT = "CanvasRenderer is not yet implemented";
const NO_RENDERER_HINT = "no available renderer";
const PIXI_RENDERER_INIT_TIMEOUT_MS = 8_000;
const WEBCAM_MEDIA_ELEMENT_READY_TIMEOUT_MS = 5_000;
function isCanvasRenderer(application: Application): boolean {
const rendererName = application?.renderer?.constructor?.name?.toLowerCase();
return Boolean(rendererName && (rendererName.includes("canvasrenderer") || rendererName.includes("canvas")));
return Boolean(
rendererName &&
(rendererName.includes("canvasrenderer") || rendererName.includes("canvas")),
);
}
function toErrorMessage(error: unknown): string {
@@ -681,7 +685,8 @@ export class FrameRenderer {
backend,
);
const elapsed = Math.round(
(typeof performance === "undefined" ? Date.now() : performance.now()) - initStarted,
(typeof performance === "undefined" ? Date.now() : performance.now()) -
initStarted,
);
if (isCanvasRenderer(app)) {
throw new Error(
@@ -691,7 +696,8 @@ export class FrameRenderer {
return { app, backend };
} catch (error) {
const elapsed = Math.round(
(typeof performance === "undefined" ? Date.now() : performance.now()) - initStarted,
(typeof performance === "undefined" ? Date.now() : performance.now()) -
initStarted,
);
failures.push({
backend,
@@ -894,7 +900,9 @@ export class FrameRenderer {
}
const cachedTimestamp =
kind === "scene" ? this.retainedSceneBitmapTimestamp : this.retainedBackgroundBitmapTimestamp;
kind === "scene"
? this.retainedSceneBitmapTimestamp
: this.retainedBackgroundBitmapTimestamp;
const cachedBitmap =
kind === "scene" ? this.retainedSceneBitmap : this.retainedBackgroundBitmap;
if (cachedTimestamp === frame.timestamp && cachedBitmap) {
@@ -924,8 +932,8 @@ export class FrameRenderer {
private resolveRetainedVideoFrameSource(
frame: VideoFrame,
kind: "scene" | "background" | "webcam",
_fallbackWidth: number,
_fallbackHeight: number,
fallbackWidth: number,
fallbackHeight: number,
): CanvasImageSource | VideoFrame {
if (this.rendererBackend !== "webgpu") {
return frame;
@@ -948,12 +956,7 @@ export class FrameRenderer {
`[ModernFrameRenderer] Failed to retain ${kind} VideoFrame, falling back to staging canvas:`,
error,
);
return this.stageVideoFrameForTexture(
frame,
"scene",
this.config.videoWidth,
this.config.videoHeight,
);
return this.stageVideoFrameOnCanvas(frame, kind, fallbackWidth, fallbackHeight);
}
}
@@ -1010,21 +1013,12 @@ export class FrameRenderer {
return { canvas, context };
}
private stageVideoFrameForTexture(
private stageVideoFrameOnCanvas(
frame: VideoFrame,
kind: "scene" | "background" | "webcam",
fallbackWidth: number,
fallbackHeight: number,
): CanvasImageSource | VideoFrame {
if (this.rendererBackend === "webgpu") {
return this.resolveRetainedVideoFrameSource(
frame,
kind,
fallbackWidth,
fallbackHeight,
);
}
const width = Math.max(1, frame.displayWidth || fallbackWidth);
const height = Math.max(1, frame.displayHeight || fallbackHeight);
const staging = this.ensureVideoFrameStagingCanvas(kind, width, height);
@@ -1037,6 +1031,19 @@ export class FrameRenderer {
return staging.canvas;
}
private stageVideoFrameForTexture(
frame: VideoFrame,
kind: "scene" | "background" | "webcam",
fallbackWidth: number,
fallbackHeight: number,
): CanvasImageSource | VideoFrame {
if (this.rendererBackend === "webgpu") {
return this.resolveRetainedVideoFrameSource(frame, kind, fallbackWidth, fallbackHeight);
}
return this.stageVideoFrameOnCanvas(frame, kind, fallbackWidth, fallbackHeight);
}
private replaceSpriteTexture(
sprite: Sprite,
source: CanvasImageSource | VideoFrame,
@@ -2085,6 +2092,99 @@ export class FrameRenderer {
return getRenderableAssetUrl(wallpaperAsset);
}
private disposeWebcamMediaElement(video: HTMLVideoElement): void {
try {
video.pause();
video.src = "";
video.load();
} catch {
// Ignore media element teardown errors during export fallback.
}
}
private clearWebcamMediaElement(): void {
if (this.webcamVideoElement) {
this.disposeWebcamMediaElement(this.webcamVideoElement);
}
this.webcamVideoElement = null;
this.webcamSeekPromise = null;
this.cleanupWebcamSource?.();
this.cleanupWebcamSource = null;
}
private async loadWebcamMediaElementSource(webcamUrl: string): Promise<boolean> {
this.clearWebcamMediaElement();
let webcamSource: Awaited<ReturnType<typeof resolveMediaElementSource>>;
try {
webcamSource = await resolveMediaElementSource(webcamUrl);
} catch (error) {
console.warn("[FrameRenderer] Unable to resolve webcam media element source:", error);
return false;
}
this.cleanupWebcamSource = webcamSource.revoke;
const video = document.createElement("video");
video.src = webcamSource.src;
video.muted = true;
video.preload = "auto";
video.playsInline = true;
video.load();
const ready = await new Promise<boolean>((resolve) => {
let readyTimeout: number | null = null;
const onReady = () => {
if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) {
return;
}
cleanup();
resolve(true);
};
const onError = () => {
cleanup();
resolve(false);
};
const cleanup = () => {
video.removeEventListener("loadeddata", onReady);
video.removeEventListener("canplay", onReady);
video.removeEventListener("canplaythrough", onReady);
video.removeEventListener("error", onError);
if (readyTimeout !== null) {
window.clearTimeout(readyTimeout);
readyTimeout = null;
}
};
if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {
resolve(true);
return;
}
video.addEventListener("loadeddata", onReady, { once: true });
video.addEventListener("canplay", onReady, { once: true });
video.addEventListener("canplaythrough", onReady, { once: true });
video.addEventListener("error", onError, { once: true });
readyTimeout = window.setTimeout(() => {
console.warn(
`[FrameRenderer] Webcam media element fallback did not become ready within ${WEBCAM_MEDIA_ELEMENT_READY_TIMEOUT_MS}ms`,
);
onError();
}, WEBCAM_MEDIA_ELEMENT_READY_TIMEOUT_MS);
});
if (ready && video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {
this.webcamVideoElement = video;
this.lastSyncedWebcamTime = null;
return true;
}
console.warn("[FrameRenderer] Webcam overlay unavailable during export");
this.disposeWebcamMediaElement(video);
this.clearWebcamMediaElement();
return false;
}
private async setupWebcamSource(): Promise<void> {
const webcamUrl = this.config.webcamUrl;
if (!this.config.webcam?.enabled || !webcamUrl) {
@@ -2092,9 +2192,7 @@ export class FrameRenderer {
void this.webcamForwardFrameSource?.destroy();
this.webcamForwardFrameSource = null;
this.closeWebcamDecodedFrame();
this.cleanupWebcamSource?.();
this.cleanupWebcamSource = null;
this.webcamVideoElement = null;
this.clearWebcamMediaElement();
this.webcamFrameCacheCanvas = null;
this.webcamFrameCacheCtx = null;
this.lastSyncedWebcamTime = null;
@@ -2108,8 +2206,7 @@ export class FrameRenderer {
void this.webcamForwardFrameSource?.destroy();
this.webcamForwardFrameSource = null;
this.closeWebcamDecodedFrame();
this.cleanupWebcamSource?.();
this.cleanupWebcamSource = null;
this.clearWebcamMediaElement();
this.webcamFrameCacheCanvas = null;
this.webcamFrameCacheCtx = null;
this.lastWebcamCacheRefreshTime = null;
@@ -2132,55 +2229,7 @@ export class FrameRenderer {
);
}
const webcamSource = await resolveMediaElementSource(webcamUrl);
this.cleanupWebcamSource = webcamSource.revoke;
const video = document.createElement("video");
video.src = webcamSource.src;
video.muted = true;
video.preload = "auto";
video.playsInline = true;
video.load();
await new Promise<void>((resolve, reject) => {
const onReady = () => {
if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) {
return;
}
cleanup();
resolve();
};
const onError = () => {
cleanup();
reject(new Error("Failed to load webcam source for export"));
};
const cleanup = () => {
video.removeEventListener("loadeddata", onReady);
video.removeEventListener("canplay", onReady);
video.removeEventListener("canplaythrough", onReady);
video.removeEventListener("error", onError);
};
if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {
resolve();
return;
}
video.addEventListener("loadeddata", onReady, { once: true });
video.addEventListener("canplay", onReady, { once: true });
video.addEventListener("canplaythrough", onReady, { once: true });
video.addEventListener("error", onError, { once: true });
}).catch((error) => {
console.warn("[FrameRenderer] Webcam overlay unavailable during export:", error);
this.webcamVideoElement = null;
});
if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {
this.webcamVideoElement = video;
return;
}
this.webcamVideoElement = null;
await this.loadWebcamMediaElementSource(webcamUrl);
this.lastSyncedWebcamTime = null;
}
@@ -2457,13 +2506,30 @@ export class FrameRenderer {
if (this.webcamForwardFrameSource) {
const clampedTime = clampMediaTimeToDuration(webcamTargetTime, null);
const decodedFrame = await this.webcamForwardFrameSource.getFrameAtTime(clampedTime);
this.closeWebcamDecodedFrame();
this.webcamDecodedFrame = decodedFrame;
if (decodedFrame) {
this.lastSyncedWebcamTime = clampedTime;
try {
const decodedFrame =
await this.webcamForwardFrameSource.getFrameAtTime(clampedTime);
this.closeWebcamDecodedFrame();
this.webcamDecodedFrame = decodedFrame;
if (decodedFrame) {
this.lastSyncedWebcamTime = clampedTime;
}
return;
} catch (error) {
console.warn(
"[FrameRenderer] Decoder-backed webcam source failed during export; falling back to media element sync:",
error,
);
this.webcamForwardFrameSource.cancel();
void this.webcamForwardFrameSource.destroy();
this.webcamForwardFrameSource = null;
this.closeWebcamDecodedFrame();
this.lastSyncedWebcamTime = null;
const webcamUrl = this.config.webcamUrl;
if (!webcamUrl || !(await this.loadWebcamMediaElementSource(webcamUrl))) {
return;
}
}
return;
}
const webcamVideo = this.webcamVideoElement;