lee/Recordly
1
0
Fork 0
mirror of https://github.com/webadderallorg/Recordly.git synced 2026-09-25 15:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix video wallpaper export stutter

Browse Source
This commit is contained in:
webadderall
2026-04-27 17:32:43 +10:00
parent ae060e83fe
commit a4f475228a
4 changed files with 196 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/exporter/frameRenderer.test.ts
+50
View File
@@ -1,6 +1,14 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { DEFAULT_WEBCAM_OVERLAY } from "../../components/video-editor/types";
const { initializeForwardFrameSourceMock, resolveMediaElementSourceMock } = vi.hoisted(() => ({
initializeForwardFrameSourceMock: vi.fn(async () => undefined),
resolveMediaElementSourceMock: vi.fn(async () => ({
src: "blob:background",
revoke: vi.fn(),
})),
}));
vi.mock("pixi.js", () => ({
Application: vi.fn(),
Container: vi.fn(),
@@ -58,6 +66,16 @@ vi.mock("@/components/video-editor/videoPlayback/cursorRenderer", () => ({
preloadCursorAssets: vi.fn(async () => undefined),
}));
vi.mock("./forwardFrameSource", () => ({
ForwardFrameSource: class {
initialize = initializeForwardFrameSourceMock;
},
}));
vi.mock("./localMediaSource", () => ({
resolveMediaElementSource: resolveMediaElementSourceMock,
}));
import { FrameRenderer } from "./frameRenderer";
type MockFunction = ReturnType<typeof vi.fn>;
@@ -424,4 +442,36 @@ describe("FrameRenderer webcam export path", () => {
expect(createdCanvases).toHaveLength(2);
});
it("prefers decoder-backed video wallpapers during export", async () => {
const renderer = new FrameRenderer({
width: 1920,
height: 1080,
wallpaper: "/wallpapers/wispysky.mp4",
zoomRegions: [],
showShadow: false,
shadowIntensity: 0,
backgroundBlur: 0,
cropRegion: { x: 0, y: 0, width: 1, height: 1 },
webcam: {
...DEFAULT_WEBCAM_OVERLAY,
enabled: false,
},
videoWidth: 1920,
videoHeight: 1080,
}) as unknown as {
setupBackground: () => Promise<void>;
backgroundForwardFrameSource: unknown;
backgroundVideoElement: FakeVideoElement | null;
backgroundSprite: MockCanvas | null;
};
await renderer.setupBackground();
expect(initializeForwardFrameSourceMock).toHaveBeenCalledWith("wallpapers/wispysky.mp4");
expect(resolveMediaElementSourceMock).not.toHaveBeenCalled();
expect(renderer.backgroundForwardFrameSource).toBeTruthy();
expect(renderer.backgroundVideoElement).toBeNull();
expect(renderer.backgroundSprite).toBeTruthy();
});
});
src/lib/exporter/frameRenderer.ts
+90 -13
View File
@@ -18,7 +18,6 @@ import {
BASE_PREVIEW_WIDTH,
ZOOM_DEPTH_SCALES,
} from "@/components/video-editor/types";
import { computePaddedLayout } from "@/components/video-editor/videoPlayback/layoutUtils";
import { DEFAULT_FOCUS } from "@/components/video-editor/videoPlayback/constants";
import {
type CursorFollowCameraState,
@@ -31,6 +30,7 @@ import {
PixiCursorOverlay,
preloadCursorAssets,
} from "@/components/video-editor/videoPlayback/cursorRenderer";
import { computePaddedLayout } from "@/components/video-editor/videoPlayback/layoutUtils";
import {
createSpringState,
getZoomSpringConfig,
@@ -38,6 +38,7 @@ import {
type SpringState,
stepSpringValue,
} from "@/components/video-editor/videoPlayback/motionSmoothing";
import { getWebcamMediaTargetTimeSeconds } from "@/components/video-editor/videoPlayback/webcamSync";
import { findDominantRegion } from "@/components/video-editor/videoPlayback/zoomRegionUtils";
import {
applyZoomTransform,
@@ -50,12 +51,7 @@ import {
getWebcamOverlayPosition,
getWebcamOverlaySizePx,
} from "@/components/video-editor/webcamOverlay";
import { getWebcamMediaTargetTimeSeconds } from "@/components/video-editor/videoPlayback/webcamSync";
import {
getAssetPath,
getExportableVideoUrl,
getRenderableAssetUrl,
} from "@/lib/assetPath";
import { getAssetPath, getExportableVideoUrl, getRenderableAssetUrl } from "@/lib/assetPath";
import { extensionHost } from "@/lib/extensions";
import {
mapCursorToCanvasNormalized,
@@ -194,6 +190,8 @@ export class FrameRenderer {
private shadowCtx: CanvasRenderingContext2D | null = null;
private compositeCanvas: HTMLCanvasElement | null = null;
private compositeCtx: CanvasRenderingContext2D | null = null;
private backgroundForwardFrameSource: ForwardFrameSource | null = null;
private backgroundDecodedFrame: VideoFrame | null = null;
private backgroundVideoElement: HTMLVideoElement | null = null;
private backgroundCtx: CanvasRenderingContext2D | null = null;
private backgroundSeekPromise: Promise<void> | null = null;
@@ -370,6 +368,19 @@ export class FrameRenderer {
}
this.backgroundCtx = bgCtx;
this.backgroundForwardFrameSource?.cancel();
void this.backgroundForwardFrameSource?.destroy();
this.backgroundForwardFrameSource = null;
this.closeBackgroundDecodedFrame();
this.cleanupBackgroundSource?.();
this.cleanupBackgroundSource = null;
if (this.backgroundVideoElement) {
this.backgroundVideoElement.pause();
this.backgroundVideoElement.src = "";
this.backgroundVideoElement.load();
this.backgroundVideoElement = null;
}
this.backgroundSeekPromise = null;
try {
// Check for video wallpaper first
@@ -379,6 +390,19 @@ export class FrameRenderer {
videoSrc = await getAssetPath(wallpaper.replace(/^\//, ""));
}
try {
const frameSource = new ForwardFrameSource();
await frameSource.initialize(videoSrc);
this.backgroundForwardFrameSource = frameSource;
this.backgroundSprite = bgCanvas;
return;
} catch (error) {
console.warn(
"[FrameRenderer] Decoder-backed video wallpaper unavailable during export; falling back to media element sync:",
error,
);
}
const backgroundSource = await resolveMediaElementSource(videoSrc);
this.cleanupBackgroundSource = backgroundSource.revoke;
@@ -540,12 +564,24 @@ export class FrameRenderer {
private drawVideoFrameToBackground(): void {
const video = this.backgroundVideoElement;
if (!video) return;
this.drawBackgroundSourceToCanvas(video, video.videoWidth, video.videoHeight);
}
private drawBackgroundSourceToCanvas(
source: CanvasImageSource,
sourceWidth: number,
sourceHeight: number,
): void {
const ctx = this.backgroundCtx;
if (!video || !ctx) return;
if (!ctx) return;
const w = this.config.width;
const h = this.config.height;
const videoAspect = video.videoWidth / video.videoHeight;
const safeSourceWidth = Math.max(1, sourceWidth);
const safeSourceHeight = Math.max(1, sourceHeight);
const videoAspect = safeSourceWidth / safeSourceHeight;
const canvasAspect = w / h;
let drawWidth: number, drawHeight: number, drawX: number, drawY: number;
@@ -562,7 +598,36 @@ export class FrameRenderer {
}
ctx.clearRect(0, 0, w, h);
ctx.drawImage(video, drawX, drawY, drawWidth, drawHeight);
ctx.drawImage(source, drawX, drawY, drawWidth, drawHeight);
}
private closeBackgroundDecodedFrame(): void {
if (!this.backgroundDecodedFrame) {
return;
}
this.backgroundDecodedFrame.close();
this.backgroundDecodedFrame = null;
}
private async syncBackgroundFrame(timeSeconds: number): Promise<void> {
if (this.backgroundForwardFrameSource) {
const decodedFrame = await this.backgroundForwardFrameSource.getFrameAtTime(
Math.max(0, timeSeconds),
);
this.closeBackgroundDecodedFrame();
this.backgroundDecodedFrame = decodedFrame;
if (decodedFrame) {
this.drawBackgroundSourceToCanvas(
decodedFrame,
decodedFrame.displayWidth,
decodedFrame.displayHeight,
);
}
return;
}
await this.syncBackgroundVideo(timeSeconds);
}
private async syncBackgroundVideo(timeSeconds: number): Promise<void> {
@@ -1040,8 +1105,8 @@ export class FrameRenderer {
}
// Sync video wallpaper frame
if (this.backgroundVideoElement) {
await this.syncBackgroundVideo(this.currentVideoTime);
if (this.backgroundForwardFrameSource || this.backgroundVideoElement) {
await this.syncBackgroundFrame(this.currentVideoTime);
}
// Create or update video sprite from VideoFrame
@@ -1327,7 +1392,15 @@ export class FrameRenderer {
private updateLayout(): void {
if (!this.app || !this.videoSprite || !this.maskGraphics || !this.videoContainer) return;
const { width, height, cropRegion, borderRadius = 0, padding = 0, videoWidth, videoHeight } = this.config;
const {
width,
height,
cropRegion,
borderRadius = 0,
padding = 0,
videoWidth,
videoHeight,
} = this.config;
const layout = computePaddedLayout({
width,
@@ -1848,6 +1921,10 @@ export class FrameRenderer {
this.compositeCanvas = null;
this.compositeCtx = null;
this.backgroundCtx = null;
this.closeBackgroundDecodedFrame();
this.backgroundForwardFrameSource?.cancel();
void this.backgroundForwardFrameSource?.destroy();
this.backgroundForwardFrameSource = null;
if (this.backgroundVideoElement) {
this.backgroundVideoElement.pause();
this.backgroundVideoElement.src = "";
src/lib/exporter/modernFrameRenderer.test.ts
+39 -2
View File
@@ -1,6 +1,14 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { DEFAULT_WEBCAM_OVERLAY } from "../../components/video-editor/types";
const { initializeForwardFrameSourceMock, resolveMediaElementSourceMock } = vi.hoisted(() => ({
initializeForwardFrameSourceMock: vi.fn(async () => undefined),
resolveMediaElementSourceMock: vi.fn(async () => ({
src: "blob:background",
revoke: vi.fn(),
})),
}));
vi.mock("pixi.js", () => ({
Application: class {},
BlurFilter: class {},
@@ -73,11 +81,13 @@ vi.mock("@/components/video-editor/videoPlayback/cursorRenderer", () => ({
}));
vi.mock("./forwardFrameSource", () => ({
ForwardFrameSource: class {},
ForwardFrameSource: class {
initialize = initializeForwardFrameSourceMock;
},
}));
vi.mock("./localMediaSource", () => ({
resolveMediaElementSource: vi.fn(async () => null),
resolveMediaElementSource: resolveMediaElementSourceMock,
}));
vi.mock("./annotationRenderer", () => ({
@@ -186,4 +196,31 @@ describe("ModernFrameRenderer blur export path", () => {
expect(renderer.getCanvas()).not.toBe(sourceCanvas);
expect(renderer.capturePixelsForNativeExport()).not.toBeNull();
});
it("prefers decoder-backed video wallpapers during export", async () => {
const renderer = new FrameRenderer({
width: 1920,
height: 1080,
nativeReadbackMode: "pixels",
wallpaper: "/wallpapers/wispysky.mp4",
zoomRegions: [],
showShadow: false,
shadowIntensity: 0,
backgroundBlur: 0,
cropRegion: { x: 0, y: 0, width: 1, height: 1 },
webcam: {
...DEFAULT_WEBCAM_OVERLAY,
enabled: false,
},
videoWidth: 1920,
videoHeight: 1080,
}) as any;
await renderer.setupBackground();
expect(initializeForwardFrameSourceMock).toHaveBeenCalledWith("wallpapers/wispysky.mp4");
expect(resolveMediaElementSourceMock).not.toHaveBeenCalled();
expect(renderer.backgroundForwardFrameSource).toBeTruthy();
expect(renderer.backgroundVideoElement).toBeNull();
});
});
src/lib/exporter/modernFrameRenderer.ts
+17 -7
View File
@@ -24,7 +24,6 @@ import type {
ZoomTransitionEasing,
} from "@/components/video-editor/types";
import { getDefaultCaptionFontFamily, ZOOM_DEPTH_SCALES } from "@/components/video-editor/types";
import { computePaddedLayout } from "@/components/video-editor/videoPlayback/layoutUtils";
import { DEFAULT_FOCUS } from "@/components/video-editor/videoPlayback/constants";
import {
type CursorFollowCameraState,
@@ -37,6 +36,7 @@ import {
PixiCursorOverlay,
preloadCursorAssets,
} from "@/components/video-editor/videoPlayback/cursorRenderer";
import { computePaddedLayout } from "@/components/video-editor/videoPlayback/layoutUtils";
import {
createSpringState,
getZoomSpringConfig,
@@ -44,6 +44,7 @@ import {
type SpringState,
stepSpringValue,
} from "@/components/video-editor/videoPlayback/motionSmoothing";
import { getWebcamMediaTargetTimeSeconds } from "@/components/video-editor/videoPlayback/webcamSync";
import { findDominantRegion } from "@/components/video-editor/videoPlayback/zoomRegionUtils";
import {
applyZoomTransform,
@@ -56,12 +57,7 @@ import {
getWebcamOverlayPosition,
getWebcamOverlaySizePx,
} from "@/components/video-editor/webcamOverlay";
import { getWebcamMediaTargetTimeSeconds } from "@/components/video-editor/videoPlayback/webcamSync";
import {
getAssetPath,
getExportableVideoUrl,
getRenderableAssetUrl,
} from "@/lib/assetPath";
import { getAssetPath, getExportableVideoUrl, getRenderableAssetUrl } from "@/lib/assetPath";
import { extensionHost } from "@/lib/extensions";
import {
mapCursorToCanvasNormalized,
@@ -863,6 +859,20 @@ export class FrameRenderer {
videoSrc = await getAssetPath(wallpaper.replace(/^\//, ""));
}
try {
const frameSource = new ForwardFrameSource();
await frameSource.initialize(videoSrc);
this.backgroundForwardFrameSource = frameSource;
this.backgroundVideoElement = null;
this.backgroundSeekPromise = null;
return;
} catch (error) {
console.warn(
"[FrameRenderer] Decoder-backed video wallpaper unavailable during export; falling back to media element sync:",
error,
);
}
const backgroundSource = await resolveMediaElementSource(videoSrc);
this.cleanupBackgroundSource = backgroundSource.revoke;