lee/Recordly
1
0
Fork 0
mirror of https://github.com/webadderallorg/Recordly.git synced 2026-09-25 15:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #452 from meiiie/fix/export-background-preview-audio

Browse Source 
fix(export): recover video background fallback and smooth preview audio
This commit is contained in:
Phạm Thị Minh Hồng
2026-05-08 22:13:53 +07:00
committed by GitHub
parent d4dc0afc46 31cb818dfd
commit 144698c4b5
5 changed files with 463 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/components/video-editor/VideoEditor.tsx
+75 -56
View File
@@ -339,6 +339,11 @@ async function writeSmokeExportReport(
const SMOKE_EXPORT_READY_TIMEOUT_MS = 30_000;
const DEFAULT_MP4_EXPORT_FRAME_RATE: ExportMp4FrameRate = 30;
const SOURCE_AUDIO_FALLBACK_TOAST_ID = "source-audio-fallback-error";
const SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS = 0.18;
const SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS = 0.01;
const SOURCE_AUDIO_PREVIEW_RATE_TOLERANCE_SECONDS = 0.08;
const SOURCE_AUDIO_PREVIEW_RATE_CORRECTION_WINDOW_SECONDS = 8;
const SOURCE_AUDIO_PREVIEW_MAX_RATE_ADJUSTMENT = 0.015;
const PROJECT_AUTOSAVE_DELAY_MS = 1000;
const EXPORT_ERROR_TOAST_DURATION_MS = 20000;
@@ -439,7 +444,9 @@ function getSmokeExportConfig(search: string): SmokeExportConfig {
: enabled && params.get("smokeBackendPreference") === "breeze"
? "breeze"
: undefined,
renderBackend: enabled ? parseSmokeRenderBackend(params.get("smokeRenderBackend")) : undefined,
renderBackend: enabled
? parseSmokeRenderBackend(params.get("smokeRenderBackend"))
: undefined,
maxEncodeQueue: enabled
? parseSmokeExportNumber(params.get("smokeMaxEncodeQueue"))
: undefined,
@@ -595,9 +602,7 @@ export default function VideoEditor() {
);
const devOpenRecordingConfig = useMemo(
() =>
getDevOpenRecordingConfig(
typeof window === "undefined" ? "" : window.location.search,
),
getDevOpenRecordingConfig(typeof window === "undefined" ? "" : window.location.search),
[],
);
const [appPlatform, setAppPlatform] = useState<string>(
@@ -1730,16 +1735,16 @@ export default function VideoEditor() {
}, [activeEffectSection]);
const buildPersistedEditorState = useCallback(
(
editor: Partial<{
wallpaper: string;
shadowIntensity: number;
backgroundBlur: number;
zoomMotionBlur: number;
zoomMotionBlurTuning: ZoomMotionBlurTuning;
zoomTemporalMotionBlur: number;
zoomMotionBlurSampleCount: number | null;
zoomMotionBlurShutterFraction: number | null;
(
editor: Partial<{
wallpaper: string;
shadowIntensity: number;
backgroundBlur: number;
zoomMotionBlur: number;
zoomMotionBlurTuning: ZoomMotionBlurTuning;
zoomTemporalMotionBlur: number;
zoomMotionBlurSampleCount: number | null;
zoomMotionBlurShutterFraction: number | null;
connectZooms: boolean;
zoomInDurationMs: number;
zoomInOverlapMs: number;
@@ -2135,15 +2140,16 @@ export default function VideoEditor() {
preserveProjectPath: Boolean(path),
},
);
} else {
await window.electronAPI.setCurrentVideoPath(sourcePath, {
preserveProjectPath: Boolean(path),
});
}
const sessionResult = await window.electronAPI.getCurrentRecordingSession?.();
applySessionPresentation(sessionResult?.success ? sessionResult.session : null);
} else {
await window.electronAPI.setCurrentVideoPath(sourcePath, {
preserveProjectPath: Boolean(path),
});
applySessionPresentation(null);
}
setWallpaper(normalizedEditor.wallpaper);
setWallpaper(normalizedEditor.wallpaper);
setShadowIntensity(normalizedEditor.shadowIntensity);
setBackgroundBlur(normalizedEditor.backgroundBlur);
setZoomMotionBlur(normalizedEditor.zoomMotionBlur);
@@ -2157,10 +2163,10 @@ export default function VideoEditor() {
setZoomOutDurationMs(normalizedEditor.zoomOutDurationMs);
setConnectedZoomGapMs(normalizedEditor.connectedZoomGapMs);
setConnectedZoomDurationMs(normalizedEditor.connectedZoomDurationMs);
setZoomInEasing(normalizedEditor.zoomInEasing);
setZoomOutEasing(normalizedEditor.zoomOutEasing);
setConnectedZoomEasing(normalizedEditor.connectedZoomEasing);
setShowCursor(normalizedEditor.showCursor);
setZoomInEasing(normalizedEditor.zoomInEasing);
setZoomOutEasing(normalizedEditor.zoomOutEasing);
setConnectedZoomEasing(normalizedEditor.connectedZoomEasing);
setShowCursor(normalizedEditor.showCursor);
setLoopCursor(normalizedEditor.loopCursor);
setCursorStyle(normalizedEditor.cursorStyle);
setCursorSize(normalizedEditor.cursorSize);
@@ -2249,7 +2255,12 @@ export default function VideoEditor() {
await refreshProjectLibrary();
return true;
},
[buildPersistedEditorState, refreshProjectLibrary, syncHistoryButtons],
[
applySessionPresentation,
buildPersistedEditorState,
refreshProjectLibrary,
syncHistoryButtons,
],
);
const currentProjectSnapshot = useMemo(() => {
@@ -3907,7 +3918,7 @@ export default function VideoEditor() {
),
);
},
[applySessionPresentation],
[],
);
const handleAnnotationDelete = useCallback(
@@ -4326,7 +4337,9 @@ export default function VideoEditor() {
const previousTimelineTime = lastSourceAudioSyncTimeRef.current;
const timelineJumped =
previousTimelineTime === null || Math.abs(currentTime - previousTimelineTime) > 0.25;
const driftThreshold = isPlaying ? 0.35 : 0.01;
const driftThreshold = isPlaying
? SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS
: SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS;
for (const audio of sourceAudioElementsRef.current.values()) {
enablePitchPreservingPlayback(audio);
@@ -4354,6 +4367,9 @@ export default function VideoEditor() {
basePlaybackRate: targetPlaybackRate,
currentTime: audio.currentTime,
targetTime,
toleranceSeconds: SOURCE_AUDIO_PREVIEW_RATE_TOLERANCE_SECONDS,
correctionWindowSeconds: SOURCE_AUDIO_PREVIEW_RATE_CORRECTION_WINDOW_SECONDS,
maxAdjustment: SOURCE_AUDIO_PREVIEW_MAX_RATE_ADJUSTMENT,
});
if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) {
audio.playbackRate = syncedPlaybackRate;
@@ -4606,8 +4622,7 @@ export default function VideoEditor() {
? (smokeExportConfig.fps ?? settings.mp4FrameRate ?? mp4FrameRate)
: (settings.mp4FrameRate ?? mp4FrameRate);
const pipelineModel = smokeExportConfig.enabled
? (smokeExportConfig.pipelineModel ??
"modern")
? (smokeExportConfig.pipelineModel ?? "modern")
: (settings.pipelineModel ?? exportPipelineModel);
const useExperimentalNativeExport =
pipelineModel === "modern" &&
@@ -4615,12 +4630,12 @@ export default function VideoEditor() {
const backendPreference =
pipelineModel === "legacy"
? "webcodecs"
: useExperimentalNativeExport
? "auto"
: smokeExportConfig.enabled
? (smokeExportConfig.backendPreference ??
(smokeExportConfig.useNativeExport ? "breeze" : "webcodecs"))
: (settings.backendPreference ?? exportBackendPreference);
: useExperimentalNativeExport
? "auto"
: (settings.backendPreference ?? exportBackendPreference);
const supportedSourceDimensions =
await ensureSupportedMp4SourceDimensions(selectedMp4FrameRate);
const { width: exportWidth, height: exportHeight } =
@@ -5368,31 +5383,35 @@ export default function VideoEditor() {
? isExportPreparing
? t("editor.exportStatus.preparing", "Preparing export...")
: isExportSaving
? t("editor.exportStatus.saving", "Opening save dialog...")
: isRenderingAudio
? t("editor.exportStatus.renderingAudio", "Rendering audio {{percent}}%", {
percent: Math.round((exportProgress.audioProgress ?? 0) * 100),
})
: isExportFinalizing
? exportFormat === "mp4" && exportPipelineModel === "modern"
? isExportFinalSaveIndeterminate
? t(
"editor.exportStatus.muxingAndSaving",
"Muxing audio and saving file...",
)
: t(
"editor.exportStatus.muxingAndSavingPercent",
"Muxing and saving {{percent}}%",
{
percent: exportFinalizingPercent ?? 100,
},
)
: t("editor.exportStatus.finalizingPercent", "Finalizing {{percent}}%", {
percent: exportFinalizingPercent ?? 100,
})
: t("editor.exportStatus.completePercent", "{{percent}}% complete", {
percent: Math.round(exportProgress.percentage),
? t("editor.exportStatus.saving", "Opening save dialog...")
: isRenderingAudio
? t("editor.exportStatus.renderingAudio", "Rendering audio {{percent}}%", {
percent: Math.round((exportProgress.audioProgress ?? 0) * 100),
})
: isExportFinalizing
? exportFormat === "mp4" && exportPipelineModel === "modern"
? isExportFinalSaveIndeterminate
? t(
"editor.exportStatus.muxingAndSaving",
"Muxing audio and saving file...",
)
: t(
"editor.exportStatus.muxingAndSavingPercent",
"Muxing and saving {{percent}}%",
{
percent: exportFinalizingPercent ?? 100,
},
)
: t(
"editor.exportStatus.finalizingPercent",
"Finalizing {{percent}}%",
{
percent: exportFinalizingPercent ?? 100,
},
)
: t("editor.exportStatus.completePercent", "{{percent}}% complete", {
percent: Math.round(exportProgress.percentage),
})
: t("editor.exportStatus.preparing", "Preparing export...");
const projectBrowser = (
src/lib/exporter/frameRenderer.test.ts
+56 -1
View File
@@ -1,7 +1,16 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { DEFAULT_WEBCAM_OVERLAY } from "../../components/video-editor/types";
const { initializeForwardFrameSourceMock, resolveMediaElementSourceMock } = vi.hoisted(() => ({
const {
cancelForwardFrameSourceMock,
destroyForwardFrameSourceMock,
getForwardFrameAtTimeMock,
initializeForwardFrameSourceMock,
resolveMediaElementSourceMock,
} = vi.hoisted(() => ({
cancelForwardFrameSourceMock: vi.fn(),
destroyForwardFrameSourceMock: vi.fn(async () => undefined),
getForwardFrameAtTimeMock: vi.fn(async () => null),
initializeForwardFrameSourceMock: vi.fn(async () => undefined),
resolveMediaElementSourceMock: vi.fn(async () => ({
src: "blob:background",
@@ -68,6 +77,9 @@ vi.mock("@/components/video-editor/videoPlayback/cursorRenderer", () => ({
vi.mock("./forwardFrameSource", () => ({
ForwardFrameSource: class {
cancel = cancelForwardFrameSourceMock;
destroy = destroyForwardFrameSourceMock;
getFrameAtTime = getForwardFrameAtTimeMock;
initialize = initializeForwardFrameSourceMock;
},
}));
@@ -449,6 +461,7 @@ describe("FrameRenderer webcam export path", () => {
});
it("prefers decoder-backed sync for video wallpapers during export", async () => {
vi.clearAllMocks();
const renderer = new FrameRenderer({
width: 1920,
height: 1080,
@@ -479,4 +492,46 @@ describe("FrameRenderer webcam export path", () => {
expect(renderer.backgroundVideoElement).toBeNull();
expect(renderer.backgroundSprite).toBeTruthy();
});
it("falls back to media-element sync when video wallpaper packet streaming fails", async () => {
vi.clearAllMocks();
initializeForwardFrameSourceMock.mockResolvedValue(undefined);
getForwardFrameAtTimeMock.mockRejectedValueOnce(
new Error("readAVPacket pipeline failed: Failed after 3 attempts"),
);
resolveMediaElementSourceMock.mockResolvedValueOnce({
src: "blob:background-video",
revoke: vi.fn(),
});
const renderer = new FrameRenderer({
width: 1920,
height: 1080,
wallpaper: "/wallpapers/wispysky.mp4",
zoomRegions: [],
showShadow: false,
shadowIntensity: 0,
backgroundBlur: 0,
cropRegion: { x: 0, y: 0, width: 1, height: 1 },
webcam: {
...DEFAULT_WEBCAM_OVERLAY,
enabled: false,
},
videoWidth: 1920,
videoHeight: 1080,
}) as unknown as {
setupBackground: () => Promise<void>;
syncBackgroundFrame: (timeSeconds: number) => Promise<void>;
backgroundForwardFrameSource: unknown;
backgroundVideoElement: FakeVideoElement | null;
};
await renderer.setupBackground();
await expect(renderer.syncBackgroundFrame(1)).resolves.toBeUndefined();
expect(cancelForwardFrameSourceMock).toHaveBeenCalled();
expect(destroyForwardFrameSourceMock).toHaveBeenCalled();
expect(resolveMediaElementSourceMock).toHaveBeenCalledWith("wallpapers/wispysky.mp4");
expect(renderer.backgroundForwardFrameSource).toBeNull();
expect(renderer.backgroundVideoElement).toBeTruthy();
});
});
src/lib/exporter/frameRenderer.ts
+150 -44
View File
@@ -153,10 +153,14 @@ type PixiRendererAttempt = {
};
const PIXI_RENDERER_INIT_TIMEOUT_MS = 8_000;
const BACKGROUND_MEDIA_ELEMENT_READY_TIMEOUT_MS = 5_000;
function isCanvasRenderer(renderer: Application): boolean {
const rendererName = renderer?.renderer?.constructor?.name?.toLowerCase();
return Boolean(rendererName && (rendererName.includes("canvasrenderer") || rendererName.includes("canvas")));
return Boolean(
rendererName &&
(rendererName.includes("canvasrenderer") || rendererName.includes("canvas")),
);
}
function toErrorMessage(error: unknown): string {
@@ -357,7 +361,8 @@ export class FrameRenderer {
backend,
);
const elapsed = Math.round(
(typeof performance === "undefined" ? Date.now() : performance.now()) - initStarted,
(typeof performance === "undefined" ? Date.now() : performance.now()) -
initStarted,
);
if (isCanvasRenderer(app)) {
throw new Error(
@@ -367,9 +372,13 @@ export class FrameRenderer {
return { app, backend };
} catch (error) {
const elapsed = Math.round(
(typeof performance === "undefined" ? Date.now() : performance.now()) - initStarted,
(typeof performance === "undefined" ? Date.now() : performance.now()) -
initStarted,
);
failures.push({ backend, message: `${toErrorMessage(error)} (after ${elapsed}ms)` });
failures.push({
backend,
message: `${toErrorMessage(error)} (after ${elapsed}ms)`,
});
console.warn(
`[FrameRenderer] ${backend} renderer unavailable after ${elapsed}ms; trying next backend.`,
error,
@@ -574,44 +583,9 @@ export class FrameRenderer {
);
}
const backgroundSource = await resolveMediaElementSource(videoSrc);
this.cleanupBackgroundSource = backgroundSource.revoke;
const video = document.createElement("video");
video.muted = true;
video.loop = true;
video.playsInline = true;
video.preload = "auto";
video.src = backgroundSource.src;
video.load();
await new Promise<void>((resolve, reject) => {
const onReady = () => {
if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) {
return;
}
cleanup();
resolve();
};
const onError = () => {
cleanup();
reject(new Error(`Failed to load video wallpaper: ${wallpaper}`));
};
const cleanup = () => {
video.removeEventListener("loadeddata", onReady);
video.removeEventListener("canplay", onReady);
video.removeEventListener("error", onError);
};
video.addEventListener("loadeddata", onReady);
video.addEventListener("canplay", onReady);
video.addEventListener("error", onError);
onReady();
});
this.backgroundVideoElement = video;
this.lastSyncedBackgroundLoopTimeSec = null;
this.drawVideoFrameToBackground();
if (!(await this.loadBackgroundMediaElementSource(videoSrc, wallpaper))) {
throw new Error(`Failed to load video wallpaper: ${wallpaper}`);
}
this.backgroundSprite = bgCanvas;
return;
}
@@ -828,8 +802,20 @@ export class FrameRenderer {
}
}
const decodedFrame =
await this.backgroundForwardFrameSource.getFrameAtTime(normalizedTargetTime);
let decodedFrame: VideoFrame | null = null;
try {
decodedFrame =
await this.backgroundForwardFrameSource.getFrameAtTime(normalizedTargetTime);
} catch (error) {
console.warn(
"[FrameRenderer] Decoder-backed video wallpaper failed during export; falling back to media element sync:",
error,
);
if (await this.fallbackBackgroundForwardFrameSourceToMediaElement()) {
await this.syncBackgroundFrame(timeSeconds);
}
return;
}
const resolvedDecodedDuration =
this.backgroundForwardFrameSource.getResolvedDurationSec();
if (
@@ -865,6 +851,10 @@ export class FrameRenderer {
"[FrameRenderer] Unable to wrap looping video wallpaper at decoded EOF during export:",
error,
);
if (await this.fallbackBackgroundForwardFrameSourceToMediaElement()) {
await this.syncBackgroundFrame(timeSeconds);
}
return;
}
}
this.closeBackgroundDecodedFrame();
@@ -883,6 +873,122 @@ export class FrameRenderer {
await this.syncBackgroundVideo(timeSeconds);
}
private async fallbackBackgroundForwardFrameSourceToMediaElement(): Promise<boolean> {
const sourceUrl = this.backgroundForwardFrameSourceUrl;
this.backgroundForwardFrameSource?.cancel();
void this.backgroundForwardFrameSource?.destroy();
this.backgroundForwardFrameSource = null;
this.backgroundForwardFrameSourceUrl = null;
this.backgroundForwardFrameDurationSec = null;
this.closeBackgroundDecodedFrame();
this.lastSyncedBackgroundLoopTimeSec = null;
return sourceUrl ? this.loadBackgroundMediaElementSource(sourceUrl, sourceUrl) : false;
}
private async loadBackgroundMediaElementSource(
videoSrc: string,
errorLabel: string,
): Promise<boolean> {
if (this.backgroundVideoElement) {
try {
this.backgroundVideoElement.pause();
this.backgroundVideoElement.src = "";
this.backgroundVideoElement.load();
} catch {
// Ignore media element teardown errors during export fallback.
}
this.backgroundVideoElement = null;
}
this.backgroundSeekPromise = null;
this.cleanupBackgroundSource?.();
this.cleanupBackgroundSource = null;
let backgroundSource: Awaited<ReturnType<typeof resolveMediaElementSource>>;
try {
backgroundSource = await resolveMediaElementSource(videoSrc);
} catch (error) {
console.warn(
"[FrameRenderer] Unable to resolve video wallpaper fallback source:",
error,
);
return false;
}
this.cleanupBackgroundSource = backgroundSource.revoke;
const video = document.createElement("video");
video.muted = true;
video.loop = true;
video.playsInline = true;
video.preload = "auto";
video.src = backgroundSource.src;
video.load();
const ready = await new Promise<boolean>((resolve) => {
let settled = false;
let timeoutId: ReturnType<typeof setTimeout> | null = null;
function cleanup() {
if (timeoutId !== null) {
clearTimeout(timeoutId);
}
video.removeEventListener("loadeddata", onReady);
video.removeEventListener("canplay", onReady);
video.removeEventListener("error", onError);
}
function settle(value: boolean) {
if (settled) {
return;
}
settled = true;
cleanup();
resolve(value);
}
function onReady() {
if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) {
return;
}
settle(true);
}
function onError() {
settle(false);
}
video.addEventListener("loadeddata", onReady);
video.addEventListener("canplay", onReady);
video.addEventListener("error", onError);
timeoutId = setTimeout(() => {
console.warn(
`[FrameRenderer] Video wallpaper media element fallback did not become ready within ${BACKGROUND_MEDIA_ELEMENT_READY_TIMEOUT_MS}ms`,
);
settle(false);
}, BACKGROUND_MEDIA_ELEMENT_READY_TIMEOUT_MS);
onReady();
});
if (!ready) {
console.warn(`[FrameRenderer] Failed to load video wallpaper: ${errorLabel}`);
try {
video.pause();
video.src = "";
video.load();
} catch {
// Ignore media element teardown errors on failed fallback.
}
backgroundSource.revoke();
if (this.cleanupBackgroundSource === backgroundSource.revoke) {
this.cleanupBackgroundSource = null;
}
return false;
}
this.backgroundVideoElement = video;
this.lastSyncedBackgroundLoopTimeSec = null;
this.drawVideoFrameToBackground();
return true;
}
private async syncBackgroundVideo(timeSeconds: number): Promise<void> {
const video = this.backgroundVideoElement;
if (!video) return;
src/lib/exporter/modernFrameRenderer.test.ts
+44
View File
@@ -183,6 +183,11 @@ describe("ModernFrameRenderer blur export path", () => {
beforeEach(() => {
Object.assign(globalThis, {
window: globalThis,
requestAnimationFrame: (callback: FrameRequestCallback) => {
callback(0);
return 1;
},
cancelAnimationFrame: vi.fn(),
HTMLMediaElement: {
HAVE_CURRENT_DATA: 2,
},
@@ -232,6 +237,7 @@ describe("ModernFrameRenderer blur export path", () => {
});
it("prefers decoder-backed sync for video wallpapers during export", async () => {
vi.clearAllMocks();
const renderer = new FrameRenderer({
width: 1920,
height: 1080,
@@ -257,6 +263,44 @@ describe("ModernFrameRenderer blur export path", () => {
expect(renderer.backgroundForwardFrameSource).toBeTruthy();
expect(renderer.backgroundVideoElement).toBeNull();
});
it("falls back to media-element sync when video wallpaper packet streaming fails", async () => {
vi.clearAllMocks();
initializeForwardFrameSourceMock.mockResolvedValue(undefined);
getForwardFrameAtTimeMock.mockRejectedValueOnce(
new Error("readAVPacket pipeline failed: Failed after 3 attempts"),
);
resolveMediaElementSourceMock.mockResolvedValueOnce({
src: "blob:background-video",
revoke: vi.fn(),
});
const renderer = new FrameRenderer({
width: 1920,
height: 1080,
nativeReadbackMode: "pixels",
wallpaper: "/wallpapers/wispysky.mp4",
zoomRegions: [],
showShadow: false,
shadowIntensity: 0,
backgroundBlur: 0,
cropRegion: { x: 0, y: 0, width: 1, height: 1 },
webcam: {
...DEFAULT_WEBCAM_OVERLAY,
enabled: false,
},
videoWidth: 1920,
videoHeight: 1080,
}) as any;
await renderer.setupBackground();
await expect(renderer.syncBackgroundFrame(1)).resolves.toBeUndefined();
expect(cancelForwardFrameSourceMock).toHaveBeenCalled();
expect(destroyForwardFrameSourceMock).toHaveBeenCalled();
expect(resolveMediaElementSourceMock).toHaveBeenCalledWith("wallpapers/wispysky.mp4");
expect(renderer.backgroundForwardFrameSource).toBeNull();
expect(renderer.backgroundVideoElement).toBeTruthy();
});
});
describe("ModernFrameRenderer webcam frame cache", () => {
src/lib/exporter/modernFrameRenderer.ts
+138 -40
View File
@@ -246,6 +246,7 @@ type PixiRendererAttempt = {
const CANVAS_RENDERER_NOT_IMPLEMENTED_HINT = "CanvasRenderer is not yet implemented";
const NO_RENDERER_HINT = "no available renderer";
const PIXI_RENDERER_INIT_TIMEOUT_MS = 8_000;
const BACKGROUND_MEDIA_ELEMENT_READY_TIMEOUT_MS = 5_000;
const WEBCAM_MEDIA_ELEMENT_READY_TIMEOUT_MS = 5_000;
function isCanvasRenderer(application: Application): boolean {
@@ -1144,44 +1145,9 @@ export class FrameRenderer {
);
}
const backgroundSource = await resolveMediaElementSource(videoSrc);
this.cleanupBackgroundSource = backgroundSource.revoke;
const video = document.createElement("video");
video.muted = true;
video.loop = true;
video.playsInline = true;
video.preload = "auto";
video.src = backgroundSource.src;
video.load();
await new Promise<void>((resolve, reject) => {
const onReady = () => {
if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) {
return;
}
cleanup();
resolve();
};
const onError = () => {
cleanup();
reject(new Error(`Failed to load video wallpaper: ${wallpaper}`));
};
const cleanup = () => {
video.removeEventListener("loadeddata", onReady);
video.removeEventListener("canplay", onReady);
video.removeEventListener("error", onError);
};
video.addEventListener("loadeddata", onReady);
video.addEventListener("canplay", onReady);
video.addEventListener("error", onError);
onReady();
});
this.backgroundVideoElement = video;
this.lastSyncedBackgroundLoopTimeSec = null;
this.ensureBackgroundSprite(video, video.videoWidth, video.videoHeight);
if (!(await this.loadBackgroundMediaElementSource(videoSrc, wallpaper))) {
throw new Error(`Failed to load video wallpaper: ${wallpaper}`);
}
return;
}
@@ -1853,8 +1819,20 @@ export class FrameRenderer {
}
}
const decodedFrame =
await this.backgroundForwardFrameSource.getFrameAtTime(normalizedTargetTime);
let decodedFrame: VideoFrame | null = null;
try {
decodedFrame =
await this.backgroundForwardFrameSource.getFrameAtTime(normalizedTargetTime);
} catch (error) {
console.warn(
"[FrameRenderer] Decoder-backed video wallpaper failed during export; falling back to media element sync:",
error,
);
if (await this.fallbackBackgroundForwardFrameSourceToMediaElement()) {
await this.syncBackgroundFrame(timeSeconds);
}
return;
}
const resolvedDecodedDuration =
this.backgroundForwardFrameSource.getResolvedDurationSec();
if (
@@ -1897,6 +1875,10 @@ export class FrameRenderer {
"[FrameRenderer] Unable to wrap looping video wallpaper at decoded EOF during export:",
error,
);
if (await this.fallbackBackgroundForwardFrameSourceToMediaElement()) {
await this.syncBackgroundFrame(timeSeconds);
}
return;
}
}
this.closeBackgroundDecodedFrame();
@@ -2092,6 +2074,122 @@ export class FrameRenderer {
return getRenderableAssetUrl(wallpaperAsset);
}
private async fallbackBackgroundForwardFrameSourceToMediaElement(): Promise<boolean> {
const sourceUrl = this.backgroundForwardFrameSourceUrl;
this.backgroundForwardFrameSource?.cancel();
void this.backgroundForwardFrameSource?.destroy();
this.backgroundForwardFrameSource = null;
this.backgroundForwardFrameSourceUrl = null;
this.backgroundForwardFrameDurationSec = null;
this.closeBackgroundDecodedFrame();
this.lastSyncedBackgroundLoopTimeSec = null;
return sourceUrl ? this.loadBackgroundMediaElementSource(sourceUrl, sourceUrl) : false;
}
private async loadBackgroundMediaElementSource(
videoSrc: string,
errorLabel: string,
): Promise<boolean> {
if (this.backgroundVideoElement) {
try {
this.backgroundVideoElement.pause();
this.backgroundVideoElement.src = "";
this.backgroundVideoElement.load();
} catch {
// Ignore media element teardown errors during export fallback.
}
this.backgroundVideoElement = null;
}
this.backgroundSeekPromise = null;
this.cleanupBackgroundSource?.();
this.cleanupBackgroundSource = null;
let backgroundSource: Awaited<ReturnType<typeof resolveMediaElementSource>>;
try {
backgroundSource = await resolveMediaElementSource(videoSrc);
} catch (error) {
console.warn(
"[FrameRenderer] Unable to resolve video wallpaper fallback source:",
error,
);
return false;
}
this.cleanupBackgroundSource = backgroundSource.revoke;
const video = document.createElement("video");
video.muted = true;
video.loop = true;
video.playsInline = true;
video.preload = "auto";
video.src = backgroundSource.src;
video.load();
const ready = await new Promise<boolean>((resolve) => {
let settled = false;
let timeoutId: ReturnType<typeof setTimeout> | null = null;
function cleanup() {
if (timeoutId !== null) {
clearTimeout(timeoutId);
}
video.removeEventListener("loadeddata", onReady);
video.removeEventListener("canplay", onReady);
video.removeEventListener("error", onError);
}
function settle(value: boolean) {
if (settled) {
return;
}
settled = true;
cleanup();
resolve(value);
}
function onReady() {
if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) {
return;
}
settle(true);
}
function onError() {
settle(false);
}
video.addEventListener("loadeddata", onReady);
video.addEventListener("canplay", onReady);
video.addEventListener("error", onError);
timeoutId = setTimeout(() => {
console.warn(
`[FrameRenderer] Video wallpaper media element fallback did not become ready within ${BACKGROUND_MEDIA_ELEMENT_READY_TIMEOUT_MS}ms`,
);
settle(false);
}, BACKGROUND_MEDIA_ELEMENT_READY_TIMEOUT_MS);
onReady();
});
if (!ready) {
console.warn(`[FrameRenderer] Failed to load video wallpaper: ${errorLabel}`);
try {
video.pause();
video.src = "";
video.load();
} catch {
// Ignore media element teardown errors on failed fallback.
}
backgroundSource.revoke();
if (this.cleanupBackgroundSource === backgroundSource.revoke) {
this.cleanupBackgroundSource = null;
}
return false;
}
this.backgroundVideoElement = video;
this.lastSyncedBackgroundLoopTimeSec = null;
await this.ensureBackgroundSprite(video, video.videoWidth, video.videoHeight);
return true;
}
private disposeWebcamMediaElement(video: HTMLVideoElement): void {
try {
video.pause();