fix(export): stabilize Lightning webcam rendering and round Windows icon
|
Before Width: | Height: | Size: 226 KiB After Width: | Height: | Size: 108 KiB |
|
Before Width: | Height: | Size: 13 KiB After Width: | Height: | Size: 12 KiB |
|
Before Width: | Height: | Size: 740 B After Width: | Height: | Size: 795 B |
|
Before Width: | Height: | Size: 1.2 KiB After Width: | Height: | Size: 1.3 KiB |
|
Before Width: | Height: | Size: 34 KiB After Width: | Height: | Size: 26 KiB |
|
Before Width: | Height: | Size: 1.9 KiB After Width: | Height: | Size: 1.9 KiB |
|
Before Width: | Height: | Size: 3.4 KiB After Width: | Height: | Size: 3.2 KiB |
|
Before Width: | Height: | Size: 86 KiB After Width: | Height: | Size: 62 KiB |
|
Before Width: | Height: | Size: 5.1 KiB After Width: | Height: | Size: 4.8 KiB |
|
Before Width: | Height: | Size: 264 KiB After Width: | Height: | Size: 50 KiB |
@@ -304,6 +304,28 @@ describe("ModernFrameRenderer blur export path", () => {
|
||||
});
|
||||
|
||||
describe("ModernFrameRenderer webcam frame cache", () => {
|
||||
it("stages webcam video frames on WebGPU instead of using retained frame uploads", () => {
|
||||
const renderer = createRenderer() as any;
|
||||
renderer.rendererBackend = "webgpu";
|
||||
|
||||
const frame = {
|
||||
displayWidth: 320,
|
||||
displayHeight: 180,
|
||||
timestamp: 0,
|
||||
} as VideoFrame;
|
||||
|
||||
const result = renderer.stageVideoFrameForTexture(frame, "webcam", 640, 360);
|
||||
|
||||
expect(result).toBe(renderer.webcamVideoFrameStagingCanvas);
|
||||
expect(renderer.webcamVideoFrameStagingCtx.drawImage).toHaveBeenCalledWith(
|
||||
frame,
|
||||
0,
|
||||
0,
|
||||
320,
|
||||
180,
|
||||
);
|
||||
});
|
||||
|
||||
it("uses staging canvas instead of recursing when WebGPU frame retention fails", () => {
|
||||
const renderer = createRenderer() as any;
|
||||
const originalVideoFrame = (globalThis as any).VideoFrame;
|
||||
@@ -355,7 +377,12 @@ describe("ModernFrameRenderer webcam frame cache", () => {
|
||||
it("bypasses the refresh throttle for cropped webcam regions", () => {
|
||||
const renderer = createRenderer() as any;
|
||||
|
||||
renderer.config.webcam.cropRegion = { x: 0.25, y: 0, width: 0.5, height: 1 };
|
||||
renderer.config.webcam.cropRegion = {
|
||||
x: 0.25,
|
||||
y: 0,
|
||||
width: 0.5,
|
||||
height: 1,
|
||||
};
|
||||
renderer.webcamFrameCacheCanvas = { width: 640, height: 720 };
|
||||
renderer.lastWebcamCacheRefreshTime = 10;
|
||||
renderer.currentVideoTime = 10.1;
|
||||
@@ -506,4 +533,221 @@ describe("ModernFrameRenderer webcam export fallback", () => {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps the webcam live when sync uses an offset timeline", () => {
|
||||
const renderer = createRenderer() as any;
|
||||
renderer.config.webcam = {
|
||||
...DEFAULT_WEBCAM_OVERLAY,
|
||||
enabled: true,
|
||||
timeOffsetMs: 500,
|
||||
};
|
||||
renderer.currentVideoTime = 10;
|
||||
renderer.lastSyncedWebcamTime = 9.5;
|
||||
renderer.webcamVideoElement = {
|
||||
readyState: 2,
|
||||
seeking: false,
|
||||
videoWidth: 640,
|
||||
videoHeight: 360,
|
||||
duration: Number.NaN,
|
||||
};
|
||||
renderer.webcamRootContainer = {
|
||||
visible: false,
|
||||
position: { set: vi.fn() },
|
||||
};
|
||||
renderer.webcamContainer = {
|
||||
addChildAt: vi.fn(),
|
||||
};
|
||||
renderer.webcamMaskGraphics = {
|
||||
clear: vi.fn(),
|
||||
moveTo: vi.fn(),
|
||||
lineTo: vi.fn(),
|
||||
closePath: vi.fn(),
|
||||
fill: vi.fn(),
|
||||
};
|
||||
renderer.webcamShadowLayers = [];
|
||||
renderer.animationState = {
|
||||
appliedScale: 1,
|
||||
};
|
||||
|
||||
renderer.updateWebcamOverlay();
|
||||
|
||||
expect(renderer.webcamRootContainer.visible).toBe(true);
|
||||
expect(renderer.webcamSprite).toBeTruthy();
|
||||
});
|
||||
|
||||
it("keeps the webcam live when the media element time is current but lastSyncedWebcamTime is stale", () => {
|
||||
const renderer = createRenderer() as any;
|
||||
renderer.config.webcam = {
|
||||
...DEFAULT_WEBCAM_OVERLAY,
|
||||
enabled: true,
|
||||
timeOffsetMs: 500,
|
||||
};
|
||||
renderer.currentVideoTime = 10;
|
||||
renderer.lastSyncedWebcamTime = 8;
|
||||
renderer.webcamVideoElement = {
|
||||
currentTime: 9.5,
|
||||
readyState: 2,
|
||||
seeking: false,
|
||||
videoWidth: 640,
|
||||
videoHeight: 360,
|
||||
duration: Number.NaN,
|
||||
};
|
||||
renderer.webcamRootContainer = {
|
||||
visible: false,
|
||||
position: { set: vi.fn() },
|
||||
};
|
||||
renderer.webcamContainer = {
|
||||
addChildAt: vi.fn(),
|
||||
};
|
||||
renderer.webcamMaskGraphics = {
|
||||
clear: vi.fn(),
|
||||
moveTo: vi.fn(),
|
||||
lineTo: vi.fn(),
|
||||
closePath: vi.fn(),
|
||||
fill: vi.fn(),
|
||||
};
|
||||
renderer.webcamShadowLayers = [];
|
||||
renderer.animationState = {
|
||||
appliedScale: 1,
|
||||
};
|
||||
|
||||
renderer.updateWebcamOverlay();
|
||||
|
||||
expect(renderer.webcamRootContainer.visible).toBe(true);
|
||||
expect(renderer.webcamSprite).toBeTruthy();
|
||||
});
|
||||
|
||||
it("snapshots media-element webcam frames into the cache before rendering", () => {
|
||||
const renderer = createRenderer() as any;
|
||||
renderer.config.webcam = {
|
||||
...DEFAULT_WEBCAM_OVERLAY,
|
||||
enabled: true,
|
||||
cropRegion: { x: 0, y: 0, width: 1, height: 1 },
|
||||
};
|
||||
renderer.currentVideoTime = 4;
|
||||
const previousHtmlVideoElement = (
|
||||
globalThis as typeof globalThis & { HTMLVideoElement?: unknown }
|
||||
).HTMLVideoElement;
|
||||
class MockHtmlVideoElement {
|
||||
currentTime = 4;
|
||||
readyState = 2;
|
||||
seeking = false;
|
||||
videoWidth = 640;
|
||||
videoHeight = 360;
|
||||
duration = Number.NaN;
|
||||
}
|
||||
Object.assign(globalThis, {
|
||||
HTMLVideoElement: MockHtmlVideoElement,
|
||||
});
|
||||
|
||||
const webcamVideoElement = new MockHtmlVideoElement();
|
||||
|
||||
try {
|
||||
const renderableSource = renderer.resolveRenderableWebcamSource(
|
||||
webcamVideoElement,
|
||||
640,
|
||||
360,
|
||||
true,
|
||||
);
|
||||
|
||||
expect(renderableSource?.source).toBe(renderer.webcamFrameCacheCanvas);
|
||||
expect(renderer.webcamFrameCacheCtx.drawImage).toHaveBeenCalledWith(
|
||||
webcamVideoElement,
|
||||
0,
|
||||
0,
|
||||
640,
|
||||
360,
|
||||
0,
|
||||
0,
|
||||
640,
|
||||
360,
|
||||
);
|
||||
} finally {
|
||||
Object.assign(globalThis, {
|
||||
HTMLVideoElement: previousHtmlVideoElement,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it("renders decoder-backed webcam frames directly for the default crop region", () => {
|
||||
const renderer = createRenderer() as any;
|
||||
renderer.config.webcam = {
|
||||
...DEFAULT_WEBCAM_OVERLAY,
|
||||
enabled: true,
|
||||
cropRegion: { x: 0, y: 0, width: 1, height: 1 },
|
||||
};
|
||||
|
||||
const previousVideoFrame = (globalThis as typeof globalThis & { VideoFrame?: unknown })
|
||||
.VideoFrame;
|
||||
class MockVideoFrame {}
|
||||
Object.assign(globalThis, {
|
||||
VideoFrame: MockVideoFrame,
|
||||
});
|
||||
|
||||
try {
|
||||
const webcamFrame = new MockVideoFrame();
|
||||
const renderableSource = renderer.resolveRenderableWebcamSource(
|
||||
webcamFrame,
|
||||
640,
|
||||
360,
|
||||
true,
|
||||
);
|
||||
|
||||
expect(renderableSource?.source).toBe(webcamFrame);
|
||||
expect(renderer.webcamFrameCacheCanvas).toBeNull();
|
||||
} finally {
|
||||
Object.assign(globalThis, {
|
||||
VideoFrame: previousVideoFrame,
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("ModernFrameRenderer temporal webcam sync", () => {
|
||||
it("pins webcam sync to the frame center during temporal blur sampling", async () => {
|
||||
const renderer = createRenderer() as any;
|
||||
renderer.config.annotationRegions = [];
|
||||
renderer.config.zoomTemporalMotionBlur = 1;
|
||||
renderer.config.zoomMotionBlurSampleCount = 3;
|
||||
renderer.config.zoomMotionBlurShutterFraction = 0.5;
|
||||
renderer.app = { canvas: createMockCanvas() };
|
||||
renderer.updateCaptionLayer = vi.fn();
|
||||
renderer.renderSceneSample = vi.fn(async (sampleTimestamp: number) => ({
|
||||
timeMs: sampleTimestamp / 1000,
|
||||
cursorTimeMs: sampleTimestamp / 1000,
|
||||
backgroundTimelineTimeMs: sampleTimestamp / 1000,
|
||||
sceneTransform: { scale: 1, x: 0, y: 0 },
|
||||
zoom: { scale: 1, focusX: 0.5, focusY: 0.5, progress: 0 },
|
||||
}));
|
||||
|
||||
await renderer.renderTemporalMotionBlurFrame(
|
||||
1_000_000,
|
||||
1_000_000,
|
||||
1_000_000,
|
||||
33_333,
|
||||
{
|
||||
stageSize: { width: 1920, height: 1080 },
|
||||
videoSize: { width: 1920, height: 1080 },
|
||||
baseScale: 1,
|
||||
baseOffset: { x: 0, y: 0 },
|
||||
maskRect: {
|
||||
x: 0,
|
||||
y: 0,
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
sourceCrop: { x: 0, y: 0, width: 1, height: 1 },
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
expect(renderer.renderSceneSample).toHaveBeenCalledTimes(3);
|
||||
expect(renderer.renderSceneSample.mock.calls.map((call: unknown[]) => call[6])).toEqual([
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
]);
|
||||
expect(
|
||||
new Set(renderer.renderSceneSample.mock.calls.map((call: unknown[]) => call[0])).size,
|
||||
).toBeGreaterThan(1);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -450,9 +450,15 @@ export class FrameRenderer {
|
||||
private captionRenderKey: string | null = null;
|
||||
private frameSprite: Sprite | null = null;
|
||||
private frameImage: HTMLImageElement | null = null;
|
||||
private frameDraw: ((ctx: CanvasRenderingContext2D, width: number, height: number) => void) | null =
|
||||
null;
|
||||
private frameInsets: { top: number; right: number; bottom: number; left: number } | null = null;
|
||||
private frameDraw:
|
||||
| ((ctx: CanvasRenderingContext2D, width: number, height: number) => void)
|
||||
| null = null;
|
||||
private frameInsets: {
|
||||
top: number;
|
||||
right: number;
|
||||
bottom: number;
|
||||
left: number;
|
||||
} | null = null;
|
||||
private frameRasterCanvas: HTMLCanvasElement | null = null;
|
||||
private frameRasterWidth = 0;
|
||||
private frameRasterHeight = 0;
|
||||
@@ -534,7 +540,9 @@ export class FrameRenderer {
|
||||
canvas.height = this.config.height;
|
||||
|
||||
try {
|
||||
const exportCanvas = canvas as HTMLCanvasElement & { colorSpace?: string };
|
||||
const exportCanvas = canvas as HTMLCanvasElement & {
|
||||
colorSpace?: string;
|
||||
};
|
||||
if ("colorSpace" in exportCanvas) {
|
||||
exportCanvas.colorSpace = "srgb";
|
||||
}
|
||||
@@ -628,7 +636,10 @@ export class FrameRenderer {
|
||||
this.setupCaptionResources();
|
||||
|
||||
if (this.shouldUseZoomMotionBlur()) {
|
||||
this.zoomBlurFilter = new ZoomBlurFilter({ strength: 0, maxKernelSize: 13 });
|
||||
this.zoomBlurFilter = new ZoomBlurFilter({
|
||||
strength: 0,
|
||||
maxKernelSize: 13,
|
||||
});
|
||||
this.motionBlurFilter = new MotionBlurFilter([0, 0], 5, 0);
|
||||
}
|
||||
|
||||
@@ -732,7 +743,11 @@ export class FrameRenderer {
|
||||
|
||||
private createShadowLayers(
|
||||
parent: Container,
|
||||
configs: ReadonlyArray<{ offsetScale: number; alphaScale: number; blurScale: number }>,
|
||||
configs: ReadonlyArray<{
|
||||
offsetScale: number;
|
||||
alphaScale: number;
|
||||
blurScale: number;
|
||||
}>,
|
||||
): ShadowLayer[] {
|
||||
return configs.map((config) => {
|
||||
const container = new Container();
|
||||
@@ -1050,6 +1065,13 @@ export class FrameRenderer {
|
||||
fallbackWidth: number,
|
||||
fallbackHeight: number,
|
||||
): CanvasImageSource | VideoFrame {
|
||||
// Keep webcam uploads on the older canvas-staged path. The newer
|
||||
// retained-VideoFrame upload path is fine for the main scene/background,
|
||||
// but it has produced unstable webcam overlays in Lightning exports.
|
||||
if (kind === "webcam") {
|
||||
return this.stageVideoFrameOnCanvas(frame, kind, fallbackWidth, fallbackHeight);
|
||||
}
|
||||
|
||||
if (this.rendererBackend === "webgpu") {
|
||||
return this.resolveRetainedVideoFrameSource(frame, kind, fallbackWidth, fallbackHeight);
|
||||
}
|
||||
@@ -2004,8 +2026,12 @@ export class FrameRenderer {
|
||||
}
|
||||
};
|
||||
|
||||
video.addEventListener("seeked", waitForPresentedFrame, { once: true });
|
||||
video.addEventListener("loadeddata", handleMediaReady, { once: true });
|
||||
video.addEventListener("seeked", waitForPresentedFrame, {
|
||||
once: true,
|
||||
});
|
||||
video.addEventListener("loadeddata", handleMediaReady, {
|
||||
once: true,
|
||||
});
|
||||
video.addEventListener("canplay", handleMediaReady, { once: true });
|
||||
video.addEventListener("error", finish, { once: true });
|
||||
|
||||
@@ -2111,7 +2137,11 @@ export class FrameRenderer {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
image.onload = () => resolve();
|
||||
image.onerror = () =>
|
||||
reject(new Error(`[ModernFrameRenderer] Failed to load device frame image: ${frameId}`));
|
||||
reject(
|
||||
new Error(
|
||||
`[ModernFrameRenderer] Failed to load device frame image: ${frameId}`,
|
||||
),
|
||||
);
|
||||
image.src = frame.filePath;
|
||||
});
|
||||
this.frameImage = image;
|
||||
@@ -2474,6 +2504,7 @@ export class FrameRenderer {
|
||||
source: CanvasImageSource | VideoFrame,
|
||||
width: number,
|
||||
height: number,
|
||||
referenceTimeSeconds = this.currentVideoTime,
|
||||
): boolean {
|
||||
const sourceRect = getWebcamCropSourceRect(this.config.webcam?.cropRegion, width, height);
|
||||
if (!this.ensureWebcamFrameCache(sourceRect.sw, sourceRect.sh)) {
|
||||
@@ -2501,7 +2532,7 @@ export class FrameRenderer {
|
||||
this.webcamFrameCacheCanvas.width,
|
||||
this.webcamFrameCacheCanvas.height,
|
||||
);
|
||||
this.lastWebcamCacheRefreshTime = this.currentVideoTime;
|
||||
this.lastWebcamCacheRefreshTime = referenceTimeSeconds;
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -2527,18 +2558,29 @@ export class FrameRenderer {
|
||||
liveSourceWidth: number,
|
||||
liveSourceHeight: number,
|
||||
canUseLiveSource: boolean,
|
||||
referenceTimeSeconds = this.currentVideoTime,
|
||||
): WebcamRenderSource | null {
|
||||
if (canUseLiveSource && liveSource && liveSourceWidth > 0 && liveSourceHeight > 0) {
|
||||
if (this.shouldRefreshWebcamFrameCache(liveSourceWidth, liveSourceHeight)) {
|
||||
this.refreshWebcamFrameCache(liveSource, liveSourceWidth, liveSourceHeight);
|
||||
}
|
||||
if (!isWebcamCropRegionDefault(this.config.webcam?.cropRegion)) {
|
||||
const usesDefaultCropRegion = isWebcamCropRegionDefault(this.config.webcam?.cropRegion);
|
||||
const needsCacheBackedSource =
|
||||
!usesDefaultCropRegion ||
|
||||
(typeof HTMLVideoElement !== "undefined" &&
|
||||
liveSource instanceof HTMLVideoElement);
|
||||
|
||||
if (needsCacheBackedSource) {
|
||||
this.refreshWebcamFrameCache(
|
||||
liveSource,
|
||||
liveSourceWidth,
|
||||
liveSourceHeight,
|
||||
referenceTimeSeconds,
|
||||
);
|
||||
const cachedSource = this.getCachedWebcamRenderSource();
|
||||
if (cachedSource) {
|
||||
this.setWebcamRenderMode("live");
|
||||
return cachedSource;
|
||||
}
|
||||
}
|
||||
|
||||
this.setWebcamRenderMode("live");
|
||||
return {
|
||||
source: liveSource,
|
||||
@@ -2636,14 +2678,18 @@ export class FrameRenderer {
|
||||
this.webcamLayoutCache = { ...nextLayout };
|
||||
}
|
||||
|
||||
private async syncWebcamFrame(targetTime: number): Promise<void> {
|
||||
const webcamTargetTime = getWebcamMediaTargetTimeSeconds({
|
||||
private getExpectedWebcamTargetTimeSeconds(targetTime: number): number {
|
||||
return getWebcamMediaTargetTimeSeconds({
|
||||
currentTime: targetTime,
|
||||
webcamDuration: Number.isFinite(this.webcamVideoElement?.duration)
|
||||
? this.webcamVideoElement?.duration
|
||||
: null,
|
||||
timeOffsetMs: this.config.webcam?.timeOffsetMs,
|
||||
});
|
||||
}
|
||||
|
||||
private async syncWebcamFrame(targetTime: number): Promise<void> {
|
||||
const webcamTargetTime = this.getExpectedWebcamTargetTimeSeconds(targetTime);
|
||||
|
||||
if (this.webcamForwardFrameSource) {
|
||||
const clampedTime = clampMediaTimeToDuration(webcamTargetTime, null);
|
||||
@@ -2779,8 +2825,12 @@ export class FrameRenderer {
|
||||
}
|
||||
};
|
||||
|
||||
webcamVideo.addEventListener("seeked", waitForPresentedFrame, { once: true });
|
||||
webcamVideo.addEventListener("loadeddata", handleMediaReady, { once: true });
|
||||
webcamVideo.addEventListener("seeked", waitForPresentedFrame, {
|
||||
once: true,
|
||||
});
|
||||
webcamVideo.addEventListener("loadeddata", handleMediaReady, {
|
||||
once: true,
|
||||
});
|
||||
webcamVideo.addEventListener("canplay", handleMediaReady, { once: true });
|
||||
webcamVideo.addEventListener("error", finish, { once: true });
|
||||
|
||||
@@ -2811,7 +2861,7 @@ export class FrameRenderer {
|
||||
}
|
||||
}
|
||||
|
||||
private updateWebcamOverlay(): void {
|
||||
private updateWebcamOverlay(referenceTimeSeconds = this.currentVideoTime): void {
|
||||
const webcam = this.config.webcam;
|
||||
if (!webcam?.enabled || !this.webcamRootContainer || !this.webcamMaskGraphics) {
|
||||
if (this.webcamRootContainer) {
|
||||
@@ -2828,10 +2878,16 @@ export class FrameRenderer {
|
||||
: { width: 0, height: 0 };
|
||||
const activeWebcamVideoElement =
|
||||
webcamSource === this.webcamVideoElement ? this.webcamVideoElement : null;
|
||||
const expectedWebcamTargetTime =
|
||||
this.getExpectedWebcamTargetTimeSeconds(referenceTimeSeconds);
|
||||
const measuredWebcamTime =
|
||||
activeWebcamVideoElement && Number.isFinite(activeWebcamVideoElement.currentTime)
|
||||
? activeWebcamVideoElement.currentTime
|
||||
: this.lastSyncedWebcamTime;
|
||||
const webcamTimeDrift =
|
||||
this.lastSyncedWebcamTime === null
|
||||
measuredWebcamTime === null
|
||||
? 0
|
||||
: Math.abs(this.lastSyncedWebcamTime - this.currentVideoTime);
|
||||
: Math.abs(measuredWebcamTime - expectedWebcamTargetTime);
|
||||
const canUseLiveSource =
|
||||
!!webcamSource &&
|
||||
liveSourceDimensions.width > 0 &&
|
||||
@@ -2845,6 +2901,7 @@ export class FrameRenderer {
|
||||
liveSourceDimensions.width,
|
||||
liveSourceDimensions.height,
|
||||
canUseLiveSource,
|
||||
referenceTimeSeconds,
|
||||
);
|
||||
|
||||
if (!renderableWebcamSource) {
|
||||
@@ -2910,15 +2967,20 @@ export class FrameRenderer {
|
||||
layoutCache: LayoutCache,
|
||||
useVelocityMotionBlur: boolean,
|
||||
includeOverlayLayers = true,
|
||||
webcamTimeSecondsOverride?: number,
|
||||
): Promise<RenderSnapshot> {
|
||||
if (!this.app || !this.cameraContainer || !this.videoMaskGraphics) {
|
||||
throw new Error("Renderer not initialized");
|
||||
}
|
||||
|
||||
this.currentVideoTime = timestamp / 1_000_000;
|
||||
const webcamRenderTimeSeconds = Math.max(
|
||||
0,
|
||||
webcamTimeSecondsOverride ?? this.currentVideoTime,
|
||||
);
|
||||
|
||||
if (this.webcamForwardFrameSource || this.webcamVideoElement) {
|
||||
await this.syncWebcamFrame(Math.max(0, this.currentVideoTime));
|
||||
await this.syncWebcamFrame(webcamRenderTimeSeconds);
|
||||
}
|
||||
|
||||
if (this.backgroundForwardFrameSource || this.backgroundVideoElement) {
|
||||
@@ -2966,7 +3028,7 @@ export class FrameRenderer {
|
||||
this.updateAnnotationLayer(timeMs);
|
||||
this.updateCaptionLayer(timeMs);
|
||||
}
|
||||
this.updateWebcamOverlay();
|
||||
this.updateWebcamOverlay(webcamRenderTimeSeconds);
|
||||
|
||||
const annotationContainerVisible = this.annotationContainer?.visible ?? true;
|
||||
const captionContainerVisible = this.captionContainer?.visible ?? true;
|
||||
@@ -3033,6 +3095,7 @@ export class FrameRenderer {
|
||||
}
|
||||
|
||||
const samplePlan = buildTemporalSamplePlanUs(frameDurationUs, blurConfig);
|
||||
const webcamReferenceTimeSeconds = Math.max(0, timestamp / 1_000_000);
|
||||
|
||||
compositeState.context.clearRect(
|
||||
0,
|
||||
@@ -3058,6 +3121,7 @@ export class FrameRenderer {
|
||||
layoutCache,
|
||||
false,
|
||||
false,
|
||||
webcamReferenceTimeSeconds,
|
||||
);
|
||||
lastSnapshot = snapshot;
|
||||
if (Math.abs(sampleOffsetUs) < 0.0001) {
|
||||
|
||||
@@ -117,4 +117,57 @@ describe("ModernVideoExporter native fallback routing", () => {
|
||||
expect(initializeEncoder).toHaveBeenCalledTimes(1);
|
||||
expect(mocks.muxerFinalize).toHaveBeenCalledTimes(1);
|
||||
}, 15_000);
|
||||
|
||||
it("retries the main decode path once with a readable file-backed source", async () => {
|
||||
const { ModernVideoExporter } = await import("./modernVideoExporter");
|
||||
mocks.streamingDecoderGetEffectiveDuration.mockReturnValue(1);
|
||||
mocks.streamingDecoderDecodeAll
|
||||
.mockRejectedValueOnce(
|
||||
new Error("readAVPacket pipeline failed: Failed after 3 attempts"),
|
||||
)
|
||||
.mockResolvedValueOnce(undefined);
|
||||
|
||||
const exporter = new ModernVideoExporter({
|
||||
videoUrl: "file:///recording.mp4",
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
frameRate: 30,
|
||||
bitrate: 8_000_000,
|
||||
wallpaper: "#101010",
|
||||
padding: 0,
|
||||
borderRadius: 0,
|
||||
backgroundBlur: 0,
|
||||
shadowIntensity: 0,
|
||||
showShadow: false,
|
||||
cropRegion: { x: 0, y: 0, width: 1, height: 1 },
|
||||
backendPreference: "webcodecs",
|
||||
} as never) as unknown as {
|
||||
export: () => Promise<{ success: boolean; blob?: Blob; error?: string }>;
|
||||
initializeEncoder: () => Promise<unknown>;
|
||||
};
|
||||
|
||||
vi.spyOn(exporter, "initializeEncoder").mockResolvedValue({
|
||||
codec: "avc1.640034",
|
||||
hardwareAcceleration: "prefer-hardware",
|
||||
});
|
||||
|
||||
const result = await exporter.export();
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(mocks.streamingDecoderLoadMetadata).toHaveBeenCalledTimes(2);
|
||||
expect(mocks.streamingDecoderLoadMetadata.mock.calls[0]).toEqual([
|
||||
"file:///recording.mp4",
|
||||
{
|
||||
forceReadableFileSource: false,
|
||||
},
|
||||
]);
|
||||
expect(mocks.streamingDecoderLoadMetadata.mock.calls[1]).toEqual([
|
||||
"file:///recording.mp4",
|
||||
{
|
||||
forceReadableFileSource: true,
|
||||
},
|
||||
]);
|
||||
expect(mocks.streamingDecoderDecodeAll).toHaveBeenCalledTimes(2);
|
||||
expect(mocks.muxerFinalize).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
import { WebDemuxer } from "web-demuxer";
|
||||
import type { SpeedRegion, TrimRegion } from "@/components/video-editor/types";
|
||||
import { getEffectiveVideoStreamDurationSeconds } from "@/lib/mediaTiming";
|
||||
import {
|
||||
createReadableMediaResourceFile,
|
||||
resolveMediaResourceUrl,
|
||||
} from "./localMediaSource";
|
||||
import { createReadableMediaResourceFile, resolveMediaResourceUrl } from "./localMediaSource";
|
||||
|
||||
const DEFAULT_MAX_DECODE_QUEUE = 12;
|
||||
const DEFAULT_MAX_PENDING_FRAMES = 32;
|
||||
@@ -26,6 +23,10 @@ export interface DecodedVideoInfo {
|
||||
audioSampleRate?: number;
|
||||
}
|
||||
|
||||
interface StreamingVideoDecoderLoadOptions {
|
||||
forceReadableFileSource?: boolean;
|
||||
}
|
||||
|
||||
/** Decoder retains ownership of the VideoFrame and closes it after use. */
|
||||
type OnFrameCallback = (
|
||||
frame: VideoFrame,
|
||||
@@ -76,7 +77,10 @@ export class StreamingVideoDecoder {
|
||||
private readonly maxDecodeQueue: number;
|
||||
private readonly maxPendingFrames: number;
|
||||
|
||||
constructor(options?: { maxDecodeQueue?: number; maxPendingFrames?: number }) {
|
||||
constructor(options?: {
|
||||
maxDecodeQueue?: number;
|
||||
maxPendingFrames?: number;
|
||||
}) {
|
||||
this.maxDecodeQueue = Math.max(
|
||||
1,
|
||||
Math.floor(options?.maxDecodeQueue ?? DEFAULT_MAX_DECODE_QUEUE),
|
||||
@@ -87,7 +91,10 @@ export class StreamingVideoDecoder {
|
||||
);
|
||||
}
|
||||
|
||||
async loadMetadata(videoUrl: string): Promise<DecodedVideoInfo> {
|
||||
async loadMetadata(
|
||||
videoUrl: string,
|
||||
options: StreamingVideoDecoderLoadOptions = {},
|
||||
): Promise<DecodedVideoInfo> {
|
||||
if (this.decoder) {
|
||||
try {
|
||||
if (this.decoder.state === "configured") {
|
||||
@@ -119,22 +126,26 @@ export class StreamingVideoDecoder {
|
||||
};
|
||||
|
||||
let mediaInfo;
|
||||
try {
|
||||
mediaInfo = await loadMediaInfo(resourceUrl);
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
"[StreamingVideoDecoder] Direct source load failed, retrying with file fallback:",
|
||||
error,
|
||||
);
|
||||
const currentDemuxer = this.demuxer;
|
||||
if (currentDemuxer) {
|
||||
try {
|
||||
(currentDemuxer as unknown as { destroy: () => void }).destroy();
|
||||
} catch {
|
||||
// Ignore cleanup errors before fallback re-init.
|
||||
}
|
||||
}
|
||||
if (options.forceReadableFileSource) {
|
||||
mediaInfo = await loadMediaInfo(await createReadableMediaResourceFile(videoUrl));
|
||||
} else {
|
||||
try {
|
||||
mediaInfo = await loadMediaInfo(resourceUrl);
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
"[StreamingVideoDecoder] Direct source load failed, retrying with file fallback:",
|
||||
error,
|
||||
);
|
||||
const currentDemuxer = this.demuxer;
|
||||
if (currentDemuxer) {
|
||||
try {
|
||||
(currentDemuxer as unknown as { destroy: () => void }).destroy();
|
||||
} catch {
|
||||
// Ignore cleanup errors before fallback re-init.
|
||||
}
|
||||
}
|
||||
mediaInfo = await loadMediaInfo(await createReadableMediaResourceFile(videoUrl));
|
||||
}
|
||||
}
|
||||
|
||||
const videoStream = mediaInfo.streams.find((s) => s.codec_type_string === "video");
|
||||
@@ -628,7 +639,11 @@ export class StreamingVideoDecoder {
|
||||
}
|
||||
const effectiveStart = Math.max(cursor, srStart);
|
||||
if (srEnd > effectiveStart) {
|
||||
result.push({ startSec: effectiveStart, endSec: srEnd, speed: sr.speed });
|
||||
result.push({
|
||||
startSec: effectiveStart,
|
||||
endSec: srEnd,
|
||||
speed: sr.speed,
|
||||
});
|
||||
}
|
||||
cursor = Math.max(cursor, srEnd);
|
||||
}
|
||||
|
||||