Speed up Lightning export and relax local media reads

This commit is contained in:
webadderall
2026-05-08 11:32:04 +10:00
parent cea93cb9fd
commit 25cd1f2f75
23 changed files with 689 additions and 343 deletions
+4
View File
@@ -208,6 +208,10 @@ interface Window {
sessionId: string,
frameData: Uint8Array,
) => Promise<{ success: boolean; error?: string }>;
nativeVideoExportWriteFrames: (
sessionId: string,
frameDataList: Uint8Array[],
) => Promise<{ success: boolean; error?: string }>;
nativeVideoExportFinish: (
sessionId: string,
options?: {
+18
View File
@@ -278,6 +278,24 @@ export async function enqueueNativeVideoExportFrameWrite(
await writePromise;
}
export async function enqueueNativeVideoExportFrameWrites(
session: NativeVideoExportSession,
frameDataList: Array<Uint8Array | ArrayBuffer>,
) {
const writePromise = session.writeSequence.then(async () => {
if (session.terminating) {
throw new Error("Native video export session was cancelled");
}
for (const frameData of frameDataList) {
await writeNativeVideoExportFrame(session, frameData);
}
});
session.writeSequence = writePromise.catch(() => undefined);
await writePromise;
}
export async function getAvailableNativeVideoEncoders(ffmpegPath: string) {
const { stdout } = await execFileAsync(ffmpegPath, ["-hide_banner", "-encoders"], {
timeout: 15000,
+12 -13
View File
@@ -5,9 +5,18 @@ import { pathToFileURL } from "node:url";
import { ipcMain } from "electron";
import { USER_DATA_PATH } from "../../appPaths";
import { normalizePath } from "../utils";
import { isAllowedLocalReadPath, getAssetRootPath } from "../project/manager";
import { getAssetRootPath } from "../project/manager";
export function registerAssetHandlers() {
async function resolveReadableLocalFilePath(filePath: string) {
const normalizedPath = normalizePath(filePath)
const resolvedPath = await fs.realpath(normalizedPath).catch(() => normalizedPath)
const stats = await fs.stat(resolvedPath)
if (!stats.isFile()) {
throw new Error('Path is not a readable file')
}
return normalizePath(resolvedPath)
}
// Generate a tiny thumbnail for a wallpaper image and cache it in userData.
// Returns the cached thumbnail as raw JPEG bytes for fast grid rendering.
@@ -18,13 +27,7 @@ export function registerAssetHandlers() {
ipcMain.handle('generate-wallpaper-thumbnail', async (_, filePath: string) => {
try {
const resolved = normalizePath(filePath)
// isAllowedLocalReadPath now canonicalizes via realpath internally and
// requires both the lexical and real paths to satisfy the policy, so a
// single check covers symlinks under allowed prefixes.
if (!isAllowedLocalReadPath(resolved)) {
return { success: false, error: 'Access denied' }
}
const resolved = await resolveReadableLocalFilePath(filePath)
// Deterministic cache key from file path + mtime
const stat = await fs.stat(resolved)
@@ -106,11 +109,7 @@ export function registerAssetHandlers() {
ipcMain.handle('read-local-file', async (_, filePath: string) => {
try {
const resolved = normalizePath(filePath)
if (!isAllowedLocalReadPath(resolved)) {
console.warn(`[read-local-file] Blocked read outside allowed directories: ${resolved}`)
return { success: false, error: 'Access denied: path outside allowed directories' }
}
const resolved = await resolveReadableLocalFilePath(filePath)
const data = await fs.readFile(resolved)
return { success: true, data }
+75 -1
View File
@@ -15,6 +15,7 @@ import {
} from "../export/exportStream";
import {
enqueueNativeVideoExportFrameWrite,
enqueueNativeVideoExportFrameWrites,
flushNativeVideoExportPendingWriteRequests,
getNativeVideoExportMaxQueuedWriteBytes,
getNativeVideoExportSessionError,
@@ -132,7 +133,7 @@ export function registerExportHandlers() {
inputMode,
maxQueuedWriteBytes:
inputMode === "h264-stream"
? 8 * 1024 * 1024
? 32 * 1024 * 1024
: getNativeVideoExportMaxQueuedWriteBytes(inputByteSize),
stderrOutput: "",
encoderName,
@@ -218,6 +219,79 @@ export function registerExportHandlers() {
},
);
ipcMain.on(
"native-video-export-write-frames-async",
(
event,
payload: {
sessionId: string;
requestId: number;
frameDataList: Uint8Array[];
},
) => {
const sessionId = payload?.sessionId;
const requestId = payload?.requestId;
const frameDataList = payload?.frameDataList;
if (
typeof sessionId !== "string" ||
typeof requestId !== "number" ||
!Array.isArray(frameDataList) ||
frameDataList.length === 0
) {
return;
}
const session = nativeVideoExportSessions.get(sessionId);
if (!session) {
sendNativeVideoExportWriteFrameResult(event.sender, sessionId, requestId, {
success: false,
error: "Invalid native export session",
});
return;
}
session.sender = event.sender;
session.pendingWriteRequestIds.add(requestId);
if (session.terminating) {
settleNativeVideoExportWriteFrameRequest(sessionId, session, requestId, {
success: false,
error: "Native video export session was cancelled",
});
return;
}
if (
session.inputMode !== "h264-stream" &&
frameDataList.some((frameData) => frameData.byteLength !== session.inputByteSize)
) {
settleNativeVideoExportWriteFrameRequest(sessionId, session, requestId, {
success: false,
error: "Native video export batch contained invalid frame sizes",
});
return;
}
void enqueueNativeVideoExportFrameWrites(session, frameDataList)
.then(() => {
settleNativeVideoExportWriteFrameRequest(sessionId, session, requestId, {
success: true,
});
})
.catch((error) => {
session.stdinError = error instanceof Error ? error : new Error(String(error));
settleNativeVideoExportWriteFrameRequest(sessionId, session, requestId, {
success: false,
error: getNativeVideoExportSessionError(
session,
session.stdinError.message,
),
});
});
},
);
ipcMain.on(
"native-video-export-write-frame-async",
(
+64 -2
View File
@@ -112,6 +112,7 @@ process.env.APP_ROOT = path.join(__dirname, "..");
export const VITE_DEV_SERVER_URL = process.env["VITE_DEV_SERVER_URL"];
export const MAIN_DIST = path.join(process.env.APP_ROOT, "dist-electron");
export const RENDERER_DIST = path.join(process.env.APP_ROOT, "dist");
const IS_DEV = Boolean(VITE_DEV_SERVER_URL);
process.env.VITE_PUBLIC = VITE_DEV_SERVER_URL
? path.join(process.env.APP_ROOT, "public")
@@ -125,9 +126,14 @@ let trayContextMenu: Menu | null = null;
let selectedSourceName = "";
let editorHasUnsavedChanges = false;
let isForceClosing = false;
let isCreatingMainWindow = false;
let isCreatingEditorWindow = false;
let activeUpdateNotification: Notification | null = null;
let activeUpdateNotificationKey: string | null = null;
const hasSingleInstanceLock = app.requestSingleInstanceLock();
const shouldEnforceSingleInstanceLock = !IS_DEV;
const hasSingleInstanceLock = shouldEnforceSingleInstanceLock
? app.requestSingleInstanceLock()
: true;
if (!hasSingleInstanceLock) {
app.quit();
@@ -167,6 +173,14 @@ function restoreWindowSafely(window: BrowserWindow | null) {
window.focus();
}
function getExistingEditorWindow(): BrowserWindow | null {
return (
BrowserWindow.getAllWindows().find(
(window) => !window.isDestroyed() && isEditorWindow(window),
) ?? null
);
}
// Tray Icons (lazily created after app is ready to avoid accessing Electron APIs too early)
let defaultTrayIcon: ReturnType<typeof getTrayIcon> | null = null;
let recordingTrayIcon: ReturnType<typeof getTrayIcon> | null = null;
@@ -222,7 +236,31 @@ function createWindow() {
return;
}
mainWindow = createHudOverlayWindow();
if (isCreatingMainWindow) {
return;
}
if (mainWindow && !mainWindow.isDestroyed()) {
restoreWindowSafely(mainWindow);
return;
}
const existingHudWindow = getHudOverlayWindow();
if (existingHudWindow) {
mainWindow = existingHudWindow;
restoreWindowSafely(existingHudWindow);
return;
}
isCreatingMainWindow = true;
const createdHudWindow = createHudOverlayWindow();
mainWindow = createdHudWindow;
createdHudWindow.once("closed", () => {
if (mainWindow === createdHudWindow) {
mainWindow = null;
}
});
isCreatingMainWindow = false;
}
function focusOrCreateMainWindow() {
@@ -719,6 +757,27 @@ function updateTrayMenu(recording: boolean = false) {
}
function createEditorWindowWrapper() {
const existingEditorWindow = getExistingEditorWindow();
if (existingEditorWindow) {
mainWindow = existingEditorWindow;
restoreWindowSafely(existingEditorWindow);
return existingEditorWindow;
}
if (isCreatingEditorWindow) {
const currentWindow = mainWindow;
if (currentWindow && !currentWindow.isDestroyed()) {
return currentWindow;
}
const currentEditorWindow = getExistingEditorWindow();
if (currentEditorWindow) {
mainWindow = currentEditorWindow;
return currentEditorWindow;
}
}
isCreatingEditorWindow = true;
const previousWindow = mainWindow;
if (previousWindow && !previousWindow.isDestroyed()) {
const closingEditorWindow = isEditorWindow(previousWindow);
@@ -738,6 +797,7 @@ function createEditorWindowWrapper() {
if (mainWindow === editorWindow) {
mainWindow = null;
}
isCreatingEditorWindow = false;
isForceClosing = false;
editorHasUnsavedChanges = false;
});
@@ -770,6 +830,8 @@ function createEditorWindowWrapper() {
closeEditorWindowBypassingUnsavedPrompt(editorWindow);
}
});
return editorWindow;
}
function createSourceSelectorWindowWrapper() {
+17
View File
@@ -135,6 +135,23 @@ contextBridge.exposeInMainWorld("electronAPI", {
});
});
},
nativeVideoExportWriteFrames: (sessionId: string, frameDataList: Uint8Array[]) => {
ensureNativeVideoExportWriteResultListener();
return new Promise<NativeVideoExportWriteResult>((resolve) => {
const requestId = nextNativeVideoExportWriteRequestId++;
nativeVideoExportWriteRequests.set(requestId, {
sessionId,
resolve,
});
ipcRenderer.send("native-video-export-write-frames-async", {
sessionId,
requestId,
frameDataList,
});
});
},
nativeVideoExportFinish: (
sessionId: string,
options?: {
+14 -3
View File
@@ -330,6 +330,7 @@ async function writeSmokeExportReport(
const DEFAULT_MP4_EXPORT_FRAME_RATE: ExportMp4FrameRate = 30;
const SOURCE_AUDIO_FALLBACK_TOAST_ID = "source-audio-fallback-error";
const PROJECT_AUTOSAVE_DELAY_MS = 1000;
const EXPORT_ERROR_TOAST_DURATION_MS = 20000;
function getEncodingModeBitrateMultiplier(encodingMode: ExportEncodingMode): number {
switch (encodingMode) {
@@ -352,6 +353,14 @@ function summarizeErrorMessage(message: string): string {
return firstLine ?? message;
}
function showExportErrorToast(message: string) {
const summary = summarizeErrorMessage(message);
toast.error(summary, {
description: summary === message ? undefined : message,
duration: EXPORT_ERROR_TOAST_DURATION_MS,
});
}
function cloneStructured<T>(value: T): T {
return globalThis.structuredClone(value);
}
@@ -4805,7 +4814,7 @@ export default function VideoEditor() {
});
}
setExportError(saveResult.message || "Failed to save video");
toast.error(saveResult.message || "Failed to save video");
showExportErrorToast(saveResult.message || "Failed to save video");
// Keep the pending-save entry so the user can retry without
// re-rendering. The temp file is still on disk (the main
// process only moves/deletes it on success) and the
@@ -4837,7 +4846,8 @@ export default function VideoEditor() {
});
}
setExportError(result.error || "Export failed");
toast.error(summarizeErrorMessage(result.error || "Export failed"));
showExportErrorToast(result.error || "Export failed");
keepExportDialogOpen = true;
if (smokeExportConfig.enabled) {
window.close();
return;
@@ -4866,7 +4876,8 @@ export default function VideoEditor() {
});
}
setExportError(errorMessage);
toast.error(`Export failed: ${summarizeErrorMessage(errorMessage)}`);
showExportErrorToast(`Export failed: ${errorMessage}`);
keepExportDialogOpen = true;
if (smokeExportConfig.enabled) {
window.close();
}
@@ -69,9 +69,14 @@ describe("editorPreferences", () => {
expect(DEFAULT_EDITOR_PREFERENCES.exportQuality).toBe("source");
});
it("defaults cursor preferences to macOS at 2.5x", () => {
expect(DEFAULT_EDITOR_PREFERENCES.cursorStyle).toBe("macos");
it("defaults cursor preferences to Tahoe at 2.5x with lighter sway", () => {
expect(DEFAULT_EDITOR_PREFERENCES.cursorStyle).toBe("tahoe");
expect(DEFAULT_EDITOR_PREFERENCES.cursorSize).toBe(2.5);
expect(DEFAULT_EDITOR_PREFERENCES.cursorSway).toBe(0.25);
});
it("defaults MP4 exports to the Lightning pipeline", () => {
expect(DEFAULT_EDITOR_PREFERENCES.exportPipelineModel).toBe("modern");
});
it("loads stored editor control preferences", () => {
@@ -174,7 +174,7 @@ export function normalizeExportPipelineModel(value: unknown): ExportPipelineMode
return value;
}
return "legacy";
return "modern";
}
export function normalizeExportMp4FrameRate(value: unknown): ExportMp4FrameRate {
+2 -2
View File
@@ -51,7 +51,7 @@ export interface CursorVisualSettings {
}
export type CursorStyle = "macos" | "tahoe" | "tahoe-inverted" | "dot" | "figma" | (string & {}); // extension-contributed cursor styles
export const DEFAULT_CURSOR_STYLE: CursorStyle = "macos";
export const DEFAULT_CURSOR_STYLE: CursorStyle = "tahoe";
export type EditorEffectSection =
| "scene"
@@ -100,7 +100,7 @@ export const DEFAULT_CURSOR_SMOOTHING = 0.67;
export const DEFAULT_CURSOR_MOTION_BLUR = 0.4;
export const DEFAULT_CURSOR_CLICK_BOUNCE = 2.5;
export const DEFAULT_CURSOR_CLICK_BOUNCE_DURATION = 350;
export const DEFAULT_CURSOR_SWAY = 0.4;
export const DEFAULT_CURSOR_SWAY = 0.25;
export const DEFAULT_ZOOM_SMOOTHNESS = 0.5;
export const DEFAULT_ZOOM_MOTION_BLUR = 0.35;
export interface ZoomMotionBlurTuning {
+51 -16
View File
@@ -357,6 +357,24 @@ export class AudioProcessor {
let encodeError: Error | null = null;
let muxError: Error | null = null;
let pendingMuxing = Promise.resolve();
const capacityWaiters = new Set<() => void>();
const notifyCapacityAvailable = () => {
if (capacityWaiters.size === 0) {
return;
}
const waiters = [...capacityWaiters];
capacityWaiters.clear();
for (const resolve of waiters) {
resolve();
}
};
const waitForCapacity = () =>
new Promise<void>((resolve) => {
capacityWaiters.add(resolve);
});
const failIfNeeded = () => {
if (decodeError) throw decodeError;
@@ -380,6 +398,7 @@ export class AudioProcessor {
encoder.encode(frame);
frame.close();
notifyCapacityAvailable();
}
};
@@ -416,10 +435,13 @@ export class AudioProcessor {
})
.catch((error) => {
muxError = error instanceof Error ? error : new Error(String(error));
notifyCapacityAvailable();
});
notifyCapacityAvailable();
},
error: (error: DOMException) => {
encodeError = new Error(`[AudioProcessor] Encode error: ${error.message}`);
notifyCapacityAvailable();
},
});
@@ -445,9 +467,11 @@ export class AudioProcessor {
}
pendingFrames.push(transformed);
notifyCapacityAvailable();
},
error: (error: DOMException) => {
decodeError = new Error(`[AudioProcessor] Decode error: ${error.message}`);
notifyCapacityAvailable();
},
});
decoder.configure(audioConfig);
@@ -477,7 +501,7 @@ export class AudioProcessor {
) {
failIfNeeded();
pumpEncodedFrames();
await new Promise((resolve) => setTimeout(resolve, 1));
await waitForCapacity();
}
}
@@ -489,7 +513,7 @@ export class AudioProcessor {
failIfNeeded();
pumpEncodedFrames();
if (pendingFrames.length > 0 || encoder.encodeQueueSize > 0) {
await new Promise((resolve) => setTimeout(resolve, 1));
await waitForCapacity();
}
}
@@ -502,6 +526,7 @@ export class AudioProcessor {
await pendingMuxing;
failIfNeeded();
} finally {
notifyCapacityAvailable();
if (reader) {
try {
await reader.cancel();
@@ -914,16 +939,9 @@ export class AudioProcessor {
let demuxer: WebDemuxer | null = null;
try {
const response = await fetch(source.src);
const blob = await response.blob();
const filename = url.split("/").pop() || "audio";
const file = new File([blob], filename, {
type: blob.type || "video/mp4",
});
const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href;
demuxer = new WebDemuxer({ wasmFilePath: wasmUrl });
await demuxer.load(file);
await demuxer.load(source.src);
let audioConfig: AudioDecoderConfig;
try {
@@ -939,6 +957,24 @@ export class AudioProcessor {
const channelChunks: Float32Array[][] = Array.from({ length: numChannels }, () => []);
let totalFrames = 0;
let decodeError: Error | null = null;
const decodeCapacityWaiters = new Set<() => void>();
const notifyDecodeCapacityAvailable = () => {
if (decodeCapacityWaiters.size === 0) {
return;
}
const waiters = [...decodeCapacityWaiters];
decodeCapacityWaiters.clear();
for (const resolve of waiters) {
resolve();
}
};
const waitForDecodeCapacity = () =>
new Promise<void>((resolve) => {
decodeCapacityWaiters.add(resolve);
});
const decoder = new AudioDecoder({
output: (data: AudioData) => {
@@ -986,10 +1022,12 @@ export class AudioProcessor {
totalFrames += frames;
} finally {
data.close();
notifyDecodeCapacityAvailable();
}
},
error: (err: DOMException) => {
decodeError = new Error(`Streaming audio decode error: ${err.message}`);
notifyDecodeCapacityAvailable();
},
});
@@ -1008,7 +1046,7 @@ export class AudioProcessor {
while (decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT && !this.cancelled) {
if (decodeError) throw decodeError;
await new Promise((r) => setTimeout(r, 1));
await waitForDecodeCapacity();
}
}
@@ -1017,6 +1055,7 @@ export class AudioProcessor {
}
if (decodeError) throw decodeError;
} finally {
notifyDecodeCapacityAvailable();
try {
await reader.cancel();
} catch {
@@ -1376,13 +1415,9 @@ export class AudioProcessor {
try {
const source = await resolveMediaElementSource(audioPath);
try {
const response = await fetch(source.src);
const blob = await response.blob();
const filename = audioPath.split("/").pop() || "sidecar-audio";
const file = new File([blob], filename, { type: blob.type || "audio/webm" });
const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href;
const demuxer = new WebDemuxer({ wasmFilePath: wasmUrl });
await demuxer.load(file);
await demuxer.load(source.src);
return demuxer;
} finally {
source.revoke();
+3 -3
View File
@@ -10,10 +10,10 @@ describe("backendPolicy", () => {
expect(normalizeLightningRuntimePlatform("unknown")).toBe("unknown");
});
it("keeps auto backend WebCodecs-first on every platform", () => {
expect(shouldPreferNativeAutoBackend("win32")).toBe(false);
it("prefers native auto backend on desktop platforms with the fastest native path", () => {
expect(shouldPreferNativeAutoBackend("win32")).toBe(true);
expect(shouldPreferNativeAutoBackend("linux")).toBe(false);
expect(shouldPreferNativeAutoBackend("darwin")).toBe(false);
expect(shouldPreferNativeAutoBackend("darwin")).toBe(true);
expect(shouldPreferNativeAutoBackend("unknown")).toBe(false);
});
});
+1 -1
View File
@@ -23,5 +23,5 @@ export function normalizeLightningRuntimePlatform(
}
export function shouldPreferNativeAutoBackend(_platform: LightningRuntimePlatform): boolean {
return false;
return _platform === "darwin" || _platform === "win32";
}
+8 -8
View File
@@ -50,9 +50,9 @@ describe("exportTuning", () => {
expect(webCodecsProfile.maxDecodeQueue).toBe(12);
expect(webCodecsProfile.maxPendingFrames).toBe(32);
expect(breezeProfile.name).toBe("breeze-balanced-plus");
expect(breezeProfile.maxDecodeQueue).toBe(10);
expect(breezeProfile.maxPendingFrames).toBe(24);
expect(breezeProfile.maxInFlightNativeWrites).toBe(4);
expect(breezeProfile.maxDecodeQueue).toBe(14);
expect(breezeProfile.maxPendingFrames).toBe(40);
expect(breezeProfile.maxInFlightNativeWrites).toBe(8);
});
it("falls back to conservative native settings on low-core or very heavy workloads", () => {
@@ -72,12 +72,12 @@ describe("exportTuning", () => {
});
expect(breezeLowCoreProfile.name).toBe("breeze-conservative");
expect(breezeLowCoreProfile.maxDecodeQueue).toBe(6);
expect(breezeLowCoreProfile.maxPendingFrames).toBe(12);
expect(breezeLowCoreProfile.maxInFlightNativeWrites).toBe(1);
expect(breezeLowCoreProfile.maxDecodeQueue).toBe(8);
expect(breezeLowCoreProfile.maxPendingFrames).toBe(16);
expect(breezeLowCoreProfile.maxInFlightNativeWrites).toBe(2);
expect(breezeHeavyProfile.name).toBe("breeze-conservative");
expect(breezeHeavyProfile.maxDecodeQueue).toBe(6);
expect(breezeHeavyProfile.maxPendingFrames).toBe(12);
expect(breezeHeavyProfile.maxDecodeQueue).toBe(8);
expect(breezeHeavyProfile.maxPendingFrames).toBe(16);
});
});
+9 -9
View File
@@ -130,9 +130,9 @@ export function getExportBackpressureProfile(
return {
name: "breeze-conservative",
maxEncodeQueue,
maxDecodeQueue: 6,
maxPendingFrames: 12,
maxInFlightNativeWrites: 1,
maxDecodeQueue: 8,
maxPendingFrames: 16,
maxInFlightNativeWrites: 2,
};
}
@@ -140,18 +140,18 @@ export function getExportBackpressureProfile(
return {
name: "breeze-balanced-plus",
maxEncodeQueue,
maxDecodeQueue: 10,
maxPendingFrames: 24,
maxInFlightNativeWrites: 4,
maxDecodeQueue: 14,
maxPendingFrames: 40,
maxInFlightNativeWrites: 8,
};
}
return {
name: "breeze-balanced",
maxEncodeQueue,
maxDecodeQueue: 8,
maxPendingFrames: 16,
maxInFlightNativeWrites: 2,
maxDecodeQueue: 12,
maxPendingFrames: 28,
maxInFlightNativeWrites: 4,
};
}
+56 -77
View File
@@ -1,7 +1,10 @@
import { WebDemuxer } from "web-demuxer";
import { getEffectiveVideoStreamDurationSeconds } from "@/lib/mediaTiming";
import { getDecodedFrameTimelineOffsetUs } from "./streamingDecoder";
import { getLocalFilePath } from "./localMediaSource";
import {
createReadableMediaResourceFile,
resolveMediaResourceUrl,
} from "./localMediaSource";
const DEFAULT_MAX_DECODE_QUEUE = 12;
const DEFAULT_MAX_PENDING_FRAMES = 32;
@@ -40,85 +43,36 @@ export class ForwardFrameSource {
private resolvedDecodedDurationSec: number | null = null;
private firstFrameTimestampUs: number | null = null;
private frameTimelineOffsetUs = 0;
private inferMimeType(fileName: string): string {
const extension = fileName.split(".").pop()?.toLowerCase();
switch (extension) {
case "mov":
return "video/quicktime";
case "webm":
return "video/webm";
case "mkv":
return "video/x-matroska";
case "avi":
return "video/x-msvideo";
case "mp4":
default:
return "video/mp4";
}
}
private async loadVideoFile(resourceUrl: string): Promise<File> {
const localFilePath = getLocalFilePath(resourceUrl);
const filename =
(localFilePath ?? resourceUrl).split(/[\\/]/).pop()?.split("?")[0] || "video";
if (localFilePath) {
const result = await window.electronAPI.readLocalFile(localFilePath);
if (!result.success || !result.data) {
throw new Error(result.error || "Failed to read local video file");
}
const bytes =
result.data instanceof Uint8Array ? result.data : new Uint8Array(result.data);
const arrayBuffer = bytes.buffer.slice(
bytes.byteOffset,
bytes.byteOffset + bytes.byteLength,
) as ArrayBuffer;
return new File([arrayBuffer], filename, {
type: this.inferMimeType(filename),
});
}
const response = await fetch(resourceUrl);
if (!response.ok) {
throw new Error(
`Failed to load video resource: ${response.status} ${response.statusText}`,
);
}
const blob = await response.blob();
return new File([blob], filename, {
type: blob.type || this.inferMimeType(filename),
});
}
private resolveVideoResourceUrl(videoUrl: string): string {
if (/^(blob:|data:|https?:|file:)/i.test(videoUrl)) {
return videoUrl;
}
if (/^[A-Za-z]:[\\/]/.test(videoUrl)) {
const normalized = videoUrl.replace(/\\/g, "/");
return `file:///${encodeURI(normalized)}`;
}
if (videoUrl.startsWith("/")) {
return `file://${encodeURI(videoUrl)}`;
}
return videoUrl;
}
private decodeCapacityWaiters = new Set<() => void>();
async initialize(videoUrl: string): Promise<ForwardFrameSourceMetadata> {
const resourceUrl = this.resolveVideoResourceUrl(videoUrl);
const resourceUrl = await resolveMediaResourceUrl(videoUrl);
const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href;
this.demuxer = new WebDemuxer({ wasmFilePath: wasmUrl });
const loadMediaInfo = async (source: string | File) => {
this.demuxer = new WebDemuxer({ wasmFilePath: wasmUrl });
await this.demuxer.load(source);
return this.demuxer.getMediaInfo();
};
const file = await this.loadVideoFile(resourceUrl);
await this.demuxer.load(file);
let mediaInfo;
try {
mediaInfo = await loadMediaInfo(resourceUrl);
} catch (error) {
console.warn(
"[ForwardFrameSource] Direct source load failed, retrying with file fallback:",
error,
);
const currentDemuxer = this.demuxer;
if (currentDemuxer) {
try {
(currentDemuxer as unknown as { destroy: () => void }).destroy();
} catch {
// Ignore cleanup errors before fallback re-init.
}
}
mediaInfo = await loadMediaInfo(await createReadableMediaResourceFile(videoUrl));
}
const mediaInfo = await this.demuxer.getMediaInfo();
const videoStream = mediaInfo.streams.find(
(stream) => stream.codec_type_string === "video",
);
@@ -166,6 +120,7 @@ export class ForwardFrameSource {
} else {
this.pendingFrames.push(frame);
}
this.notifyDecoderCapacityAvailable();
},
error: (error: DOMException) => {
this.decodeError = new Error(`VideoDecoder error: ${error.message}`);
@@ -174,6 +129,7 @@ export class ForwardFrameSource {
this.frameResolve = null;
resolve(null);
}
this.notifyDecoderCapacityAvailable();
},
});
@@ -214,7 +170,7 @@ export class ForwardFrameSource {
this.pendingFrames.length > DEFAULT_MAX_PENDING_FRAMES) &&
!this.cancelled
) {
await new Promise((resolve) => setTimeout(resolve, 1));
await this.waitForDecoderCapacity();
}
if (this.cancelled) {
@@ -231,6 +187,7 @@ export class ForwardFrameSource {
this.decodeError = error instanceof Error ? error : new Error(String(error));
} finally {
this.decodeDone = true;
this.notifyDecoderCapacityAvailable();
if (this.frameResolve) {
const resolve = this.frameResolve;
this.frameResolve = null;
@@ -246,7 +203,9 @@ export class ForwardFrameSource {
}
if (this.pendingFrames.length > 0) {
return Promise.resolve(this.pendingFrames.shift()!);
const frame = this.pendingFrames.shift()!;
this.notifyDecoderCapacityAvailable();
return Promise.resolve(frame);
}
if (this.decodeDone) {
@@ -262,6 +221,24 @@ export class ForwardFrameSource {
});
}
private waitForDecoderCapacity(): Promise<void> {
return new Promise((resolve) => {
this.decodeCapacityWaiters.add(resolve);
});
}
private notifyDecoderCapacityAvailable(): void {
if (this.decodeCapacityWaiters.size === 0) {
return;
}
const waiters = [...this.decodeCapacityWaiters];
this.decodeCapacityWaiters.clear();
for (const resolve of waiters) {
resolve();
}
}
async getFrameAtTime(targetTimeSec: number): Promise<VideoFrame | null> {
if (!this.metadata) {
throw new Error("Frame source not initialized");
@@ -345,6 +322,7 @@ export class ForwardFrameSource {
cancel(): void {
this.cancelled = true;
this.notifyDecoderCapacityAvailable();
if (this.frameResolve) {
const resolve = this.frameResolve;
this.frameResolve = null;
@@ -416,5 +394,6 @@ export class ForwardFrameSource {
this.resolvedDecodedDurationSec = null;
this.firstFrameTimestampUs = null;
this.frameTimelineOffsetUs = 0;
this.decodeCapacityWaiters.clear();
}
}
-2
View File
@@ -262,7 +262,6 @@ export class GifExporter {
this.config.speedRegions,
async (videoFrame, _exportTimestampUs, sourceTimestampMs, cursorTimestampMs) => {
if (this.cancelled) {
videoFrame.close();
return;
}
@@ -275,7 +274,6 @@ export class GifExporter {
frameDurationUs,
frameIndex * frameDurationUs,
);
videoFrame.close();
this.addRenderedGifFrame(frameDelay);
frameIndex++;
+22 -39
View File
@@ -2,8 +2,6 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
import { resolveMediaElementSource } from "./localMediaSource";
const NativeURL = URL;
describe("resolveMediaElementSource", () => {
beforeEach(() => {
vi.restoreAllMocks();
@@ -11,60 +9,45 @@ describe("resolveMediaElementSource", () => {
window: {
electronAPI: {
readLocalFile: vi.fn(),
getLocalMediaUrl: vi.fn(async (filePath: string) => ({
success: true,
url: `http://127.0.0.1:4321/video?path=${encodeURIComponent(filePath)}`,
})),
},
},
});
class MockURL extends NativeURL {}
Object.assign(MockURL, {
createObjectURL: vi.fn(() => "blob:mock-local-media"),
revokeObjectURL: vi.fn(),
});
vi.stubGlobal("URL", MockURL);
});
it("reads file URLs through Electron IPC and returns an object URL", async () => {
const readLocalFile = vi.fn(async () => ({
success: true,
data: new Uint8Array([1, 2, 3]),
}));
(window as any).electronAPI.readLocalFile = readLocalFile;
it("resolves file URLs through the local media server for media elements", async () => {
const result = await resolveMediaElementSource("file:///tmp/example.mp4");
expect(readLocalFile).toHaveBeenCalledWith("/tmp/example.mp4");
expect(URL.createObjectURL).toHaveBeenCalledOnce();
expect(result.src).toBe("blob:mock-local-media");
result.revoke();
expect(URL.revokeObjectURL).toHaveBeenCalledWith("blob:mock-local-media");
expect((window as any).electronAPI.readLocalFile).not.toHaveBeenCalled();
expect((window as any).electronAPI.getLocalMediaUrl).toHaveBeenCalledWith(
"/tmp/example.mp4",
);
expect(result.src).toBe("http://127.0.0.1:4321/video?path=%2Ftmp%2Fexample.mp4");
});
it("reads absolute local paths through Electron IPC and normalizes fallback URLs", async () => {
const readLocalFile = vi.fn(async () => ({
success: true,
data: new Uint8Array([4, 5, 6]),
}));
(window as any).electronAPI.readLocalFile = readLocalFile;
it("resolves absolute local paths through the local media server without copying them into blobs", async () => {
const result = await resolveMediaElementSource("/tmp/example.wav");
expect(readLocalFile).toHaveBeenCalledWith("/tmp/example.wav");
expect(result.src).toBe("blob:mock-local-media");
expect((window as any).electronAPI.readLocalFile).not.toHaveBeenCalled();
expect((window as any).electronAPI.getLocalMediaUrl).toHaveBeenCalledWith(
"/tmp/example.wav",
);
expect(result.src).toBe("http://127.0.0.1:4321/video?path=%2Ftmp%2Fexample.wav");
});
it("reads loopback media-server URLs through Electron IPC", async () => {
const readLocalFile = vi.fn(async () => ({
success: true,
data: new Uint8Array([7, 8, 9]),
}));
(window as any).electronAPI.readLocalFile = readLocalFile;
it("preserves loopback media-server URLs instead of materializing them through IPC", async () => {
const result = await resolveMediaElementSource(
"http://127.0.0.1:43123/video?path=%2Ftmp%2Fexample%20clip.mp4",
);
expect(readLocalFile).toHaveBeenCalledWith("/tmp/example clip.mp4");
expect(result.src).toBe("blob:mock-local-media");
expect((window as any).electronAPI.readLocalFile).not.toHaveBeenCalled();
expect((window as any).electronAPI.getLocalMediaUrl).not.toHaveBeenCalled();
expect(result.src).toBe(
"http://127.0.0.1:43123/video?path=%2Ftmp%2Fexample%20clip.mp4",
);
});
it("leaves remote URLs untouched", async () => {
+59 -26
View File
@@ -60,7 +60,7 @@ function isRemoteMediaResource(resource: string) {
return REMOTE_MEDIA_URL_PATTERN.test(resource) && !isLocalMediaServerUrl(resource);
}
function getNormalizedResourceUrl(resource: string) {
export function getNormalizedMediaResourceUrl(resource: string) {
const localFilePath = getLocalFilePath(resource);
if (!localFilePath) {
return resource;
@@ -92,6 +92,60 @@ function inferMimeType(filePath: string) {
return "application/octet-stream";
}
export async function resolveMediaResourceUrl(resource: string): Promise<string> {
const localFilePath = getLocalFilePath(resource);
if (!localFilePath) {
return resource;
}
if (isLocalMediaServerUrl(resource)) {
return resource;
}
if (typeof window !== "undefined" && window.electronAPI?.getLocalMediaUrl) {
try {
const result = await window.electronAPI.getLocalMediaUrl(localFilePath);
if (result.success) {
return result.url;
}
} catch {
// Fall through to a file URL when the local media server is unavailable.
}
}
return /^file:\/\//i.test(resource) ? resource : toFileUrl(localFilePath);
}
export async function createReadableMediaResourceFile(resource: string): Promise<File> {
const localFilePath = getLocalFilePath(resource);
const filename = (localFilePath ?? resource).split(/[\\/]/).pop()?.split("?")[0] || "media";
if (localFilePath && typeof window !== "undefined" && window.electronAPI?.readLocalFile) {
const result = await window.electronAPI.readLocalFile(localFilePath);
if (!result.success || !result.data) {
throw new Error(result.error || "Failed to read local media file");
}
const bytes = result.data instanceof Uint8Array ? result.data : new Uint8Array(result.data);
const arrayBuffer = bytes.buffer.slice(
bytes.byteOffset,
bytes.byteOffset + bytes.byteLength,
) as ArrayBuffer;
return new File([arrayBuffer], filename, { type: inferMimeType(filename) });
}
const resourceUrl = await resolveMediaResourceUrl(resource);
const response = await fetch(resourceUrl);
if (!response.ok) {
throw new Error(
`Failed to load media resource: ${response.status} ${response.statusText}`,
);
}
const blob = await response.blob();
return new File([blob], filename, { type: blob.type || inferMimeType(filename) });
}
export async function resolveMediaElementSource(resource: string): Promise<{
src: string;
revoke: () => void;
@@ -100,29 +154,8 @@ export async function resolveMediaElementSource(resource: string): Promise<{
return { src: resource, revoke: NOOP };
}
const normalizedResource = getNormalizedResourceUrl(resource);
const localFilePath = getLocalFilePath(resource) ?? getLocalFilePath(normalizedResource);
if (!localFilePath || typeof window === "undefined" || !window.electronAPI?.readLocalFile) {
return { src: normalizedResource, revoke: NOOP };
}
try {
const result = await window.electronAPI.readLocalFile(localFilePath);
if (!result.success || !result.data) {
return { src: normalizedResource, revoke: NOOP };
}
const bytes = result.data instanceof Uint8Array ? result.data : new Uint8Array(result.data);
const blob = new Blob([Uint8Array.from(bytes)], { type: inferMimeType(localFilePath) });
const objectUrl = URL.createObjectURL(blob);
return {
src: objectUrl,
revoke: () => {
URL.revokeObjectURL(objectUrl);
},
};
} catch {
return { src: normalizedResource, revoke: NOOP };
}
return {
src: await resolveMediaResourceUrl(resource),
revoke: NOOP,
};
}
+118 -49
View File
@@ -16,7 +16,7 @@ import type {
} from "@/components/video-editor/types";
import { extensionHost } from "@/lib/extensions";
import { AudioProcessor, isAacAudioEncodingSupported } from "./audioEncoder";
import { normalizeLightningRuntimePlatform } from "./backendPolicy";
import { normalizeLightningRuntimePlatform, shouldPreferNativeAutoBackend } from "./backendPolicy";
import { buildEditedTrackSourceSegments, classifyEditedTrackStrategy } from "./editedTrackStrategy";
import {
type ExportBackpressureProfile,
@@ -134,7 +134,9 @@ const NATIVE_EXPORT_ENGINE_NAME = "Breeze";
const LIGHTNING_PIPELINE_NAME = "Lightning (Beta)";
export class ModernVideoExporter {
private static readonly NATIVE_ENCODER_QUEUE_LIMIT = 32;
private static readonly NATIVE_ENCODER_QUEUE_LIMIT = 64;
private static readonly NATIVE_WRITE_BATCH_MAX_CHUNKS = 12;
private static readonly NATIVE_WRITE_BATCH_MAX_BYTES = 2 * 1024 * 1024;
private config: VideoExporterConfig;
private streamingDecoder: StreamingVideoDecoder | null = null;
@@ -157,9 +159,10 @@ export class ModernVideoExporter {
private encoderName: string | null = null;
private backpressureProfile: ExportBackpressureProfile | null = null;
private nativeExportSessionId: string | null = null;
private nativePendingWrite: Promise<void> = Promise.resolve();
private nativeWritePromises = new Set<Promise<void>>();
private nativeWriteError: Error | null = null;
private pendingNativeWriteChunks: Uint8Array[] = [];
private pendingNativeWriteBytes = 0;
private maxNativeWriteInFlight = 1;
private lastNativeExportError: string | null = null;
private nativeH264Encoder: VideoEncoder | null = null;
@@ -182,6 +185,7 @@ export class ModernVideoExporter {
private finalizationTimeMs = 0;
private finalizationStageMs: ExportFinalizationStageMetrics = {};
private processedFrameCount = 0;
private encodeCapacityWaiters = new Set<() => void>();
private activeFinalizationProgressWatchdog: FinalizationProgressWatchdog | null = null;
private lastFinalizationRenderProgress = INITIAL_FINALIZATION_PROGRESS_STATE.lastRenderProgress;
private lastFinalizationAudioProgress = INITIAL_FINALIZATION_PROGRESS_STATE.lastAudioProgress;
@@ -200,6 +204,7 @@ export class ModernVideoExporter {
this.nativeEncoderError = null;
this.totalExportStartTimeMs = this.getNowMs();
const backendPreference = this.config.backendPreference ?? "auto";
const runtimePlatform = this.getRuntimePlatform();
let useNativeEncoder = false;
this.lastNativeExportError = null;
@@ -213,6 +218,22 @@ export class ModernVideoExporter {
`${NATIVE_EXPORT_ENGINE_NAME} export is unavailable for this output profile on this system.`,
);
}
} else if (
backendPreference === "auto" &&
shouldPreferNativeAutoBackend(runtimePlatform)
) {
stageStartedAt = this.getNowMs();
useNativeEncoder = await this.tryStartNativeVideoExport();
this.nativeSessionStartTimeMs = this.getNowMs() - stageStartedAt;
if (!useNativeEncoder) {
console.warn(
`[VideoExporter] ${NATIVE_EXPORT_ENGINE_NAME} auto-preferred native export was unavailable; falling back to WebCodecs.`,
this.lastNativeExportError,
);
stageStartedAt = this.getNowMs();
await this.initializeEncoder();
}
} else {
try {
const configuredWebCodecsPath = await this.initializeEncoder();
@@ -307,7 +328,7 @@ export class ModernVideoExporter {
this.renderer = new ModernFrameRenderer({
width: this.config.width,
height: this.config.height,
preferredRenderBackend: useNativeEncoder ? "webgl" : undefined,
preferredRenderBackend: undefined,
wallpaper: this.config.wallpaper,
zoomRegions: this.config.zoomRegions,
showShadow: this.config.showShadow,
@@ -392,7 +413,6 @@ export class ModernVideoExporter {
async (videoFrame, _exportTimestampUs, sourceTimestampMs, cursorTimestampMs) => {
const callbackStartedAt = this.getNowMs();
if (this.cancelled) {
videoFrame.close();
return;
}
@@ -408,7 +428,6 @@ export class ModernVideoExporter {
timestamp,
);
this.renderFrameTimeMs += this.getNowMs() - renderStartedAt;
videoFrame.close();
if (this.cancelled) {
return;
@@ -609,12 +628,7 @@ export class ModernVideoExporter {
}
private getPlatformLabel(): string {
if (typeof navigator === "undefined") {
return "Unknown";
}
const platformHint = navigator.platform || navigator.userAgent || "";
switch (normalizeLightningRuntimePlatform(platformHint)) {
switch (this.getRuntimePlatform()) {
case "win32":
return "Windows";
case "linux":
@@ -622,10 +636,22 @@ export class ModernVideoExporter {
case "darwin":
return "macOS";
default:
return platformHint || "Unknown";
if (typeof navigator === "undefined") {
return "Unknown";
}
return navigator.platform || navigator.userAgent || "Unknown";
}
}
private getRuntimePlatform() {
if (typeof navigator === "undefined") {
return "unknown";
}
return normalizeLightningRuntimePlatform(navigator.platform || navigator.userAgent || "");
}
private getLightningErrorGuidance(message: string): string[] {
const guidance = new Set<string>();
const platform = this.getPlatformLabel();
@@ -948,7 +974,8 @@ export class ModernVideoExporter {
this.lastNativeExportError = null;
this.encodeBackend = "ffmpeg";
this.encoderName = "h264-stream-copy";
this.nativePendingWrite = Promise.resolve();
this.pendingNativeWriteChunks = [];
this.pendingNativeWriteBytes = 0;
const sessionId = result.sessionId;
const encoder = new VideoEncoder({
@@ -959,40 +986,11 @@ export class ModernVideoExporter {
const buffer = new ArrayBuffer(chunk.byteLength);
chunk.copyTo(buffer);
const writePromise = this.nativePendingWrite
.then(() =>
window.electronAPI.nativeVideoExportWriteFrame(
sessionId,
new Uint8Array(buffer),
),
)
.then((writeResult) => {
if (!writeResult.success && !this.cancelled) {
throw new Error(
writeResult.error ||
"Failed to write H.264 chunk to native encoder",
);
}
})
.catch((error) => {
if (!this.cancelled) {
const resolvedError =
error instanceof Error ? error : new Error(String(error));
if (!this.nativeEncoderError) {
this.nativeEncoderError = resolvedError;
}
if (!this.nativeWriteError) {
this.nativeWriteError = resolvedError;
}
}
throw error;
});
this.nativePendingWrite = writePromise;
this.trackNativeWritePromise(writePromise);
this.queueNativeWriteChunk(sessionId, new Uint8Array(buffer));
},
error: (error) => {
this.nativeEncoderError = error;
this.notifyEncodeCapacityAvailable();
},
});
@@ -1043,7 +1041,7 @@ export class ModernVideoExporter {
while (
this.nativeH264Encoder.encodeQueueSize >= ModernVideoExporter.NATIVE_ENCODER_QUEUE_LIMIT
) {
await new Promise<void>((r) => setTimeout(r, 2));
await this.waitForEncodeCapacity();
if (this.cancelled) return;
if (this.nativeEncoderError) throw this.nativeEncoderError;
}
@@ -1100,6 +1098,7 @@ export class ModernVideoExporter {
encoderName: this.encoderName ?? "unknown",
});
this.flushPendingNativeWriteBatch(sessionId);
await this.awaitPendingNativeWrites();
const result = await this.measureFinalizationStage("nativeExportFinalizeMs", async () =>
@@ -1138,7 +1137,6 @@ export class ModernVideoExporter {
this.finalizationStageMs.ffmpegAudioMuxBreakdown = result.metrics;
}
this.nativeExportSessionId = null;
this.nativePendingWrite = Promise.resolve();
if (!result.success) {
return {
@@ -1317,7 +1315,7 @@ export class ModernVideoExporter {
) {
const encodeWaitStartedAt = this.getNowMs();
this.encodeWaitEvents++;
await new Promise((resolve) => setTimeout(resolve, 2));
await this.waitForEncodeCapacity();
this.encodeWaitTimeMs += this.getNowMs() - encodeWaitStartedAt;
}
@@ -1368,6 +1366,72 @@ export class ModernVideoExporter {
this.reportProgress(totalFrames, totalFrames, "finalizing", renderProgress, audioProgress);
}
private queueNativeWriteChunk(sessionId: string, chunk: Uint8Array): void {
this.pendingNativeWriteChunks.push(chunk);
this.pendingNativeWriteBytes += chunk.byteLength;
if (
this.pendingNativeWriteChunks.length >=
ModernVideoExporter.NATIVE_WRITE_BATCH_MAX_CHUNKS ||
this.pendingNativeWriteBytes >= ModernVideoExporter.NATIVE_WRITE_BATCH_MAX_BYTES
) {
this.flushPendingNativeWriteBatch(sessionId);
}
}
private flushPendingNativeWriteBatch(sessionId: string): void {
if (this.pendingNativeWriteChunks.length === 0) {
return;
}
const chunks = this.pendingNativeWriteChunks;
this.pendingNativeWriteChunks = [];
this.pendingNativeWriteBytes = 0;
const writePromise = window.electronAPI
.nativeVideoExportWriteFrames(sessionId, chunks)
.then((writeResult) => {
if (!writeResult.success && !this.cancelled) {
throw new Error(
writeResult.error || "Failed to write H.264 chunks to native encoder",
);
}
})
.catch((error) => {
if (!this.cancelled) {
const resolvedError =
error instanceof Error ? error : new Error(String(error));
if (!this.nativeEncoderError) {
this.nativeEncoderError = resolvedError;
}
if (!this.nativeWriteError) {
this.nativeWriteError = resolvedError;
}
}
throw error;
});
this.trackNativeWritePromise(writePromise);
this.notifyEncodeCapacityAvailable();
}
private waitForEncodeCapacity(): Promise<void> {
return new Promise((resolve) => {
this.encodeCapacityWaiters.add(resolve);
});
}
private notifyEncodeCapacityAvailable(): void {
if (this.encodeCapacityWaiters.size === 0) {
return;
}
const waiters = [...this.encodeCapacityWaiters];
this.encodeCapacityWaiters.clear();
for (const resolve of waiters) {
resolve();
}
}
private reportProgress(
currentFrame: number,
totalFrames: number,
@@ -1656,6 +1720,7 @@ export class ModernVideoExporter {
}
});
this.encodeQueue--;
this.notifyEncodeCapacityAvailable();
},
error: (error) => {
console.error(
@@ -1664,6 +1729,7 @@ export class ModernVideoExporter {
);
this.encoderError = error instanceof Error ? error : new Error(String(error));
this.cancelled = true;
this.notifyEncodeCapacityAvailable();
},
});
@@ -1831,9 +1897,12 @@ export class ModernVideoExporter {
this.lastProgressSampleTimeMs = 0;
this.lastProgressSampleFrame = 0;
this.nativeWritePromises = new Set();
this.nativePendingWrite = Promise.resolve();
this.nativeWriteError = null;
this.pendingNativeWriteChunks = [];
this.pendingNativeWriteBytes = 0;
this.maxNativeWriteInFlight = 1;
this.notifyEncodeCapacityAvailable();
this.encodeCapacityWaiters.clear();
this.videoDescription = undefined;
this.videoColorSpace = undefined;
this.renderBackend = null;
+83 -11
View File
@@ -5,35 +5,107 @@ import {
StreamingVideoDecoder,
} from "./streamingDecoder";
const {
mockDemuxerLoad,
mockDemuxerGetMediaInfo,
mockDemuxerDestroy,
mockDemuxerGetDecoderConfig,
} = vi.hoisted(() => ({
mockDemuxerLoad: vi.fn(),
mockDemuxerGetMediaInfo: vi.fn(async () => ({
duration: 4,
start_time: 0,
streams: [
{
codec_type_string: "video",
width: 1920,
height: 1080,
avg_frame_rate: "30/1",
codec_string: "avc1.640034",
start_time: 0,
duration: 4,
},
],
})),
mockDemuxerDestroy: vi.fn(),
mockDemuxerGetDecoderConfig: vi.fn(),
}));
vi.mock("web-demuxer", () => ({
WebDemuxer: class MockWebDemuxer {
load = mockDemuxerLoad;
getMediaInfo = mockDemuxerGetMediaInfo;
destroy = mockDemuxerDestroy;
getDecoderConfig = mockDemuxerGetDecoderConfig;
},
}));
describe("StreamingVideoDecoder local media loading", () => {
beforeEach(() => {
vi.restoreAllMocks();
mockDemuxerLoad.mockReset();
mockDemuxerGetMediaInfo.mockClear();
mockDemuxerDestroy.mockClear();
mockDemuxerGetDecoderConfig.mockClear();
Object.assign(globalThis, {
window: {
location: {
href: "http://localhost:5173/",
},
electronAPI: {
readLocalFile: vi.fn(),
getLocalMediaUrl: vi.fn(async (filePath: string) => ({
success: true,
url: `http://127.0.0.1:4321/video?path=${encodeURIComponent(filePath)}`,
})),
},
},
});
});
it("loads loopback media-server URLs through Electron IPC instead of fetch", async () => {
const readLocalFile = vi.fn(async () => ({
it("loads loopback media-server URLs directly into WebDemuxer", async () => {
const decoder = new StreamingVideoDecoder();
await decoder.loadMetadata("http://127.0.0.1:43123/video?path=%2Ftmp%2Fcapture.mp4");
expect((window as any).electronAPI.readLocalFile).not.toHaveBeenCalled();
expect(mockDemuxerLoad).toHaveBeenCalledWith(
"http://127.0.0.1:43123/video?path=%2Ftmp%2Fcapture.mp4",
);
});
it("normalizes absolute local paths to file URLs before loading them", async () => {
const decoder = new StreamingVideoDecoder();
await decoder.loadMetadata("/tmp/capture.mp4");
expect((window as any).electronAPI.getLocalMediaUrl).toHaveBeenCalledWith(
"/tmp/capture.mp4",
);
expect(mockDemuxerLoad).toHaveBeenCalledWith(
"http://127.0.0.1:4321/video?path=%2Ftmp%2Fcapture.mp4",
);
});
it("falls back to a readable File when direct loading fails", async () => {
mockDemuxerLoad.mockReset();
mockDemuxerLoad
.mockRejectedValueOnce(new Error("get_media_info failed: Failed after 3 attempts"))
.mockResolvedValueOnce(undefined);
(window as any).electronAPI.readLocalFile = vi.fn(async () => ({
success: true,
data: new Uint8Array([1, 2, 3]),
}));
(window as any).electronAPI.readLocalFile = readLocalFile;
const fetchSpy = vi.fn();
vi.stubGlobal("fetch", fetchSpy);
const decoder = new StreamingVideoDecoder();
const file = await (decoder as any).loadVideoFile(
"http://127.0.0.1:43123/video?path=%2Ftmp%2Fcapture.mp4",
);
await decoder.loadMetadata("/tmp/fallback.mp4");
expect(readLocalFile).toHaveBeenCalledWith("/tmp/capture.mp4");
expect(fetchSpy).not.toHaveBeenCalled();
expect(file.name).toBe("capture.mp4");
expect(mockDemuxerLoad).toHaveBeenNthCalledWith(
1,
"http://127.0.0.1:4321/video?path=%2Ftmp%2Ffallback.mp4",
);
expect(mockDemuxerLoad.mock.calls[1]?.[0]).toBeInstanceOf(File);
expect((window as any).electronAPI.readLocalFile).toHaveBeenCalledWith(
"/tmp/fallback.mp4",
);
});
});
+65 -76
View File
@@ -1,10 +1,16 @@
import { WebDemuxer } from "web-demuxer";
import type { SpeedRegion, TrimRegion } from "@/components/video-editor/types";
import { getEffectiveVideoStreamDurationSeconds } from "@/lib/mediaTiming";
import { getLocalFilePath } from "./localMediaSource";
import {
createReadableMediaResourceFile,
resolveMediaResourceUrl,
} from "./localMediaSource";
const DEFAULT_MAX_DECODE_QUEUE = 12;
const DEFAULT_MAX_PENDING_FRAMES = 32;
const STARTUP_STABILIZATION_SECONDS = 1.25;
const STARTUP_MAX_DECODE_QUEUE = 12;
const STARTUP_MAX_PENDING_FRAMES = 28;
export interface DecodedVideoInfo {
width: number;
@@ -20,7 +26,7 @@ export interface DecodedVideoInfo {
audioSampleRate?: number;
}
/** Caller must close the VideoFrame after use. */
/** Decoder retains ownership of the VideoFrame and closes it after use. */
type OnFrameCallback = (
frame: VideoFrame,
exportTimestampUs: number,
@@ -81,66 +87,6 @@ export class StreamingVideoDecoder {
);
}
private inferMimeType(fileName: string): string {
const extension = fileName.split(".").pop()?.toLowerCase();
switch (extension) {
case "mov":
return "video/quicktime";
case "webm":
return "video/webm";
case "mkv":
return "video/x-matroska";
case "avi":
return "video/x-msvideo";
case "mp4":
default:
return "video/mp4";
}
}
private async loadVideoFile(resourceUrl: string): Promise<File> {
const localFilePath = getLocalFilePath(resourceUrl);
const filename =
(localFilePath ?? resourceUrl).split(/[\\/]/).pop()?.split("?")[0] || "video";
if (localFilePath) {
const result = await window.electronAPI.readLocalFile(localFilePath);
if (!result.success || !result.data) {
throw new Error(result.error || "Failed to read local video file");
}
const bytes =
result.data instanceof Uint8Array ? result.data : new Uint8Array(result.data);
const arrayBuffer = bytes.buffer.slice(
bytes.byteOffset,
bytes.byteOffset + bytes.byteLength,
) as ArrayBuffer;
return new File([arrayBuffer], filename, { type: this.inferMimeType(filename) });
}
const response = await fetch(resourceUrl);
if (!response.ok) {
throw new Error(
`Failed to load video resource: ${response.status} ${response.statusText}`,
);
}
const blob = await response.blob();
return new File([blob], filename, { type: blob.type || this.inferMimeType(filename) });
}
private resolveVideoResourceUrl(videoUrl: string): string {
if (/^(blob:|data:|https?:|file:)/i.test(videoUrl)) {
return videoUrl;
}
if (videoUrl.startsWith("/")) {
return `file://${encodeURI(videoUrl)}`;
}
return videoUrl;
}
async loadMetadata(videoUrl: string): Promise<DecodedVideoInfo> {
if (this.decoder) {
try {
@@ -162,15 +108,35 @@ export class StreamingVideoDecoder {
this.demuxer = null;
}
const resourceUrl = this.resolveVideoResourceUrl(videoUrl);
const resourceUrl = await resolveMediaResourceUrl(videoUrl);
// Relative URL so it resolves correctly in both dev (http) and packaged (file://) builds
const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href;
this.demuxer = new WebDemuxer({ wasmFilePath: wasmUrl });
const file = await this.loadVideoFile(resourceUrl);
await this.demuxer.load(file);
const loadMediaInfo = async (source: string | File) => {
this.demuxer = new WebDemuxer({ wasmFilePath: wasmUrl });
await this.demuxer.load(source);
return this.demuxer.getMediaInfo();
};
let mediaInfo;
try {
mediaInfo = await loadMediaInfo(resourceUrl);
} catch (error) {
console.warn(
"[StreamingVideoDecoder] Direct source load failed, retrying with file fallback:",
error,
);
const currentDemuxer = this.demuxer;
if (currentDemuxer) {
try {
(currentDemuxer as unknown as { destroy: () => void }).destroy();
} catch {
// Ignore cleanup errors before fallback re-init.
}
}
mediaInfo = await loadMediaInfo(await createReadableMediaResourceFile(videoUrl));
}
const mediaInfo = await this.demuxer.getMediaInfo();
const videoStream = mediaInfo.streams.find((s) => s.codec_type_string === "video");
const audioStream = mediaInfo.streams.find((s) => s.codec_type_string === "audio");
const mediaStartTime =
@@ -245,13 +211,31 @@ export class StreamingVideoDecoder {
);
const frameDurationUs = 1_000_000 / targetFrameRate;
const epsilonSec = 0.001;
const startupStabilizationSeconds = 3;
const startupStabilizationSeconds = STARTUP_STABILIZATION_SECONDS;
const startupFrameBudget = Math.max(
1,
Math.round(targetFrameRate * startupStabilizationSeconds),
);
let exportFrameIndex = 0;
let loggedSteadyStateBackpressure = false;
const backpressureWaiters = new Set<() => void>();
const notifyBackpressureProgress = () => {
if (backpressureWaiters.size === 0) {
return;
}
const waiters = [...backpressureWaiters];
backpressureWaiters.clear();
for (const resolve of waiters) {
resolve();
}
};
const waitForBackpressureProgress = () =>
new Promise<void>((resolve) => {
backpressureWaiters.add(resolve);
});
console.log(
`[StreamingVideoDecoder] Startup-safe decode backpressure active for first ${startupStabilizationSeconds}s (${startupFrameBudget} frames)`,
@@ -275,6 +259,7 @@ export class StreamingVideoDecoder {
} else {
pendingFrames.push(frame);
}
notifyBackpressureProgress();
},
error: (e: DOMException) => {
decodeError = new Error(`VideoDecoder error: ${e.message}`);
@@ -283,6 +268,7 @@ export class StreamingVideoDecoder {
frameResolve = null;
resolve(null);
}
notifyBackpressureProgress();
},
});
const preferredDecoderConfig = shouldPreferSoftwareDecode
@@ -304,7 +290,11 @@ export class StreamingVideoDecoder {
const getNextFrame = (): Promise<VideoFrame | null> => {
if (decodeError) throw decodeError;
if (pendingFrames.length > 0) return Promise.resolve(pendingFrames.shift()!);
if (pendingFrames.length > 0) {
const frame = pendingFrames.shift()!;
notifyBackpressureProgress();
return Promise.resolve(frame);
}
if (decodeDone) return Promise.resolve(null);
return new Promise((resolve) => {
frameResolve = resolve;
@@ -337,11 +327,11 @@ export class StreamingVideoDecoder {
const decodeQueueLimit =
exportFrameIndex < startupFrameBudget
? Math.min(this.maxDecodeQueue, 10)
? Math.min(this.maxDecodeQueue, STARTUP_MAX_DECODE_QUEUE)
: this.maxDecodeQueue;
const pendingFrameLimit =
exportFrameIndex < startupFrameBudget
? Math.min(this.maxPendingFrames, 24)
? Math.min(this.maxPendingFrames, STARTUP_MAX_PENDING_FRAMES)
: this.maxPendingFrames;
// Backpressure on both decode queue and decoded frame backlog.
@@ -350,7 +340,7 @@ export class StreamingVideoDecoder {
pendingFrames.length > pendingFrameLimit) &&
!this.cancelled
) {
await new Promise((resolve) => setTimeout(resolve, 1));
await waitForBackpressureProgress();
}
if (this.cancelled) break;
@@ -369,6 +359,7 @@ export class StreamingVideoDecoder {
frameResolve = null;
resolve(null);
}
notifyBackpressureProgress();
}
})();
@@ -393,10 +384,9 @@ export class StreamingVideoDecoder {
segment.startSec + (segmentFrameIndex / segmentFrameCount) * segmentDurationSec;
if (sourceTimeSec >= segment.endSec - epsilonSec) return false;
const clone = new VideoFrame(heldFrame, { timestamp: heldFrame.timestamp });
const sourceTimestampMs = sourceTimeSec * 1000;
await onFrame(
clone,
heldFrame,
exportFrameIndex * frameDurationUs,
sourceTimestampMs,
sourceTimestampMs,
@@ -489,10 +479,9 @@ export class StreamingVideoDecoder {
break;
}
const clone = new VideoFrame(heldFrame, { timestamp: heldFrame.timestamp });
const sourceTimestampMs = sourceTimeSec * 1000;
await onFrame(
clone,
heldFrame,
exportFrameIndex * frameDurationUs,
sourceTimestampMs,
sourceTimestampMs,
-2
View File
@@ -284,7 +284,6 @@ export class VideoExporter {
this.config.speedRegions,
async (videoFrame, _exportTimestampUs, sourceTimestampMs, cursorTimestampMs) => {
if (this.cancelled) {
videoFrame.close();
return;
}
@@ -298,7 +297,6 @@ export class VideoExporter {
frameDuration,
timestamp,
);
videoFrame.close();
if (useNativeEncoder) {
await this.encodeRenderedFrameNative(timestamp, frameDuration, frameIndex);