mirror of
https://github.com/webadderallorg/Recordly.git
synced 2026-09-27 00:05:39 +00:00
Speed up Lightning export and relax local media reads
This commit is contained in:
Vendored
+4
@@ -208,6 +208,10 @@ interface Window {
|
||||
sessionId: string,
|
||||
frameData: Uint8Array,
|
||||
) => Promise<{ success: boolean; error?: string }>;
|
||||
nativeVideoExportWriteFrames: (
|
||||
sessionId: string,
|
||||
frameDataList: Uint8Array[],
|
||||
) => Promise<{ success: boolean; error?: string }>;
|
||||
nativeVideoExportFinish: (
|
||||
sessionId: string,
|
||||
options?: {
|
||||
|
||||
@@ -278,6 +278,24 @@ export async function enqueueNativeVideoExportFrameWrite(
|
||||
await writePromise;
|
||||
}
|
||||
|
||||
export async function enqueueNativeVideoExportFrameWrites(
|
||||
session: NativeVideoExportSession,
|
||||
frameDataList: Array<Uint8Array | ArrayBuffer>,
|
||||
) {
|
||||
const writePromise = session.writeSequence.then(async () => {
|
||||
if (session.terminating) {
|
||||
throw new Error("Native video export session was cancelled");
|
||||
}
|
||||
|
||||
for (const frameData of frameDataList) {
|
||||
await writeNativeVideoExportFrame(session, frameData);
|
||||
}
|
||||
});
|
||||
|
||||
session.writeSequence = writePromise.catch(() => undefined);
|
||||
await writePromise;
|
||||
}
|
||||
|
||||
export async function getAvailableNativeVideoEncoders(ffmpegPath: string) {
|
||||
const { stdout } = await execFileAsync(ffmpegPath, ["-hide_banner", "-encoders"], {
|
||||
timeout: 15000,
|
||||
|
||||
@@ -5,9 +5,18 @@ import { pathToFileURL } from "node:url";
|
||||
import { ipcMain } from "electron";
|
||||
import { USER_DATA_PATH } from "../../appPaths";
|
||||
import { normalizePath } from "../utils";
|
||||
import { isAllowedLocalReadPath, getAssetRootPath } from "../project/manager";
|
||||
import { getAssetRootPath } from "../project/manager";
|
||||
|
||||
export function registerAssetHandlers() {
|
||||
async function resolveReadableLocalFilePath(filePath: string) {
|
||||
const normalizedPath = normalizePath(filePath)
|
||||
const resolvedPath = await fs.realpath(normalizedPath).catch(() => normalizedPath)
|
||||
const stats = await fs.stat(resolvedPath)
|
||||
if (!stats.isFile()) {
|
||||
throw new Error('Path is not a readable file')
|
||||
}
|
||||
return normalizePath(resolvedPath)
|
||||
}
|
||||
|
||||
// Generate a tiny thumbnail for a wallpaper image and cache it in userData.
|
||||
// Returns the cached thumbnail as raw JPEG bytes for fast grid rendering.
|
||||
@@ -18,13 +27,7 @@ export function registerAssetHandlers() {
|
||||
|
||||
ipcMain.handle('generate-wallpaper-thumbnail', async (_, filePath: string) => {
|
||||
try {
|
||||
const resolved = normalizePath(filePath)
|
||||
// isAllowedLocalReadPath now canonicalizes via realpath internally and
|
||||
// requires both the lexical and real paths to satisfy the policy, so a
|
||||
// single check covers symlinks under allowed prefixes.
|
||||
if (!isAllowedLocalReadPath(resolved)) {
|
||||
return { success: false, error: 'Access denied' }
|
||||
}
|
||||
const resolved = await resolveReadableLocalFilePath(filePath)
|
||||
|
||||
// Deterministic cache key from file path + mtime
|
||||
const stat = await fs.stat(resolved)
|
||||
@@ -106,11 +109,7 @@ export function registerAssetHandlers() {
|
||||
|
||||
ipcMain.handle('read-local-file', async (_, filePath: string) => {
|
||||
try {
|
||||
const resolved = normalizePath(filePath)
|
||||
if (!isAllowedLocalReadPath(resolved)) {
|
||||
console.warn(`[read-local-file] Blocked read outside allowed directories: ${resolved}`)
|
||||
return { success: false, error: 'Access denied: path outside allowed directories' }
|
||||
}
|
||||
const resolved = await resolveReadableLocalFilePath(filePath)
|
||||
|
||||
const data = await fs.readFile(resolved)
|
||||
return { success: true, data }
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
} from "../export/exportStream";
|
||||
import {
|
||||
enqueueNativeVideoExportFrameWrite,
|
||||
enqueueNativeVideoExportFrameWrites,
|
||||
flushNativeVideoExportPendingWriteRequests,
|
||||
getNativeVideoExportMaxQueuedWriteBytes,
|
||||
getNativeVideoExportSessionError,
|
||||
@@ -132,7 +133,7 @@ export function registerExportHandlers() {
|
||||
inputMode,
|
||||
maxQueuedWriteBytes:
|
||||
inputMode === "h264-stream"
|
||||
? 8 * 1024 * 1024
|
||||
? 32 * 1024 * 1024
|
||||
: getNativeVideoExportMaxQueuedWriteBytes(inputByteSize),
|
||||
stderrOutput: "",
|
||||
encoderName,
|
||||
@@ -218,6 +219,79 @@ export function registerExportHandlers() {
|
||||
},
|
||||
);
|
||||
|
||||
ipcMain.on(
|
||||
"native-video-export-write-frames-async",
|
||||
(
|
||||
event,
|
||||
payload: {
|
||||
sessionId: string;
|
||||
requestId: number;
|
||||
frameDataList: Uint8Array[];
|
||||
},
|
||||
) => {
|
||||
const sessionId = payload?.sessionId;
|
||||
const requestId = payload?.requestId;
|
||||
const frameDataList = payload?.frameDataList;
|
||||
|
||||
if (
|
||||
typeof sessionId !== "string" ||
|
||||
typeof requestId !== "number" ||
|
||||
!Array.isArray(frameDataList) ||
|
||||
frameDataList.length === 0
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const session = nativeVideoExportSessions.get(sessionId);
|
||||
if (!session) {
|
||||
sendNativeVideoExportWriteFrameResult(event.sender, sessionId, requestId, {
|
||||
success: false,
|
||||
error: "Invalid native export session",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
session.sender = event.sender;
|
||||
session.pendingWriteRequestIds.add(requestId);
|
||||
|
||||
if (session.terminating) {
|
||||
settleNativeVideoExportWriteFrameRequest(sessionId, session, requestId, {
|
||||
success: false,
|
||||
error: "Native video export session was cancelled",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
session.inputMode !== "h264-stream" &&
|
||||
frameDataList.some((frameData) => frameData.byteLength !== session.inputByteSize)
|
||||
) {
|
||||
settleNativeVideoExportWriteFrameRequest(sessionId, session, requestId, {
|
||||
success: false,
|
||||
error: "Native video export batch contained invalid frame sizes",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
void enqueueNativeVideoExportFrameWrites(session, frameDataList)
|
||||
.then(() => {
|
||||
settleNativeVideoExportWriteFrameRequest(sessionId, session, requestId, {
|
||||
success: true,
|
||||
});
|
||||
})
|
||||
.catch((error) => {
|
||||
session.stdinError = error instanceof Error ? error : new Error(String(error));
|
||||
settleNativeVideoExportWriteFrameRequest(sessionId, session, requestId, {
|
||||
success: false,
|
||||
error: getNativeVideoExportSessionError(
|
||||
session,
|
||||
session.stdinError.message,
|
||||
),
|
||||
});
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
ipcMain.on(
|
||||
"native-video-export-write-frame-async",
|
||||
(
|
||||
|
||||
+64
-2
@@ -112,6 +112,7 @@ process.env.APP_ROOT = path.join(__dirname, "..");
|
||||
export const VITE_DEV_SERVER_URL = process.env["VITE_DEV_SERVER_URL"];
|
||||
export const MAIN_DIST = path.join(process.env.APP_ROOT, "dist-electron");
|
||||
export const RENDERER_DIST = path.join(process.env.APP_ROOT, "dist");
|
||||
const IS_DEV = Boolean(VITE_DEV_SERVER_URL);
|
||||
|
||||
process.env.VITE_PUBLIC = VITE_DEV_SERVER_URL
|
||||
? path.join(process.env.APP_ROOT, "public")
|
||||
@@ -125,9 +126,14 @@ let trayContextMenu: Menu | null = null;
|
||||
let selectedSourceName = "";
|
||||
let editorHasUnsavedChanges = false;
|
||||
let isForceClosing = false;
|
||||
let isCreatingMainWindow = false;
|
||||
let isCreatingEditorWindow = false;
|
||||
let activeUpdateNotification: Notification | null = null;
|
||||
let activeUpdateNotificationKey: string | null = null;
|
||||
const hasSingleInstanceLock = app.requestSingleInstanceLock();
|
||||
const shouldEnforceSingleInstanceLock = !IS_DEV;
|
||||
const hasSingleInstanceLock = shouldEnforceSingleInstanceLock
|
||||
? app.requestSingleInstanceLock()
|
||||
: true;
|
||||
|
||||
if (!hasSingleInstanceLock) {
|
||||
app.quit();
|
||||
@@ -167,6 +173,14 @@ function restoreWindowSafely(window: BrowserWindow | null) {
|
||||
window.focus();
|
||||
}
|
||||
|
||||
function getExistingEditorWindow(): BrowserWindow | null {
|
||||
return (
|
||||
BrowserWindow.getAllWindows().find(
|
||||
(window) => !window.isDestroyed() && isEditorWindow(window),
|
||||
) ?? null
|
||||
);
|
||||
}
|
||||
|
||||
// Tray Icons (lazily created after app is ready to avoid accessing Electron APIs too early)
|
||||
let defaultTrayIcon: ReturnType<typeof getTrayIcon> | null = null;
|
||||
let recordingTrayIcon: ReturnType<typeof getTrayIcon> | null = null;
|
||||
@@ -222,7 +236,31 @@ function createWindow() {
|
||||
return;
|
||||
}
|
||||
|
||||
mainWindow = createHudOverlayWindow();
|
||||
if (isCreatingMainWindow) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (mainWindow && !mainWindow.isDestroyed()) {
|
||||
restoreWindowSafely(mainWindow);
|
||||
return;
|
||||
}
|
||||
|
||||
const existingHudWindow = getHudOverlayWindow();
|
||||
if (existingHudWindow) {
|
||||
mainWindow = existingHudWindow;
|
||||
restoreWindowSafely(existingHudWindow);
|
||||
return;
|
||||
}
|
||||
|
||||
isCreatingMainWindow = true;
|
||||
const createdHudWindow = createHudOverlayWindow();
|
||||
mainWindow = createdHudWindow;
|
||||
createdHudWindow.once("closed", () => {
|
||||
if (mainWindow === createdHudWindow) {
|
||||
mainWindow = null;
|
||||
}
|
||||
});
|
||||
isCreatingMainWindow = false;
|
||||
}
|
||||
|
||||
function focusOrCreateMainWindow() {
|
||||
@@ -719,6 +757,27 @@ function updateTrayMenu(recording: boolean = false) {
|
||||
}
|
||||
|
||||
function createEditorWindowWrapper() {
|
||||
const existingEditorWindow = getExistingEditorWindow();
|
||||
if (existingEditorWindow) {
|
||||
mainWindow = existingEditorWindow;
|
||||
restoreWindowSafely(existingEditorWindow);
|
||||
return existingEditorWindow;
|
||||
}
|
||||
|
||||
if (isCreatingEditorWindow) {
|
||||
const currentWindow = mainWindow;
|
||||
if (currentWindow && !currentWindow.isDestroyed()) {
|
||||
return currentWindow;
|
||||
}
|
||||
|
||||
const currentEditorWindow = getExistingEditorWindow();
|
||||
if (currentEditorWindow) {
|
||||
mainWindow = currentEditorWindow;
|
||||
return currentEditorWindow;
|
||||
}
|
||||
}
|
||||
|
||||
isCreatingEditorWindow = true;
|
||||
const previousWindow = mainWindow;
|
||||
if (previousWindow && !previousWindow.isDestroyed()) {
|
||||
const closingEditorWindow = isEditorWindow(previousWindow);
|
||||
@@ -738,6 +797,7 @@ function createEditorWindowWrapper() {
|
||||
if (mainWindow === editorWindow) {
|
||||
mainWindow = null;
|
||||
}
|
||||
isCreatingEditorWindow = false;
|
||||
isForceClosing = false;
|
||||
editorHasUnsavedChanges = false;
|
||||
});
|
||||
@@ -770,6 +830,8 @@ function createEditorWindowWrapper() {
|
||||
closeEditorWindowBypassingUnsavedPrompt(editorWindow);
|
||||
}
|
||||
});
|
||||
|
||||
return editorWindow;
|
||||
}
|
||||
|
||||
function createSourceSelectorWindowWrapper() {
|
||||
|
||||
@@ -135,6 +135,23 @@ contextBridge.exposeInMainWorld("electronAPI", {
|
||||
});
|
||||
});
|
||||
},
|
||||
nativeVideoExportWriteFrames: (sessionId: string, frameDataList: Uint8Array[]) => {
|
||||
ensureNativeVideoExportWriteResultListener();
|
||||
|
||||
return new Promise<NativeVideoExportWriteResult>((resolve) => {
|
||||
const requestId = nextNativeVideoExportWriteRequestId++;
|
||||
nativeVideoExportWriteRequests.set(requestId, {
|
||||
sessionId,
|
||||
resolve,
|
||||
});
|
||||
|
||||
ipcRenderer.send("native-video-export-write-frames-async", {
|
||||
sessionId,
|
||||
requestId,
|
||||
frameDataList,
|
||||
});
|
||||
});
|
||||
},
|
||||
nativeVideoExportFinish: (
|
||||
sessionId: string,
|
||||
options?: {
|
||||
|
||||
@@ -330,6 +330,7 @@ async function writeSmokeExportReport(
|
||||
const DEFAULT_MP4_EXPORT_FRAME_RATE: ExportMp4FrameRate = 30;
|
||||
const SOURCE_AUDIO_FALLBACK_TOAST_ID = "source-audio-fallback-error";
|
||||
const PROJECT_AUTOSAVE_DELAY_MS = 1000;
|
||||
const EXPORT_ERROR_TOAST_DURATION_MS = 20000;
|
||||
|
||||
function getEncodingModeBitrateMultiplier(encodingMode: ExportEncodingMode): number {
|
||||
switch (encodingMode) {
|
||||
@@ -352,6 +353,14 @@ function summarizeErrorMessage(message: string): string {
|
||||
return firstLine ?? message;
|
||||
}
|
||||
|
||||
function showExportErrorToast(message: string) {
|
||||
const summary = summarizeErrorMessage(message);
|
||||
toast.error(summary, {
|
||||
description: summary === message ? undefined : message,
|
||||
duration: EXPORT_ERROR_TOAST_DURATION_MS,
|
||||
});
|
||||
}
|
||||
|
||||
function cloneStructured<T>(value: T): T {
|
||||
return globalThis.structuredClone(value);
|
||||
}
|
||||
@@ -4805,7 +4814,7 @@ export default function VideoEditor() {
|
||||
});
|
||||
}
|
||||
setExportError(saveResult.message || "Failed to save video");
|
||||
toast.error(saveResult.message || "Failed to save video");
|
||||
showExportErrorToast(saveResult.message || "Failed to save video");
|
||||
// Keep the pending-save entry so the user can retry without
|
||||
// re-rendering. The temp file is still on disk (the main
|
||||
// process only moves/deletes it on success) and the
|
||||
@@ -4837,7 +4846,8 @@ export default function VideoEditor() {
|
||||
});
|
||||
}
|
||||
setExportError(result.error || "Export failed");
|
||||
toast.error(summarizeErrorMessage(result.error || "Export failed"));
|
||||
showExportErrorToast(result.error || "Export failed");
|
||||
keepExportDialogOpen = true;
|
||||
if (smokeExportConfig.enabled) {
|
||||
window.close();
|
||||
return;
|
||||
@@ -4866,7 +4876,8 @@ export default function VideoEditor() {
|
||||
});
|
||||
}
|
||||
setExportError(errorMessage);
|
||||
toast.error(`Export failed: ${summarizeErrorMessage(errorMessage)}`);
|
||||
showExportErrorToast(`Export failed: ${errorMessage}`);
|
||||
keepExportDialogOpen = true;
|
||||
if (smokeExportConfig.enabled) {
|
||||
window.close();
|
||||
}
|
||||
|
||||
@@ -69,9 +69,14 @@ describe("editorPreferences", () => {
|
||||
expect(DEFAULT_EDITOR_PREFERENCES.exportQuality).toBe("source");
|
||||
});
|
||||
|
||||
it("defaults cursor preferences to macOS at 2.5x", () => {
|
||||
expect(DEFAULT_EDITOR_PREFERENCES.cursorStyle).toBe("macos");
|
||||
it("defaults cursor preferences to Tahoe at 2.5x with lighter sway", () => {
|
||||
expect(DEFAULT_EDITOR_PREFERENCES.cursorStyle).toBe("tahoe");
|
||||
expect(DEFAULT_EDITOR_PREFERENCES.cursorSize).toBe(2.5);
|
||||
expect(DEFAULT_EDITOR_PREFERENCES.cursorSway).toBe(0.25);
|
||||
});
|
||||
|
||||
it("defaults MP4 exports to the Lightning pipeline", () => {
|
||||
expect(DEFAULT_EDITOR_PREFERENCES.exportPipelineModel).toBe("modern");
|
||||
});
|
||||
|
||||
it("loads stored editor control preferences", () => {
|
||||
|
||||
@@ -174,7 +174,7 @@ export function normalizeExportPipelineModel(value: unknown): ExportPipelineMode
|
||||
return value;
|
||||
}
|
||||
|
||||
return "legacy";
|
||||
return "modern";
|
||||
}
|
||||
|
||||
export function normalizeExportMp4FrameRate(value: unknown): ExportMp4FrameRate {
|
||||
|
||||
@@ -51,7 +51,7 @@ export interface CursorVisualSettings {
|
||||
}
|
||||
|
||||
export type CursorStyle = "macos" | "tahoe" | "tahoe-inverted" | "dot" | "figma" | (string & {}); // extension-contributed cursor styles
|
||||
export const DEFAULT_CURSOR_STYLE: CursorStyle = "macos";
|
||||
export const DEFAULT_CURSOR_STYLE: CursorStyle = "tahoe";
|
||||
|
||||
export type EditorEffectSection =
|
||||
| "scene"
|
||||
@@ -100,7 +100,7 @@ export const DEFAULT_CURSOR_SMOOTHING = 0.67;
|
||||
export const DEFAULT_CURSOR_MOTION_BLUR = 0.4;
|
||||
export const DEFAULT_CURSOR_CLICK_BOUNCE = 2.5;
|
||||
export const DEFAULT_CURSOR_CLICK_BOUNCE_DURATION = 350;
|
||||
export const DEFAULT_CURSOR_SWAY = 0.4;
|
||||
export const DEFAULT_CURSOR_SWAY = 0.25;
|
||||
export const DEFAULT_ZOOM_SMOOTHNESS = 0.5;
|
||||
export const DEFAULT_ZOOM_MOTION_BLUR = 0.35;
|
||||
export interface ZoomMotionBlurTuning {
|
||||
|
||||
@@ -357,6 +357,24 @@ export class AudioProcessor {
|
||||
let encodeError: Error | null = null;
|
||||
let muxError: Error | null = null;
|
||||
let pendingMuxing = Promise.resolve();
|
||||
const capacityWaiters = new Set<() => void>();
|
||||
|
||||
const notifyCapacityAvailable = () => {
|
||||
if (capacityWaiters.size === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const waiters = [...capacityWaiters];
|
||||
capacityWaiters.clear();
|
||||
for (const resolve of waiters) {
|
||||
resolve();
|
||||
}
|
||||
};
|
||||
|
||||
const waitForCapacity = () =>
|
||||
new Promise<void>((resolve) => {
|
||||
capacityWaiters.add(resolve);
|
||||
});
|
||||
|
||||
const failIfNeeded = () => {
|
||||
if (decodeError) throw decodeError;
|
||||
@@ -380,6 +398,7 @@ export class AudioProcessor {
|
||||
|
||||
encoder.encode(frame);
|
||||
frame.close();
|
||||
notifyCapacityAvailable();
|
||||
}
|
||||
};
|
||||
|
||||
@@ -416,10 +435,13 @@ export class AudioProcessor {
|
||||
})
|
||||
.catch((error) => {
|
||||
muxError = error instanceof Error ? error : new Error(String(error));
|
||||
notifyCapacityAvailable();
|
||||
});
|
||||
notifyCapacityAvailable();
|
||||
},
|
||||
error: (error: DOMException) => {
|
||||
encodeError = new Error(`[AudioProcessor] Encode error: ${error.message}`);
|
||||
notifyCapacityAvailable();
|
||||
},
|
||||
});
|
||||
|
||||
@@ -445,9 +467,11 @@ export class AudioProcessor {
|
||||
}
|
||||
|
||||
pendingFrames.push(transformed);
|
||||
notifyCapacityAvailable();
|
||||
},
|
||||
error: (error: DOMException) => {
|
||||
decodeError = new Error(`[AudioProcessor] Decode error: ${error.message}`);
|
||||
notifyCapacityAvailable();
|
||||
},
|
||||
});
|
||||
decoder.configure(audioConfig);
|
||||
@@ -477,7 +501,7 @@ export class AudioProcessor {
|
||||
) {
|
||||
failIfNeeded();
|
||||
pumpEncodedFrames();
|
||||
await new Promise((resolve) => setTimeout(resolve, 1));
|
||||
await waitForCapacity();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -489,7 +513,7 @@ export class AudioProcessor {
|
||||
failIfNeeded();
|
||||
pumpEncodedFrames();
|
||||
if (pendingFrames.length > 0 || encoder.encodeQueueSize > 0) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 1));
|
||||
await waitForCapacity();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -502,6 +526,7 @@ export class AudioProcessor {
|
||||
await pendingMuxing;
|
||||
failIfNeeded();
|
||||
} finally {
|
||||
notifyCapacityAvailable();
|
||||
if (reader) {
|
||||
try {
|
||||
await reader.cancel();
|
||||
@@ -914,16 +939,9 @@ export class AudioProcessor {
|
||||
let demuxer: WebDemuxer | null = null;
|
||||
|
||||
try {
|
||||
const response = await fetch(source.src);
|
||||
const blob = await response.blob();
|
||||
const filename = url.split("/").pop() || "audio";
|
||||
const file = new File([blob], filename, {
|
||||
type: blob.type || "video/mp4",
|
||||
});
|
||||
|
||||
const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href;
|
||||
demuxer = new WebDemuxer({ wasmFilePath: wasmUrl });
|
||||
await demuxer.load(file);
|
||||
await demuxer.load(source.src);
|
||||
|
||||
let audioConfig: AudioDecoderConfig;
|
||||
try {
|
||||
@@ -939,6 +957,24 @@ export class AudioProcessor {
|
||||
const channelChunks: Float32Array[][] = Array.from({ length: numChannels }, () => []);
|
||||
let totalFrames = 0;
|
||||
let decodeError: Error | null = null;
|
||||
const decodeCapacityWaiters = new Set<() => void>();
|
||||
|
||||
const notifyDecodeCapacityAvailable = () => {
|
||||
if (decodeCapacityWaiters.size === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const waiters = [...decodeCapacityWaiters];
|
||||
decodeCapacityWaiters.clear();
|
||||
for (const resolve of waiters) {
|
||||
resolve();
|
||||
}
|
||||
};
|
||||
|
||||
const waitForDecodeCapacity = () =>
|
||||
new Promise<void>((resolve) => {
|
||||
decodeCapacityWaiters.add(resolve);
|
||||
});
|
||||
|
||||
const decoder = new AudioDecoder({
|
||||
output: (data: AudioData) => {
|
||||
@@ -986,10 +1022,12 @@ export class AudioProcessor {
|
||||
totalFrames += frames;
|
||||
} finally {
|
||||
data.close();
|
||||
notifyDecodeCapacityAvailable();
|
||||
}
|
||||
},
|
||||
error: (err: DOMException) => {
|
||||
decodeError = new Error(`Streaming audio decode error: ${err.message}`);
|
||||
notifyDecodeCapacityAvailable();
|
||||
},
|
||||
});
|
||||
|
||||
@@ -1008,7 +1046,7 @@ export class AudioProcessor {
|
||||
|
||||
while (decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT && !this.cancelled) {
|
||||
if (decodeError) throw decodeError;
|
||||
await new Promise((r) => setTimeout(r, 1));
|
||||
await waitForDecodeCapacity();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1017,6 +1055,7 @@ export class AudioProcessor {
|
||||
}
|
||||
if (decodeError) throw decodeError;
|
||||
} finally {
|
||||
notifyDecodeCapacityAvailable();
|
||||
try {
|
||||
await reader.cancel();
|
||||
} catch {
|
||||
@@ -1376,13 +1415,9 @@ export class AudioProcessor {
|
||||
try {
|
||||
const source = await resolveMediaElementSource(audioPath);
|
||||
try {
|
||||
const response = await fetch(source.src);
|
||||
const blob = await response.blob();
|
||||
const filename = audioPath.split("/").pop() || "sidecar-audio";
|
||||
const file = new File([blob], filename, { type: blob.type || "audio/webm" });
|
||||
const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href;
|
||||
const demuxer = new WebDemuxer({ wasmFilePath: wasmUrl });
|
||||
await demuxer.load(file);
|
||||
await demuxer.load(source.src);
|
||||
return demuxer;
|
||||
} finally {
|
||||
source.revoke();
|
||||
|
||||
@@ -10,10 +10,10 @@ describe("backendPolicy", () => {
|
||||
expect(normalizeLightningRuntimePlatform("unknown")).toBe("unknown");
|
||||
});
|
||||
|
||||
it("keeps auto backend WebCodecs-first on every platform", () => {
|
||||
expect(shouldPreferNativeAutoBackend("win32")).toBe(false);
|
||||
it("prefers native auto backend on desktop platforms with the fastest native path", () => {
|
||||
expect(shouldPreferNativeAutoBackend("win32")).toBe(true);
|
||||
expect(shouldPreferNativeAutoBackend("linux")).toBe(false);
|
||||
expect(shouldPreferNativeAutoBackend("darwin")).toBe(false);
|
||||
expect(shouldPreferNativeAutoBackend("darwin")).toBe(true);
|
||||
expect(shouldPreferNativeAutoBackend("unknown")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -23,5 +23,5 @@ export function normalizeLightningRuntimePlatform(
|
||||
}
|
||||
|
||||
export function shouldPreferNativeAutoBackend(_platform: LightningRuntimePlatform): boolean {
|
||||
return false;
|
||||
return _platform === "darwin" || _platform === "win32";
|
||||
}
|
||||
|
||||
@@ -50,9 +50,9 @@ describe("exportTuning", () => {
|
||||
expect(webCodecsProfile.maxDecodeQueue).toBe(12);
|
||||
expect(webCodecsProfile.maxPendingFrames).toBe(32);
|
||||
expect(breezeProfile.name).toBe("breeze-balanced-plus");
|
||||
expect(breezeProfile.maxDecodeQueue).toBe(10);
|
||||
expect(breezeProfile.maxPendingFrames).toBe(24);
|
||||
expect(breezeProfile.maxInFlightNativeWrites).toBe(4);
|
||||
expect(breezeProfile.maxDecodeQueue).toBe(14);
|
||||
expect(breezeProfile.maxPendingFrames).toBe(40);
|
||||
expect(breezeProfile.maxInFlightNativeWrites).toBe(8);
|
||||
});
|
||||
|
||||
it("falls back to conservative native settings on low-core or very heavy workloads", () => {
|
||||
@@ -72,12 +72,12 @@ describe("exportTuning", () => {
|
||||
});
|
||||
|
||||
expect(breezeLowCoreProfile.name).toBe("breeze-conservative");
|
||||
expect(breezeLowCoreProfile.maxDecodeQueue).toBe(6);
|
||||
expect(breezeLowCoreProfile.maxPendingFrames).toBe(12);
|
||||
expect(breezeLowCoreProfile.maxInFlightNativeWrites).toBe(1);
|
||||
expect(breezeLowCoreProfile.maxDecodeQueue).toBe(8);
|
||||
expect(breezeLowCoreProfile.maxPendingFrames).toBe(16);
|
||||
expect(breezeLowCoreProfile.maxInFlightNativeWrites).toBe(2);
|
||||
|
||||
expect(breezeHeavyProfile.name).toBe("breeze-conservative");
|
||||
expect(breezeHeavyProfile.maxDecodeQueue).toBe(6);
|
||||
expect(breezeHeavyProfile.maxPendingFrames).toBe(12);
|
||||
expect(breezeHeavyProfile.maxDecodeQueue).toBe(8);
|
||||
expect(breezeHeavyProfile.maxPendingFrames).toBe(16);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -130,9 +130,9 @@ export function getExportBackpressureProfile(
|
||||
return {
|
||||
name: "breeze-conservative",
|
||||
maxEncodeQueue,
|
||||
maxDecodeQueue: 6,
|
||||
maxPendingFrames: 12,
|
||||
maxInFlightNativeWrites: 1,
|
||||
maxDecodeQueue: 8,
|
||||
maxPendingFrames: 16,
|
||||
maxInFlightNativeWrites: 2,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -140,18 +140,18 @@ export function getExportBackpressureProfile(
|
||||
return {
|
||||
name: "breeze-balanced-plus",
|
||||
maxEncodeQueue,
|
||||
maxDecodeQueue: 10,
|
||||
maxPendingFrames: 24,
|
||||
maxInFlightNativeWrites: 4,
|
||||
maxDecodeQueue: 14,
|
||||
maxPendingFrames: 40,
|
||||
maxInFlightNativeWrites: 8,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
name: "breeze-balanced",
|
||||
maxEncodeQueue,
|
||||
maxDecodeQueue: 8,
|
||||
maxPendingFrames: 16,
|
||||
maxInFlightNativeWrites: 2,
|
||||
maxDecodeQueue: 12,
|
||||
maxPendingFrames: 28,
|
||||
maxInFlightNativeWrites: 4,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
import { WebDemuxer } from "web-demuxer";
|
||||
import { getEffectiveVideoStreamDurationSeconds } from "@/lib/mediaTiming";
|
||||
import { getDecodedFrameTimelineOffsetUs } from "./streamingDecoder";
|
||||
import { getLocalFilePath } from "./localMediaSource";
|
||||
import {
|
||||
createReadableMediaResourceFile,
|
||||
resolveMediaResourceUrl,
|
||||
} from "./localMediaSource";
|
||||
|
||||
const DEFAULT_MAX_DECODE_QUEUE = 12;
|
||||
const DEFAULT_MAX_PENDING_FRAMES = 32;
|
||||
@@ -40,85 +43,36 @@ export class ForwardFrameSource {
|
||||
private resolvedDecodedDurationSec: number | null = null;
|
||||
private firstFrameTimestampUs: number | null = null;
|
||||
private frameTimelineOffsetUs = 0;
|
||||
|
||||
private inferMimeType(fileName: string): string {
|
||||
const extension = fileName.split(".").pop()?.toLowerCase();
|
||||
switch (extension) {
|
||||
case "mov":
|
||||
return "video/quicktime";
|
||||
case "webm":
|
||||
return "video/webm";
|
||||
case "mkv":
|
||||
return "video/x-matroska";
|
||||
case "avi":
|
||||
return "video/x-msvideo";
|
||||
case "mp4":
|
||||
default:
|
||||
return "video/mp4";
|
||||
}
|
||||
}
|
||||
|
||||
private async loadVideoFile(resourceUrl: string): Promise<File> {
|
||||
const localFilePath = getLocalFilePath(resourceUrl);
|
||||
const filename =
|
||||
(localFilePath ?? resourceUrl).split(/[\\/]/).pop()?.split("?")[0] || "video";
|
||||
|
||||
if (localFilePath) {
|
||||
const result = await window.electronAPI.readLocalFile(localFilePath);
|
||||
if (!result.success || !result.data) {
|
||||
throw new Error(result.error || "Failed to read local video file");
|
||||
}
|
||||
|
||||
const bytes =
|
||||
result.data instanceof Uint8Array ? result.data : new Uint8Array(result.data);
|
||||
const arrayBuffer = bytes.buffer.slice(
|
||||
bytes.byteOffset,
|
||||
bytes.byteOffset + bytes.byteLength,
|
||||
) as ArrayBuffer;
|
||||
return new File([arrayBuffer], filename, {
|
||||
type: this.inferMimeType(filename),
|
||||
});
|
||||
}
|
||||
|
||||
const response = await fetch(resourceUrl);
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`Failed to load video resource: ${response.status} ${response.statusText}`,
|
||||
);
|
||||
}
|
||||
|
||||
const blob = await response.blob();
|
||||
return new File([blob], filename, {
|
||||
type: blob.type || this.inferMimeType(filename),
|
||||
});
|
||||
}
|
||||
|
||||
private resolveVideoResourceUrl(videoUrl: string): string {
|
||||
if (/^(blob:|data:|https?:|file:)/i.test(videoUrl)) {
|
||||
return videoUrl;
|
||||
}
|
||||
|
||||
if (/^[A-Za-z]:[\\/]/.test(videoUrl)) {
|
||||
const normalized = videoUrl.replace(/\\/g, "/");
|
||||
return `file:///${encodeURI(normalized)}`;
|
||||
}
|
||||
|
||||
if (videoUrl.startsWith("/")) {
|
||||
return `file://${encodeURI(videoUrl)}`;
|
||||
}
|
||||
|
||||
return videoUrl;
|
||||
}
|
||||
private decodeCapacityWaiters = new Set<() => void>();
|
||||
|
||||
async initialize(videoUrl: string): Promise<ForwardFrameSourceMetadata> {
|
||||
const resourceUrl = this.resolveVideoResourceUrl(videoUrl);
|
||||
const resourceUrl = await resolveMediaResourceUrl(videoUrl);
|
||||
const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href;
|
||||
this.demuxer = new WebDemuxer({ wasmFilePath: wasmUrl });
|
||||
const loadMediaInfo = async (source: string | File) => {
|
||||
this.demuxer = new WebDemuxer({ wasmFilePath: wasmUrl });
|
||||
await this.demuxer.load(source);
|
||||
return this.demuxer.getMediaInfo();
|
||||
};
|
||||
|
||||
const file = await this.loadVideoFile(resourceUrl);
|
||||
await this.demuxer.load(file);
|
||||
let mediaInfo;
|
||||
try {
|
||||
mediaInfo = await loadMediaInfo(resourceUrl);
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
"[ForwardFrameSource] Direct source load failed, retrying with file fallback:",
|
||||
error,
|
||||
);
|
||||
const currentDemuxer = this.demuxer;
|
||||
if (currentDemuxer) {
|
||||
try {
|
||||
(currentDemuxer as unknown as { destroy: () => void }).destroy();
|
||||
} catch {
|
||||
// Ignore cleanup errors before fallback re-init.
|
||||
}
|
||||
}
|
||||
mediaInfo = await loadMediaInfo(await createReadableMediaResourceFile(videoUrl));
|
||||
}
|
||||
|
||||
const mediaInfo = await this.demuxer.getMediaInfo();
|
||||
const videoStream = mediaInfo.streams.find(
|
||||
(stream) => stream.codec_type_string === "video",
|
||||
);
|
||||
@@ -166,6 +120,7 @@ export class ForwardFrameSource {
|
||||
} else {
|
||||
this.pendingFrames.push(frame);
|
||||
}
|
||||
this.notifyDecoderCapacityAvailable();
|
||||
},
|
||||
error: (error: DOMException) => {
|
||||
this.decodeError = new Error(`VideoDecoder error: ${error.message}`);
|
||||
@@ -174,6 +129,7 @@ export class ForwardFrameSource {
|
||||
this.frameResolve = null;
|
||||
resolve(null);
|
||||
}
|
||||
this.notifyDecoderCapacityAvailable();
|
||||
},
|
||||
});
|
||||
|
||||
@@ -214,7 +170,7 @@ export class ForwardFrameSource {
|
||||
this.pendingFrames.length > DEFAULT_MAX_PENDING_FRAMES) &&
|
||||
!this.cancelled
|
||||
) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 1));
|
||||
await this.waitForDecoderCapacity();
|
||||
}
|
||||
|
||||
if (this.cancelled) {
|
||||
@@ -231,6 +187,7 @@ export class ForwardFrameSource {
|
||||
this.decodeError = error instanceof Error ? error : new Error(String(error));
|
||||
} finally {
|
||||
this.decodeDone = true;
|
||||
this.notifyDecoderCapacityAvailable();
|
||||
if (this.frameResolve) {
|
||||
const resolve = this.frameResolve;
|
||||
this.frameResolve = null;
|
||||
@@ -246,7 +203,9 @@ export class ForwardFrameSource {
|
||||
}
|
||||
|
||||
if (this.pendingFrames.length > 0) {
|
||||
return Promise.resolve(this.pendingFrames.shift()!);
|
||||
const frame = this.pendingFrames.shift()!;
|
||||
this.notifyDecoderCapacityAvailable();
|
||||
return Promise.resolve(frame);
|
||||
}
|
||||
|
||||
if (this.decodeDone) {
|
||||
@@ -262,6 +221,24 @@ export class ForwardFrameSource {
|
||||
});
|
||||
}
|
||||
|
||||
private waitForDecoderCapacity(): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
this.decodeCapacityWaiters.add(resolve);
|
||||
});
|
||||
}
|
||||
|
||||
private notifyDecoderCapacityAvailable(): void {
|
||||
if (this.decodeCapacityWaiters.size === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const waiters = [...this.decodeCapacityWaiters];
|
||||
this.decodeCapacityWaiters.clear();
|
||||
for (const resolve of waiters) {
|
||||
resolve();
|
||||
}
|
||||
}
|
||||
|
||||
async getFrameAtTime(targetTimeSec: number): Promise<VideoFrame | null> {
|
||||
if (!this.metadata) {
|
||||
throw new Error("Frame source not initialized");
|
||||
@@ -345,6 +322,7 @@ export class ForwardFrameSource {
|
||||
|
||||
cancel(): void {
|
||||
this.cancelled = true;
|
||||
this.notifyDecoderCapacityAvailable();
|
||||
if (this.frameResolve) {
|
||||
const resolve = this.frameResolve;
|
||||
this.frameResolve = null;
|
||||
@@ -416,5 +394,6 @@ export class ForwardFrameSource {
|
||||
this.resolvedDecodedDurationSec = null;
|
||||
this.firstFrameTimestampUs = null;
|
||||
this.frameTimelineOffsetUs = 0;
|
||||
this.decodeCapacityWaiters.clear();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -262,7 +262,6 @@ export class GifExporter {
|
||||
this.config.speedRegions,
|
||||
async (videoFrame, _exportTimestampUs, sourceTimestampMs, cursorTimestampMs) => {
|
||||
if (this.cancelled) {
|
||||
videoFrame.close();
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -275,7 +274,6 @@ export class GifExporter {
|
||||
frameDurationUs,
|
||||
frameIndex * frameDurationUs,
|
||||
);
|
||||
videoFrame.close();
|
||||
|
||||
this.addRenderedGifFrame(frameDelay);
|
||||
frameIndex++;
|
||||
|
||||
@@ -2,8 +2,6 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { resolveMediaElementSource } from "./localMediaSource";
|
||||
|
||||
const NativeURL = URL;
|
||||
|
||||
describe("resolveMediaElementSource", () => {
|
||||
beforeEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
@@ -11,60 +9,45 @@ describe("resolveMediaElementSource", () => {
|
||||
window: {
|
||||
electronAPI: {
|
||||
readLocalFile: vi.fn(),
|
||||
getLocalMediaUrl: vi.fn(async (filePath: string) => ({
|
||||
success: true,
|
||||
url: `http://127.0.0.1:4321/video?path=${encodeURIComponent(filePath)}`,
|
||||
})),
|
||||
},
|
||||
},
|
||||
});
|
||||
class MockURL extends NativeURL {}
|
||||
Object.assign(MockURL, {
|
||||
createObjectURL: vi.fn(() => "blob:mock-local-media"),
|
||||
revokeObjectURL: vi.fn(),
|
||||
});
|
||||
vi.stubGlobal("URL", MockURL);
|
||||
});
|
||||
|
||||
it("reads file URLs through Electron IPC and returns an object URL", async () => {
|
||||
const readLocalFile = vi.fn(async () => ({
|
||||
success: true,
|
||||
data: new Uint8Array([1, 2, 3]),
|
||||
}));
|
||||
(window as any).electronAPI.readLocalFile = readLocalFile;
|
||||
|
||||
it("resolves file URLs through the local media server for media elements", async () => {
|
||||
const result = await resolveMediaElementSource("file:///tmp/example.mp4");
|
||||
|
||||
expect(readLocalFile).toHaveBeenCalledWith("/tmp/example.mp4");
|
||||
expect(URL.createObjectURL).toHaveBeenCalledOnce();
|
||||
expect(result.src).toBe("blob:mock-local-media");
|
||||
|
||||
result.revoke();
|
||||
expect(URL.revokeObjectURL).toHaveBeenCalledWith("blob:mock-local-media");
|
||||
expect((window as any).electronAPI.readLocalFile).not.toHaveBeenCalled();
|
||||
expect((window as any).electronAPI.getLocalMediaUrl).toHaveBeenCalledWith(
|
||||
"/tmp/example.mp4",
|
||||
);
|
||||
expect(result.src).toBe("http://127.0.0.1:4321/video?path=%2Ftmp%2Fexample.mp4");
|
||||
});
|
||||
|
||||
it("reads absolute local paths through Electron IPC and normalizes fallback URLs", async () => {
|
||||
const readLocalFile = vi.fn(async () => ({
|
||||
success: true,
|
||||
data: new Uint8Array([4, 5, 6]),
|
||||
}));
|
||||
(window as any).electronAPI.readLocalFile = readLocalFile;
|
||||
|
||||
it("resolves absolute local paths through the local media server without copying them into blobs", async () => {
|
||||
const result = await resolveMediaElementSource("/tmp/example.wav");
|
||||
|
||||
expect(readLocalFile).toHaveBeenCalledWith("/tmp/example.wav");
|
||||
expect(result.src).toBe("blob:mock-local-media");
|
||||
expect((window as any).electronAPI.readLocalFile).not.toHaveBeenCalled();
|
||||
expect((window as any).electronAPI.getLocalMediaUrl).toHaveBeenCalledWith(
|
||||
"/tmp/example.wav",
|
||||
);
|
||||
expect(result.src).toBe("http://127.0.0.1:4321/video?path=%2Ftmp%2Fexample.wav");
|
||||
});
|
||||
|
||||
it("reads loopback media-server URLs through Electron IPC", async () => {
|
||||
const readLocalFile = vi.fn(async () => ({
|
||||
success: true,
|
||||
data: new Uint8Array([7, 8, 9]),
|
||||
}));
|
||||
(window as any).electronAPI.readLocalFile = readLocalFile;
|
||||
|
||||
it("preserves loopback media-server URLs instead of materializing them through IPC", async () => {
|
||||
const result = await resolveMediaElementSource(
|
||||
"http://127.0.0.1:43123/video?path=%2Ftmp%2Fexample%20clip.mp4",
|
||||
);
|
||||
|
||||
expect(readLocalFile).toHaveBeenCalledWith("/tmp/example clip.mp4");
|
||||
expect(result.src).toBe("blob:mock-local-media");
|
||||
expect((window as any).electronAPI.readLocalFile).not.toHaveBeenCalled();
|
||||
expect((window as any).electronAPI.getLocalMediaUrl).not.toHaveBeenCalled();
|
||||
expect(result.src).toBe(
|
||||
"http://127.0.0.1:43123/video?path=%2Ftmp%2Fexample%20clip.mp4",
|
||||
);
|
||||
});
|
||||
|
||||
it("leaves remote URLs untouched", async () => {
|
||||
|
||||
@@ -60,7 +60,7 @@ function isRemoteMediaResource(resource: string) {
|
||||
return REMOTE_MEDIA_URL_PATTERN.test(resource) && !isLocalMediaServerUrl(resource);
|
||||
}
|
||||
|
||||
function getNormalizedResourceUrl(resource: string) {
|
||||
export function getNormalizedMediaResourceUrl(resource: string) {
|
||||
const localFilePath = getLocalFilePath(resource);
|
||||
if (!localFilePath) {
|
||||
return resource;
|
||||
@@ -92,6 +92,60 @@ function inferMimeType(filePath: string) {
|
||||
return "application/octet-stream";
|
||||
}
|
||||
|
||||
export async function resolveMediaResourceUrl(resource: string): Promise<string> {
|
||||
const localFilePath = getLocalFilePath(resource);
|
||||
if (!localFilePath) {
|
||||
return resource;
|
||||
}
|
||||
|
||||
if (isLocalMediaServerUrl(resource)) {
|
||||
return resource;
|
||||
}
|
||||
|
||||
if (typeof window !== "undefined" && window.electronAPI?.getLocalMediaUrl) {
|
||||
try {
|
||||
const result = await window.electronAPI.getLocalMediaUrl(localFilePath);
|
||||
if (result.success) {
|
||||
return result.url;
|
||||
}
|
||||
} catch {
|
||||
// Fall through to a file URL when the local media server is unavailable.
|
||||
}
|
||||
}
|
||||
|
||||
return /^file:\/\//i.test(resource) ? resource : toFileUrl(localFilePath);
|
||||
}
|
||||
|
||||
export async function createReadableMediaResourceFile(resource: string): Promise<File> {
|
||||
const localFilePath = getLocalFilePath(resource);
|
||||
const filename = (localFilePath ?? resource).split(/[\\/]/).pop()?.split("?")[0] || "media";
|
||||
|
||||
if (localFilePath && typeof window !== "undefined" && window.electronAPI?.readLocalFile) {
|
||||
const result = await window.electronAPI.readLocalFile(localFilePath);
|
||||
if (!result.success || !result.data) {
|
||||
throw new Error(result.error || "Failed to read local media file");
|
||||
}
|
||||
|
||||
const bytes = result.data instanceof Uint8Array ? result.data : new Uint8Array(result.data);
|
||||
const arrayBuffer = bytes.buffer.slice(
|
||||
bytes.byteOffset,
|
||||
bytes.byteOffset + bytes.byteLength,
|
||||
) as ArrayBuffer;
|
||||
return new File([arrayBuffer], filename, { type: inferMimeType(filename) });
|
||||
}
|
||||
|
||||
const resourceUrl = await resolveMediaResourceUrl(resource);
|
||||
const response = await fetch(resourceUrl);
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`Failed to load media resource: ${response.status} ${response.statusText}`,
|
||||
);
|
||||
}
|
||||
|
||||
const blob = await response.blob();
|
||||
return new File([blob], filename, { type: blob.type || inferMimeType(filename) });
|
||||
}
|
||||
|
||||
export async function resolveMediaElementSource(resource: string): Promise<{
|
||||
src: string;
|
||||
revoke: () => void;
|
||||
@@ -100,29 +154,8 @@ export async function resolveMediaElementSource(resource: string): Promise<{
|
||||
return { src: resource, revoke: NOOP };
|
||||
}
|
||||
|
||||
const normalizedResource = getNormalizedResourceUrl(resource);
|
||||
const localFilePath = getLocalFilePath(resource) ?? getLocalFilePath(normalizedResource);
|
||||
if (!localFilePath || typeof window === "undefined" || !window.electronAPI?.readLocalFile) {
|
||||
return { src: normalizedResource, revoke: NOOP };
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await window.electronAPI.readLocalFile(localFilePath);
|
||||
if (!result.success || !result.data) {
|
||||
return { src: normalizedResource, revoke: NOOP };
|
||||
}
|
||||
|
||||
const bytes = result.data instanceof Uint8Array ? result.data : new Uint8Array(result.data);
|
||||
const blob = new Blob([Uint8Array.from(bytes)], { type: inferMimeType(localFilePath) });
|
||||
const objectUrl = URL.createObjectURL(blob);
|
||||
|
||||
return {
|
||||
src: objectUrl,
|
||||
revoke: () => {
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
},
|
||||
};
|
||||
} catch {
|
||||
return { src: normalizedResource, revoke: NOOP };
|
||||
}
|
||||
return {
|
||||
src: await resolveMediaResourceUrl(resource),
|
||||
revoke: NOOP,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ import type {
|
||||
} from "@/components/video-editor/types";
|
||||
import { extensionHost } from "@/lib/extensions";
|
||||
import { AudioProcessor, isAacAudioEncodingSupported } from "./audioEncoder";
|
||||
import { normalizeLightningRuntimePlatform } from "./backendPolicy";
|
||||
import { normalizeLightningRuntimePlatform, shouldPreferNativeAutoBackend } from "./backendPolicy";
|
||||
import { buildEditedTrackSourceSegments, classifyEditedTrackStrategy } from "./editedTrackStrategy";
|
||||
import {
|
||||
type ExportBackpressureProfile,
|
||||
@@ -134,7 +134,9 @@ const NATIVE_EXPORT_ENGINE_NAME = "Breeze";
|
||||
const LIGHTNING_PIPELINE_NAME = "Lightning (Beta)";
|
||||
|
||||
export class ModernVideoExporter {
|
||||
private static readonly NATIVE_ENCODER_QUEUE_LIMIT = 32;
|
||||
private static readonly NATIVE_ENCODER_QUEUE_LIMIT = 64;
|
||||
private static readonly NATIVE_WRITE_BATCH_MAX_CHUNKS = 12;
|
||||
private static readonly NATIVE_WRITE_BATCH_MAX_BYTES = 2 * 1024 * 1024;
|
||||
|
||||
private config: VideoExporterConfig;
|
||||
private streamingDecoder: StreamingVideoDecoder | null = null;
|
||||
@@ -157,9 +159,10 @@ export class ModernVideoExporter {
|
||||
private encoderName: string | null = null;
|
||||
private backpressureProfile: ExportBackpressureProfile | null = null;
|
||||
private nativeExportSessionId: string | null = null;
|
||||
private nativePendingWrite: Promise<void> = Promise.resolve();
|
||||
private nativeWritePromises = new Set<Promise<void>>();
|
||||
private nativeWriteError: Error | null = null;
|
||||
private pendingNativeWriteChunks: Uint8Array[] = [];
|
||||
private pendingNativeWriteBytes = 0;
|
||||
private maxNativeWriteInFlight = 1;
|
||||
private lastNativeExportError: string | null = null;
|
||||
private nativeH264Encoder: VideoEncoder | null = null;
|
||||
@@ -182,6 +185,7 @@ export class ModernVideoExporter {
|
||||
private finalizationTimeMs = 0;
|
||||
private finalizationStageMs: ExportFinalizationStageMetrics = {};
|
||||
private processedFrameCount = 0;
|
||||
private encodeCapacityWaiters = new Set<() => void>();
|
||||
private activeFinalizationProgressWatchdog: FinalizationProgressWatchdog | null = null;
|
||||
private lastFinalizationRenderProgress = INITIAL_FINALIZATION_PROGRESS_STATE.lastRenderProgress;
|
||||
private lastFinalizationAudioProgress = INITIAL_FINALIZATION_PROGRESS_STATE.lastAudioProgress;
|
||||
@@ -200,6 +204,7 @@ export class ModernVideoExporter {
|
||||
this.nativeEncoderError = null;
|
||||
this.totalExportStartTimeMs = this.getNowMs();
|
||||
const backendPreference = this.config.backendPreference ?? "auto";
|
||||
const runtimePlatform = this.getRuntimePlatform();
|
||||
let useNativeEncoder = false;
|
||||
this.lastNativeExportError = null;
|
||||
|
||||
@@ -213,6 +218,22 @@ export class ModernVideoExporter {
|
||||
`${NATIVE_EXPORT_ENGINE_NAME} export is unavailable for this output profile on this system.`,
|
||||
);
|
||||
}
|
||||
} else if (
|
||||
backendPreference === "auto" &&
|
||||
shouldPreferNativeAutoBackend(runtimePlatform)
|
||||
) {
|
||||
stageStartedAt = this.getNowMs();
|
||||
useNativeEncoder = await this.tryStartNativeVideoExport();
|
||||
this.nativeSessionStartTimeMs = this.getNowMs() - stageStartedAt;
|
||||
|
||||
if (!useNativeEncoder) {
|
||||
console.warn(
|
||||
`[VideoExporter] ${NATIVE_EXPORT_ENGINE_NAME} auto-preferred native export was unavailable; falling back to WebCodecs.`,
|
||||
this.lastNativeExportError,
|
||||
);
|
||||
stageStartedAt = this.getNowMs();
|
||||
await this.initializeEncoder();
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
const configuredWebCodecsPath = await this.initializeEncoder();
|
||||
@@ -307,7 +328,7 @@ export class ModernVideoExporter {
|
||||
this.renderer = new ModernFrameRenderer({
|
||||
width: this.config.width,
|
||||
height: this.config.height,
|
||||
preferredRenderBackend: useNativeEncoder ? "webgl" : undefined,
|
||||
preferredRenderBackend: undefined,
|
||||
wallpaper: this.config.wallpaper,
|
||||
zoomRegions: this.config.zoomRegions,
|
||||
showShadow: this.config.showShadow,
|
||||
@@ -392,7 +413,6 @@ export class ModernVideoExporter {
|
||||
async (videoFrame, _exportTimestampUs, sourceTimestampMs, cursorTimestampMs) => {
|
||||
const callbackStartedAt = this.getNowMs();
|
||||
if (this.cancelled) {
|
||||
videoFrame.close();
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -408,7 +428,6 @@ export class ModernVideoExporter {
|
||||
timestamp,
|
||||
);
|
||||
this.renderFrameTimeMs += this.getNowMs() - renderStartedAt;
|
||||
videoFrame.close();
|
||||
|
||||
if (this.cancelled) {
|
||||
return;
|
||||
@@ -609,12 +628,7 @@ export class ModernVideoExporter {
|
||||
}
|
||||
|
||||
private getPlatformLabel(): string {
|
||||
if (typeof navigator === "undefined") {
|
||||
return "Unknown";
|
||||
}
|
||||
|
||||
const platformHint = navigator.platform || navigator.userAgent || "";
|
||||
switch (normalizeLightningRuntimePlatform(platformHint)) {
|
||||
switch (this.getRuntimePlatform()) {
|
||||
case "win32":
|
||||
return "Windows";
|
||||
case "linux":
|
||||
@@ -622,10 +636,22 @@ export class ModernVideoExporter {
|
||||
case "darwin":
|
||||
return "macOS";
|
||||
default:
|
||||
return platformHint || "Unknown";
|
||||
if (typeof navigator === "undefined") {
|
||||
return "Unknown";
|
||||
}
|
||||
|
||||
return navigator.platform || navigator.userAgent || "Unknown";
|
||||
}
|
||||
}
|
||||
|
||||
private getRuntimePlatform() {
|
||||
if (typeof navigator === "undefined") {
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
return normalizeLightningRuntimePlatform(navigator.platform || navigator.userAgent || "");
|
||||
}
|
||||
|
||||
private getLightningErrorGuidance(message: string): string[] {
|
||||
const guidance = new Set<string>();
|
||||
const platform = this.getPlatformLabel();
|
||||
@@ -948,7 +974,8 @@ export class ModernVideoExporter {
|
||||
this.lastNativeExportError = null;
|
||||
this.encodeBackend = "ffmpeg";
|
||||
this.encoderName = "h264-stream-copy";
|
||||
this.nativePendingWrite = Promise.resolve();
|
||||
this.pendingNativeWriteChunks = [];
|
||||
this.pendingNativeWriteBytes = 0;
|
||||
|
||||
const sessionId = result.sessionId;
|
||||
const encoder = new VideoEncoder({
|
||||
@@ -959,40 +986,11 @@ export class ModernVideoExporter {
|
||||
|
||||
const buffer = new ArrayBuffer(chunk.byteLength);
|
||||
chunk.copyTo(buffer);
|
||||
const writePromise = this.nativePendingWrite
|
||||
.then(() =>
|
||||
window.electronAPI.nativeVideoExportWriteFrame(
|
||||
sessionId,
|
||||
new Uint8Array(buffer),
|
||||
),
|
||||
)
|
||||
.then((writeResult) => {
|
||||
if (!writeResult.success && !this.cancelled) {
|
||||
throw new Error(
|
||||
writeResult.error ||
|
||||
"Failed to write H.264 chunk to native encoder",
|
||||
);
|
||||
}
|
||||
})
|
||||
.catch((error) => {
|
||||
if (!this.cancelled) {
|
||||
const resolvedError =
|
||||
error instanceof Error ? error : new Error(String(error));
|
||||
if (!this.nativeEncoderError) {
|
||||
this.nativeEncoderError = resolvedError;
|
||||
}
|
||||
if (!this.nativeWriteError) {
|
||||
this.nativeWriteError = resolvedError;
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
this.nativePendingWrite = writePromise;
|
||||
|
||||
this.trackNativeWritePromise(writePromise);
|
||||
this.queueNativeWriteChunk(sessionId, new Uint8Array(buffer));
|
||||
},
|
||||
error: (error) => {
|
||||
this.nativeEncoderError = error;
|
||||
this.notifyEncodeCapacityAvailable();
|
||||
},
|
||||
});
|
||||
|
||||
@@ -1043,7 +1041,7 @@ export class ModernVideoExporter {
|
||||
while (
|
||||
this.nativeH264Encoder.encodeQueueSize >= ModernVideoExporter.NATIVE_ENCODER_QUEUE_LIMIT
|
||||
) {
|
||||
await new Promise<void>((r) => setTimeout(r, 2));
|
||||
await this.waitForEncodeCapacity();
|
||||
if (this.cancelled) return;
|
||||
if (this.nativeEncoderError) throw this.nativeEncoderError;
|
||||
}
|
||||
@@ -1100,6 +1098,7 @@ export class ModernVideoExporter {
|
||||
encoderName: this.encoderName ?? "unknown",
|
||||
});
|
||||
|
||||
this.flushPendingNativeWriteBatch(sessionId);
|
||||
await this.awaitPendingNativeWrites();
|
||||
|
||||
const result = await this.measureFinalizationStage("nativeExportFinalizeMs", async () =>
|
||||
@@ -1138,7 +1137,6 @@ export class ModernVideoExporter {
|
||||
this.finalizationStageMs.ffmpegAudioMuxBreakdown = result.metrics;
|
||||
}
|
||||
this.nativeExportSessionId = null;
|
||||
this.nativePendingWrite = Promise.resolve();
|
||||
|
||||
if (!result.success) {
|
||||
return {
|
||||
@@ -1317,7 +1315,7 @@ export class ModernVideoExporter {
|
||||
) {
|
||||
const encodeWaitStartedAt = this.getNowMs();
|
||||
this.encodeWaitEvents++;
|
||||
await new Promise((resolve) => setTimeout(resolve, 2));
|
||||
await this.waitForEncodeCapacity();
|
||||
this.encodeWaitTimeMs += this.getNowMs() - encodeWaitStartedAt;
|
||||
}
|
||||
|
||||
@@ -1368,6 +1366,72 @@ export class ModernVideoExporter {
|
||||
this.reportProgress(totalFrames, totalFrames, "finalizing", renderProgress, audioProgress);
|
||||
}
|
||||
|
||||
private queueNativeWriteChunk(sessionId: string, chunk: Uint8Array): void {
|
||||
this.pendingNativeWriteChunks.push(chunk);
|
||||
this.pendingNativeWriteBytes += chunk.byteLength;
|
||||
|
||||
if (
|
||||
this.pendingNativeWriteChunks.length >=
|
||||
ModernVideoExporter.NATIVE_WRITE_BATCH_MAX_CHUNKS ||
|
||||
this.pendingNativeWriteBytes >= ModernVideoExporter.NATIVE_WRITE_BATCH_MAX_BYTES
|
||||
) {
|
||||
this.flushPendingNativeWriteBatch(sessionId);
|
||||
}
|
||||
}
|
||||
|
||||
private flushPendingNativeWriteBatch(sessionId: string): void {
|
||||
if (this.pendingNativeWriteChunks.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const chunks = this.pendingNativeWriteChunks;
|
||||
this.pendingNativeWriteChunks = [];
|
||||
this.pendingNativeWriteBytes = 0;
|
||||
const writePromise = window.electronAPI
|
||||
.nativeVideoExportWriteFrames(sessionId, chunks)
|
||||
.then((writeResult) => {
|
||||
if (!writeResult.success && !this.cancelled) {
|
||||
throw new Error(
|
||||
writeResult.error || "Failed to write H.264 chunks to native encoder",
|
||||
);
|
||||
}
|
||||
})
|
||||
.catch((error) => {
|
||||
if (!this.cancelled) {
|
||||
const resolvedError =
|
||||
error instanceof Error ? error : new Error(String(error));
|
||||
if (!this.nativeEncoderError) {
|
||||
this.nativeEncoderError = resolvedError;
|
||||
}
|
||||
if (!this.nativeWriteError) {
|
||||
this.nativeWriteError = resolvedError;
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
|
||||
this.trackNativeWritePromise(writePromise);
|
||||
this.notifyEncodeCapacityAvailable();
|
||||
}
|
||||
|
||||
private waitForEncodeCapacity(): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
this.encodeCapacityWaiters.add(resolve);
|
||||
});
|
||||
}
|
||||
|
||||
private notifyEncodeCapacityAvailable(): void {
|
||||
if (this.encodeCapacityWaiters.size === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const waiters = [...this.encodeCapacityWaiters];
|
||||
this.encodeCapacityWaiters.clear();
|
||||
for (const resolve of waiters) {
|
||||
resolve();
|
||||
}
|
||||
}
|
||||
|
||||
private reportProgress(
|
||||
currentFrame: number,
|
||||
totalFrames: number,
|
||||
@@ -1656,6 +1720,7 @@ export class ModernVideoExporter {
|
||||
}
|
||||
});
|
||||
this.encodeQueue--;
|
||||
this.notifyEncodeCapacityAvailable();
|
||||
},
|
||||
error: (error) => {
|
||||
console.error(
|
||||
@@ -1664,6 +1729,7 @@ export class ModernVideoExporter {
|
||||
);
|
||||
this.encoderError = error instanceof Error ? error : new Error(String(error));
|
||||
this.cancelled = true;
|
||||
this.notifyEncodeCapacityAvailable();
|
||||
},
|
||||
});
|
||||
|
||||
@@ -1831,9 +1897,12 @@ export class ModernVideoExporter {
|
||||
this.lastProgressSampleTimeMs = 0;
|
||||
this.lastProgressSampleFrame = 0;
|
||||
this.nativeWritePromises = new Set();
|
||||
this.nativePendingWrite = Promise.resolve();
|
||||
this.nativeWriteError = null;
|
||||
this.pendingNativeWriteChunks = [];
|
||||
this.pendingNativeWriteBytes = 0;
|
||||
this.maxNativeWriteInFlight = 1;
|
||||
this.notifyEncodeCapacityAvailable();
|
||||
this.encodeCapacityWaiters.clear();
|
||||
this.videoDescription = undefined;
|
||||
this.videoColorSpace = undefined;
|
||||
this.renderBackend = null;
|
||||
|
||||
@@ -5,35 +5,107 @@ import {
|
||||
StreamingVideoDecoder,
|
||||
} from "./streamingDecoder";
|
||||
|
||||
const {
|
||||
mockDemuxerLoad,
|
||||
mockDemuxerGetMediaInfo,
|
||||
mockDemuxerDestroy,
|
||||
mockDemuxerGetDecoderConfig,
|
||||
} = vi.hoisted(() => ({
|
||||
mockDemuxerLoad: vi.fn(),
|
||||
mockDemuxerGetMediaInfo: vi.fn(async () => ({
|
||||
duration: 4,
|
||||
start_time: 0,
|
||||
streams: [
|
||||
{
|
||||
codec_type_string: "video",
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
avg_frame_rate: "30/1",
|
||||
codec_string: "avc1.640034",
|
||||
start_time: 0,
|
||||
duration: 4,
|
||||
},
|
||||
],
|
||||
})),
|
||||
mockDemuxerDestroy: vi.fn(),
|
||||
mockDemuxerGetDecoderConfig: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("web-demuxer", () => ({
|
||||
WebDemuxer: class MockWebDemuxer {
|
||||
load = mockDemuxerLoad;
|
||||
getMediaInfo = mockDemuxerGetMediaInfo;
|
||||
destroy = mockDemuxerDestroy;
|
||||
getDecoderConfig = mockDemuxerGetDecoderConfig;
|
||||
},
|
||||
}));
|
||||
|
||||
describe("StreamingVideoDecoder local media loading", () => {
|
||||
beforeEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
mockDemuxerLoad.mockReset();
|
||||
mockDemuxerGetMediaInfo.mockClear();
|
||||
mockDemuxerDestroy.mockClear();
|
||||
mockDemuxerGetDecoderConfig.mockClear();
|
||||
Object.assign(globalThis, {
|
||||
window: {
|
||||
location: {
|
||||
href: "http://localhost:5173/",
|
||||
},
|
||||
electronAPI: {
|
||||
readLocalFile: vi.fn(),
|
||||
getLocalMediaUrl: vi.fn(async (filePath: string) => ({
|
||||
success: true,
|
||||
url: `http://127.0.0.1:4321/video?path=${encodeURIComponent(filePath)}`,
|
||||
})),
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("loads loopback media-server URLs through Electron IPC instead of fetch", async () => {
|
||||
const readLocalFile = vi.fn(async () => ({
|
||||
it("loads loopback media-server URLs directly into WebDemuxer", async () => {
|
||||
const decoder = new StreamingVideoDecoder();
|
||||
await decoder.loadMetadata("http://127.0.0.1:43123/video?path=%2Ftmp%2Fcapture.mp4");
|
||||
|
||||
expect((window as any).electronAPI.readLocalFile).not.toHaveBeenCalled();
|
||||
expect(mockDemuxerLoad).toHaveBeenCalledWith(
|
||||
"http://127.0.0.1:43123/video?path=%2Ftmp%2Fcapture.mp4",
|
||||
);
|
||||
});
|
||||
|
||||
it("normalizes absolute local paths to file URLs before loading them", async () => {
|
||||
const decoder = new StreamingVideoDecoder();
|
||||
await decoder.loadMetadata("/tmp/capture.mp4");
|
||||
|
||||
expect((window as any).electronAPI.getLocalMediaUrl).toHaveBeenCalledWith(
|
||||
"/tmp/capture.mp4",
|
||||
);
|
||||
expect(mockDemuxerLoad).toHaveBeenCalledWith(
|
||||
"http://127.0.0.1:4321/video?path=%2Ftmp%2Fcapture.mp4",
|
||||
);
|
||||
});
|
||||
|
||||
it("falls back to a readable File when direct loading fails", async () => {
|
||||
mockDemuxerLoad.mockReset();
|
||||
mockDemuxerLoad
|
||||
.mockRejectedValueOnce(new Error("get_media_info failed: Failed after 3 attempts"))
|
||||
.mockResolvedValueOnce(undefined);
|
||||
(window as any).electronAPI.readLocalFile = vi.fn(async () => ({
|
||||
success: true,
|
||||
data: new Uint8Array([1, 2, 3]),
|
||||
}));
|
||||
(window as any).electronAPI.readLocalFile = readLocalFile;
|
||||
const fetchSpy = vi.fn();
|
||||
vi.stubGlobal("fetch", fetchSpy);
|
||||
|
||||
const decoder = new StreamingVideoDecoder();
|
||||
const file = await (decoder as any).loadVideoFile(
|
||||
"http://127.0.0.1:43123/video?path=%2Ftmp%2Fcapture.mp4",
|
||||
);
|
||||
await decoder.loadMetadata("/tmp/fallback.mp4");
|
||||
|
||||
expect(readLocalFile).toHaveBeenCalledWith("/tmp/capture.mp4");
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
expect(file.name).toBe("capture.mp4");
|
||||
expect(mockDemuxerLoad).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
"http://127.0.0.1:4321/video?path=%2Ftmp%2Ffallback.mp4",
|
||||
);
|
||||
expect(mockDemuxerLoad.mock.calls[1]?.[0]).toBeInstanceOf(File);
|
||||
expect((window as any).electronAPI.readLocalFile).toHaveBeenCalledWith(
|
||||
"/tmp/fallback.mp4",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -1,10 +1,16 @@
|
||||
import { WebDemuxer } from "web-demuxer";
|
||||
import type { SpeedRegion, TrimRegion } from "@/components/video-editor/types";
|
||||
import { getEffectiveVideoStreamDurationSeconds } from "@/lib/mediaTiming";
|
||||
import { getLocalFilePath } from "./localMediaSource";
|
||||
import {
|
||||
createReadableMediaResourceFile,
|
||||
resolveMediaResourceUrl,
|
||||
} from "./localMediaSource";
|
||||
|
||||
const DEFAULT_MAX_DECODE_QUEUE = 12;
|
||||
const DEFAULT_MAX_PENDING_FRAMES = 32;
|
||||
const STARTUP_STABILIZATION_SECONDS = 1.25;
|
||||
const STARTUP_MAX_DECODE_QUEUE = 12;
|
||||
const STARTUP_MAX_PENDING_FRAMES = 28;
|
||||
|
||||
export interface DecodedVideoInfo {
|
||||
width: number;
|
||||
@@ -20,7 +26,7 @@ export interface DecodedVideoInfo {
|
||||
audioSampleRate?: number;
|
||||
}
|
||||
|
||||
/** Caller must close the VideoFrame after use. */
|
||||
/** Decoder retains ownership of the VideoFrame and closes it after use. */
|
||||
type OnFrameCallback = (
|
||||
frame: VideoFrame,
|
||||
exportTimestampUs: number,
|
||||
@@ -81,66 +87,6 @@ export class StreamingVideoDecoder {
|
||||
);
|
||||
}
|
||||
|
||||
private inferMimeType(fileName: string): string {
|
||||
const extension = fileName.split(".").pop()?.toLowerCase();
|
||||
switch (extension) {
|
||||
case "mov":
|
||||
return "video/quicktime";
|
||||
case "webm":
|
||||
return "video/webm";
|
||||
case "mkv":
|
||||
return "video/x-matroska";
|
||||
case "avi":
|
||||
return "video/x-msvideo";
|
||||
case "mp4":
|
||||
default:
|
||||
return "video/mp4";
|
||||
}
|
||||
}
|
||||
|
||||
private async loadVideoFile(resourceUrl: string): Promise<File> {
|
||||
const localFilePath = getLocalFilePath(resourceUrl);
|
||||
const filename =
|
||||
(localFilePath ?? resourceUrl).split(/[\\/]/).pop()?.split("?")[0] || "video";
|
||||
|
||||
if (localFilePath) {
|
||||
const result = await window.electronAPI.readLocalFile(localFilePath);
|
||||
if (!result.success || !result.data) {
|
||||
throw new Error(result.error || "Failed to read local video file");
|
||||
}
|
||||
|
||||
const bytes =
|
||||
result.data instanceof Uint8Array ? result.data : new Uint8Array(result.data);
|
||||
const arrayBuffer = bytes.buffer.slice(
|
||||
bytes.byteOffset,
|
||||
bytes.byteOffset + bytes.byteLength,
|
||||
) as ArrayBuffer;
|
||||
return new File([arrayBuffer], filename, { type: this.inferMimeType(filename) });
|
||||
}
|
||||
|
||||
const response = await fetch(resourceUrl);
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`Failed to load video resource: ${response.status} ${response.statusText}`,
|
||||
);
|
||||
}
|
||||
|
||||
const blob = await response.blob();
|
||||
return new File([blob], filename, { type: blob.type || this.inferMimeType(filename) });
|
||||
}
|
||||
|
||||
private resolveVideoResourceUrl(videoUrl: string): string {
|
||||
if (/^(blob:|data:|https?:|file:)/i.test(videoUrl)) {
|
||||
return videoUrl;
|
||||
}
|
||||
|
||||
if (videoUrl.startsWith("/")) {
|
||||
return `file://${encodeURI(videoUrl)}`;
|
||||
}
|
||||
|
||||
return videoUrl;
|
||||
}
|
||||
|
||||
async loadMetadata(videoUrl: string): Promise<DecodedVideoInfo> {
|
||||
if (this.decoder) {
|
||||
try {
|
||||
@@ -162,15 +108,35 @@ export class StreamingVideoDecoder {
|
||||
this.demuxer = null;
|
||||
}
|
||||
|
||||
const resourceUrl = this.resolveVideoResourceUrl(videoUrl);
|
||||
const resourceUrl = await resolveMediaResourceUrl(videoUrl);
|
||||
|
||||
// Relative URL so it resolves correctly in both dev (http) and packaged (file://) builds
|
||||
const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href;
|
||||
this.demuxer = new WebDemuxer({ wasmFilePath: wasmUrl });
|
||||
const file = await this.loadVideoFile(resourceUrl);
|
||||
await this.demuxer.load(file);
|
||||
const loadMediaInfo = async (source: string | File) => {
|
||||
this.demuxer = new WebDemuxer({ wasmFilePath: wasmUrl });
|
||||
await this.demuxer.load(source);
|
||||
return this.demuxer.getMediaInfo();
|
||||
};
|
||||
|
||||
let mediaInfo;
|
||||
try {
|
||||
mediaInfo = await loadMediaInfo(resourceUrl);
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
"[StreamingVideoDecoder] Direct source load failed, retrying with file fallback:",
|
||||
error,
|
||||
);
|
||||
const currentDemuxer = this.demuxer;
|
||||
if (currentDemuxer) {
|
||||
try {
|
||||
(currentDemuxer as unknown as { destroy: () => void }).destroy();
|
||||
} catch {
|
||||
// Ignore cleanup errors before fallback re-init.
|
||||
}
|
||||
}
|
||||
mediaInfo = await loadMediaInfo(await createReadableMediaResourceFile(videoUrl));
|
||||
}
|
||||
|
||||
const mediaInfo = await this.demuxer.getMediaInfo();
|
||||
const videoStream = mediaInfo.streams.find((s) => s.codec_type_string === "video");
|
||||
const audioStream = mediaInfo.streams.find((s) => s.codec_type_string === "audio");
|
||||
const mediaStartTime =
|
||||
@@ -245,13 +211,31 @@ export class StreamingVideoDecoder {
|
||||
);
|
||||
const frameDurationUs = 1_000_000 / targetFrameRate;
|
||||
const epsilonSec = 0.001;
|
||||
const startupStabilizationSeconds = 3;
|
||||
const startupStabilizationSeconds = STARTUP_STABILIZATION_SECONDS;
|
||||
const startupFrameBudget = Math.max(
|
||||
1,
|
||||
Math.round(targetFrameRate * startupStabilizationSeconds),
|
||||
);
|
||||
let exportFrameIndex = 0;
|
||||
let loggedSteadyStateBackpressure = false;
|
||||
const backpressureWaiters = new Set<() => void>();
|
||||
|
||||
const notifyBackpressureProgress = () => {
|
||||
if (backpressureWaiters.size === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const waiters = [...backpressureWaiters];
|
||||
backpressureWaiters.clear();
|
||||
for (const resolve of waiters) {
|
||||
resolve();
|
||||
}
|
||||
};
|
||||
|
||||
const waitForBackpressureProgress = () =>
|
||||
new Promise<void>((resolve) => {
|
||||
backpressureWaiters.add(resolve);
|
||||
});
|
||||
|
||||
console.log(
|
||||
`[StreamingVideoDecoder] Startup-safe decode backpressure active for first ${startupStabilizationSeconds}s (${startupFrameBudget} frames)`,
|
||||
@@ -275,6 +259,7 @@ export class StreamingVideoDecoder {
|
||||
} else {
|
||||
pendingFrames.push(frame);
|
||||
}
|
||||
notifyBackpressureProgress();
|
||||
},
|
||||
error: (e: DOMException) => {
|
||||
decodeError = new Error(`VideoDecoder error: ${e.message}`);
|
||||
@@ -283,6 +268,7 @@ export class StreamingVideoDecoder {
|
||||
frameResolve = null;
|
||||
resolve(null);
|
||||
}
|
||||
notifyBackpressureProgress();
|
||||
},
|
||||
});
|
||||
const preferredDecoderConfig = shouldPreferSoftwareDecode
|
||||
@@ -304,7 +290,11 @@ export class StreamingVideoDecoder {
|
||||
|
||||
const getNextFrame = (): Promise<VideoFrame | null> => {
|
||||
if (decodeError) throw decodeError;
|
||||
if (pendingFrames.length > 0) return Promise.resolve(pendingFrames.shift()!);
|
||||
if (pendingFrames.length > 0) {
|
||||
const frame = pendingFrames.shift()!;
|
||||
notifyBackpressureProgress();
|
||||
return Promise.resolve(frame);
|
||||
}
|
||||
if (decodeDone) return Promise.resolve(null);
|
||||
return new Promise((resolve) => {
|
||||
frameResolve = resolve;
|
||||
@@ -337,11 +327,11 @@ export class StreamingVideoDecoder {
|
||||
|
||||
const decodeQueueLimit =
|
||||
exportFrameIndex < startupFrameBudget
|
||||
? Math.min(this.maxDecodeQueue, 10)
|
||||
? Math.min(this.maxDecodeQueue, STARTUP_MAX_DECODE_QUEUE)
|
||||
: this.maxDecodeQueue;
|
||||
const pendingFrameLimit =
|
||||
exportFrameIndex < startupFrameBudget
|
||||
? Math.min(this.maxPendingFrames, 24)
|
||||
? Math.min(this.maxPendingFrames, STARTUP_MAX_PENDING_FRAMES)
|
||||
: this.maxPendingFrames;
|
||||
|
||||
// Backpressure on both decode queue and decoded frame backlog.
|
||||
@@ -350,7 +340,7 @@ export class StreamingVideoDecoder {
|
||||
pendingFrames.length > pendingFrameLimit) &&
|
||||
!this.cancelled
|
||||
) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 1));
|
||||
await waitForBackpressureProgress();
|
||||
}
|
||||
if (this.cancelled) break;
|
||||
|
||||
@@ -369,6 +359,7 @@ export class StreamingVideoDecoder {
|
||||
frameResolve = null;
|
||||
resolve(null);
|
||||
}
|
||||
notifyBackpressureProgress();
|
||||
}
|
||||
})();
|
||||
|
||||
@@ -393,10 +384,9 @@ export class StreamingVideoDecoder {
|
||||
segment.startSec + (segmentFrameIndex / segmentFrameCount) * segmentDurationSec;
|
||||
if (sourceTimeSec >= segment.endSec - epsilonSec) return false;
|
||||
|
||||
const clone = new VideoFrame(heldFrame, { timestamp: heldFrame.timestamp });
|
||||
const sourceTimestampMs = sourceTimeSec * 1000;
|
||||
await onFrame(
|
||||
clone,
|
||||
heldFrame,
|
||||
exportFrameIndex * frameDurationUs,
|
||||
sourceTimestampMs,
|
||||
sourceTimestampMs,
|
||||
@@ -489,10 +479,9 @@ export class StreamingVideoDecoder {
|
||||
break;
|
||||
}
|
||||
|
||||
const clone = new VideoFrame(heldFrame, { timestamp: heldFrame.timestamp });
|
||||
const sourceTimestampMs = sourceTimeSec * 1000;
|
||||
await onFrame(
|
||||
clone,
|
||||
heldFrame,
|
||||
exportFrameIndex * frameDurationUs,
|
||||
sourceTimestampMs,
|
||||
sourceTimestampMs,
|
||||
|
||||
@@ -284,7 +284,6 @@ export class VideoExporter {
|
||||
this.config.speedRegions,
|
||||
async (videoFrame, _exportTimestampUs, sourceTimestampMs, cursorTimestampMs) => {
|
||||
if (this.cancelled) {
|
||||
videoFrame.close();
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -298,7 +297,6 @@ export class VideoExporter {
|
||||
frameDuration,
|
||||
timestamp,
|
||||
);
|
||||
videoFrame.close();
|
||||
|
||||
if (useNativeEncoder) {
|
||||
await this.encodeRenderedFrameNative(timestamp, frameDuration, frameIndex);
|
||||
|
||||
Reference in New Issue
Block a user