fix(export): merge upstream audio timing and cross-platform path normalization

This commit is contained in:
webadderall committed 2026-03-15 12:01:45 +11:00
1 parent 931d95bd20
commit 7404301bee
7 files changed
+463 -56

No files matched your search

+27 -20
View File
@@ -6,6 +6,7 @@ import fs from 'node:fs/promises'
import { constants as fsConstants } from 'node:fs'
import { createRequire } from 'node:module'
import path from 'node:path'
import { fileURLToPath, pathToFileURL } from 'node:url'
import { promisify } from 'node:util'
import { RECORDINGS_DIR } from '../main'
@@ -182,21 +183,25 @@ async function persistRecordingsDirectorySetting(nextDir: string) {
)
}
function normalizeVideoPathInput(videoPath: string) {
if (!videoPath.startsWith('file:')) {
return videoPath
function normalizeVideoSourcePath(videoPath?: string | null): string | null {
if (typeof videoPath !== 'string') {
return null
}
try {
const url = new URL(videoPath)
let filePath = decodeURIComponent(url.pathname)
if (/^\/[A-Za-z]:/.test(filePath)) {
filePath = filePath.slice(1)
}
return filePath
} catch {
return videoPath.replace(/^file:\/\//, '')
const trimmed = videoPath.trim()
if (!trimmed) {
return null
}
if (/^file:\/\//i.test(trimmed)) {
try {
return fileURLToPath(trimmed)
} catch {
// Fall through and keep best-effort string path below.
}
}
return trimmed
}
async function hasSiblingProjectFile(videoPath: string) {
@@ -2223,7 +2228,7 @@ export function registerIpcHandlers(
})
ipcMain.handle('get-cursor-telemetry', async (_, videoPath?: string) => {
const targetVideoPath = videoPath ? normalizeVideoPathInput(videoPath) : currentVideoPath
const targetVideoPath = normalizeVideoSourcePath(videoPath ?? currentVideoPath)
if (!targetVideoPath) {
return { success: true, samples: [] }
}
@@ -2361,9 +2366,11 @@ export function registerIpcHandlers(
ipcMain.handle('get-asset-base-path', () => {
try {
if (app.isPackaged) {
return path.join(process.resourcesPath, 'assets')
const assetPath = path.join(process.resourcesPath, 'assets')
return pathToFileURL(`${assetPath}${path.sep}`).toString()
}
return path.join(app.getAppPath(), 'public')
const assetPath = path.join(app.getAppPath(), 'public')
return pathToFileURL(`${assetPath}${path.sep}`).toString()
} catch (err) {
console.error('Failed to resolve asset base path:', err)
return null
@@ -2614,7 +2621,7 @@ export function registerIpcHandlers(
const project = JSON.parse(content)
currentProjectPath = filePath
if (project && typeof project === 'object' && typeof project.videoPath === 'string') {
currentVideoPath = project.videoPath
currentVideoPath = normalizeVideoSourcePath(project.videoPath) ?? project.videoPath
}
return {
@@ -2641,7 +2648,7 @@ export function registerIpcHandlers(
const content = await fs.readFile(currentProjectPath, 'utf-8')
const project = JSON.parse(content)
if (project && typeof project === 'object' && typeof project.videoPath === 'string') {
currentVideoPath = project.videoPath
currentVideoPath = normalizeVideoSourcePath(project.videoPath) ?? project.videoPath
}
return {
success: true,
@@ -2658,10 +2665,10 @@ export function registerIpcHandlers(
}
})
ipcMain.handle('set-current-video-path', (_, path: string) => {
currentVideoPath = path;
currentVideoPath = normalizeVideoSourcePath(path) ?? path
currentProjectPath = null
return { success: true };
});
return { success: true }
})
ipcMain.handle('get-current-video-path', () => {
return currentVideoPath ? { success: true, path: currentVideoPath } : { success: false };
+4 -3
View File
@@ -216,7 +216,7 @@ export default function VideoEditor() {
}
const project = candidate;
const sourcePath = project.videoPath;
const sourcePath = fromFileUrl(project.videoPath);
const normalizedEditor = normalizeProjectEditor(project.editor);
try {
@@ -387,8 +387,9 @@ export default function VideoEditor() {
const result = await window.electronAPI.getCurrentVideoPath();
if (result.success && result.path) {
setVideoSourcePath(result.path);
setVideoPath(toFileUrl(result.path));
const sourcePath = fromFileUrl(result.path);
setVideoSourcePath(sourcePath);
setVideoPath(toFileUrl(sourcePath));
setCurrentProjectPath(null);
setLastSavedSnapshot(null);
} else {
@@ -63,28 +63,71 @@ function clamp(value: number, min: number, max: number) {
return Math.min(max, Math.max(min, value));
}
function isFileUrl(value: string): boolean {
return /^file:\/\//i.test(value);
}
function encodePathSegments(pathname: string, keepWindowsDrive = false): string {
return pathname
.split("/")
.map((segment, index) => {
if (!segment) return "";
if (keepWindowsDrive && index === 1 && /^[a-zA-Z]:$/.test(segment)) {
return segment;
}
return encodeURIComponent(segment);
})
.join("/");
}
export function toFileUrl(filePath: string): string {
const normalized = filePath.replace(/\\/g, "/");
if (normalized.match(/^[a-zA-Z]:/)) {
return `file:///${normalized}`;
// Windows drive path: C:/Users/...
if (/^[a-zA-Z]:\//.test(normalized)) {
return `file://${encodePathSegments(`/${normalized}`, true)}`;
}
return `file://${normalized}`;
// UNC path: //server/share/...
if (normalized.startsWith("//")) {
const [host, ...pathParts] = normalized.replace(/^\/+/, "").split("/");
const encodedPath = pathParts.map((part) => encodeURIComponent(part)).join("/");
return encodedPath ? `file://${host}/${encodedPath}` : `file://${host}/`;
}
const absolutePath = normalized.startsWith("/") ? normalized : `/${normalized}`;
return `file://${encodePathSegments(absolutePath)}`;
}
export function fromFileUrl(fileUrl: string): string {
if (!fileUrl.startsWith("file://")) {
const value = fileUrl.trim();
if (!isFileUrl(value)) {
return fileUrl;
}
try {
const url = new URL(fileUrl);
let filePath = decodeURIComponent(url.pathname);
if (/^\/[A-Za-z]:/.test(filePath)) {
filePath = filePath.slice(1);
const url = new URL(value);
const pathname = decodeURIComponent(url.pathname);
if (url.host && url.host !== "localhost") {
const uncPath = `//${url.host}${pathname.startsWith("/") ? pathname : `/${pathname}`}`;
return uncPath.replace(/\//g, "\\");
}
return filePath;
if (/^\/[A-Za-z]:/.test(pathname)) {
return pathname.slice(1);
}
return pathname;
} catch {
return fileUrl.replace(/^file:\/\//, "");
const rawFallbackPath = value.replace(/^file:\/\//i, "");
let fallbackPath = rawFallbackPath;
try {
fallbackPath = decodeURIComponent(rawFallbackPath);
} catch {
// Keep raw best-effort path if percent decoding fails.
}
return fallbackPath.replace(/^\/([a-zA-Z]:)/, "$1");
}
}
+18 -4
View File
@@ -1,16 +1,30 @@
function encodeRelativeAssetPath(relativePath: string): string {
return relativePath
.replace(/^\/+/, '')
.split('/')
.filter(Boolean)
.map((part) => encodeURIComponent(part))
.join('/')
}
function ensureTrailingSlash(value: string): string {
return value.endsWith('/') ? value : `${value}/`
}
export async function getAssetPath(relativePath: string): Promise<string> {
const encodedRelativePath = encodeRelativeAssetPath(relativePath)
try {
if (typeof window !== 'undefined') {
// If running in a dev server (http/https), prefer the web-served path
if (window.location && window.location.protocol && window.location.protocol.startsWith('http')) {
return `/${relativePath.replace(/^\//, '')}`
return `/${encodedRelativePath}`
}
if ((window as any).electronAPI && typeof (window as any).electronAPI.getAssetBasePath === 'function') {
const base = await (window as any).electronAPI.getAssetBasePath()
if (base) {
const normalized = base.replace(/\\/g, '/')
return `file://${normalized}/${relativePath}`
return new URL(encodedRelativePath, ensureTrailingSlash(base)).toString()
}
}
}
@@ -19,7 +33,7 @@ export async function getAssetPath(relativePath: string): Promise<string> {
}
// Fallback for web/dev server: public/wallpapers are served at '/wallpapers/...'
return `/${relativePath.replace(/^\//, '')}`
return `/${encodedRelativePath}`
}
const BASE64_CHUNK_SIZE = 0x8000
+328 -5
View File
@@ -1,14 +1,58 @@
import type { WebDemuxer } from 'web-demuxer'
import type { TrimRegion } from '@/components/video-editor/types'
import { WebDemuxer } from 'web-demuxer'
import type { SpeedRegion, TrimRegion } from '@/components/video-editor/types'
import type { VideoMuxer } from './muxer'
const AUDIO_BITRATE = 128_000
const DECODE_BACKPRESSURE_LIMIT = 20
const MIN_SPEED_REGION_DELTA_MS = 0.0001
export class AudioProcessor {
private cancelled = false
async process(demuxer: WebDemuxer, muxer: VideoMuxer, trimRegions?: TrimRegion[]): Promise<void> {
/**
* Audio export has two modes:
* 1) no speed regions -> fast WebCodecs trim-only pipeline
* 2) speed regions present -> pitch-preserving rendered timeline pipeline
*/
async process(
demuxer: WebDemuxer,
muxer: VideoMuxer,
videoUrl: string,
trimRegions?: TrimRegion[],
speedRegions?: SpeedRegion[],
readEndSec?: number,
): Promise<void> {
const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) : []
const sortedSpeedRegions = speedRegions
? [...speedRegions]
.filter((region) => region.endMs - region.startMs > MIN_SPEED_REGION_DELTA_MS)
.sort((a, b) => a.startMs - b.startMs)
: []
// Speed edits must use timeline playback to preserve pitch.
if (sortedSpeedRegions.length > 0) {
const renderedAudioBlob = await this.renderPitchPreservedTimelineAudio(
videoUrl,
sortedTrims,
sortedSpeedRegions,
)
if (!this.cancelled) {
await this.muxRenderedAudioBlob(renderedAudioBlob, muxer)
return
}
}
// No speed edits: keep the original demux/decode/encode path with trim timestamp remap.
await this.processTrimOnlyAudio(demuxer, muxer, sortedTrims, readEndSec)
}
// Legacy trim-only path used when no speed regions are configured.
private async processTrimOnlyAudio(
demuxer: WebDemuxer,
muxer: VideoMuxer,
sortedTrims: TrimRegion[],
readEndSec?: number,
): Promise<void> {
let audioConfig: AudioDecoderConfig
try {
audioConfig = (await demuxer.getDecoderConfig('audio')) as AudioDecoderConfig
@@ -23,7 +67,6 @@ export class AudioProcessor {
return
}
const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) : []
const decodedFrames: AudioData[] = []
const decoder = new AudioDecoder({
@@ -32,7 +75,10 @@ export class AudioProcessor {
})
decoder.configure(audioConfig)
const reader = (demuxer.read('audio') as ReadableStream<EncodedAudioChunk>).getReader()
const audioStream = typeof readEndSec === 'number'
? demuxer.read('audio', 0, readEndSec)
: demuxer.read('audio')
const reader = (audioStream as ReadableStream<EncodedAudioChunk>).getReader()
while (!this.cancelled) {
const { done, value: chunk } = await reader.read()
@@ -112,6 +158,283 @@ export class AudioProcessor {
}
}
// Speed-aware path that mirrors preview semantics (trim skipping + playbackRate regions)
// and preserves pitch through browser media playback behavior.
private async renderPitchPreservedTimelineAudio(
videoUrl: string,
trimRegions: TrimRegion[],
speedRegions: SpeedRegion[],
): Promise<Blob> {
const media = document.createElement('audio')
media.src = videoUrl
media.preload = 'auto'
const pitchMedia = media as HTMLMediaElement & {
preservesPitch?: boolean
mozPreservesPitch?: boolean
webkitPreservesPitch?: boolean
}
pitchMedia.preservesPitch = true
pitchMedia.mozPreservesPitch = true
pitchMedia.webkitPreservesPitch = true
await this.waitForLoadedMetadata(media)
if (this.cancelled) {
throw new Error('Export cancelled')
}
const audioContext = new AudioContext()
const sourceNode = audioContext.createMediaElementSource(media)
const destinationNode = audioContext.createMediaStreamDestination()
sourceNode.connect(destinationNode)
const { recorder, recordedBlobPromise } = this.startAudioRecording(destinationNode.stream)
let rafId: number | null = null
try {
if (audioContext.state === 'suspended') {
await audioContext.resume()
}
await this.seekTo(media, 0)
await media.play()
await new Promise<void>((resolve, reject) => {
const cleanup = () => {
if (rafId !== null) {
cancelAnimationFrame(rafId)
rafId = null
}
media.removeEventListener('error', onError)
media.removeEventListener('ended', onEnded)
}
const onError = () => {
cleanup()
reject(new Error('Failed while rendering speed-adjusted audio timeline'))
}
const onEnded = () => {
cleanup()
resolve()
}
const tick = () => {
if (this.cancelled) {
cleanup()
resolve()
return
}
const currentTimeMs = media.currentTime * 1000
const activeTrimRegion = this.findActiveTrimRegion(currentTimeMs, trimRegions)
if (activeTrimRegion && !media.paused && !media.ended) {
const skipToTime = activeTrimRegion.endMs / 1000
if (skipToTime >= media.duration) {
media.pause()
cleanup()
resolve()
return
}
media.currentTime = skipToTime
} else {
const activeSpeedRegion = this.findActiveSpeedRegion(currentTimeMs, speedRegions)
const playbackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1
if (Math.abs(media.playbackRate - playbackRate) > 0.0001) {
media.playbackRate = playbackRate
}
}
if (!media.paused && !media.ended) {
rafId = requestAnimationFrame(tick)
} else {
cleanup()
resolve()
}
}
media.addEventListener('error', onError, { once: true })
media.addEventListener('ended', onEnded, { once: true })
rafId = requestAnimationFrame(tick)
})
} finally {
if (rafId !== null) {
cancelAnimationFrame(rafId)
}
media.pause()
if (recorder.state !== 'inactive') {
recorder.stop()
}
destinationNode.stream.getTracks().forEach((track) => track.stop())
sourceNode.disconnect()
destinationNode.disconnect()
await audioContext.close()
media.src = ''
media.load()
}
const recordedBlob = await recordedBlobPromise
if (this.cancelled) {
throw new Error('Export cancelled')
}
return recordedBlob
}
// Demuxes the rendered speed-adjusted blob and feeds encoded chunks into the MP4 muxer.
private async muxRenderedAudioBlob(blob: Blob, muxer: VideoMuxer): Promise<void> {
if (this.cancelled) return
const file = new File([blob], 'speed-audio.webm', { type: blob.type || 'audio/webm' })
const wasmUrl = new URL('./wasm/web-demuxer.wasm', window.location.href).href
const demuxer = new WebDemuxer({ wasmFilePath: wasmUrl })
try {
await demuxer.load(file)
const audioConfig = (await demuxer.getDecoderConfig('audio')) as AudioDecoderConfig
const reader = (demuxer.read('audio') as ReadableStream<EncodedAudioChunk>).getReader()
let isFirstChunk = true
try {
while (!this.cancelled) {
const { done, value: chunk } = await reader.read()
if (done || !chunk) break
if (isFirstChunk) {
await muxer.addAudioChunk(chunk, { decoderConfig: audioConfig })
isFirstChunk = false
} else {
await muxer.addAudioChunk(chunk)
}
}
} finally {
try {
await reader.cancel()
} catch {
// reader already closed
}
}
} finally {
try {
demuxer.destroy()
} catch {
// ignore
}
}
}
private startAudioRecording(stream: MediaStream): {
recorder: MediaRecorder
recordedBlobPromise: Promise<Blob>
} {
const mimeType = this.getSupportedAudioMimeType()
const options: MediaRecorderOptions = {
audioBitsPerSecond: AUDIO_BITRATE,
...(mimeType ? { mimeType } : {}),
}
const recorder = new MediaRecorder(stream, options)
const chunks: Blob[] = []
const recordedBlobPromise = new Promise<Blob>((resolve, reject) => {
recorder.ondataavailable = (event: BlobEvent) => {
if (event.data && event.data.size > 0) {
chunks.push(event.data)
}
}
recorder.onerror = () => {
reject(new Error('MediaRecorder failed while capturing speed-adjusted audio'))
}
recorder.onstop = () => {
const type = mimeType || chunks[0]?.type || 'audio/webm'
resolve(new Blob(chunks, { type }))
}
})
recorder.start()
return { recorder, recordedBlobPromise }
}
private getSupportedAudioMimeType(): string | undefined {
const candidates = ['audio/webm;codecs=opus', 'audio/webm']
for (const candidate of candidates) {
if (MediaRecorder.isTypeSupported(candidate)) {
return candidate
}
}
return undefined
}
private waitForLoadedMetadata(media: HTMLMediaElement): Promise<void> {
if (Number.isFinite(media.duration) && media.readyState >= HTMLMediaElement.HAVE_METADATA) {
return Promise.resolve()
}
return new Promise<void>((resolve, reject) => {
const onLoaded = () => {
cleanup()
resolve()
}
const onError = () => {
cleanup()
reject(new Error('Failed to load media metadata for speed-adjusted audio'))
}
const cleanup = () => {
media.removeEventListener('loadedmetadata', onLoaded)
media.removeEventListener('error', onError)
}
media.addEventListener('loadedmetadata', onLoaded)
media.addEventListener('error', onError, { once: true })
})
}
private seekTo(media: HTMLMediaElement, targetSec: number): Promise<void> {
if (Math.abs(media.currentTime - targetSec) < 0.0001) {
return Promise.resolve()
}
return new Promise<void>((resolve, reject) => {
const onSeeked = () => {
cleanup()
resolve()
}
const onError = () => {
cleanup()
reject(new Error('Failed to seek media for speed-adjusted audio'))
}
const cleanup = () => {
media.removeEventListener('seeked', onSeeked)
media.removeEventListener('error', onError)
}
media.addEventListener('seeked', onSeeked, { once: true })
media.addEventListener('error', onError, { once: true })
media.currentTime = targetSec
})
}
private findActiveTrimRegion(
currentTimeMs: number,
trimRegions: TrimRegion[],
): TrimRegion | null {
return (
trimRegions.find(
(region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs,
) || null
)
}
private findActiveSpeedRegion(
currentTimeMs: number,
speedRegions: SpeedRegion[],
): SpeedRegion | null {
return (
speedRegions.find(
(region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs,
) || null
)
}
private cloneWithTimestamp(src: AudioData, newTimestamp: number): AudioData {
const isPlanar = src.format?.includes('planar') ?? false
const numPlanes = isPlanar ? src.numberOfChannels : 1
+26 -13
View File
@@ -2,7 +2,7 @@ import { Application, Container, Sprite, Graphics, BlurFilter, Texture } from 'p
import { MotionBlurFilter } from 'pixi-filters/motion-blur';
import type { ZoomRegion, CropRegion, AnnotationRegion, SpeedRegion, CursorTelemetryPoint } from '@/components/video-editor/types';
import { ZOOM_DEPTH_SCALES } from '@/components/video-editor/types';
import { getRenderableAssetUrl } from '@/lib/assetPath';
import { getAssetPath, getRenderableAssetUrl } from '@/lib/assetPath';
import { findDominantRegion } from '@/components/video-editor/videoPlayback/zoomRegionUtils';
import { applyZoomTransform, computeFocusFromTransform, computeZoomTransform, createMotionBlurState, type MotionBlurState } from '@/components/video-editor/videoPlayback/zoomTransform';
import { DEFAULT_FOCUS, ZOOM_SCALE_DEADZONE, ZOOM_TRANSLATION_DEADZONE_PX } from '@/components/video-editor/videoPlayback/constants';
@@ -198,18 +198,14 @@ export class FrameRenderer {
if (wallpaper.startsWith('file://') || wallpaper.startsWith('data:') || wallpaper.startsWith('/') || wallpaper.startsWith('http')) {
// Image background
const img = new Image();
// Don't set crossOrigin for same-origin images to avoid CORS taint
// Only set it for cross-origin URLs
let imageUrl: string;
if (wallpaper.startsWith('http')) {
imageUrl = wallpaper;
if (!imageUrl.startsWith(window.location.origin)) {
img.crossOrigin = 'anonymous';
}
} else if (wallpaper.startsWith('file://') || wallpaper.startsWith('data:')) {
imageUrl = wallpaper;
} else {
imageUrl = window.location.origin + wallpaper;
const imageUrl = await this.resolveWallpaperImageUrl(wallpaper);
// Don't set crossOrigin for same-origin images to avoid CORS taint.
if (
imageUrl.startsWith('http')
&& window.location.origin
&& !imageUrl.startsWith(window.location.origin)
) {
img.crossOrigin = 'anonymous';
}
await new Promise<void>((resolve, reject) => {
@@ -301,6 +297,23 @@ export class FrameRenderer {
this.backgroundSprite = bgCanvas as any;
}
private async resolveWallpaperImageUrl(wallpaper: string): Promise<string> {
if (
wallpaper.startsWith('file://')
|| wallpaper.startsWith('data:')
|| wallpaper.startsWith('http')
) {
return wallpaper;
}
const resolved = await getAssetPath(wallpaper.replace(/^\/+/, ''));
if (resolved.startsWith('/') && window.location.protocol.startsWith('http')) {
return `${window.location.origin}${resolved}`;
}
return resolved;
}
private async resolveWallpaperForExport(wallpaper: string): Promise<string> {
if (!wallpaper) {
return wallpaper;
+7 -1
View File
@@ -151,7 +151,13 @@ export class VideoExporter {
if (demuxer) {
this.audioProcessor = new AudioProcessor();
await this.awaitWithWindowsTimeout(
this.audioProcessor.process(demuxer, this.muxer!, this.config.trimRegions),
this.audioProcessor.process(
demuxer,
this.muxer!,
this.config.videoUrl,
this.config.trimRegions,
this.config.speedRegions,
),
'audio processing',
);
}