From 7404301bee5360d3bc73e259a4fa20e7a3ef0afd Mon Sep 17 00:00:00 2001 From: webadderall <131426131+webadderall@users.noreply.github.com> Date: Sun, 15 Mar 2026 10:56:32 +1100 Subject: [PATCH] fix(export): merge upstream audio timing and cross-platform path normalization --- electron/ipc/handlers.ts | 47 +-- src/components/video-editor/VideoEditor.tsx | 7 +- .../video-editor/projectPersistence.ts | 63 +++- src/lib/assetPath.ts | 22 +- src/lib/exporter/audioEncoder.ts | 333 +++++++++++++++++- src/lib/exporter/frameRenderer.ts | 39 +- src/lib/exporter/videoExporter.ts | 8 +- 7 files changed, 463 insertions(+), 56 deletions(-) diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 20b8119a..ef1d22d9 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -6,6 +6,7 @@ import fs from 'node:fs/promises' import { constants as fsConstants } from 'node:fs' import { createRequire } from 'node:module' import path from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' import { promisify } from 'node:util' import { RECORDINGS_DIR } from '../main' @@ -182,21 +183,25 @@ async function persistRecordingsDirectorySetting(nextDir: string) { ) } -function normalizeVideoPathInput(videoPath: string) { - if (!videoPath.startsWith('file:')) { - return videoPath +function normalizeVideoSourcePath(videoPath?: string | null): string | null { + if (typeof videoPath !== 'string') { + return null } - try { - const url = new URL(videoPath) - let filePath = decodeURIComponent(url.pathname) - if (/^\/[A-Za-z]:/.test(filePath)) { - filePath = filePath.slice(1) - } - return filePath - } catch { - return videoPath.replace(/^file:\/\//, '') + const trimmed = videoPath.trim() + if (!trimmed) { + return null } + + if (/^file:\/\//i.test(trimmed)) { + try { + return fileURLToPath(trimmed) + } catch { + // Fall through and keep best-effort string path below. + } + } + + return trimmed } async function hasSiblingProjectFile(videoPath: string) { @@ -2223,7 +2228,7 @@ export function registerIpcHandlers( }) ipcMain.handle('get-cursor-telemetry', async (_, videoPath?: string) => { - const targetVideoPath = videoPath ? normalizeVideoPathInput(videoPath) : currentVideoPath + const targetVideoPath = normalizeVideoSourcePath(videoPath ?? currentVideoPath) if (!targetVideoPath) { return { success: true, samples: [] } } @@ -2361,9 +2366,11 @@ export function registerIpcHandlers( ipcMain.handle('get-asset-base-path', () => { try { if (app.isPackaged) { - return path.join(process.resourcesPath, 'assets') + const assetPath = path.join(process.resourcesPath, 'assets') + return pathToFileURL(`${assetPath}${path.sep}`).toString() } - return path.join(app.getAppPath(), 'public') + const assetPath = path.join(app.getAppPath(), 'public') + return pathToFileURL(`${assetPath}${path.sep}`).toString() } catch (err) { console.error('Failed to resolve asset base path:', err) return null @@ -2614,7 +2621,7 @@ export function registerIpcHandlers( const project = JSON.parse(content) currentProjectPath = filePath if (project && typeof project === 'object' && typeof project.videoPath === 'string') { - currentVideoPath = project.videoPath + currentVideoPath = normalizeVideoSourcePath(project.videoPath) ?? project.videoPath } return { @@ -2641,7 +2648,7 @@ export function registerIpcHandlers( const content = await fs.readFile(currentProjectPath, 'utf-8') const project = JSON.parse(content) if (project && typeof project === 'object' && typeof project.videoPath === 'string') { - currentVideoPath = project.videoPath + currentVideoPath = normalizeVideoSourcePath(project.videoPath) ?? project.videoPath } return { success: true, @@ -2658,10 +2665,10 @@ export function registerIpcHandlers( } }) ipcMain.handle('set-current-video-path', (_, path: string) => { - currentVideoPath = path; + currentVideoPath = normalizeVideoSourcePath(path) ?? path currentProjectPath = null - return { success: true }; - }); + return { success: true } + }) ipcMain.handle('get-current-video-path', () => { return currentVideoPath ? { success: true, path: currentVideoPath } : { success: false }; diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index fd7b3207..595b34e8 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -216,7 +216,7 @@ export default function VideoEditor() { } const project = candidate; - const sourcePath = project.videoPath; + const sourcePath = fromFileUrl(project.videoPath); const normalizedEditor = normalizeProjectEditor(project.editor); try { @@ -387,8 +387,9 @@ export default function VideoEditor() { const result = await window.electronAPI.getCurrentVideoPath(); if (result.success && result.path) { - setVideoSourcePath(result.path); - setVideoPath(toFileUrl(result.path)); + const sourcePath = fromFileUrl(result.path); + setVideoSourcePath(sourcePath); + setVideoPath(toFileUrl(sourcePath)); setCurrentProjectPath(null); setLastSavedSnapshot(null); } else { diff --git a/src/components/video-editor/projectPersistence.ts b/src/components/video-editor/projectPersistence.ts index 81309d84..693c3429 100644 --- a/src/components/video-editor/projectPersistence.ts +++ b/src/components/video-editor/projectPersistence.ts @@ -63,28 +63,71 @@ function clamp(value: number, min: number, max: number) { return Math.min(max, Math.max(min, value)); } +function isFileUrl(value: string): boolean { + return /^file:\/\//i.test(value); +} + +function encodePathSegments(pathname: string, keepWindowsDrive = false): string { + return pathname + .split("/") + .map((segment, index) => { + if (!segment) return ""; + if (keepWindowsDrive && index === 1 && /^[a-zA-Z]:$/.test(segment)) { + return segment; + } + return encodeURIComponent(segment); + }) + .join("/"); +} + export function toFileUrl(filePath: string): string { const normalized = filePath.replace(/\\/g, "/"); - if (normalized.match(/^[a-zA-Z]:/)) { - return `file:///${normalized}`; + + // Windows drive path: C:/Users/... + if (/^[a-zA-Z]:\//.test(normalized)) { + return `file://${encodePathSegments(`/${normalized}`, true)}`; } - return `file://${normalized}`; + + // UNC path: //server/share/... + if (normalized.startsWith("//")) { + const [host, ...pathParts] = normalized.replace(/^\/+/, "").split("/"); + const encodedPath = pathParts.map((part) => encodeURIComponent(part)).join("/"); + return encodedPath ? `file://${host}/${encodedPath}` : `file://${host}/`; + } + + const absolutePath = normalized.startsWith("/") ? normalized : `/${normalized}`; + return `file://${encodePathSegments(absolutePath)}`; } export function fromFileUrl(fileUrl: string): string { - if (!fileUrl.startsWith("file://")) { + const value = fileUrl.trim(); + if (!isFileUrl(value)) { return fileUrl; } try { - const url = new URL(fileUrl); - let filePath = decodeURIComponent(url.pathname); - if (/^\/[A-Za-z]:/.test(filePath)) { - filePath = filePath.slice(1); + const url = new URL(value); + const pathname = decodeURIComponent(url.pathname); + + if (url.host && url.host !== "localhost") { + const uncPath = `//${url.host}${pathname.startsWith("/") ? pathname : `/${pathname}`}`; + return uncPath.replace(/\//g, "\\"); } - return filePath; + + if (/^\/[A-Za-z]:/.test(pathname)) { + return pathname.slice(1); + } + + return pathname; } catch { - return fileUrl.replace(/^file:\/\//, ""); + const rawFallbackPath = value.replace(/^file:\/\//i, ""); + let fallbackPath = rawFallbackPath; + try { + fallbackPath = decodeURIComponent(rawFallbackPath); + } catch { + // Keep raw best-effort path if percent decoding fails. + } + return fallbackPath.replace(/^\/([a-zA-Z]:)/, "$1"); } } diff --git a/src/lib/assetPath.ts b/src/lib/assetPath.ts index 1ab0b9e0..229e2ea4 100644 --- a/src/lib/assetPath.ts +++ b/src/lib/assetPath.ts @@ -1,16 +1,30 @@ +function encodeRelativeAssetPath(relativePath: string): string { + return relativePath + .replace(/^\/+/, '') + .split('/') + .filter(Boolean) + .map((part) => encodeURIComponent(part)) + .join('/') +} + +function ensureTrailingSlash(value: string): string { + return value.endsWith('/') ? value : `${value}/` +} + export async function getAssetPath(relativePath: string): Promise { + const encodedRelativePath = encodeRelativeAssetPath(relativePath) + try { if (typeof window !== 'undefined') { // If running in a dev server (http/https), prefer the web-served path if (window.location && window.location.protocol && window.location.protocol.startsWith('http')) { - return `/${relativePath.replace(/^\//, '')}` + return `/${encodedRelativePath}` } if ((window as any).electronAPI && typeof (window as any).electronAPI.getAssetBasePath === 'function') { const base = await (window as any).electronAPI.getAssetBasePath() if (base) { - const normalized = base.replace(/\\/g, '/') - return `file://${normalized}/${relativePath}` + return new URL(encodedRelativePath, ensureTrailingSlash(base)).toString() } } } @@ -19,7 +33,7 @@ export async function getAssetPath(relativePath: string): Promise { } // Fallback for web/dev server: public/wallpapers are served at '/wallpapers/...' - return `/${relativePath.replace(/^\//, '')}` + return `/${encodedRelativePath}` } const BASE64_CHUNK_SIZE = 0x8000 diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 85d09b6a..09407801 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -1,14 +1,58 @@ -import type { WebDemuxer } from 'web-demuxer' -import type { TrimRegion } from '@/components/video-editor/types' +import { WebDemuxer } from 'web-demuxer' +import type { SpeedRegion, TrimRegion } from '@/components/video-editor/types' import type { VideoMuxer } from './muxer' const AUDIO_BITRATE = 128_000 const DECODE_BACKPRESSURE_LIMIT = 20 +const MIN_SPEED_REGION_DELTA_MS = 0.0001 export class AudioProcessor { private cancelled = false - async process(demuxer: WebDemuxer, muxer: VideoMuxer, trimRegions?: TrimRegion[]): Promise { + /** + * Audio export has two modes: + * 1) no speed regions -> fast WebCodecs trim-only pipeline + * 2) speed regions present -> pitch-preserving rendered timeline pipeline + */ + async process( + demuxer: WebDemuxer, + muxer: VideoMuxer, + videoUrl: string, + trimRegions?: TrimRegion[], + speedRegions?: SpeedRegion[], + readEndSec?: number, + ): Promise { + const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) : [] + const sortedSpeedRegions = speedRegions + ? [...speedRegions] + .filter((region) => region.endMs - region.startMs > MIN_SPEED_REGION_DELTA_MS) + .sort((a, b) => a.startMs - b.startMs) + : [] + + // Speed edits must use timeline playback to preserve pitch. + if (sortedSpeedRegions.length > 0) { + const renderedAudioBlob = await this.renderPitchPreservedTimelineAudio( + videoUrl, + sortedTrims, + sortedSpeedRegions, + ) + if (!this.cancelled) { + await this.muxRenderedAudioBlob(renderedAudioBlob, muxer) + return + } + } + + // No speed edits: keep the original demux/decode/encode path with trim timestamp remap. + await this.processTrimOnlyAudio(demuxer, muxer, sortedTrims, readEndSec) + } + + // Legacy trim-only path used when no speed regions are configured. + private async processTrimOnlyAudio( + demuxer: WebDemuxer, + muxer: VideoMuxer, + sortedTrims: TrimRegion[], + readEndSec?: number, + ): Promise { let audioConfig: AudioDecoderConfig try { audioConfig = (await demuxer.getDecoderConfig('audio')) as AudioDecoderConfig @@ -23,7 +67,6 @@ export class AudioProcessor { return } - const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) : [] const decodedFrames: AudioData[] = [] const decoder = new AudioDecoder({ @@ -32,7 +75,10 @@ export class AudioProcessor { }) decoder.configure(audioConfig) - const reader = (demuxer.read('audio') as ReadableStream).getReader() + const audioStream = typeof readEndSec === 'number' + ? demuxer.read('audio', 0, readEndSec) + : demuxer.read('audio') + const reader = (audioStream as ReadableStream).getReader() while (!this.cancelled) { const { done, value: chunk } = await reader.read() @@ -112,6 +158,283 @@ export class AudioProcessor { } } + // Speed-aware path that mirrors preview semantics (trim skipping + playbackRate regions) + // and preserves pitch through browser media playback behavior. + private async renderPitchPreservedTimelineAudio( + videoUrl: string, + trimRegions: TrimRegion[], + speedRegions: SpeedRegion[], + ): Promise { + const media = document.createElement('audio') + media.src = videoUrl + media.preload = 'auto' + + const pitchMedia = media as HTMLMediaElement & { + preservesPitch?: boolean + mozPreservesPitch?: boolean + webkitPreservesPitch?: boolean + } + pitchMedia.preservesPitch = true + pitchMedia.mozPreservesPitch = true + pitchMedia.webkitPreservesPitch = true + + await this.waitForLoadedMetadata(media) + if (this.cancelled) { + throw new Error('Export cancelled') + } + + const audioContext = new AudioContext() + const sourceNode = audioContext.createMediaElementSource(media) + const destinationNode = audioContext.createMediaStreamDestination() + sourceNode.connect(destinationNode) + + const { recorder, recordedBlobPromise } = this.startAudioRecording(destinationNode.stream) + let rafId: number | null = null + + try { + if (audioContext.state === 'suspended') { + await audioContext.resume() + } + + await this.seekTo(media, 0) + await media.play() + + await new Promise((resolve, reject) => { + const cleanup = () => { + if (rafId !== null) { + cancelAnimationFrame(rafId) + rafId = null + } + media.removeEventListener('error', onError) + media.removeEventListener('ended', onEnded) + } + + const onError = () => { + cleanup() + reject(new Error('Failed while rendering speed-adjusted audio timeline')) + } + + const onEnded = () => { + cleanup() + resolve() + } + + const tick = () => { + if (this.cancelled) { + cleanup() + resolve() + return + } + + const currentTimeMs = media.currentTime * 1000 + const activeTrimRegion = this.findActiveTrimRegion(currentTimeMs, trimRegions) + + if (activeTrimRegion && !media.paused && !media.ended) { + const skipToTime = activeTrimRegion.endMs / 1000 + if (skipToTime >= media.duration) { + media.pause() + cleanup() + resolve() + return + } + media.currentTime = skipToTime + } else { + const activeSpeedRegion = this.findActiveSpeedRegion(currentTimeMs, speedRegions) + const playbackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1 + if (Math.abs(media.playbackRate - playbackRate) > 0.0001) { + media.playbackRate = playbackRate + } + } + + if (!media.paused && !media.ended) { + rafId = requestAnimationFrame(tick) + } else { + cleanup() + resolve() + } + } + + media.addEventListener('error', onError, { once: true }) + media.addEventListener('ended', onEnded, { once: true }) + rafId = requestAnimationFrame(tick) + }) + } finally { + if (rafId !== null) { + cancelAnimationFrame(rafId) + } + media.pause() + if (recorder.state !== 'inactive') { + recorder.stop() + } + destinationNode.stream.getTracks().forEach((track) => track.stop()) + sourceNode.disconnect() + destinationNode.disconnect() + await audioContext.close() + media.src = '' + media.load() + } + + const recordedBlob = await recordedBlobPromise + if (this.cancelled) { + throw new Error('Export cancelled') + } + return recordedBlob + } + + // Demuxes the rendered speed-adjusted blob and feeds encoded chunks into the MP4 muxer. + private async muxRenderedAudioBlob(blob: Blob, muxer: VideoMuxer): Promise { + if (this.cancelled) return + + const file = new File([blob], 'speed-audio.webm', { type: blob.type || 'audio/webm' }) + const wasmUrl = new URL('./wasm/web-demuxer.wasm', window.location.href).href + const demuxer = new WebDemuxer({ wasmFilePath: wasmUrl }) + + try { + await demuxer.load(file) + const audioConfig = (await demuxer.getDecoderConfig('audio')) as AudioDecoderConfig + const reader = (demuxer.read('audio') as ReadableStream).getReader() + let isFirstChunk = true + + try { + while (!this.cancelled) { + const { done, value: chunk } = await reader.read() + if (done || !chunk) break + if (isFirstChunk) { + await muxer.addAudioChunk(chunk, { decoderConfig: audioConfig }) + isFirstChunk = false + } else { + await muxer.addAudioChunk(chunk) + } + } + } finally { + try { + await reader.cancel() + } catch { + // reader already closed + } + } + } finally { + try { + demuxer.destroy() + } catch { + // ignore + } + } + } + + private startAudioRecording(stream: MediaStream): { + recorder: MediaRecorder + recordedBlobPromise: Promise + } { + const mimeType = this.getSupportedAudioMimeType() + const options: MediaRecorderOptions = { + audioBitsPerSecond: AUDIO_BITRATE, + ...(mimeType ? { mimeType } : {}), + } + + const recorder = new MediaRecorder(stream, options) + const chunks: Blob[] = [] + + const recordedBlobPromise = new Promise((resolve, reject) => { + recorder.ondataavailable = (event: BlobEvent) => { + if (event.data && event.data.size > 0) { + chunks.push(event.data) + } + } + recorder.onerror = () => { + reject(new Error('MediaRecorder failed while capturing speed-adjusted audio')) + } + recorder.onstop = () => { + const type = mimeType || chunks[0]?.type || 'audio/webm' + resolve(new Blob(chunks, { type })) + } + }) + + recorder.start() + return { recorder, recordedBlobPromise } + } + + private getSupportedAudioMimeType(): string | undefined { + const candidates = ['audio/webm;codecs=opus', 'audio/webm'] + for (const candidate of candidates) { + if (MediaRecorder.isTypeSupported(candidate)) { + return candidate + } + } + return undefined + } + + private waitForLoadedMetadata(media: HTMLMediaElement): Promise { + if (Number.isFinite(media.duration) && media.readyState >= HTMLMediaElement.HAVE_METADATA) { + return Promise.resolve() + } + + return new Promise((resolve, reject) => { + const onLoaded = () => { + cleanup() + resolve() + } + const onError = () => { + cleanup() + reject(new Error('Failed to load media metadata for speed-adjusted audio')) + } + const cleanup = () => { + media.removeEventListener('loadedmetadata', onLoaded) + media.removeEventListener('error', onError) + } + + media.addEventListener('loadedmetadata', onLoaded) + media.addEventListener('error', onError, { once: true }) + }) + } + + private seekTo(media: HTMLMediaElement, targetSec: number): Promise { + if (Math.abs(media.currentTime - targetSec) < 0.0001) { + return Promise.resolve() + } + + return new Promise((resolve, reject) => { + const onSeeked = () => { + cleanup() + resolve() + } + const onError = () => { + cleanup() + reject(new Error('Failed to seek media for speed-adjusted audio')) + } + const cleanup = () => { + media.removeEventListener('seeked', onSeeked) + media.removeEventListener('error', onError) + } + + media.addEventListener('seeked', onSeeked, { once: true }) + media.addEventListener('error', onError, { once: true }) + media.currentTime = targetSec + }) + } + + private findActiveTrimRegion( + currentTimeMs: number, + trimRegions: TrimRegion[], + ): TrimRegion | null { + return ( + trimRegions.find( + (region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs, + ) || null + ) + } + + private findActiveSpeedRegion( + currentTimeMs: number, + speedRegions: SpeedRegion[], + ): SpeedRegion | null { + return ( + speedRegions.find( + (region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs, + ) || null + ) + } + private cloneWithTimestamp(src: AudioData, newTimestamp: number): AudioData { const isPlanar = src.format?.includes('planar') ?? false const numPlanes = isPlanar ? src.numberOfChannels : 1 diff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.ts index 045f486c..65dc925d 100644 --- a/src/lib/exporter/frameRenderer.ts +++ b/src/lib/exporter/frameRenderer.ts @@ -2,7 +2,7 @@ import { Application, Container, Sprite, Graphics, BlurFilter, Texture } from 'p import { MotionBlurFilter } from 'pixi-filters/motion-blur'; import type { ZoomRegion, CropRegion, AnnotationRegion, SpeedRegion, CursorTelemetryPoint } from '@/components/video-editor/types'; import { ZOOM_DEPTH_SCALES } from '@/components/video-editor/types'; -import { getRenderableAssetUrl } from '@/lib/assetPath'; +import { getAssetPath, getRenderableAssetUrl } from '@/lib/assetPath'; import { findDominantRegion } from '@/components/video-editor/videoPlayback/zoomRegionUtils'; import { applyZoomTransform, computeFocusFromTransform, computeZoomTransform, createMotionBlurState, type MotionBlurState } from '@/components/video-editor/videoPlayback/zoomTransform'; import { DEFAULT_FOCUS, ZOOM_SCALE_DEADZONE, ZOOM_TRANSLATION_DEADZONE_PX } from '@/components/video-editor/videoPlayback/constants'; @@ -198,18 +198,14 @@ export class FrameRenderer { if (wallpaper.startsWith('file://') || wallpaper.startsWith('data:') || wallpaper.startsWith('/') || wallpaper.startsWith('http')) { // Image background const img = new Image(); - // Don't set crossOrigin for same-origin images to avoid CORS taint - // Only set it for cross-origin URLs - let imageUrl: string; - if (wallpaper.startsWith('http')) { - imageUrl = wallpaper; - if (!imageUrl.startsWith(window.location.origin)) { - img.crossOrigin = 'anonymous'; - } - } else if (wallpaper.startsWith('file://') || wallpaper.startsWith('data:')) { - imageUrl = wallpaper; - } else { - imageUrl = window.location.origin + wallpaper; + const imageUrl = await this.resolveWallpaperImageUrl(wallpaper); + // Don't set crossOrigin for same-origin images to avoid CORS taint. + if ( + imageUrl.startsWith('http') + && window.location.origin + && !imageUrl.startsWith(window.location.origin) + ) { + img.crossOrigin = 'anonymous'; } await new Promise((resolve, reject) => { @@ -301,6 +297,23 @@ export class FrameRenderer { this.backgroundSprite = bgCanvas as any; } + private async resolveWallpaperImageUrl(wallpaper: string): Promise { + if ( + wallpaper.startsWith('file://') + || wallpaper.startsWith('data:') + || wallpaper.startsWith('http') + ) { + return wallpaper; + } + + const resolved = await getAssetPath(wallpaper.replace(/^\/+/, '')); + if (resolved.startsWith('/') && window.location.protocol.startsWith('http')) { + return `${window.location.origin}${resolved}`; + } + + return resolved; + } + private async resolveWallpaperForExport(wallpaper: string): Promise { if (!wallpaper) { return wallpaper; diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index 681df698..9eee91ea 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -151,7 +151,13 @@ export class VideoExporter { if (demuxer) { this.audioProcessor = new AudioProcessor(); await this.awaitWithWindowsTimeout( - this.audioProcessor.process(demuxer, this.muxer!, this.config.trimRegions), + this.audioProcessor.process( + demuxer, + this.muxer!, + this.config.videoUrl, + this.config.trimRegions, + this.config.speedRegions, + ), 'audio processing', ); }