diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index b7b87649..97c3a3bf 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -1569,6 +1569,7 @@ export default function VideoEditor() { cursorSmoothing, cursorMotionBlur, cursorClickBounce, + audioRegions, previewWidth, previewHeight, onProgress: (progress: ExportProgress) => { diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 09407801..bd1c20cd 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -1,5 +1,6 @@ import { WebDemuxer } from 'web-demuxer' -import type { SpeedRegion, TrimRegion } from '@/components/video-editor/types' +import type { SpeedRegion, TrimRegion, AudioRegion } from '@/components/video-editor/types' +import { toFileUrl } from '@/components/video-editor/projectPersistence' import type { VideoMuxer } from './muxer' const AUDIO_BITRATE = 128_000 @@ -21,6 +22,7 @@ export class AudioProcessor { trimRegions?: TrimRegion[], speedRegions?: SpeedRegion[], readEndSec?: number, + audioRegions?: AudioRegion[], ): Promise { const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) : [] const sortedSpeedRegions = speedRegions @@ -28,13 +30,17 @@ export class AudioProcessor { .filter((region) => region.endMs - region.startMs > MIN_SPEED_REGION_DELTA_MS) .sort((a, b) => a.startMs - b.startMs) : [] + const sortedAudioRegions = audioRegions + ? [...audioRegions].sort((a, b) => a.startMs - b.startMs) + : [] - // Speed edits must use timeline playback to preserve pitch. - if (sortedSpeedRegions.length > 0) { - const renderedAudioBlob = await this.renderPitchPreservedTimelineAudio( + // When audio regions or speed edits are present, use AudioContext mixing path. + if (sortedSpeedRegions.length > 0 || sortedAudioRegions.length > 0) { + const renderedAudioBlob = await this.renderMixedTimelineAudio( videoUrl, sortedTrims, sortedSpeedRegions, + sortedAudioRegions, ) if (!this.cancelled) { await this.muxRenderedAudioBlob(renderedAudioBlob, muxer) @@ -42,7 +48,7 @@ export class AudioProcessor { } } - // No speed edits: keep the original demux/decode/encode path with trim timestamp remap. + // No speed edits or audio regions: keep the original demux/decode/encode path with trim timestamp remap. await this.processTrimOnlyAudio(demuxer, muxer, sortedTrims, readEndSec) } @@ -158,12 +164,13 @@ export class AudioProcessor { } } - // Speed-aware path that mirrors preview semantics (trim skipping + playbackRate regions) - // and preserves pitch through browser media playback behavior. - private async renderPitchPreservedTimelineAudio( + // Renders mixed audio: original video audio (with speed/trim) + external audio regions. + // Uses AudioContext to mix all sources into a single recorded stream. + private async renderMixedTimelineAudio( videoUrl: string, trimRegions: TrimRegion[], speedRegions: SpeedRegion[], + audioRegions: AudioRegion[], ): Promise { const media = document.createElement('audio') media.src = videoUrl @@ -184,10 +191,41 @@ export class AudioProcessor { } const audioContext = new AudioContext() - const sourceNode = audioContext.createMediaElementSource(media) const destinationNode = audioContext.createMediaStreamDestination() + + // Connect original video audio + const sourceNode = audioContext.createMediaElementSource(media) sourceNode.connect(destinationNode) + // Prepare external audio region elements + const audioRegionElements: { + media: HTMLAudioElement + sourceNode: MediaElementAudioSourceNode + gainNode: GainNode + region: AudioRegion + }[] = [] + + for (const region of audioRegions) { + const audioEl = document.createElement('audio') + audioEl.src = toFileUrl(region.audioPath) + audioEl.preload = 'auto' + try { + await this.waitForLoadedMetadata(audioEl) + } catch { + console.warn('[AudioProcessor] Failed to load audio region:', region.audioPath) + continue + } + if (this.cancelled) throw new Error('Export cancelled') + + const regionSource = audioContext.createMediaElementSource(audioEl) + const gainNode = audioContext.createGain() + gainNode.gain.value = Math.max(0, Math.min(1, region.volume)) + regionSource.connect(gainNode) + gainNode.connect(destinationNode) + + audioRegionElements.push({ media: audioEl, sourceNode: regionSource, gainNode, region }) + } + const { recorder, recordedBlobPromise } = this.startAudioRecording(destinationNode.stream) let rafId: number | null = null @@ -211,7 +249,7 @@ export class AudioProcessor { const onError = () => { cleanup() - reject(new Error('Failed while rendering speed-adjusted audio timeline')) + reject(new Error('Failed while rendering mixed audio timeline')) } const onEnded = () => { @@ -246,6 +284,26 @@ export class AudioProcessor { } } + // Sync external audio regions with the video timeline position + for (const entry of audioRegionElements) { + const { media: audioEl, region } = entry + const isInRegion = currentTimeMs >= region.startMs && currentTimeMs < region.endMs + + if (isInRegion) { + const audioOffset = (currentTimeMs - region.startMs) / 1000 + if (audioEl.paused) { + audioEl.currentTime = audioOffset + audioEl.play().catch(() => {}) + } else if (Math.abs(audioEl.currentTime - audioOffset) > 0.3) { + audioEl.currentTime = audioOffset + } + } else { + if (!audioEl.paused) { + audioEl.pause() + } + } + } + if (!media.paused && !media.ended) { rafId = requestAnimationFrame(tick) } else { @@ -263,6 +321,13 @@ export class AudioProcessor { cancelAnimationFrame(rafId) } media.pause() + for (const entry of audioRegionElements) { + entry.media.pause() + entry.sourceNode.disconnect() + entry.gainNode.disconnect() + entry.media.src = '' + entry.media.load() + } if (recorder.state !== 'inactive') { recorder.stop() } diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index 9eee91ea..cc266e62 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -3,7 +3,7 @@ import { AudioProcessor } from './audioEncoder'; import { StreamingVideoDecoder } from './streamingDecoder'; import { FrameRenderer } from './frameRenderer'; import { VideoMuxer } from './muxer'; -import type { ZoomRegion, CropRegion, TrimRegion, AnnotationRegion, SpeedRegion, CursorTelemetryPoint } from '@/components/video-editor/types'; +import type { ZoomRegion, CropRegion, TrimRegion, AnnotationRegion, SpeedRegion, AudioRegion, CursorTelemetryPoint } from '@/components/video-editor/types'; interface VideoExporterConfig extends ExportConfig { videoUrl: string; @@ -27,6 +27,7 @@ interface VideoExporterConfig extends ExportConfig { cursorSmoothing?: number; cursorMotionBlur?: number; cursorClickBounce?: number; + audioRegions?: AudioRegion[]; previewWidth?: number; previewHeight?: number; onProgress?: (progress: ExportProgress) => void; @@ -94,7 +95,8 @@ export class VideoExporter { // Initialize video encoder await this.initializeEncoder(); - const hasAudio = videoInfo.hasAudio; + const hasAudioRegions = (this.config.audioRegions ?? []).length > 0; + const hasAudio = videoInfo.hasAudio || hasAudioRegions; // Initialize muxer this.muxer = new VideoMuxer(this.config, hasAudio); @@ -148,15 +150,17 @@ export class VideoExporter { if (hasAudio && !this.cancelled) { const demuxer = this.streamingDecoder.getDemuxer(); - if (demuxer) { + if (demuxer || hasAudioRegions) { this.audioProcessor = new AudioProcessor(); await this.awaitWithWindowsTimeout( this.audioProcessor.process( - demuxer, + demuxer!, this.muxer!, this.config.videoUrl, this.config.trimRegions, this.config.speedRegions, + undefined, + this.config.audioRegions, ), 'audio processing', );