mirror of
https://github.com/webadderallorg/Recordly.git
synced 2026-09-25 15:25:44 +00:00
Add sidecar audio fallback for silent recordings
This commit is contained in:
webadderall
Vendored
+3
@@ -137,6 +137,9 @@ interface Window {
|
||||
readLocalFile: (
|
||||
filePath: string,
|
||||
) => Promise<{ success: boolean; data?: Uint8Array; error?: string }>;
|
||||
getVideoAudioFallbackPaths: (
|
||||
videoPath: string,
|
||||
) => Promise<{ success: boolean; paths: string[]; error?: string }>;
|
||||
setRecordingState: (recording: boolean) => Promise<void>;
|
||||
getCursorTelemetry: (videoPath?: string) => Promise<{
|
||||
success: boolean;
|
||||
|
||||
+93
-22
@@ -34,6 +34,10 @@ const RECORDING_SESSION_MANIFEST_SUFFIX = '.recordly-session.json'
|
||||
const WHISPER_MODEL_DOWNLOAD_URL = 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.bin'
|
||||
const WHISPER_MODEL_DIR = path.join(USER_DATA_PATH, 'whisper')
|
||||
const WHISPER_SMALL_MODEL_PATH = path.join(WHISPER_MODEL_DIR, 'ggml-small.bin')
|
||||
const COMPANION_AUDIO_LAYOUTS = [
|
||||
{ platform: 'mac' as const, systemSuffix: '.system.m4a', micSuffix: '.mic.m4a' },
|
||||
{ platform: 'win' as const, systemSuffix: '.system.wav', micSuffix: '.mic.wav' },
|
||||
]
|
||||
|
||||
function getAssetRootPath() {
|
||||
if (app.isPackaged) {
|
||||
@@ -309,6 +313,77 @@ function parseFfmpegDurationSeconds(stderr: string) {
|
||||
return hours * 3600 + minutes * 60 + seconds
|
||||
}
|
||||
|
||||
type CompanionAudioCandidate = {
|
||||
platform: (typeof COMPANION_AUDIO_LAYOUTS)[number]['platform']
|
||||
systemPath: string
|
||||
micPath: string
|
||||
usablePaths: string[]
|
||||
}
|
||||
|
||||
async function getUsableCompanionAudioCandidates(videoPath: string): Promise<CompanionAudioCandidate[]> {
|
||||
const basePath = videoPath.replace(/\.[^.]+$/u, '')
|
||||
const candidates: CompanionAudioCandidate[] = []
|
||||
|
||||
for (const layout of COMPANION_AUDIO_LAYOUTS) {
|
||||
const systemPath = `${basePath}${layout.systemSuffix}`
|
||||
const micPath = `${basePath}${layout.micSuffix}`
|
||||
const usablePaths: string[] = []
|
||||
|
||||
for (const companionPath of [systemPath, micPath]) {
|
||||
try {
|
||||
const stat = await fs.stat(companionPath)
|
||||
if (stat.size > 0) {
|
||||
usablePaths.push(companionPath)
|
||||
}
|
||||
} catch {
|
||||
// Missing companion audio is expected for many recordings.
|
||||
}
|
||||
}
|
||||
|
||||
if (usablePaths.length > 0) {
|
||||
candidates.push({
|
||||
platform: layout.platform,
|
||||
systemPath,
|
||||
micPath,
|
||||
usablePaths,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return candidates
|
||||
}
|
||||
|
||||
async function hasEmbeddedAudioStream(videoPath: string) {
|
||||
const ffmpegPath = getFfmpegBinaryPath()
|
||||
let stderr = ''
|
||||
|
||||
try {
|
||||
const result = await execFileAsync(
|
||||
ffmpegPath,
|
||||
['-hide_banner', '-i', videoPath, '-map', '0:a:0', '-frames:a', '1', '-f', 'null', '-'],
|
||||
{ timeout: 20000, maxBuffer: 10 * 1024 * 1024 },
|
||||
)
|
||||
stderr = result.stderr
|
||||
} catch (error) {
|
||||
stderr = (error as NodeJS.ErrnoException & { stderr?: string }).stderr ?? ''
|
||||
}
|
||||
|
||||
return /Stream #.*Audio:/i.test(stderr)
|
||||
}
|
||||
|
||||
async function getCompanionAudioFallbackPaths(videoPath: string) {
|
||||
const companionCandidates = await getUsableCompanionAudioCandidates(videoPath)
|
||||
if (companionCandidates.length === 0) {
|
||||
return []
|
||||
}
|
||||
|
||||
if (await hasEmbeddedAudioStream(videoPath)) {
|
||||
return []
|
||||
}
|
||||
|
||||
return companionCandidates.flatMap((candidate) => candidate.usablePaths)
|
||||
}
|
||||
|
||||
async function validateRecordedVideo(videoPath: string) {
|
||||
const stat = await fs.stat(videoPath)
|
||||
if (!stat.isFile()) {
|
||||
@@ -2804,32 +2879,15 @@ function snapshotCursorTelemetryForPersistence() {
|
||||
async function finalizeStoredVideo(videoPath: string) {
|
||||
// Safety net: if companion audio files still exist, the mux was skipped — attempt it now
|
||||
if (videoPath.endsWith('.mp4')) {
|
||||
const base = videoPath.replace(/\.mp4$/i, '')
|
||||
// macOS uses .m4a, Windows uses .wav
|
||||
const candidates = [
|
||||
{ system: `${base}.system.m4a`, mic: `${base}.mic.m4a`, platform: 'mac' as const },
|
||||
{ system: `${base}.system.wav`, mic: `${base}.mic.wav`, platform: 'win' as const },
|
||||
]
|
||||
for (const { system, mic, platform } of candidates) {
|
||||
let hasUnmuxedAudio = false
|
||||
for (const siblingPath of [system, mic]) {
|
||||
try {
|
||||
const stat = await fs.stat(siblingPath)
|
||||
if (stat.size > 0) {
|
||||
hasUnmuxedAudio = true
|
||||
break
|
||||
}
|
||||
} catch {
|
||||
// file doesn't exist — expected if mux already succeeded or audio wasn't enabled
|
||||
}
|
||||
}
|
||||
if (hasUnmuxedAudio) {
|
||||
const companionCandidates = await getUsableCompanionAudioCandidates(videoPath)
|
||||
for (const { systemPath, micPath, platform } of companionCandidates) {
|
||||
if (platform === 'mac' || platform === 'win') {
|
||||
console.log(`[finalize] Detected un-muxed ${platform} audio files alongside video — attempting safety-net mux`)
|
||||
try {
|
||||
if (platform === 'win') {
|
||||
await muxNativeWindowsVideoWithAudio(videoPath, system, mic)
|
||||
await muxNativeWindowsVideoWithAudio(videoPath, systemPath, micPath)
|
||||
} else {
|
||||
await muxNativeMacRecordingWithAudio(videoPath, system, mic)
|
||||
await muxNativeMacRecordingWithAudio(videoPath, systemPath, micPath)
|
||||
}
|
||||
console.log('[finalize] Safety-net mux completed successfully')
|
||||
} catch (error) {
|
||||
@@ -4095,6 +4153,19 @@ body{background:transparent;overflow:hidden;width:100vw;height:100vh}
|
||||
return { success: true, diagnostics: lastNativeCaptureDiagnostics }
|
||||
})
|
||||
|
||||
ipcMain.handle('get-video-audio-fallback-paths', async (_event, videoPath: string) => {
|
||||
if (!videoPath) {
|
||||
return { success: true, paths: [] }
|
||||
}
|
||||
|
||||
try {
|
||||
return { success: true, paths: await getCompanionAudioFallbackPaths(videoPath) }
|
||||
} catch (error) {
|
||||
console.error('Failed to resolve companion audio fallback paths:', error)
|
||||
return { success: false, paths: [], error: String(error) }
|
||||
}
|
||||
})
|
||||
|
||||
ipcMain.handle('mux-native-windows-recording', async (_event, pauseSegments?: PauseSegment[]) => {
|
||||
const videoPath = windowsPendingVideoPath
|
||||
windowsPendingVideoPath = null
|
||||
|
||||
@@ -37,6 +37,9 @@ contextBridge.exposeInMainWorld("electronAPI", {
|
||||
readLocalFile: (filePath: string) => {
|
||||
return ipcRenderer.invoke("read-local-file", filePath);
|
||||
},
|
||||
getVideoAudioFallbackPaths: (videoPath: string) => {
|
||||
return ipcRenderer.invoke("get-video-audio-fallback-paths", videoPath);
|
||||
},
|
||||
getSources: async (opts: Electron.SourcesOptions) => {
|
||||
return await ipcRenderer.invoke("get-sources", opts);
|
||||
},
|
||||
|
||||
@@ -43,6 +43,7 @@ import {
|
||||
VideoExporter,
|
||||
} from "@/lib/exporter";
|
||||
import { resolveMediaElementSource } from "@/lib/exporter/localMediaSource";
|
||||
import { clampMediaTimeToDuration } from "@/lib/mediaTiming";
|
||||
import { matchesShortcut } from "@/lib/shortcuts";
|
||||
import { type AspectRatio, getAspectRatioValue } from "@/utils/aspectRatioUtils";
|
||||
import { resolveAutoCaptionSourcePath } from "./autoCaptionSource";
|
||||
@@ -398,6 +399,7 @@ export default function VideoEditor() {
|
||||
const [exportError, setExportError] = useState<string | null>(null);
|
||||
const [showExportDropdown, setShowExportDropdown] = useState(false);
|
||||
const [previewVolume, setPreviewVolume] = useState(1);
|
||||
const [sourceAudioFallbackPaths, setSourceAudioFallbackPaths] = useState<string[]>([]);
|
||||
const [aspectRatio, setAspectRatio] = useState<AspectRatio>(initialEditorPreferences.aspectRatio);
|
||||
const [activeEffectSection, setActiveEffectSection] = useState<EditorEffectSection>("scene");
|
||||
const [exportQuality, setExportQuality] = useState<ExportQuality>(
|
||||
@@ -881,6 +883,36 @@ export default function VideoEditor() {
|
||||
() => videoSourcePath ?? (videoPath ? fromFileUrl(videoPath) : null),
|
||||
[videoPath, videoSourcePath],
|
||||
);
|
||||
const hasSourceAudioFallback = sourceAudioFallbackPaths.length > 0;
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
setSourceAudioFallbackPaths([]);
|
||||
|
||||
if (!currentSourcePath) {
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}
|
||||
|
||||
void (async () => {
|
||||
try {
|
||||
const result = await window.electronAPI.getVideoAudioFallbackPaths(currentSourcePath);
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
setSourceAudioFallbackPaths(result.success ? (result.paths ?? []) : []);
|
||||
} catch {
|
||||
if (!cancelled) {
|
||||
setSourceAudioFallbackPaths([]);
|
||||
}
|
||||
}
|
||||
})();
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [currentSourcePath]);
|
||||
|
||||
const projectDisplayName = useMemo(() => {
|
||||
const fileName =
|
||||
@@ -2374,6 +2406,10 @@ export default function VideoEditor() {
|
||||
const audioElementsRef = useRef<Map<string, HTMLAudioElement>>(new Map());
|
||||
const audioElementRevokersRef = useRef<Map<string, () => void>>(new Map());
|
||||
const audioElementResourcesRef = useRef<Map<string, string>>(new Map());
|
||||
const sourceAudioElementsRef = useRef<Map<string, HTMLAudioElement>>(new Map());
|
||||
const sourceAudioElementRevokersRef = useRef<Map<string, () => void>>(new Map());
|
||||
const sourceAudioElementResourcesRef = useRef<Map<string, string>>(new Map());
|
||||
const lastSourceAudioSyncTimeRef = useRef<number | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
@@ -2434,6 +2470,67 @@ export default function VideoEditor() {
|
||||
};
|
||||
}, [audioRegions, previewVolume]);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
const existing = sourceAudioElementsRef.current;
|
||||
const currentIds = new Set(sourceAudioFallbackPaths);
|
||||
|
||||
for (const [id, audio] of existing) {
|
||||
if (!currentIds.has(id)) {
|
||||
audio.pause();
|
||||
audio.src = "";
|
||||
sourceAudioElementRevokersRef.current.get(id)?.();
|
||||
sourceAudioElementRevokersRef.current.delete(id);
|
||||
sourceAudioElementResourcesRef.current.delete(id);
|
||||
existing.delete(id);
|
||||
}
|
||||
}
|
||||
|
||||
for (const audioPath of sourceAudioFallbackPaths) {
|
||||
let audio = existing.get(audioPath);
|
||||
if (!audio) {
|
||||
audio = new Audio();
|
||||
audio.preload = "auto";
|
||||
existing.set(audioPath, audio);
|
||||
}
|
||||
|
||||
if (sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath) {
|
||||
audio.pause();
|
||||
audio.src = "";
|
||||
sourceAudioElementRevokersRef.current.get(audioPath)?.();
|
||||
sourceAudioElementRevokersRef.current.delete(audioPath);
|
||||
sourceAudioElementResourcesRef.current.set(audioPath, audioPath);
|
||||
|
||||
void (async () => {
|
||||
const resolved = await resolveMediaElementSource(audioPath);
|
||||
const latestAudio = existing.get(audioPath);
|
||||
|
||||
if (
|
||||
cancelled ||
|
||||
latestAudio !== audio ||
|
||||
sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath
|
||||
) {
|
||||
resolved.revoke();
|
||||
return;
|
||||
}
|
||||
|
||||
sourceAudioElementRevokersRef.current.set(audioPath, resolved.revoke);
|
||||
latestAudio.src = resolved.src;
|
||||
})();
|
||||
}
|
||||
|
||||
audio.volume = Math.max(0, Math.min(1, previewVolume));
|
||||
}
|
||||
|
||||
if (sourceAudioFallbackPaths.length === 0) {
|
||||
lastSourceAudioSyncTimeRef.current = null;
|
||||
}
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [previewVolume, sourceAudioFallbackPaths]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
for (const audio of audioElementsRef.current.values()) {
|
||||
@@ -2446,6 +2543,17 @@ export default function VideoEditor() {
|
||||
audioElementsRef.current.clear();
|
||||
audioElementRevokersRef.current.clear();
|
||||
audioElementResourcesRef.current.clear();
|
||||
for (const audio of sourceAudioElementsRef.current.values()) {
|
||||
audio.pause();
|
||||
audio.src = "";
|
||||
}
|
||||
for (const revoke of sourceAudioElementRevokersRef.current.values()) {
|
||||
revoke();
|
||||
}
|
||||
sourceAudioElementsRef.current.clear();
|
||||
sourceAudioElementRevokersRef.current.clear();
|
||||
sourceAudioElementResourcesRef.current.clear();
|
||||
lastSourceAudioSyncTimeRef.current = null;
|
||||
};
|
||||
}, []);
|
||||
|
||||
@@ -2475,6 +2583,50 @@ export default function VideoEditor() {
|
||||
}
|
||||
}, [isPlaying, currentTime, audioRegions]);
|
||||
|
||||
useEffect(() => {
|
||||
if (sourceAudioFallbackPaths.length === 0) {
|
||||
lastSourceAudioSyncTimeRef.current = null;
|
||||
return;
|
||||
}
|
||||
|
||||
const activeSpeedRegion = speedRegions.find(
|
||||
(region) => currentTime * 1000 >= region.startMs && currentTime * 1000 < region.endMs,
|
||||
);
|
||||
const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1;
|
||||
const previousTimelineTime = lastSourceAudioSyncTimeRef.current;
|
||||
const timelineJumped =
|
||||
previousTimelineTime === null || Math.abs(currentTime - previousTimelineTime) > 0.25;
|
||||
const driftThreshold = isPlaying ? 0.35 : 0.01;
|
||||
|
||||
for (const audio of sourceAudioElementsRef.current.values()) {
|
||||
const targetTime = clampMediaTimeToDuration(
|
||||
currentTime,
|
||||
Number.isFinite(audio.duration) ? audio.duration : null,
|
||||
);
|
||||
|
||||
if (Math.abs(audio.playbackRate - targetPlaybackRate) > 0.001) {
|
||||
audio.playbackRate = targetPlaybackRate;
|
||||
}
|
||||
|
||||
if (timelineJumped || Math.abs(audio.currentTime - targetTime) > driftThreshold) {
|
||||
try {
|
||||
audio.currentTime = targetTime;
|
||||
} catch {
|
||||
// no-op
|
||||
}
|
||||
}
|
||||
|
||||
const atEnd = Number.isFinite(audio.duration) && targetTime >= audio.duration;
|
||||
if (isPlaying && !atEnd) {
|
||||
audio.play().catch(() => undefined);
|
||||
} else if (!audio.paused) {
|
||||
audio.pause();
|
||||
}
|
||||
}
|
||||
|
||||
lastSourceAudioSyncTimeRef.current = currentTime;
|
||||
}, [currentTime, isPlaying, sourceAudioFallbackPaths, speedRegions]);
|
||||
|
||||
const showExportSuccessToast = useCallback((filePath: string) => {
|
||||
toast.success(`Exported successfully to ${filePath}`, {
|
||||
action: {
|
||||
@@ -2683,6 +2835,7 @@ export default function VideoEditor() {
|
||||
cursorClickBounceDuration,
|
||||
cursorSway,
|
||||
audioRegions,
|
||||
sourceAudioFallbackPaths,
|
||||
previewWidth,
|
||||
previewHeight,
|
||||
onProgress: (progress: ExportProgress) => {
|
||||
@@ -3346,7 +3499,7 @@ export default function VideoEditor() {
|
||||
cursorClickBounce={cursorClickBounce}
|
||||
cursorClickBounceDuration={cursorClickBounceDuration}
|
||||
cursorSway={cursorSway}
|
||||
volume={previewVolume}
|
||||
volume={hasSourceAudioFallback ? 0 : previewVolume}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -18,13 +18,14 @@ export class AudioProcessor {
|
||||
* 2) speed regions present -> pitch-preserving rendered timeline pipeline
|
||||
*/
|
||||
async process(
|
||||
demuxer: WebDemuxer,
|
||||
demuxer: WebDemuxer | null,
|
||||
muxer: VideoMuxer,
|
||||
videoUrl: string,
|
||||
trimRegions?: TrimRegion[],
|
||||
speedRegions?: SpeedRegion[],
|
||||
readEndSec?: number,
|
||||
audioRegions?: AudioRegion[],
|
||||
sourceAudioFallbackPaths?: string[],
|
||||
): Promise<void> {
|
||||
const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) : []
|
||||
const sortedSpeedRegions = speedRegions
|
||||
@@ -35,14 +36,22 @@ export class AudioProcessor {
|
||||
const sortedAudioRegions = audioRegions
|
||||
? [...audioRegions].sort((a, b) => a.startMs - b.startMs)
|
||||
: []
|
||||
const sortedSourceAudioFallbackPaths = sourceAudioFallbackPaths
|
||||
? sourceAudioFallbackPaths.filter((audioPath) => typeof audioPath === 'string' && audioPath.trim().length > 0)
|
||||
: []
|
||||
|
||||
// When audio regions or speed edits are present, use AudioContext mixing path.
|
||||
if (sortedSpeedRegions.length > 0 || sortedAudioRegions.length > 0) {
|
||||
if (
|
||||
sortedSpeedRegions.length > 0
|
||||
|| sortedAudioRegions.length > 0
|
||||
|| sortedSourceAudioFallbackPaths.length > 0
|
||||
) {
|
||||
const renderedAudioBlob = await this.renderMixedTimelineAudio(
|
||||
videoUrl,
|
||||
sortedTrims,
|
||||
sortedSpeedRegions,
|
||||
sortedAudioRegions,
|
||||
sortedSourceAudioFallbackPaths,
|
||||
)
|
||||
if (!this.cancelled) {
|
||||
await this.muxRenderedAudioBlob(renderedAudioBlob, muxer)
|
||||
@@ -51,6 +60,11 @@ export class AudioProcessor {
|
||||
}
|
||||
|
||||
// No speed edits or audio regions: keep the original demux/decode/encode path with trim timestamp remap.
|
||||
if (!demuxer) {
|
||||
console.warn('[AudioProcessor] No demuxer available, skipping audio')
|
||||
return
|
||||
}
|
||||
|
||||
await this.processTrimOnlyAudio(demuxer, muxer, sortedTrims, readEndSec)
|
||||
}
|
||||
|
||||
@@ -279,13 +293,15 @@ export class AudioProcessor {
|
||||
trimRegions: TrimRegion[],
|
||||
speedRegions: SpeedRegion[],
|
||||
audioRegions: AudioRegion[],
|
||||
sourceAudioFallbackPaths: string[] = [],
|
||||
): Promise<Blob> {
|
||||
const mediaSource = await resolveMediaElementSource(videoUrl)
|
||||
const media = document.createElement('audio')
|
||||
media.src = mediaSource.src
|
||||
media.preload = 'auto'
|
||||
const timelineMediaSource = await resolveMediaElementSource(videoUrl)
|
||||
const timelineMedia = document.createElement('video')
|
||||
timelineMedia.src = timelineMediaSource.src
|
||||
timelineMedia.preload = 'auto'
|
||||
timelineMedia.playsInline = true
|
||||
|
||||
const pitchMedia = media as HTMLMediaElement & {
|
||||
const pitchMedia = timelineMedia as HTMLMediaElement & {
|
||||
preservesPitch?: boolean
|
||||
mozPreservesPitch?: boolean
|
||||
webkitPreservesPitch?: boolean
|
||||
@@ -294,7 +310,7 @@ export class AudioProcessor {
|
||||
pitchMedia.mozPreservesPitch = true
|
||||
pitchMedia.webkitPreservesPitch = true
|
||||
|
||||
await this.waitForLoadedMetadata(media)
|
||||
await this.waitForLoadedMetadata(timelineMedia)
|
||||
if (this.cancelled) {
|
||||
throw new Error('Export cancelled')
|
||||
}
|
||||
@@ -302,9 +318,41 @@ export class AudioProcessor {
|
||||
const audioContext = new AudioContext()
|
||||
const destinationNode = audioContext.createMediaStreamDestination()
|
||||
|
||||
// Connect original video audio
|
||||
const sourceNode = audioContext.createMediaElementSource(media)
|
||||
sourceNode.connect(destinationNode)
|
||||
let timelineAudioSourceNode: MediaElementAudioSourceNode | null = null
|
||||
if (sourceAudioFallbackPaths.length === 0) {
|
||||
timelineAudioSourceNode = audioContext.createMediaElementSource(timelineMedia)
|
||||
timelineAudioSourceNode.connect(destinationNode)
|
||||
}
|
||||
|
||||
const sourceAudioElements: {
|
||||
media: HTMLAudioElement
|
||||
sourceNode: MediaElementAudioSourceNode
|
||||
cleanup: () => void
|
||||
}[] = []
|
||||
|
||||
for (const sourceAudioPath of sourceAudioFallbackPaths) {
|
||||
const sourceFileSource = await resolveMediaElementSource(sourceAudioPath)
|
||||
const audioEl = document.createElement('audio')
|
||||
audioEl.src = sourceFileSource.src
|
||||
audioEl.preload = 'auto'
|
||||
try {
|
||||
await this.waitForLoadedMetadata(audioEl)
|
||||
} catch {
|
||||
sourceFileSource.revoke()
|
||||
console.warn('[AudioProcessor] Failed to load source audio fallback:', sourceAudioPath)
|
||||
continue
|
||||
}
|
||||
if (this.cancelled) throw new Error('Export cancelled')
|
||||
|
||||
const sourceNode = audioContext.createMediaElementSource(audioEl)
|
||||
sourceNode.connect(destinationNode)
|
||||
|
||||
sourceAudioElements.push({
|
||||
media: audioEl,
|
||||
sourceNode,
|
||||
cleanup: sourceFileSource.revoke,
|
||||
})
|
||||
}
|
||||
|
||||
// Prepare external audio region elements
|
||||
const audioRegionElements: {
|
||||
@@ -352,8 +400,8 @@ export class AudioProcessor {
|
||||
await audioContext.resume()
|
||||
}
|
||||
|
||||
await this.seekTo(media, 0)
|
||||
await media.play()
|
||||
await this.seekTo(timelineMedia, 0)
|
||||
await timelineMedia.play()
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const cleanup = () => {
|
||||
@@ -361,8 +409,8 @@ export class AudioProcessor {
|
||||
cancelAnimationFrame(rafId)
|
||||
rafId = null
|
||||
}
|
||||
media.removeEventListener('error', onError)
|
||||
media.removeEventListener('ended', onEnded)
|
||||
timelineMedia.removeEventListener('error', onError)
|
||||
timelineMedia.removeEventListener('ended', onEnded)
|
||||
}
|
||||
|
||||
const onError = () => {
|
||||
@@ -382,23 +430,54 @@ export class AudioProcessor {
|
||||
return
|
||||
}
|
||||
|
||||
const currentTimeMs = media.currentTime * 1000
|
||||
let currentTimeMs = timelineMedia.currentTime * 1000
|
||||
const activeTrimRegion = this.findActiveTrimRegion(currentTimeMs, trimRegions)
|
||||
|
||||
if (activeTrimRegion && !media.paused && !media.ended) {
|
||||
if (activeTrimRegion && !timelineMedia.paused && !timelineMedia.ended) {
|
||||
const skipToTime = activeTrimRegion.endMs / 1000
|
||||
if (skipToTime >= media.duration) {
|
||||
media.pause()
|
||||
if (skipToTime >= timelineMedia.duration) {
|
||||
timelineMedia.pause()
|
||||
cleanup()
|
||||
resolve()
|
||||
return
|
||||
}
|
||||
media.currentTime = skipToTime
|
||||
} else {
|
||||
const activeSpeedRegion = this.findActiveSpeedRegion(currentTimeMs, speedRegions)
|
||||
const playbackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1
|
||||
if (Math.abs(media.playbackRate - playbackRate) > 0.0001) {
|
||||
media.playbackRate = playbackRate
|
||||
timelineMedia.currentTime = skipToTime
|
||||
currentTimeMs = skipToTime * 1000
|
||||
}
|
||||
|
||||
const activeSpeedRegion = this.findActiveSpeedRegion(currentTimeMs, speedRegions)
|
||||
const playbackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1
|
||||
if (Math.abs(timelineMedia.playbackRate - playbackRate) > 0.0001) {
|
||||
timelineMedia.playbackRate = playbackRate
|
||||
}
|
||||
|
||||
for (const entry of sourceAudioElements) {
|
||||
const audioEl = entry.media
|
||||
const targetTimeSec = Math.max(
|
||||
0,
|
||||
Math.min(
|
||||
currentTimeMs / 1000,
|
||||
Number.isFinite(audioEl.duration) ? audioEl.duration : currentTimeMs / 1000,
|
||||
),
|
||||
)
|
||||
|
||||
if (Math.abs(audioEl.playbackRate - playbackRate) > 0.0001) {
|
||||
audioEl.playbackRate = playbackRate
|
||||
}
|
||||
|
||||
const atEnd = Number.isFinite(audioEl.duration) && targetTimeSec >= audioEl.duration
|
||||
if (atEnd) {
|
||||
if (!audioEl.paused) {
|
||||
audioEl.pause()
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if (audioEl.paused) {
|
||||
audioEl.currentTime = targetTimeSec
|
||||
audioEl.play().catch(() => {})
|
||||
} else if (Math.abs(audioEl.currentTime - targetTimeSec) > 0.3) {
|
||||
audioEl.currentTime = targetTimeSec
|
||||
}
|
||||
}
|
||||
|
||||
@@ -422,7 +501,7 @@ export class AudioProcessor {
|
||||
}
|
||||
}
|
||||
|
||||
if (!media.paused && !media.ended) {
|
||||
if (!timelineMedia.paused && !timelineMedia.ended) {
|
||||
rafId = requestAnimationFrame(tick)
|
||||
} else {
|
||||
cleanup()
|
||||
@@ -430,15 +509,26 @@ export class AudioProcessor {
|
||||
}
|
||||
}
|
||||
|
||||
media.addEventListener('error', onError, { once: true })
|
||||
media.addEventListener('ended', onEnded, { once: true })
|
||||
timelineMedia.addEventListener('error', onError, { once: true })
|
||||
timelineMedia.addEventListener('ended', onEnded, { once: true })
|
||||
rafId = requestAnimationFrame(tick)
|
||||
})
|
||||
} finally {
|
||||
if (rafId !== null) {
|
||||
cancelAnimationFrame(rafId)
|
||||
}
|
||||
media.pause()
|
||||
timelineMedia.pause()
|
||||
timelineAudioSourceNode?.disconnect()
|
||||
timelineMedia.src = ''
|
||||
timelineMedia.load()
|
||||
timelineMediaSource.revoke()
|
||||
for (const entry of sourceAudioElements) {
|
||||
entry.media.pause()
|
||||
entry.sourceNode.disconnect()
|
||||
entry.media.src = ''
|
||||
entry.media.load()
|
||||
entry.cleanup()
|
||||
}
|
||||
for (const entry of audioRegionElements) {
|
||||
entry.media.pause()
|
||||
entry.sourceNode.disconnect()
|
||||
|
||||
@@ -56,6 +56,7 @@ interface VideoExporterConfig extends ExportConfig {
|
||||
cursorClickBounceDuration?: number;
|
||||
cursorSway?: number;
|
||||
audioRegions?: AudioRegion[];
|
||||
sourceAudioFallbackPaths?: string[];
|
||||
previewWidth?: number;
|
||||
previewHeight?: number;
|
||||
onProgress?: (progress: ExportProgress) => void;
|
||||
@@ -139,7 +140,8 @@ export class VideoExporter {
|
||||
await this.initializeEncoder();
|
||||
|
||||
const hasAudioRegions = (this.config.audioRegions ?? []).length > 0;
|
||||
const hasAudio = videoInfo.hasAudio || hasAudioRegions;
|
||||
const hasSourceAudioFallback = (this.config.sourceAudioFallbackPaths ?? []).length > 0;
|
||||
const hasAudio = videoInfo.hasAudio || hasAudioRegions || hasSourceAudioFallback;
|
||||
|
||||
// Initialize muxer
|
||||
this.muxer = new VideoMuxer(this.config, hasAudio);
|
||||
@@ -196,17 +198,18 @@ export class VideoExporter {
|
||||
|
||||
if (hasAudio && !this.cancelled) {
|
||||
const demuxer = this.streamingDecoder.getDemuxer();
|
||||
if (demuxer || hasAudioRegions) {
|
||||
if (demuxer || hasAudioRegions || hasSourceAudioFallback) {
|
||||
this.audioProcessor = new AudioProcessor();
|
||||
await this.awaitWithWindowsTimeout(
|
||||
this.audioProcessor.process(
|
||||
demuxer!,
|
||||
demuxer,
|
||||
this.muxer!,
|
||||
this.config.videoUrl,
|
||||
this.config.trimRegions,
|
||||
this.config.speedRegions,
|
||||
undefined,
|
||||
this.config.audioRegions,
|
||||
this.config.sourceAudioFallbackPaths,
|
||||
),
|
||||
"audio processing",
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user