lee/Recordly
1
0
Fork 0
mirror of https://github.com/webadderallorg/Recordly.git synced 2026-10-01 02:05:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add sidecar audio fallback for silent recordings

Browse Source
This commit is contained in:
webadderall
2026-04-01 15:40:40 +11:00
parent 1f1e445eb8
commit db8718cf66
6 changed files with 378 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/exporter/audioEncoder.ts
+119 -29
View File
@@ -18,13 +18,14 @@ export class AudioProcessor {
* 2) speed regions present -> pitch-preserving rendered timeline pipeline
*/
async process(
demuxer: WebDemuxer,
demuxer: WebDemuxer | null,
muxer: VideoMuxer,
videoUrl: string,
trimRegions?: TrimRegion[],
speedRegions?: SpeedRegion[],
readEndSec?: number,
audioRegions?: AudioRegion[],
sourceAudioFallbackPaths?: string[],
): Promise<void> {
const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) : []
const sortedSpeedRegions = speedRegions
@@ -35,14 +36,22 @@ export class AudioProcessor {
const sortedAudioRegions = audioRegions
? [...audioRegions].sort((a, b) => a.startMs - b.startMs)
: []
const sortedSourceAudioFallbackPaths = sourceAudioFallbackPaths
? sourceAudioFallbackPaths.filter((audioPath) => typeof audioPath === 'string' && audioPath.trim().length > 0)
: []
// When audio regions or speed edits are present, use AudioContext mixing path.
if (sortedSpeedRegions.length > 0 || sortedAudioRegions.length > 0) {
if (
sortedSpeedRegions.length > 0
|| sortedAudioRegions.length > 0
|| sortedSourceAudioFallbackPaths.length > 0
) {
const renderedAudioBlob = await this.renderMixedTimelineAudio(
videoUrl,
sortedTrims,
sortedSpeedRegions,
sortedAudioRegions,
sortedSourceAudioFallbackPaths,
)
if (!this.cancelled) {
await this.muxRenderedAudioBlob(renderedAudioBlob, muxer)
@@ -51,6 +60,11 @@ export class AudioProcessor {
}
// No speed edits or audio regions: keep the original demux/decode/encode path with trim timestamp remap.
if (!demuxer) {
console.warn('[AudioProcessor] No demuxer available, skipping audio')
return
}
await this.processTrimOnlyAudio(demuxer, muxer, sortedTrims, readEndSec)
}
@@ -279,13 +293,15 @@ export class AudioProcessor {
trimRegions: TrimRegion[],
speedRegions: SpeedRegion[],
audioRegions: AudioRegion[],
sourceAudioFallbackPaths: string[] = [],
): Promise<Blob> {
const mediaSource = await resolveMediaElementSource(videoUrl)
const media = document.createElement('audio')
media.src = mediaSource.src
media.preload = 'auto'
const timelineMediaSource = await resolveMediaElementSource(videoUrl)
const timelineMedia = document.createElement('video')
timelineMedia.src = timelineMediaSource.src
timelineMedia.preload = 'auto'
timelineMedia.playsInline = true
const pitchMedia = media as HTMLMediaElement & {
const pitchMedia = timelineMedia as HTMLMediaElement & {
preservesPitch?: boolean
mozPreservesPitch?: boolean
webkitPreservesPitch?: boolean
@@ -294,7 +310,7 @@ export class AudioProcessor {
pitchMedia.mozPreservesPitch = true
pitchMedia.webkitPreservesPitch = true
await this.waitForLoadedMetadata(media)
await this.waitForLoadedMetadata(timelineMedia)
if (this.cancelled) {
throw new Error('Export cancelled')
}
@@ -302,9 +318,41 @@ export class AudioProcessor {
const audioContext = new AudioContext()
const destinationNode = audioContext.createMediaStreamDestination()
// Connect original video audio
const sourceNode = audioContext.createMediaElementSource(media)
sourceNode.connect(destinationNode)
let timelineAudioSourceNode: MediaElementAudioSourceNode | null = null
if (sourceAudioFallbackPaths.length === 0) {
timelineAudioSourceNode = audioContext.createMediaElementSource(timelineMedia)
timelineAudioSourceNode.connect(destinationNode)
}
const sourceAudioElements: {
media: HTMLAudioElement
sourceNode: MediaElementAudioSourceNode
cleanup: () => void
}[] = []
for (const sourceAudioPath of sourceAudioFallbackPaths) {
const sourceFileSource = await resolveMediaElementSource(sourceAudioPath)
const audioEl = document.createElement('audio')
audioEl.src = sourceFileSource.src
audioEl.preload = 'auto'
try {
await this.waitForLoadedMetadata(audioEl)
} catch {
sourceFileSource.revoke()
console.warn('[AudioProcessor] Failed to load source audio fallback:', sourceAudioPath)
continue
}
if (this.cancelled) throw new Error('Export cancelled')
const sourceNode = audioContext.createMediaElementSource(audioEl)
sourceNode.connect(destinationNode)
sourceAudioElements.push({
media: audioEl,
sourceNode,
cleanup: sourceFileSource.revoke,
})
}
// Prepare external audio region elements
const audioRegionElements: {
@@ -352,8 +400,8 @@ export class AudioProcessor {
await audioContext.resume()
}
await this.seekTo(media, 0)
await media.play()
await this.seekTo(timelineMedia, 0)
await timelineMedia.play()
await new Promise<void>((resolve, reject) => {
const cleanup = () => {
@@ -361,8 +409,8 @@ export class AudioProcessor {
cancelAnimationFrame(rafId)
rafId = null
}
media.removeEventListener('error', onError)
media.removeEventListener('ended', onEnded)
timelineMedia.removeEventListener('error', onError)
timelineMedia.removeEventListener('ended', onEnded)
}
const onError = () => {
@@ -382,23 +430,54 @@ export class AudioProcessor {
return
}
const currentTimeMs = media.currentTime * 1000
let currentTimeMs = timelineMedia.currentTime * 1000
const activeTrimRegion = this.findActiveTrimRegion(currentTimeMs, trimRegions)
if (activeTrimRegion && !media.paused && !media.ended) {
if (activeTrimRegion && !timelineMedia.paused && !timelineMedia.ended) {
const skipToTime = activeTrimRegion.endMs / 1000
if (skipToTime >= media.duration) {
media.pause()
if (skipToTime >= timelineMedia.duration) {
timelineMedia.pause()
cleanup()
resolve()
return
}
media.currentTime = skipToTime
} else {
const activeSpeedRegion = this.findActiveSpeedRegion(currentTimeMs, speedRegions)
const playbackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1
if (Math.abs(media.playbackRate - playbackRate) > 0.0001) {
media.playbackRate = playbackRate
timelineMedia.currentTime = skipToTime
currentTimeMs = skipToTime * 1000
}
const activeSpeedRegion = this.findActiveSpeedRegion(currentTimeMs, speedRegions)
const playbackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1
if (Math.abs(timelineMedia.playbackRate - playbackRate) > 0.0001) {
timelineMedia.playbackRate = playbackRate
}
for (const entry of sourceAudioElements) {
const audioEl = entry.media
const targetTimeSec = Math.max(
0,
Math.min(
currentTimeMs / 1000,
Number.isFinite(audioEl.duration) ? audioEl.duration : currentTimeMs / 1000,
),
)
if (Math.abs(audioEl.playbackRate - playbackRate) > 0.0001) {
audioEl.playbackRate = playbackRate
}
const atEnd = Number.isFinite(audioEl.duration) && targetTimeSec >= audioEl.duration
if (atEnd) {
if (!audioEl.paused) {
audioEl.pause()
}
continue
}
if (audioEl.paused) {
audioEl.currentTime = targetTimeSec
audioEl.play().catch(() => {})
} else if (Math.abs(audioEl.currentTime - targetTimeSec) > 0.3) {
audioEl.currentTime = targetTimeSec
}
}
@@ -422,7 +501,7 @@ export class AudioProcessor {
}
}
if (!media.paused && !media.ended) {
if (!timelineMedia.paused && !timelineMedia.ended) {
rafId = requestAnimationFrame(tick)
} else {
cleanup()
@@ -430,15 +509,26 @@ export class AudioProcessor {
}
}
media.addEventListener('error', onError, { once: true })
media.addEventListener('ended', onEnded, { once: true })
timelineMedia.addEventListener('error', onError, { once: true })
timelineMedia.addEventListener('ended', onEnded, { once: true })
rafId = requestAnimationFrame(tick)
})
} finally {
if (rafId !== null) {
cancelAnimationFrame(rafId)
}
media.pause()
timelineMedia.pause()
timelineAudioSourceNode?.disconnect()
timelineMedia.src = ''
timelineMedia.load()
timelineMediaSource.revoke()
for (const entry of sourceAudioElements) {
entry.media.pause()
entry.sourceNode.disconnect()
entry.media.src = ''
entry.media.load()
entry.cleanup()
}
for (const entry of audioRegionElements) {
entry.media.pause()
entry.sourceNode.disconnect()
src/lib/exporter/videoExporter.ts
+6 -3
View File
@@ -56,6 +56,7 @@ interface VideoExporterConfig extends ExportConfig {
cursorClickBounceDuration?: number;
cursorSway?: number;
audioRegions?: AudioRegion[];
sourceAudioFallbackPaths?: string[];
previewWidth?: number;
previewHeight?: number;
onProgress?: (progress: ExportProgress) => void;
@@ -139,7 +140,8 @@ export class VideoExporter {
await this.initializeEncoder();
const hasAudioRegions = (this.config.audioRegions ?? []).length > 0;
const hasAudio = videoInfo.hasAudio || hasAudioRegions;
const hasSourceAudioFallback = (this.config.sourceAudioFallbackPaths ?? []).length > 0;
const hasAudio = videoInfo.hasAudio || hasAudioRegions || hasSourceAudioFallback;
// Initialize muxer
this.muxer = new VideoMuxer(this.config, hasAudio);
@@ -196,17 +198,18 @@ export class VideoExporter {
if (hasAudio && !this.cancelled) {
const demuxer = this.streamingDecoder.getDemuxer();
if (demuxer || hasAudioRegions) {
if (demuxer || hasAudioRegions || hasSourceAudioFallback) {
this.audioProcessor = new AudioProcessor();
await this.awaitWithWindowsTimeout(
this.audioProcessor.process(
demuxer!,
demuxer,
this.muxer!,
this.config.videoUrl,
this.config.trimRegions,
this.config.speedRegions,
undefined,
this.config.audioRegions,
this.config.sourceAudioFallbackPaths,
),
"audio processing",
);