mirror of
https://github.com/webadderallorg/Recordly.git
synced 2026-09-24 14:55:37 +00:00
Merge pull request #937 from webadderallorg/codex/refactor-exporter-god-classes
Refactor audio and streaming exporters into focused modules
This commit is contained in:
webadderall
+17
-1593
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,282 @@
|
||||
import { WebDemuxer } from "web-demuxer";
|
||||
import { AudioTimelineProcessor } from "./audioTimelineProcessor";
|
||||
import { DECODE_BACKPRESSURE_LIMIT, OFFLINE_AUDIO_SAMPLE_RATE } from "./audioProcessorShared";
|
||||
import { resolveMediaElementSource } from "./localMediaSource";
|
||||
|
||||
export class AudioMediaProcessor extends AudioTimelineProcessor {
|
||||
protected async decodeAudioFromUrl(url: string): Promise<AudioBuffer | null> {
|
||||
try {
|
||||
const buffer = await this.streamDecodeFromUrl(url);
|
||||
if (buffer) return buffer;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
"[AudioProcessor] Streaming decode failed, falling back to bulk decode:",
|
||||
url,
|
||||
error,
|
||||
);
|
||||
}
|
||||
return this.bulkDecodeFromUrl(url, OFFLINE_AUDIO_SAMPLE_RATE);
|
||||
}
|
||||
|
||||
// Streaming decode via WebDemuxer + AudioDecoder. Decodes audio chunk-by-chunk
|
||||
// without loading the entire compressed file into a contiguous ArrayBuffer.
|
||||
protected async streamDecodeFromUrl(url: string): Promise<AudioBuffer | null> {
|
||||
const source = await resolveMediaElementSource(url);
|
||||
let demuxer: WebDemuxer | null = null;
|
||||
|
||||
try {
|
||||
const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href;
|
||||
demuxer = new WebDemuxer({ wasmFilePath: wasmUrl });
|
||||
await demuxer.load(source.src);
|
||||
|
||||
let audioConfig: AudioDecoderConfig;
|
||||
try {
|
||||
audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig;
|
||||
} catch {
|
||||
return null; // No audio track
|
||||
}
|
||||
|
||||
const sampleRate = audioConfig.sampleRate || 48_000;
|
||||
const numChannels = Math.min(audioConfig.numberOfChannels || 2, 2);
|
||||
|
||||
// Accumulate decoded PCM per channel
|
||||
const channelChunks: Float32Array[][] = Array.from({ length: numChannels }, () => []);
|
||||
let totalFrames = 0;
|
||||
let decodeError: Error | null = null;
|
||||
const decodeCapacityWaiters = new Set<() => void>();
|
||||
|
||||
const notifyDecodeCapacityAvailable = () => {
|
||||
if (decodeCapacityWaiters.size === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const waiters = [...decodeCapacityWaiters];
|
||||
decodeCapacityWaiters.clear();
|
||||
for (const resolve of waiters) {
|
||||
resolve();
|
||||
}
|
||||
};
|
||||
|
||||
const waitForDecodeCapacity = () =>
|
||||
new Promise<void>((resolve) => {
|
||||
decodeCapacityWaiters.add(resolve);
|
||||
});
|
||||
|
||||
const decoder = new AudioDecoder({
|
||||
output: (data: AudioData) => {
|
||||
try {
|
||||
const frames = data.numberOfFrames;
|
||||
const dataChannels = Math.min(data.numberOfChannels, numChannels);
|
||||
const format = data.format;
|
||||
|
||||
if (format?.includes("planar")) {
|
||||
for (let ch = 0; ch < dataChannels; ch++) {
|
||||
const size = data.allocationSize({
|
||||
planeIndex: ch,
|
||||
});
|
||||
const bytes = new ArrayBuffer(size);
|
||||
data.copyTo(bytes, { planeIndex: ch });
|
||||
channelChunks[ch].push(this.rawToFloat32(bytes, format, frames));
|
||||
}
|
||||
} else if (format) {
|
||||
// Interleaved format — deinterleave into per-channel arrays.
|
||||
// Use data.numberOfChannels as stride (not capped dataChannels)
|
||||
// since the raw buffer contains all source channels.
|
||||
const srcChannels = data.numberOfChannels;
|
||||
const size = data.allocationSize({ planeIndex: 0 });
|
||||
const bytes = new ArrayBuffer(size);
|
||||
data.copyTo(bytes, { planeIndex: 0 });
|
||||
const interleaved = this.rawToFloat32(
|
||||
bytes,
|
||||
format,
|
||||
frames * srcChannels,
|
||||
);
|
||||
for (let ch = 0; ch < dataChannels; ch++) {
|
||||
const chData = new Float32Array(frames);
|
||||
for (let i = 0; i < frames; i++) {
|
||||
chData[i] = interleaved[i * srcChannels + ch];
|
||||
}
|
||||
channelChunks[ch].push(chData);
|
||||
}
|
||||
}
|
||||
|
||||
// Fill missing channels with silence
|
||||
for (let ch = dataChannels; ch < numChannels; ch++) {
|
||||
channelChunks[ch].push(new Float32Array(frames));
|
||||
}
|
||||
|
||||
totalFrames += frames;
|
||||
} finally {
|
||||
data.close();
|
||||
notifyDecodeCapacityAvailable();
|
||||
}
|
||||
},
|
||||
error: (err: DOMException) => {
|
||||
decodeError = new Error(`Streaming audio decode error: ${err.message}`);
|
||||
notifyDecodeCapacityAvailable();
|
||||
},
|
||||
});
|
||||
|
||||
decoder.configure(audioConfig);
|
||||
|
||||
const audioStream = demuxer.read("audio");
|
||||
const reader = (audioStream as ReadableStream<EncodedAudioChunk>).getReader();
|
||||
|
||||
try {
|
||||
while (!this.cancelled) {
|
||||
if (decodeError) throw decodeError;
|
||||
const { done, value: chunk } = await reader.read();
|
||||
if (done || !chunk) break;
|
||||
|
||||
decoder.decode(chunk);
|
||||
|
||||
while (decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT && !this.cancelled) {
|
||||
if (decodeError) throw decodeError;
|
||||
await waitForDecodeCapacity();
|
||||
}
|
||||
}
|
||||
|
||||
if (decoder.state === "configured") {
|
||||
await decoder.flush();
|
||||
}
|
||||
if (decodeError) throw decodeError;
|
||||
} finally {
|
||||
notifyDecodeCapacityAvailable();
|
||||
try {
|
||||
await reader.cancel();
|
||||
} catch {
|
||||
/* reader already closed */
|
||||
}
|
||||
if (decoder.state === "configured") {
|
||||
decoder.close();
|
||||
}
|
||||
}
|
||||
|
||||
if (totalFrames === 0) return null;
|
||||
|
||||
// Build AudioBuffer from accumulated chunks
|
||||
const audioBuffer = new AudioBuffer({
|
||||
length: totalFrames,
|
||||
numberOfChannels: numChannels,
|
||||
sampleRate,
|
||||
});
|
||||
for (let ch = 0; ch < numChannels; ch++) {
|
||||
const channelData = audioBuffer.getChannelData(ch);
|
||||
let writeOffset = 0;
|
||||
for (const chunk of channelChunks[ch]) {
|
||||
channelData.set(chunk, writeOffset);
|
||||
writeOffset += chunk.length;
|
||||
}
|
||||
}
|
||||
|
||||
return audioBuffer;
|
||||
} finally {
|
||||
source.revoke();
|
||||
try {
|
||||
demuxer?.destroy();
|
||||
} catch {
|
||||
/* cleanup */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Convert raw bytes from AudioData to Float32Array based on the sample format.
|
||||
protected rawToFloat32(bytes: ArrayBuffer, format: string, sampleCount: number): Float32Array {
|
||||
if (format.startsWith("f32")) {
|
||||
return new Float32Array(bytes);
|
||||
}
|
||||
if (format.startsWith("s16")) {
|
||||
const int16 = new Int16Array(bytes);
|
||||
const f32 = new Float32Array(sampleCount);
|
||||
for (let i = 0; i < sampleCount; i++) {
|
||||
f32[i] = int16[i] / 0x8000;
|
||||
}
|
||||
return f32;
|
||||
}
|
||||
if (format.startsWith("s32")) {
|
||||
const int32 = new Int32Array(bytes);
|
||||
const f32 = new Float32Array(sampleCount);
|
||||
for (let i = 0; i < sampleCount; i++) {
|
||||
f32[i] = int32[i] / 0x80000000;
|
||||
}
|
||||
return f32;
|
||||
}
|
||||
if (format.startsWith("u8")) {
|
||||
const uint8 = new Uint8Array(bytes);
|
||||
const f32 = new Float32Array(sampleCount);
|
||||
for (let i = 0; i < sampleCount; i++) {
|
||||
f32[i] = (uint8[i] - 128) / 128;
|
||||
}
|
||||
return f32;
|
||||
}
|
||||
// Unknown format — attempt float32 interpretation
|
||||
return new Float32Array(bytes);
|
||||
}
|
||||
|
||||
// Bulk decode fallback: loads entire file into memory and uses decodeAudioData.
|
||||
protected async bulkDecodeFromUrl(
|
||||
url: string,
|
||||
sampleRate: number,
|
||||
): Promise<AudioBuffer | null> {
|
||||
try {
|
||||
const source = await resolveMediaElementSource(url);
|
||||
try {
|
||||
const response = await fetch(source.src);
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
const tempCtx = new OfflineAudioContext(2, 1, sampleRate);
|
||||
return await tempCtx.decodeAudioData(arrayBuffer);
|
||||
} finally {
|
||||
source.revoke();
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn("[AudioProcessor] Failed to decode audio from URL:", url, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Get the duration of a media file by loading only its metadata.
|
||||
protected async getMediaDurationSec(url: string): Promise<number> {
|
||||
const source = await resolveMediaElementSource(url);
|
||||
try {
|
||||
const media = document.createElement("video");
|
||||
media.preload = "metadata";
|
||||
media.src = source.src;
|
||||
|
||||
return await new Promise<number>((resolve, reject) => {
|
||||
const timeout = setTimeout(() => {
|
||||
cleanup();
|
||||
media.src = "";
|
||||
media.load();
|
||||
reject(new Error("Timed out getting media duration (30s)"));
|
||||
}, 30_000);
|
||||
|
||||
const onLoaded = () => {
|
||||
cleanup();
|
||||
const duration = media.duration;
|
||||
media.src = "";
|
||||
media.load();
|
||||
resolve(Number.isFinite(duration) ? duration : 0);
|
||||
};
|
||||
const onError = () => {
|
||||
cleanup();
|
||||
media.src = "";
|
||||
media.load();
|
||||
reject(new Error("Failed to get media duration"));
|
||||
};
|
||||
const cleanup = () => {
|
||||
clearTimeout(timeout);
|
||||
media.removeEventListener("loadedmetadata", onLoaded);
|
||||
media.removeEventListener("error", onError);
|
||||
};
|
||||
|
||||
media.addEventListener("loadedmetadata", onLoaded);
|
||||
media.addEventListener("error", onError, { once: true });
|
||||
});
|
||||
} finally {
|
||||
source.revoke();
|
||||
}
|
||||
}
|
||||
|
||||
// Build non-overlapping timeline slices from the source timeline, excluding
|
||||
// trimmed regions and tagging each slice with its playback speed.
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
import { WebDemuxer } from "web-demuxer";
|
||||
import { resolveMediaElementSource } from "./localMediaSource";
|
||||
import type { TrimLikeRegion } from "./audioProcessorShared";
|
||||
|
||||
export class AudioProcessorBase {
|
||||
protected cancelled = false;
|
||||
protected onProgress?: (progress: number) => void;
|
||||
protected createWavHeader(
|
||||
sampleRate: number,
|
||||
numChannels: number,
|
||||
totalFrames: number,
|
||||
): ArrayBuffer {
|
||||
const bytesPerSample = 2; // 16-bit PCM
|
||||
const dataSize = totalFrames * numChannels * bytesPerSample;
|
||||
const headerSize = 44;
|
||||
const header = new ArrayBuffer(headerSize);
|
||||
const view = new DataView(header);
|
||||
|
||||
const writeString = (offset: number, str: string) => {
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
view.setUint8(offset + i, str.charCodeAt(i));
|
||||
}
|
||||
};
|
||||
|
||||
writeString(0, "RIFF");
|
||||
view.setUint32(4, headerSize - 8 + dataSize, true);
|
||||
writeString(8, "WAVE");
|
||||
writeString(12, "fmt ");
|
||||
view.setUint32(16, 16, true);
|
||||
view.setUint16(20, 1, true); // PCM format
|
||||
view.setUint16(22, numChannels, true);
|
||||
view.setUint32(24, sampleRate, true);
|
||||
view.setUint32(28, sampleRate * numChannels * bytesPerSample, true);
|
||||
view.setUint16(32, numChannels * bytesPerSample, true);
|
||||
view.setUint16(34, bytesPerSample * 8, true);
|
||||
writeString(36, "data");
|
||||
view.setUint32(40, dataSize, true);
|
||||
|
||||
return header;
|
||||
}
|
||||
|
||||
// Convert an AudioBuffer to chunked 16-bit PCM ArrayBuffers.
|
||||
// Returns small (~256KB) pieces instead of one massive allocation.
|
||||
protected audioBufferToPcmParts(buffer: AudioBuffer): ArrayBuffer[] {
|
||||
const PCM_CHUNK_FRAMES = 65536;
|
||||
const numChannels = buffer.numberOfChannels;
|
||||
const numFrames = buffer.length;
|
||||
const bytesPerSample = 2;
|
||||
const parts: ArrayBuffer[] = [];
|
||||
|
||||
const channels: Float32Array[] = [];
|
||||
for (let ch = 0; ch < numChannels; ch++) {
|
||||
channels.push(buffer.getChannelData(ch));
|
||||
}
|
||||
|
||||
for (let frameOffset = 0; frameOffset < numFrames; frameOffset += PCM_CHUNK_FRAMES) {
|
||||
const chunkFrames = Math.min(PCM_CHUNK_FRAMES, numFrames - frameOffset);
|
||||
const chunkBuffer = new ArrayBuffer(chunkFrames * numChannels * bytesPerSample);
|
||||
const view = new DataView(chunkBuffer);
|
||||
|
||||
let byteOffset = 0;
|
||||
for (let i = 0; i < chunkFrames; i++) {
|
||||
for (let ch = 0; ch < numChannels; ch++) {
|
||||
const sample = Math.max(-1, Math.min(1, channels[ch][frameOffset + i]));
|
||||
view.setInt16(byteOffset, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true);
|
||||
byteOffset += 2;
|
||||
}
|
||||
}
|
||||
|
||||
parts.push(chunkBuffer);
|
||||
}
|
||||
|
||||
return parts;
|
||||
}
|
||||
|
||||
// Loads a sidecar audio file into a WebDemuxer for direct transcoding (avoiding real-time rendering).
|
||||
protected async loadAudioFileDemuxer(audioPath: string): Promise<WebDemuxer | null> {
|
||||
try {
|
||||
const source = await resolveMediaElementSource(audioPath);
|
||||
try {
|
||||
const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href;
|
||||
const demuxer = new WebDemuxer({ wasmFilePath: wasmUrl });
|
||||
await demuxer.load(source.src);
|
||||
return demuxer;
|
||||
} finally {
|
||||
source.revoke();
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn("[AudioProcessor] Failed to create demuxer for sidecar audio:", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
protected cloneWithTimestamp(src: AudioData, newTimestamp: number): AudioData {
|
||||
const isPlanar = src.format?.includes("planar") ?? false;
|
||||
const numPlanes = isPlanar ? src.numberOfChannels : 1;
|
||||
|
||||
let totalSize = 0;
|
||||
for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) {
|
||||
totalSize += src.allocationSize({ planeIndex });
|
||||
}
|
||||
|
||||
const buffer = new ArrayBuffer(totalSize);
|
||||
let offset = 0;
|
||||
|
||||
for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) {
|
||||
const planeSize = src.allocationSize({ planeIndex });
|
||||
src.copyTo(new Uint8Array(buffer, offset, planeSize), { planeIndex });
|
||||
offset += planeSize;
|
||||
}
|
||||
|
||||
return new AudioData({
|
||||
format: src.format!,
|
||||
sampleRate: src.sampleRate,
|
||||
numberOfFrames: src.numberOfFrames,
|
||||
numberOfChannels: src.numberOfChannels,
|
||||
timestamp: newTimestamp,
|
||||
data: buffer,
|
||||
});
|
||||
}
|
||||
|
||||
protected cloneEncodedAudioChunkWithTimestamp(
|
||||
src: EncodedAudioChunk,
|
||||
newTimestamp: number,
|
||||
): EncodedAudioChunk {
|
||||
const data = new Uint8Array(src.byteLength);
|
||||
src.copyTo(data);
|
||||
|
||||
return new EncodedAudioChunk({
|
||||
type: src.type,
|
||||
timestamp: newTimestamp,
|
||||
duration: src.duration ?? undefined,
|
||||
data,
|
||||
});
|
||||
}
|
||||
|
||||
protected isInTrimRegion(timestampMs: number, trims: TrimLikeRegion[]) {
|
||||
return trims.some((trim) => timestampMs >= trim.startMs && timestampMs < trim.endMs);
|
||||
}
|
||||
|
||||
protected computeTrimOffset(timestampMs: number, trims: TrimLikeRegion[]) {
|
||||
let offset = 0;
|
||||
for (const trim of trims) {
|
||||
if (trim.endMs <= timestampMs) {
|
||||
offset += trim.endMs - trim.startMs;
|
||||
}
|
||||
}
|
||||
return offset;
|
||||
}
|
||||
|
||||
cancel() {
|
||||
this.cancelled = true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
import { SOURCE_AUDIO_NORMALIZE_GAIN } from "@/components/video-editor/audio/audioTypes";
|
||||
import type {
|
||||
AudioRegion,
|
||||
ClipRegion,
|
||||
SourceAudioTrackSettings,
|
||||
TrimRegion,
|
||||
} from "@/components/video-editor/types";
|
||||
import type { SourceTrackId } from "@/lib/exporter/audioRoutingEngine";
|
||||
|
||||
export const AUDIO_BITRATE = 128_000;
|
||||
export const DECODE_BACKPRESSURE_LIMIT = 20;
|
||||
export const ENCODE_BACKPRESSURE_LIMIT = 20;
|
||||
export const MIN_SPEED_REGION_DELTA_MS = 0.0001;
|
||||
export const MP4_AUDIO_CODEC = "mp4a.40.2";
|
||||
export const OFFLINE_AUDIO_SAMPLE_RATE = 48_000;
|
||||
export const OFFLINE_ENCODE_CHUNK_FRAMES = 1024;
|
||||
export const OFFLINE_CHUNK_DURATION_SEC = 30;
|
||||
const OFFLINE_MIX_SOFT_LIMITER_THRESHOLD = 0.9;
|
||||
const OFFLINE_MIX_SOFT_LIMITER_CEILING = 0.985;
|
||||
|
||||
function softLimitSample(sample: number): number {
|
||||
const magnitude = Math.abs(sample);
|
||||
if (magnitude <= OFFLINE_MIX_SOFT_LIMITER_THRESHOLD) {
|
||||
return sample;
|
||||
}
|
||||
|
||||
const sign = sample < 0 ? -1 : 1;
|
||||
const kneeRange = 1 - OFFLINE_MIX_SOFT_LIMITER_THRESHOLD;
|
||||
const limitedMagnitude =
|
||||
OFFLINE_MIX_SOFT_LIMITER_THRESHOLD +
|
||||
kneeRange * Math.tanh((magnitude - OFFLINE_MIX_SOFT_LIMITER_THRESHOLD) / kneeRange);
|
||||
return sign * Math.min(OFFLINE_MIX_SOFT_LIMITER_CEILING, limitedMagnitude);
|
||||
}
|
||||
|
||||
export function softLimitOfflineMixPeaksInPlace(buffer: AudioBuffer): boolean {
|
||||
let changed = false;
|
||||
for (let channel = 0; channel < buffer.numberOfChannels; channel += 1) {
|
||||
const data = buffer.getChannelData(channel);
|
||||
for (let index = 0; index < data.length; index += 1) {
|
||||
const sample = data[index];
|
||||
if (!Number.isFinite(sample)) {
|
||||
data[index] = 0;
|
||||
changed = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
const limited = softLimitSample(sample);
|
||||
if (limited !== sample) {
|
||||
data[index] = limited;
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return changed;
|
||||
}
|
||||
|
||||
export function resolveSourceTrackGain(
|
||||
sourceAudioTrackSettings: SourceAudioTrackSettings | undefined,
|
||||
trackId: "mic" | "system" | "mixed",
|
||||
) {
|
||||
const settings = sourceAudioTrackSettings?.[trackId];
|
||||
if (!settings) {
|
||||
return 1;
|
||||
}
|
||||
const normalizeGain = settings.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1;
|
||||
return Math.max(0, Math.min(2, settings.volume * normalizeGain));
|
||||
}
|
||||
|
||||
export function getSourceTrackIdFromPath(audioPath: string): SourceTrackId {
|
||||
const normalized = audioPath.toLowerCase();
|
||||
// Check for common patterns like .mic., -mic., mic.mp4, etc.
|
||||
if (
|
||||
normalized.includes(".mic.") ||
|
||||
normalized.includes("-mic.") ||
|
||||
normalized.includes("_mic_") ||
|
||||
normalized.includes("/mic.") ||
|
||||
normalized.includes("\\mic.") ||
|
||||
normalized.endsWith("mic.mp4") ||
|
||||
normalized.endsWith("mic.m4a") ||
|
||||
normalized.endsWith("mic.wav")
|
||||
) {
|
||||
return "mic";
|
||||
}
|
||||
if (
|
||||
normalized.includes(".system.") ||
|
||||
normalized.includes("-system.") ||
|
||||
normalized.includes("_system_") ||
|
||||
normalized.includes("/system.") ||
|
||||
normalized.includes("\\system.") ||
|
||||
normalized.endsWith("system.mp4") ||
|
||||
normalized.endsWith("system.m4a") ||
|
||||
normalized.endsWith("system.wav")
|
||||
) {
|
||||
return "system";
|
||||
}
|
||||
return "mixed";
|
||||
}
|
||||
|
||||
export function hasNonDefaultSourceTrackSettings(
|
||||
sourceAudioTrackSettings?: SourceAudioTrackSettings,
|
||||
) {
|
||||
if (!sourceAudioTrackSettings) {
|
||||
return false;
|
||||
}
|
||||
return Object.values(sourceAudioTrackSettings).some(
|
||||
(settings) =>
|
||||
Math.abs((settings?.volume ?? 1) - 1) > 0.0005 || Boolean(settings?.normalize),
|
||||
);
|
||||
}
|
||||
|
||||
export interface TimelineSlice {
|
||||
sourceStartMs: number;
|
||||
sourceEndMs: number;
|
||||
speed: number;
|
||||
}
|
||||
|
||||
export interface PreparedOfflineRender {
|
||||
mainBufferEntry: { buffer: AudioBuffer; gain: number } | null;
|
||||
companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number; gain: number }>;
|
||||
regionEntries: Array<{ buffer: AudioBuffer; region: AudioRegion }>;
|
||||
mutedSourceOutputRangesSec: Array<{ startSec: number; endSec: number }>;
|
||||
slices: TimelineSlice[];
|
||||
outputDurationMs: number;
|
||||
numChannels: number;
|
||||
}
|
||||
|
||||
export async function isAacAudioEncodingSupported(
|
||||
sampleRate = 48_000,
|
||||
numberOfChannels = 2,
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
const support = await AudioEncoder.isConfigSupported({
|
||||
codec: MP4_AUDIO_CODEC,
|
||||
sampleRate,
|
||||
numberOfChannels,
|
||||
bitrate: AUDIO_BITRATE,
|
||||
});
|
||||
return support.supported === true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export type TrimLikeRegion = TrimRegion | ClipRegion;
|
||||
@@ -0,0 +1,355 @@
|
||||
import type { SpeedRegion } from "@/components/video-editor/types";
|
||||
import { AudioProcessorBase } from "./audioProcessorBase";
|
||||
import type { TimelineSlice, TrimLikeRegion } from "./audioProcessorShared";
|
||||
|
||||
export class AudioTimelineProcessor extends AudioProcessorBase {
|
||||
protected buildTimelineSlices(
|
||||
sourceDurationMs: number,
|
||||
trimRegions: TrimLikeRegion[],
|
||||
speedRegions: SpeedRegion[],
|
||||
): TimelineSlice[] {
|
||||
const boundaries = new Set<number>();
|
||||
boundaries.add(0);
|
||||
boundaries.add(sourceDurationMs);
|
||||
|
||||
for (const trim of trimRegions) {
|
||||
if (trim.startMs >= 0 && trim.startMs <= sourceDurationMs) boundaries.add(trim.startMs);
|
||||
if (trim.endMs >= 0 && trim.endMs <= sourceDurationMs) boundaries.add(trim.endMs);
|
||||
}
|
||||
for (const speed of speedRegions) {
|
||||
if (speed.startMs >= 0 && speed.startMs <= sourceDurationMs)
|
||||
boundaries.add(speed.startMs);
|
||||
if (speed.endMs >= 0 && speed.endMs <= sourceDurationMs) boundaries.add(speed.endMs);
|
||||
}
|
||||
|
||||
const sorted = [...boundaries].sort((a, b) => a - b);
|
||||
const slices: TimelineSlice[] = [];
|
||||
|
||||
for (let i = 0; i < sorted.length - 1; i++) {
|
||||
const start = sorted[i];
|
||||
const end = sorted[i + 1];
|
||||
if (end - start < 0.001) continue;
|
||||
|
||||
// Skip segments entirely inside a trim region
|
||||
const midpoint = (start + end) / 2;
|
||||
if (this.isInTrimRegion(midpoint, trimRegions)) continue;
|
||||
|
||||
const speedRegion = speedRegions.find(
|
||||
(s) => midpoint >= s.startMs && midpoint < s.endMs,
|
||||
);
|
||||
|
||||
slices.push({
|
||||
sourceStartMs: start,
|
||||
sourceEndMs: end,
|
||||
speed: speedRegion?.speed ?? 1,
|
||||
});
|
||||
}
|
||||
|
||||
return slices;
|
||||
}
|
||||
|
||||
// Map a source-timeline timestamp to the corresponding output-timeline timestamp.
|
||||
protected sourceTimeToOutputTime(sourceMs: number, slices: TimelineSlice[]): number {
|
||||
let outputMs = 0;
|
||||
|
||||
for (const slice of slices) {
|
||||
if (sourceMs <= slice.sourceStartMs) {
|
||||
return outputMs;
|
||||
}
|
||||
const sliceDurationMs = slice.sourceEndMs - slice.sourceStartMs;
|
||||
if (sourceMs >= slice.sourceEndMs) {
|
||||
outputMs += sliceDurationMs / slice.speed;
|
||||
continue;
|
||||
}
|
||||
// Source time falls within this slice
|
||||
outputMs += (sourceMs - slice.sourceStartMs) / slice.speed;
|
||||
return outputMs;
|
||||
}
|
||||
|
||||
return outputMs;
|
||||
}
|
||||
|
||||
// Schedule an AudioBuffer through the timeline slices in an OfflineAudioContext.
|
||||
// Each non-trimmed segment creates an AudioBufferSourceNode with the appropriate
|
||||
// playbackRate for speed regions. When chunkOutputStartSec/chunkDurationSec are
|
||||
// provided, only sources overlapping the chunk window are scheduled.
|
||||
protected scheduleBufferThroughTimeline(
|
||||
ctx: OfflineAudioContext,
|
||||
buffer: AudioBuffer,
|
||||
slices: TimelineSlice[],
|
||||
sourceStartDelaySec: number,
|
||||
gain = 1,
|
||||
chunkOutputStartSec = 0,
|
||||
chunkDurationSec = Number.POSITIVE_INFINITY,
|
||||
mutedOutputRangesSec: Array<{ startSec: number; endSec: number }> = [],
|
||||
): void {
|
||||
let outputOffsetSec = 0;
|
||||
|
||||
for (const slice of slices) {
|
||||
const sliceSourceDurationSec = (slice.sourceEndMs - slice.sourceStartMs) / 1000;
|
||||
const sliceOutputDurationSec = sliceSourceDurationSec / slice.speed;
|
||||
|
||||
// Where in the buffer does this slice read from?
|
||||
const bufferOffsetSec = slice.sourceStartMs / 1000 - sourceStartDelaySec;
|
||||
|
||||
// Skip if slice doesn't overlap with the buffer at all
|
||||
if (
|
||||
bufferOffsetSec + sliceSourceDurationSec <= 0 ||
|
||||
bufferOffsetSec >= buffer.duration
|
||||
) {
|
||||
outputOffsetSec += sliceOutputDurationSec;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Clamp to buffer bounds
|
||||
let effectiveBufferStartSec = Math.max(0, bufferOffsetSec);
|
||||
const trimmedFromStartSec = effectiveBufferStartSec - bufferOffsetSec;
|
||||
let effectiveSourceDurationSec = Math.min(
|
||||
sliceSourceDurationSec - trimmedFromStartSec,
|
||||
buffer.duration - effectiveBufferStartSec,
|
||||
);
|
||||
|
||||
if (effectiveSourceDurationSec <= 0.001) {
|
||||
outputOffsetSec += sliceOutputDurationSec;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Calculate output position (global then chunk-local)
|
||||
let localOutputStartSec =
|
||||
outputOffsetSec + trimmedFromStartSec / slice.speed - chunkOutputStartSec;
|
||||
let localOutputEndSec = localOutputStartSec + effectiveSourceDurationSec / slice.speed;
|
||||
|
||||
// Skip if entirely outside chunk window
|
||||
if (localOutputEndSec <= 0 || localOutputStartSec >= chunkDurationSec) {
|
||||
outputOffsetSec += sliceOutputDurationSec;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Clip to chunk start
|
||||
if (localOutputStartSec < 0) {
|
||||
const skipOutputSec = -localOutputStartSec;
|
||||
const skipSourceSec = skipOutputSec * slice.speed;
|
||||
effectiveBufferStartSec += skipSourceSec;
|
||||
effectiveSourceDurationSec -= skipSourceSec;
|
||||
localOutputStartSec = 0;
|
||||
}
|
||||
|
||||
// Clip to chunk end
|
||||
if (localOutputEndSec > chunkDurationSec) {
|
||||
const excessOutputSec = localOutputEndSec - chunkDurationSec;
|
||||
effectiveSourceDurationSec -= excessOutputSec * slice.speed;
|
||||
}
|
||||
|
||||
if (effectiveSourceDurationSec <= 0.001) {
|
||||
outputOffsetSec += sliceOutputDurationSec;
|
||||
continue;
|
||||
}
|
||||
|
||||
const audibleRanges: Array<{ startSec: number; endSec: number }> = [
|
||||
{
|
||||
startSec: localOutputStartSec + chunkOutputStartSec,
|
||||
endSec:
|
||||
localOutputStartSec +
|
||||
chunkOutputStartSec +
|
||||
effectiveSourceDurationSec / slice.speed,
|
||||
},
|
||||
];
|
||||
for (const mutedRange of mutedOutputRangesSec) {
|
||||
for (let rangeIndex = audibleRanges.length - 1; rangeIndex >= 0; rangeIndex -= 1) {
|
||||
const current = audibleRanges[rangeIndex];
|
||||
const overlapStart = Math.max(current.startSec, mutedRange.startSec);
|
||||
const overlapEnd = Math.min(current.endSec, mutedRange.endSec);
|
||||
if (overlapEnd <= overlapStart) {
|
||||
continue;
|
||||
}
|
||||
audibleRanges.splice(rangeIndex, 1);
|
||||
if (current.startSec < overlapStart) {
|
||||
audibleRanges.push({ startSec: current.startSec, endSec: overlapStart });
|
||||
}
|
||||
if (overlapEnd < current.endSec) {
|
||||
audibleRanges.push({ startSec: overlapEnd, endSec: current.endSec });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const audibleRange of audibleRanges) {
|
||||
const audibleDurationSec = audibleRange.endSec - audibleRange.startSec;
|
||||
if (audibleDurationSec <= 0.001) {
|
||||
continue;
|
||||
}
|
||||
const source = ctx.createBufferSource();
|
||||
const gainNode = ctx.createGain();
|
||||
gainNode.gain.value = Math.max(0, Math.min(2, gain));
|
||||
|
||||
const sourceOffsetSec =
|
||||
effectiveBufferStartSec +
|
||||
(audibleRange.startSec - (localOutputStartSec + chunkOutputStartSec)) *
|
||||
slice.speed;
|
||||
const localStartSec = audibleRange.startSec - chunkOutputStartSec;
|
||||
const sourceDurationSec = audibleDurationSec * slice.speed;
|
||||
|
||||
const stretchedBuffer = this.stretchAudioBuffer(
|
||||
buffer,
|
||||
slice.speed,
|
||||
sourceOffsetSec,
|
||||
sourceDurationSec,
|
||||
audibleDurationSec,
|
||||
ctx,
|
||||
);
|
||||
|
||||
source.buffer = stretchedBuffer;
|
||||
source.playbackRate.value = 1;
|
||||
source.connect(gainNode);
|
||||
gainNode.connect(ctx.destination);
|
||||
|
||||
source.start(localStartSec);
|
||||
}
|
||||
|
||||
outputOffsetSec += sliceOutputDurationSec;
|
||||
}
|
||||
}
|
||||
|
||||
protected stretchAudioBuffer(
|
||||
originalBuffer: AudioBuffer,
|
||||
speed: number,
|
||||
sourceOffsetSec: number,
|
||||
sourceDurationSec: number,
|
||||
audibleDurationSec: number,
|
||||
ctx: BaseAudioContext,
|
||||
): AudioBuffer {
|
||||
const sampleRate = originalBuffer.sampleRate;
|
||||
const channels = originalBuffer.numberOfChannels;
|
||||
|
||||
const startSample = Math.max(0, Math.floor(sourceOffsetSec * sampleRate));
|
||||
const sourceSamples = Math.floor(sourceDurationSec * sampleRate);
|
||||
const endSample = Math.min(originalBuffer.length, startSample + sourceSamples);
|
||||
|
||||
const outSamples = Math.floor(audibleDurationSec * sampleRate);
|
||||
if (outSamples <= 0 || startSample >= originalBuffer.length) {
|
||||
return ctx.createBuffer(channels, 1, sampleRate);
|
||||
}
|
||||
|
||||
const outBuffer = ctx.createBuffer(channels, outSamples, sampleRate);
|
||||
|
||||
if (Math.abs(speed - 1) < 0.001) {
|
||||
const copyLength = Math.min(endSample - startSample, outSamples);
|
||||
if (copyLength > 0) {
|
||||
for (let c = 0; c < channels; c++) {
|
||||
outBuffer.copyToChannel(
|
||||
originalBuffer
|
||||
.getChannelData(c)
|
||||
.subarray(startSample, startSample + copyLength),
|
||||
c,
|
||||
);
|
||||
}
|
||||
}
|
||||
return outBuffer;
|
||||
}
|
||||
|
||||
// WSOLA uses windowing which causes fade-in at the start and fade-out at the end.
|
||||
// To avoid clicks at chunk boundaries, we render with 100ms of padding and trim it.
|
||||
const paddingSec = 0.1;
|
||||
const paddingOutSamples = Math.floor(sampleRate * paddingSec);
|
||||
const paddingInSamples = Math.floor(paddingOutSamples * speed);
|
||||
|
||||
const workStartIn = Math.max(0, startSample - paddingInSamples);
|
||||
const workEndIn = Math.min(originalBuffer.length, endSample + paddingInSamples);
|
||||
|
||||
const actualPaddingInStart = startSample - workStartIn;
|
||||
// We expect the output offset for the requested start to be roughly:
|
||||
const actualPaddingOutStart = Math.floor(actualPaddingInStart / speed);
|
||||
|
||||
const windowSize = Math.floor(sampleRate * 0.04);
|
||||
const hopOut = Math.floor(windowSize * 0.5);
|
||||
const hopIn = Math.floor(hopOut * speed);
|
||||
const searchRange = Math.floor(sampleRate * 0.015);
|
||||
|
||||
const workOutSamples = Math.floor((workEndIn - workStartIn) / speed) + windowSize * 2;
|
||||
const workOutBuffer = ctx.createBuffer(channels, workOutSamples, sampleRate);
|
||||
|
||||
const inDataByChannel = Array.from({ length: channels }, (_, c) =>
|
||||
originalBuffer.getChannelData(c),
|
||||
);
|
||||
const workOutDataByChannel = Array.from({ length: channels }, (_, c) =>
|
||||
workOutBuffer.getChannelData(c),
|
||||
);
|
||||
|
||||
const window = new Float32Array(windowSize);
|
||||
for (let i = 0; i < windowSize; i++) {
|
||||
window[i] = 0.5 * (1 - Math.cos((2 * Math.PI * i) / (windowSize - 1)));
|
||||
}
|
||||
|
||||
let inOffset = workStartIn;
|
||||
let outOffset = 0;
|
||||
|
||||
// Initial window
|
||||
for (let i = 0; i < windowSize; i++) {
|
||||
if (inOffset + i < workEndIn && outOffset + i < workOutSamples) {
|
||||
for (let c = 0; c < channels; c++) {
|
||||
workOutDataByChannel[c][outOffset + i] +=
|
||||
inDataByChannel[c][inOffset + i] * window[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
outOffset += hopOut;
|
||||
inOffset += hopIn;
|
||||
|
||||
while (outOffset + windowSize < workOutSamples && inOffset + windowSize < workEndIn) {
|
||||
let bestOffset = inOffset;
|
||||
const minSearch = Math.max(workStartIn, inOffset - searchRange);
|
||||
const maxSearch = Math.min(workEndIn - windowSize, inOffset + searchRange);
|
||||
|
||||
if (maxSearch > minSearch) {
|
||||
let maxCorr = -Infinity;
|
||||
let bestDelta = 0;
|
||||
|
||||
for (let testOffset = minSearch; testOffset <= maxSearch; testOffset += 4) {
|
||||
let corr = 0;
|
||||
for (let i = 0; i < hopOut; i += 4) {
|
||||
if (outOffset + i < workOutSamples && testOffset + i < workEndIn) {
|
||||
for (let c = 0; c < channels; c++) {
|
||||
corr +=
|
||||
workOutDataByChannel[c][outOffset + i] *
|
||||
inDataByChannel[c][testOffset + i];
|
||||
}
|
||||
}
|
||||
}
|
||||
if (corr > maxCorr) {
|
||||
maxCorr = corr;
|
||||
bestDelta = testOffset - inOffset;
|
||||
}
|
||||
}
|
||||
bestOffset = inOffset + bestDelta;
|
||||
}
|
||||
|
||||
for (let i = 0; i < windowSize; i++) {
|
||||
if (bestOffset + i < workEndIn && outOffset + i < workOutSamples) {
|
||||
for (let c = 0; c < channels; c++) {
|
||||
workOutDataByChannel[c][outOffset + i] +=
|
||||
inDataByChannel[c][bestOffset + i] * window[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
outOffset += hopOut;
|
||||
inOffset += hopIn;
|
||||
}
|
||||
|
||||
// Transfer the stable middle portion to the final buffer
|
||||
for (let c = 0; c < channels; c++) {
|
||||
const finalData = outBuffer.getChannelData(c);
|
||||
const tempData = workOutBuffer.getChannelData(c);
|
||||
for (let i = 0; i < outSamples; i++) {
|
||||
const srcIdx = actualPaddingOutStart + i;
|
||||
if (srcIdx < workOutSamples) {
|
||||
finalData[i] = tempData[srcIdx];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return outBuffer;
|
||||
}
|
||||
|
||||
// Create a WAV file header for the given audio parameters.
|
||||
}
|
||||
@@ -0,0 +1,275 @@
|
||||
import type { WebDemuxer } from "web-demuxer";
|
||||
import { OfflineAudioProcessor } from "./offlineAudioProcessor";
|
||||
import {
|
||||
AUDIO_BITRATE,
|
||||
DECODE_BACKPRESSURE_LIMIT,
|
||||
ENCODE_BACKPRESSURE_LIMIT,
|
||||
MP4_AUDIO_CODEC,
|
||||
type TrimLikeRegion,
|
||||
} from "./audioProcessorShared";
|
||||
import type { VideoMuxer } from "./muxer";
|
||||
|
||||
export class AudioTranscodeProcessor extends OfflineAudioProcessor {
|
||||
protected async processTrimOnlyAudio(
|
||||
demuxer: WebDemuxer,
|
||||
muxer: VideoMuxer,
|
||||
sortedTrims: TrimLikeRegion[],
|
||||
readEndSec?: number,
|
||||
): Promise<void> {
|
||||
let audioConfig: AudioDecoderConfig;
|
||||
try {
|
||||
audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig;
|
||||
} catch {
|
||||
console.warn("[AudioProcessor] No audio track found, skipping");
|
||||
return;
|
||||
}
|
||||
|
||||
const codecCheck = await AudioDecoder.isConfigSupported(audioConfig);
|
||||
if (!codecCheck.supported) {
|
||||
console.warn("[AudioProcessor] Audio codec not supported:", audioConfig.codec);
|
||||
return;
|
||||
}
|
||||
|
||||
const audioStream =
|
||||
typeof readEndSec === "number"
|
||||
? demuxer.read("audio", 0, readEndSec)
|
||||
: demuxer.read("audio");
|
||||
|
||||
let sourceTimestampOffsetUs: number | null = null;
|
||||
|
||||
await this.transcodeAudioStream(
|
||||
audioStream as ReadableStream<EncodedAudioChunk>,
|
||||
audioConfig,
|
||||
muxer,
|
||||
{
|
||||
observeChunkTimestampUs: (timestampUs) => {
|
||||
if (sourceTimestampOffsetUs === null) {
|
||||
sourceTimestampOffsetUs = timestampUs;
|
||||
}
|
||||
},
|
||||
shouldSkipChunk: (timestampMs) => this.isInTrimRegion(timestampMs, sortedTrims),
|
||||
transformAudioData: (data) => {
|
||||
const timestampMs = data.timestamp / 1000;
|
||||
const trimOffsetMs = this.computeTrimOffset(timestampMs, sortedTrims);
|
||||
const adjustedTimestampUs =
|
||||
data.timestamp - (sourceTimestampOffsetUs ?? 0) - trimOffsetMs * 1000;
|
||||
return this.cloneWithTimestamp(data, Math.max(0, adjustedTimestampUs));
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
protected async transcodeAudioStream(
|
||||
audioStream: ReadableStream<EncodedAudioChunk>,
|
||||
audioConfig: AudioDecoderConfig,
|
||||
muxer: VideoMuxer,
|
||||
options: {
|
||||
observeChunkTimestampUs?: (timestampUs: number) => void;
|
||||
shouldSkipChunk?: (timestampMs: number) => boolean;
|
||||
transformAudioData?: (data: AudioData) => AudioData | null;
|
||||
} = {},
|
||||
): Promise<void> {
|
||||
const pendingFrames: AudioData[] = [];
|
||||
let decodeError: Error | null = null;
|
||||
let encodeError: Error | null = null;
|
||||
let muxError: Error | null = null;
|
||||
let pendingMuxing = Promise.resolve();
|
||||
const capacityWaiters = new Set<() => void>();
|
||||
|
||||
const notifyCapacityAvailable = () => {
|
||||
if (capacityWaiters.size === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const waiters = [...capacityWaiters];
|
||||
capacityWaiters.clear();
|
||||
for (const resolve of waiters) {
|
||||
resolve();
|
||||
}
|
||||
};
|
||||
|
||||
const waitForCapacity = () =>
|
||||
new Promise<void>((resolve) => {
|
||||
capacityWaiters.add(resolve);
|
||||
});
|
||||
|
||||
const failIfNeeded = () => {
|
||||
if (decodeError) throw decodeError;
|
||||
if (encodeError) throw encodeError;
|
||||
if (muxError) throw muxError;
|
||||
};
|
||||
|
||||
const pumpEncodedFrames = () => {
|
||||
while (!this.cancelled && pendingFrames.length > 0) {
|
||||
if (encodeError || muxError) {
|
||||
break;
|
||||
}
|
||||
if (encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT) {
|
||||
break;
|
||||
}
|
||||
|
||||
const frame = pendingFrames.shift();
|
||||
if (!frame) {
|
||||
break;
|
||||
}
|
||||
|
||||
encoder.encode(frame);
|
||||
frame.close();
|
||||
notifyCapacityAvailable();
|
||||
}
|
||||
};
|
||||
|
||||
const cleanupPendingFrames = () => {
|
||||
for (const frame of pendingFrames) {
|
||||
frame.close();
|
||||
}
|
||||
pendingFrames.length = 0;
|
||||
};
|
||||
|
||||
const sampleRate = audioConfig.sampleRate || 48_000;
|
||||
const channels = audioConfig.numberOfChannels || 2;
|
||||
const encodeConfig: AudioEncoderConfig = {
|
||||
codec: MP4_AUDIO_CODEC,
|
||||
sampleRate,
|
||||
numberOfChannels: channels,
|
||||
bitrate: AUDIO_BITRATE,
|
||||
};
|
||||
|
||||
const encodeSupport = await AudioEncoder.isConfigSupported(encodeConfig);
|
||||
if (!encodeSupport.supported) {
|
||||
console.warn("[AudioProcessor] AAC encoding not supported, skipping audio");
|
||||
return;
|
||||
}
|
||||
|
||||
const encoder = new AudioEncoder({
|
||||
output: (chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) => {
|
||||
pendingMuxing = pendingMuxing
|
||||
.then(async () => {
|
||||
if (this.cancelled) {
|
||||
return;
|
||||
}
|
||||
await muxer.addAudioChunk(chunk, meta);
|
||||
})
|
||||
.catch((error) => {
|
||||
muxError = error instanceof Error ? error : new Error(String(error));
|
||||
notifyCapacityAvailable();
|
||||
});
|
||||
notifyCapacityAvailable();
|
||||
},
|
||||
error: (error: DOMException) => {
|
||||
encodeError = new Error(`[AudioProcessor] Encode error: ${error.message}`);
|
||||
notifyCapacityAvailable();
|
||||
},
|
||||
});
|
||||
|
||||
encoder.configure(encodeConfig);
|
||||
|
||||
const decoder = new AudioDecoder({
|
||||
output: (data: AudioData) => {
|
||||
if (this.cancelled || encodeError || muxError) {
|
||||
data.close();
|
||||
return;
|
||||
}
|
||||
|
||||
const transformed = options.transformAudioData
|
||||
? options.transformAudioData(data)
|
||||
: data;
|
||||
|
||||
if (transformed !== data) {
|
||||
data.close();
|
||||
}
|
||||
|
||||
if (!transformed) {
|
||||
return;
|
||||
}
|
||||
|
||||
pendingFrames.push(transformed);
|
||||
notifyCapacityAvailable();
|
||||
},
|
||||
error: (error: DOMException) => {
|
||||
decodeError = new Error(`[AudioProcessor] Decode error: ${error.message}`);
|
||||
notifyCapacityAvailable();
|
||||
},
|
||||
});
|
||||
decoder.configure(audioConfig);
|
||||
|
||||
let reader: ReadableStreamDefaultReader<EncodedAudioChunk> | null = null;
|
||||
|
||||
try {
|
||||
reader = audioStream.getReader();
|
||||
while (!this.cancelled) {
|
||||
failIfNeeded();
|
||||
|
||||
const { done, value: chunk } = await reader.read();
|
||||
if (done || !chunk) break;
|
||||
|
||||
options.observeChunkTimestampUs?.(chunk.timestamp);
|
||||
const timestampMs = chunk.timestamp / 1000;
|
||||
if (options.shouldSkipChunk?.(timestampMs)) continue;
|
||||
|
||||
decoder.decode(chunk);
|
||||
pumpEncodedFrames();
|
||||
|
||||
while (
|
||||
!this.cancelled &&
|
||||
(decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT ||
|
||||
pendingFrames.length > DECODE_BACKPRESSURE_LIMIT ||
|
||||
encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT)
|
||||
) {
|
||||
failIfNeeded();
|
||||
pumpEncodedFrames();
|
||||
await waitForCapacity();
|
||||
}
|
||||
}
|
||||
|
||||
if (decoder.state === "configured") {
|
||||
await decoder.flush();
|
||||
}
|
||||
|
||||
while (!this.cancelled && (pendingFrames.length > 0 || encoder.encodeQueueSize > 0)) {
|
||||
failIfNeeded();
|
||||
pumpEncodedFrames();
|
||||
if (pendingFrames.length > 0 || encoder.encodeQueueSize > 0) {
|
||||
await waitForCapacity();
|
||||
}
|
||||
}
|
||||
|
||||
failIfNeeded();
|
||||
|
||||
if (encoder.state === "configured") {
|
||||
await encoder.flush();
|
||||
}
|
||||
|
||||
await pendingMuxing;
|
||||
failIfNeeded();
|
||||
} finally {
|
||||
notifyCapacityAvailable();
|
||||
if (reader) {
|
||||
try {
|
||||
await reader.cancel();
|
||||
} catch {
|
||||
// reader already closed
|
||||
}
|
||||
}
|
||||
|
||||
cleanupPendingFrames();
|
||||
|
||||
if (decoder.state === "configured") {
|
||||
decoder.close();
|
||||
}
|
||||
|
||||
if (encoder.state === "configured") {
|
||||
encoder.close();
|
||||
}
|
||||
}
|
||||
|
||||
if (this.cancelled) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- Offline audio rendering pipeline ----------
|
||||
// Replaces the old real-time MediaElement+MediaRecorder approach with
|
||||
// OfflineAudioContext, which renders as fast as the CPU allows instead of
|
||||
// waiting for 1× real-time playback.
|
||||
}
|
||||
@@ -0,0 +1,444 @@
|
||||
import { SOURCE_AUDIO_NORMALIZE_GAIN } from "@/components/video-editor/audio/audioTypes";
|
||||
import type {
|
||||
AudioRegion,
|
||||
ClipRegion,
|
||||
SourceAudioTrackSettings,
|
||||
SpeedRegion,
|
||||
} from "@/components/video-editor/types";
|
||||
import { buildResolvedAudioPlan } from "@/lib/exporter/audioRoutingEngine";
|
||||
import { estimateCompanionAudioStartDelaySeconds } from "@/lib/mediaTiming";
|
||||
import { AudioMediaProcessor } from "./audioMediaProcessor";
|
||||
import {
|
||||
AUDIO_BITRATE,
|
||||
ENCODE_BACKPRESSURE_LIMIT,
|
||||
getSourceTrackIdFromPath,
|
||||
MP4_AUDIO_CODEC,
|
||||
OFFLINE_AUDIO_SAMPLE_RATE,
|
||||
OFFLINE_CHUNK_DURATION_SEC,
|
||||
OFFLINE_ENCODE_CHUNK_FRAMES,
|
||||
type PreparedOfflineRender,
|
||||
resolveSourceTrackGain,
|
||||
softLimitOfflineMixPeaksInPlace,
|
||||
type TimelineSlice,
|
||||
type TrimLikeRegion,
|
||||
} from "./audioProcessorShared";
|
||||
import type { VideoMuxer } from "./muxer";
|
||||
|
||||
export class OfflineAudioProcessor extends AudioMediaProcessor {
|
||||
protected async renderAndMuxOfflineAudio(
|
||||
videoUrl: string,
|
||||
trimRegions: TrimLikeRegion[],
|
||||
speedRegions: SpeedRegion[],
|
||||
audioRegions: AudioRegion[],
|
||||
sourceAudioFallbackPaths: string[],
|
||||
sourceAudioFallbackStartDelayMsByPath: Record<string, number> | undefined,
|
||||
sourceAudioTrackSettings: SourceAudioTrackSettings | undefined,
|
||||
clipRegions: ClipRegion[] | undefined,
|
||||
muxer: VideoMuxer,
|
||||
): Promise<void> {
|
||||
const prepared = await this.prepareOfflineRender(
|
||||
videoUrl,
|
||||
trimRegions,
|
||||
speedRegions,
|
||||
audioRegions,
|
||||
sourceAudioFallbackPaths,
|
||||
sourceAudioFallbackStartDelayMsByPath,
|
||||
sourceAudioTrackSettings,
|
||||
clipRegions,
|
||||
);
|
||||
if (this.cancelled) return;
|
||||
await this.renderAndEncodeChunked(prepared, muxer);
|
||||
}
|
||||
|
||||
protected async prepareOfflineRender(
|
||||
videoUrl: string,
|
||||
trimRegions: TrimLikeRegion[],
|
||||
speedRegions: SpeedRegion[],
|
||||
audioRegions: AudioRegion[],
|
||||
sourceAudioFallbackPaths: string[],
|
||||
sourceAudioFallbackStartDelayMsByPath?: Record<string, number>,
|
||||
sourceAudioTrackSettings?: SourceAudioTrackSettings,
|
||||
clipRegions?: ClipRegion[],
|
||||
): Promise<PreparedOfflineRender> {
|
||||
if (this.cancelled) throw new Error("Export cancelled");
|
||||
this.onProgress?.(0);
|
||||
|
||||
const resolvedPlan = buildResolvedAudioPlan({
|
||||
videoResource: videoUrl,
|
||||
sourceAudioFallbackPaths,
|
||||
audioRegions,
|
||||
sourceTrackGainById: {
|
||||
mic: resolveSourceTrackGain(sourceAudioTrackSettings, "mic"),
|
||||
system: resolveSourceTrackGain(sourceAudioTrackSettings, "system"),
|
||||
mixed: resolveSourceTrackGain(sourceAudioTrackSettings, "mixed"),
|
||||
},
|
||||
embeddedGain: Math.max(
|
||||
0,
|
||||
Math.min(
|
||||
2,
|
||||
sourceAudioTrackSettings?.mixed
|
||||
? resolveSourceTrackGain(sourceAudioTrackSettings, "mixed")
|
||||
: sourceAudioTrackSettings?.system
|
||||
? resolveSourceTrackGain(sourceAudioTrackSettings, "system")
|
||||
: 1,
|
||||
),
|
||||
),
|
||||
});
|
||||
|
||||
// Decode embedded source audio separately from companion sidecars.
|
||||
const mainBuffer = resolvedPlan.includeEmbeddedInExport
|
||||
? await this.decodeAudioFromUrl(videoUrl)
|
||||
: null;
|
||||
const mainBufferGain = resolveSourceTrackGain(sourceAudioTrackSettings, "mixed");
|
||||
const mainBufferEntry = mainBuffer ? { buffer: mainBuffer, gain: mainBufferGain } : null;
|
||||
if (this.cancelled) throw new Error("Export cancelled");
|
||||
|
||||
// Decode companion / sidecar audio files
|
||||
const companionEntries: Array<{
|
||||
buffer: AudioBuffer;
|
||||
startDelaySec: number;
|
||||
gain: number;
|
||||
}> = [];
|
||||
const refDuration =
|
||||
mainBuffer?.duration ??
|
||||
(resolvedPlan.playbackPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0);
|
||||
for (const audioPath of resolvedPlan.playbackPaths) {
|
||||
if (this.cancelled) throw new Error("Export cancelled");
|
||||
const buffer = await this.decodeAudioFromUrl(audioPath);
|
||||
if (!buffer) continue;
|
||||
|
||||
companionEntries.push({
|
||||
buffer,
|
||||
gain: resolveSourceTrackGain(
|
||||
sourceAudioTrackSettings,
|
||||
getSourceTrackIdFromPath(audioPath),
|
||||
),
|
||||
startDelaySec: estimateCompanionAudioStartDelaySeconds(
|
||||
refDuration,
|
||||
buffer.duration,
|
||||
sourceAudioFallbackStartDelayMsByPath?.[audioPath],
|
||||
),
|
||||
});
|
||||
}
|
||||
if (this.cancelled) throw new Error("Export cancelled");
|
||||
|
||||
// Decode audio region overlay files
|
||||
const regionEntries: Array<{ buffer: AudioBuffer; region: AudioRegion }> = [];
|
||||
for (const region of audioRegions) {
|
||||
if (this.cancelled) throw new Error("Export cancelled");
|
||||
const buffer = await this.decodeAudioFromUrl(region.audioPath);
|
||||
if (buffer) regionEntries.push({ buffer, region });
|
||||
}
|
||||
|
||||
this.onProgress?.(0.2);
|
||||
|
||||
// Determine source duration for timeline calculation
|
||||
const primaryBuffer = mainBufferEntry?.buffer ?? companionEntries[0]?.buffer ?? null;
|
||||
if (!primaryBuffer && regionEntries.length === 0) {
|
||||
throw new Error("No decodable audio sources found");
|
||||
}
|
||||
|
||||
let sourceDurationSec: number;
|
||||
if (mainBufferEntry?.buffer) {
|
||||
sourceDurationSec = mainBufferEntry.buffer.duration;
|
||||
} else if (resolvedPlan.playbackPaths.length > 0 || regionEntries.length > 0) {
|
||||
sourceDurationSec = await this.getMediaDurationSec(videoUrl);
|
||||
} else {
|
||||
sourceDurationSec = primaryBuffer?.duration ?? 0;
|
||||
}
|
||||
const sourceDurationMs = sourceDurationSec * 1000;
|
||||
|
||||
// Build timeline slices (non-trimmed segments with speed info)
|
||||
const slices = this.buildTimelineSlices(sourceDurationMs, trimRegions, speedRegions);
|
||||
|
||||
let outputDurationMs = 0;
|
||||
for (const slice of slices) {
|
||||
outputDurationMs += (slice.sourceEndMs - slice.sourceStartMs) / slice.speed;
|
||||
}
|
||||
|
||||
// Extend for audio regions that might exceed the video timeline
|
||||
for (const { region } of regionEntries) {
|
||||
const regionEndOutput = this.sourceTimeToOutputTime(region.endMs, slices);
|
||||
outputDurationMs = Math.max(outputDurationMs, regionEndOutput);
|
||||
}
|
||||
|
||||
const numChannels = Math.min(primaryBuffer?.numberOfChannels ?? 2, 2);
|
||||
const mutedSourceOutputRangesSec = (clipRegions ?? [])
|
||||
.filter(
|
||||
(clip) =>
|
||||
Boolean(clip.muted) &&
|
||||
Number.isFinite(clip.startMs) &&
|
||||
Number.isFinite(clip.endMs) &&
|
||||
clip.endMs > clip.startMs,
|
||||
)
|
||||
.map((clip) => ({
|
||||
startSec: Math.max(0, clip.startMs / 1000),
|
||||
endSec: Math.max(0, clip.endMs / 1000),
|
||||
}));
|
||||
|
||||
return {
|
||||
mainBufferEntry,
|
||||
companionEntries,
|
||||
regionEntries,
|
||||
mutedSourceOutputRangesSec,
|
||||
slices,
|
||||
outputDurationMs,
|
||||
numChannels,
|
||||
};
|
||||
}
|
||||
|
||||
// Render timeline in chunks and encode each chunk to the muxer immediately.
|
||||
// Memory is bounded to ~OFFLINE_CHUNK_DURATION_SEC of PCM per chunk
|
||||
// instead of holding the entire output buffer in memory.
|
||||
protected async renderAndEncodeChunked(
|
||||
prepared: PreparedOfflineRender,
|
||||
muxer: VideoMuxer,
|
||||
): Promise<void> {
|
||||
const { numChannels } = prepared;
|
||||
const totalOutputSec = Math.max(prepared.outputDurationMs / 1000, 0.01);
|
||||
|
||||
let encodeError: Error | null = null;
|
||||
let muxError: Error | null = null;
|
||||
let pendingMuxing = Promise.resolve();
|
||||
let wroteFirstChunk = false;
|
||||
|
||||
const encodeConfig: AudioEncoderConfig = {
|
||||
codec: MP4_AUDIO_CODEC,
|
||||
sampleRate: OFFLINE_AUDIO_SAMPLE_RATE,
|
||||
numberOfChannels: numChannels,
|
||||
bitrate: AUDIO_BITRATE,
|
||||
};
|
||||
|
||||
const supported = await AudioEncoder.isConfigSupported(encodeConfig);
|
||||
if (!supported.supported) {
|
||||
console.warn("[AudioProcessor] AAC encoding not supported for offline audio");
|
||||
return;
|
||||
}
|
||||
|
||||
const encoder = new AudioEncoder({
|
||||
output: (chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) => {
|
||||
pendingMuxing = pendingMuxing
|
||||
.then(async () => {
|
||||
if (this.cancelled) return;
|
||||
await muxer.addAudioChunk(chunk, !wroteFirstChunk ? meta : undefined);
|
||||
wroteFirstChunk = true;
|
||||
})
|
||||
.catch((error) => {
|
||||
muxError = error instanceof Error ? error : new Error(String(error));
|
||||
});
|
||||
},
|
||||
error: (error: DOMException) => {
|
||||
encodeError = new Error(`Audio encode error: ${error.message}`);
|
||||
},
|
||||
});
|
||||
encoder.configure(encodeConfig);
|
||||
|
||||
try {
|
||||
await this.renderChunked(
|
||||
prepared,
|
||||
totalOutputSec,
|
||||
async (rendered, outputOffsetSec) => {
|
||||
if (encodeError) throw encodeError;
|
||||
if (muxError) throw muxError;
|
||||
await this.feedBufferToEncoder(encoder, rendered, outputOffsetSec);
|
||||
},
|
||||
);
|
||||
|
||||
if (encodeError) throw encodeError;
|
||||
if (muxError) throw muxError;
|
||||
|
||||
if (encoder.state === "configured") {
|
||||
await encoder.flush();
|
||||
}
|
||||
|
||||
await pendingMuxing;
|
||||
|
||||
if (encodeError) throw encodeError;
|
||||
if (muxError) throw muxError;
|
||||
} finally {
|
||||
if (encoder.state === "configured") {
|
||||
encoder.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Render timeline to a WAV blob for the native/FFmpeg export path.
|
||||
// Processes in chunks to avoid holding the entire output in memory.
|
||||
protected async renderToWavBlobChunked(prepared: PreparedOfflineRender): Promise<Blob> {
|
||||
const totalOutputSec = Math.max(prepared.outputDurationMs / 1000, 0.01);
|
||||
const totalFrames = Math.ceil(totalOutputSec * OFFLINE_AUDIO_SAMPLE_RATE);
|
||||
const numChannels = prepared.numChannels;
|
||||
|
||||
const header = this.createWavHeader(OFFLINE_AUDIO_SAMPLE_RATE, numChannels, totalFrames);
|
||||
const pcmParts: ArrayBuffer[] = [header];
|
||||
|
||||
await this.renderChunked(prepared, totalOutputSec, async (rendered) => {
|
||||
pcmParts.push(...this.audioBufferToPcmParts(rendered));
|
||||
});
|
||||
|
||||
return new Blob(pcmParts, { type: "audio/wav" });
|
||||
}
|
||||
|
||||
// Shared chunked rendering loop. Processes the timeline in
|
||||
// OFFLINE_CHUNK_DURATION_SEC segments, calling onChunk for each rendered buffer.
|
||||
protected async renderChunked(
|
||||
prepared: PreparedOfflineRender,
|
||||
totalOutputSec: number,
|
||||
onChunk: (
|
||||
rendered: AudioBuffer,
|
||||
outputOffsetSec: number,
|
||||
chunkIndex: number,
|
||||
) => Promise<void>,
|
||||
): Promise<void> {
|
||||
const { slices, numChannels } = prepared;
|
||||
let outputOffsetSec = 0;
|
||||
const chunkCount = Math.ceil(totalOutputSec / OFFLINE_CHUNK_DURATION_SEC);
|
||||
|
||||
for (let i = 0; i < chunkCount && !this.cancelled; i++) {
|
||||
const chunkSec = Math.min(OFFLINE_CHUNK_DURATION_SEC, totalOutputSec - outputOffsetSec);
|
||||
const chunkFrames = Math.ceil(chunkSec * OFFLINE_AUDIO_SAMPLE_RATE);
|
||||
|
||||
const offlineCtx = new OfflineAudioContext(
|
||||
numChannels,
|
||||
chunkFrames,
|
||||
OFFLINE_AUDIO_SAMPLE_RATE,
|
||||
);
|
||||
|
||||
// Schedule main audio
|
||||
if (prepared.mainBufferEntry) {
|
||||
this.scheduleBufferThroughTimeline(
|
||||
offlineCtx,
|
||||
prepared.mainBufferEntry.buffer,
|
||||
slices,
|
||||
0,
|
||||
prepared.mainBufferEntry.gain,
|
||||
outputOffsetSec,
|
||||
chunkSec,
|
||||
prepared.mutedSourceOutputRangesSec,
|
||||
);
|
||||
}
|
||||
|
||||
// Schedule companion/sidecar audio
|
||||
for (const entry of prepared.companionEntries) {
|
||||
this.scheduleBufferThroughTimeline(
|
||||
offlineCtx,
|
||||
entry.buffer,
|
||||
slices,
|
||||
entry.startDelaySec,
|
||||
entry.gain,
|
||||
outputOffsetSec,
|
||||
chunkSec,
|
||||
prepared.mutedSourceOutputRangesSec,
|
||||
);
|
||||
}
|
||||
|
||||
// Schedule audio region overlays
|
||||
for (const { buffer, region } of prepared.regionEntries) {
|
||||
this.scheduleRegionForChunk(
|
||||
offlineCtx,
|
||||
buffer,
|
||||
region,
|
||||
slices,
|
||||
outputOffsetSec,
|
||||
chunkSec,
|
||||
);
|
||||
}
|
||||
|
||||
const rendered = await offlineCtx.startRendering();
|
||||
if (this.cancelled) break;
|
||||
softLimitOfflineMixPeaksInPlace(rendered);
|
||||
|
||||
await onChunk(rendered, outputOffsetSec, i);
|
||||
|
||||
outputOffsetSec += chunkSec;
|
||||
this.onProgress?.(0.3 + (outputOffsetSec / totalOutputSec) * 0.7);
|
||||
}
|
||||
}
|
||||
|
||||
// Schedule an audio region overlay clipped to a specific chunk window.
|
||||
protected scheduleRegionForChunk(
|
||||
ctx: OfflineAudioContext,
|
||||
buffer: AudioBuffer,
|
||||
region: AudioRegion,
|
||||
slices: TimelineSlice[],
|
||||
chunkOutputStartSec: number,
|
||||
chunkDurationSec: number,
|
||||
): void {
|
||||
const outputStartMs = this.sourceTimeToOutputTime(region.startMs, slices);
|
||||
const outputEndMs = this.sourceTimeToOutputTime(region.endMs, slices);
|
||||
|
||||
let localStartSec = outputStartMs / 1000 - chunkOutputStartSec;
|
||||
let localEndSec = outputEndMs / 1000 - chunkOutputStartSec;
|
||||
|
||||
// Skip if region doesn't overlap with this chunk
|
||||
if (localEndSec <= 0 || localStartSec >= chunkDurationSec) return;
|
||||
|
||||
// Clip to chunk bounds
|
||||
let bufferOffsetSec = 0;
|
||||
if (localStartSec < 0) {
|
||||
bufferOffsetSec = -localStartSec;
|
||||
localStartSec = 0;
|
||||
}
|
||||
if (localEndSec > chunkDurationSec) {
|
||||
localEndSec = chunkDurationSec;
|
||||
}
|
||||
|
||||
const duration = Math.min(localEndSec - localStartSec, buffer.duration - bufferOffsetSec);
|
||||
if (duration <= 0.001) return;
|
||||
|
||||
const gainNode = ctx.createGain();
|
||||
const normalizeGain = region.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1;
|
||||
gainNode.gain.value = Math.max(0, Math.min(1, region.volume * normalizeGain));
|
||||
gainNode.connect(ctx.destination);
|
||||
|
||||
const source = ctx.createBufferSource();
|
||||
source.buffer = buffer;
|
||||
source.connect(gainNode);
|
||||
source.start(localStartSec, bufferOffsetSec, duration);
|
||||
}
|
||||
|
||||
// Feed a rendered AudioBuffer chunk to an AudioEncoder with a timestamp offset.
|
||||
protected async feedBufferToEncoder(
|
||||
encoder: AudioEncoder,
|
||||
buffer: AudioBuffer,
|
||||
timestampOffsetSec: number,
|
||||
): Promise<void> {
|
||||
const sampleRate = buffer.sampleRate;
|
||||
const numChannels = buffer.numberOfChannels;
|
||||
const totalFrames = buffer.length;
|
||||
|
||||
for (
|
||||
let offset = 0;
|
||||
offset < totalFrames && !this.cancelled;
|
||||
offset += OFFLINE_ENCODE_CHUNK_FRAMES
|
||||
) {
|
||||
const frameCount = Math.min(OFFLINE_ENCODE_CHUNK_FRAMES, totalFrames - offset);
|
||||
|
||||
const planarData = new Float32Array(frameCount * numChannels);
|
||||
for (let ch = 0; ch < numChannels; ch++) {
|
||||
const channelData = buffer.getChannelData(ch);
|
||||
planarData.set(channelData.subarray(offset, offset + frameCount), ch * frameCount);
|
||||
}
|
||||
|
||||
const audioData = new AudioData({
|
||||
format: "f32-planar",
|
||||
sampleRate,
|
||||
numberOfFrames: frameCount,
|
||||
numberOfChannels: numChannels,
|
||||
timestamp: Math.round((offset / sampleRate + timestampOffsetSec) * 1_000_000),
|
||||
data: planarData,
|
||||
});
|
||||
|
||||
encoder.encode(audioData);
|
||||
audioData.close();
|
||||
|
||||
while (encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT && !this.cancelled) {
|
||||
await new Promise((r) => setTimeout(r, 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Decode audio from a URL using streaming WebCodecs decode with bulk fallback.
|
||||
// Streaming decode avoids holding the full compressed file in memory alongside
|
||||
// the decoded AudioBuffer, reducing peak memory for large recordings.
|
||||
}
|
||||
@@ -0,0 +1,448 @@
|
||||
import type { SpeedRegion, TrimRegion } from "@/components/video-editor/types";
|
||||
import { getEffectiveVideoStreamDurationSeconds } from "@/lib/mediaTiming";
|
||||
import type { WebDemuxer } from "web-demuxer";
|
||||
import {
|
||||
buildVideoDecodeFailure,
|
||||
getDecodedFrameTimelineOffsetUs,
|
||||
preserveFirstVideoDecodeFailure,
|
||||
type DecodedVideoInfo,
|
||||
type VideoDecodeFailureContext,
|
||||
} from "./streamingDecoderSupport";
|
||||
import { computeVideoSegments, splitVideoSegmentsBySpeed } from "./videoTimelineSegments";
|
||||
|
||||
type OnFrameCallback = (
|
||||
frame: VideoFrame,
|
||||
exportTimestampUs: number,
|
||||
sourceTimestampMs: number,
|
||||
cursorTimestampMs: number,
|
||||
) => Promise<void>;
|
||||
|
||||
interface StreamingDecodeContext {
|
||||
demuxer: WebDemuxer;
|
||||
decoder: VideoDecoder | null;
|
||||
readonly cancelled: boolean;
|
||||
readonly metadata: DecodedVideoInfo;
|
||||
pendingFrames: VideoFrame[];
|
||||
readonly maxDecodeQueue: number;
|
||||
readonly maxPendingFrames: number;
|
||||
}
|
||||
|
||||
const STARTUP_STABILIZATION_SECONDS = 1.25;
|
||||
const STARTUP_MAX_DECODE_QUEUE = 12;
|
||||
const STARTUP_MAX_PENDING_FRAMES = 28;
|
||||
|
||||
export async function decodeVideoStream(
|
||||
context: StreamingDecodeContext,
|
||||
|
||||
targetFrameRate: number,
|
||||
trimRegions: TrimRegion[] | undefined,
|
||||
speedRegions: SpeedRegion[] | undefined,
|
||||
onFrame: OnFrameCallback,
|
||||
): Promise<void> {
|
||||
if (!context.demuxer || !context.metadata) {
|
||||
throw new Error("Must call loadMetadata() before decodeAll()");
|
||||
}
|
||||
|
||||
const decoderConfig = await context.demuxer.getDecoderConfig("video");
|
||||
const codec = context.metadata.codec.toLowerCase();
|
||||
const shouldPreferSoftwareDecode = codec.includes("av01") || codec.includes("av1");
|
||||
const effectiveVideoDuration = getEffectiveVideoStreamDurationSeconds({
|
||||
duration: context.metadata.duration,
|
||||
streamDuration: context.metadata.streamDuration,
|
||||
});
|
||||
const segments = splitVideoSegmentsBySpeed(
|
||||
computeVideoSegments(effectiveVideoDuration, trimRegions),
|
||||
speedRegions,
|
||||
);
|
||||
const segmentOutputFrameCounts = segments.map((segment) =>
|
||||
Math.ceil(((segment.endSec - segment.startSec) / segment.speed) * targetFrameRate),
|
||||
);
|
||||
const expectedOutputFrames = segmentOutputFrameCounts.reduce((sum, count) => sum + count, 0);
|
||||
const frameDurationUs = 1_000_000 / targetFrameRate;
|
||||
const epsilonSec = 0.001;
|
||||
const startupStabilizationSeconds = STARTUP_STABILIZATION_SECONDS;
|
||||
const startupFrameBudget = Math.max(
|
||||
1,
|
||||
Math.round(targetFrameRate * startupStabilizationSeconds),
|
||||
);
|
||||
let exportFrameIndex = 0;
|
||||
let loggedSteadyStateBackpressure = false;
|
||||
const backpressureWaiters = new Set<() => void>();
|
||||
|
||||
const notifyBackpressureProgress = () => {
|
||||
if (backpressureWaiters.size === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const waiters = [...backpressureWaiters];
|
||||
backpressureWaiters.clear();
|
||||
for (const resolve of waiters) {
|
||||
resolve();
|
||||
}
|
||||
};
|
||||
|
||||
const waitForBackpressureProgress = () =>
|
||||
new Promise<void>((resolve) => {
|
||||
backpressureWaiters.add(resolve);
|
||||
});
|
||||
|
||||
console.log(
|
||||
`[StreamingVideoDecoder] Startup-safe decode backpressure active for first ${startupStabilizationSeconds}s (${startupFrameBudget} frames)`,
|
||||
);
|
||||
|
||||
// Async frame queue — decoder pushes, consumer pulls
|
||||
context.pendingFrames.length = 0;
|
||||
const pendingFrames = context.pendingFrames;
|
||||
let frameResolve: ((frame: VideoFrame | null) => void) | null = null;
|
||||
let decodeError: Error | null = null;
|
||||
let decodeDone = false;
|
||||
let firstDecodedFrameTimestampUs: number | null = null;
|
||||
let decodedFrameTimelineOffsetUs = 0;
|
||||
let submittedChunkCount = 0;
|
||||
let lastSubmittedChunk: EncodedVideoChunk | undefined;
|
||||
let lastSubmittedChunkIndex: number | undefined;
|
||||
const preferredDecoderConfig = shouldPreferSoftwareDecode
|
||||
? {
|
||||
...decoderConfig,
|
||||
hardwareAcceleration: "prefer-software" as const,
|
||||
}
|
||||
: decoderConfig;
|
||||
let activeDecoderConfig = preferredDecoderConfig;
|
||||
const getDecoderFailureContext = (): VideoDecodeFailureContext => ({
|
||||
decoderConfig: activeDecoderConfig,
|
||||
sourceMetadata: context.metadata ?? undefined,
|
||||
chunkIndex: lastSubmittedChunkIndex,
|
||||
chunk: lastSubmittedChunk,
|
||||
decoderState: context.decoder?.state,
|
||||
decodeQueueSize: context.decoder?.decodeQueueSize,
|
||||
});
|
||||
const recordFirstDecodeError = (error: unknown) => {
|
||||
decodeError = preserveFirstVideoDecodeFailure(
|
||||
decodeError,
|
||||
error,
|
||||
getDecoderFailureContext(),
|
||||
);
|
||||
};
|
||||
|
||||
context.decoder = new VideoDecoder({
|
||||
output: (frame: VideoFrame) => {
|
||||
if (frameResolve) {
|
||||
const resolve = frameResolve;
|
||||
frameResolve = null;
|
||||
resolve(frame);
|
||||
} else {
|
||||
pendingFrames.push(frame);
|
||||
}
|
||||
notifyBackpressureProgress();
|
||||
},
|
||||
error: (e: DOMException) => {
|
||||
recordFirstDecodeError(e);
|
||||
if (frameResolve) {
|
||||
const resolve = frameResolve;
|
||||
frameResolve = null;
|
||||
resolve(null);
|
||||
}
|
||||
notifyBackpressureProgress();
|
||||
},
|
||||
});
|
||||
try {
|
||||
context.decoder.configure(preferredDecoderConfig);
|
||||
} catch (error) {
|
||||
if (!shouldPreferSoftwareDecode) {
|
||||
throw buildVideoDecodeFailure(error, getDecoderFailureContext());
|
||||
}
|
||||
// Fall back to default decoder config if software preference is unsupported.
|
||||
activeDecoderConfig = decoderConfig;
|
||||
try {
|
||||
context.decoder.configure(decoderConfig);
|
||||
} catch (fallbackError) {
|
||||
throw buildVideoDecodeFailure(fallbackError, getDecoderFailureContext());
|
||||
}
|
||||
}
|
||||
|
||||
const getNextFrame = (): Promise<VideoFrame | null> => {
|
||||
if (decodeError) return Promise.resolve(null);
|
||||
if (pendingFrames.length > 0) {
|
||||
const frame = pendingFrames.shift()!;
|
||||
notifyBackpressureProgress();
|
||||
return Promise.resolve(frame);
|
||||
}
|
||||
if (decodeDone) return Promise.resolve(null);
|
||||
return new Promise((resolve) => {
|
||||
frameResolve = resolve;
|
||||
});
|
||||
};
|
||||
|
||||
// One forward stream through the whole file.
|
||||
// Pass explicit range because some containers are truncated when no end is provided.
|
||||
const readEndSec =
|
||||
Math.max(
|
||||
context.metadata.duration + (context.metadata.mediaStartTime ?? 0),
|
||||
(context.metadata.streamDuration ?? context.metadata.duration) +
|
||||
(context.metadata.streamStartTime ?? context.metadata.mediaStartTime ?? 0),
|
||||
) + 0.5;
|
||||
const reader = context.demuxer.read("video", 0, readEndSec).getReader();
|
||||
|
||||
// Feed chunks to decoder in background with backpressure
|
||||
const feedPromise = (async () => {
|
||||
try {
|
||||
while (!context.cancelled && !decodeError) {
|
||||
const { done, value: chunk } = await reader.read();
|
||||
if (done || !chunk) break;
|
||||
|
||||
if (!loggedSteadyStateBackpressure && exportFrameIndex >= startupFrameBudget) {
|
||||
loggedSteadyStateBackpressure = true;
|
||||
console.log(
|
||||
"[StreamingVideoDecoder] Switched to steady-state decode backpressure",
|
||||
);
|
||||
}
|
||||
|
||||
const decodeQueueLimit =
|
||||
exportFrameIndex < startupFrameBudget
|
||||
? Math.min(context.maxDecodeQueue, STARTUP_MAX_DECODE_QUEUE)
|
||||
: context.maxDecodeQueue;
|
||||
const pendingFrameLimit =
|
||||
exportFrameIndex < startupFrameBudget
|
||||
? Math.min(context.maxPendingFrames, STARTUP_MAX_PENDING_FRAMES)
|
||||
: context.maxPendingFrames;
|
||||
|
||||
// Backpressure on both decode queue and decoded frame backlog.
|
||||
while (
|
||||
!decodeError &&
|
||||
context.decoder!.state === "configured" &&
|
||||
(context.decoder!.decodeQueueSize > decodeQueueLimit ||
|
||||
pendingFrames.length > pendingFrameLimit) &&
|
||||
!context.cancelled
|
||||
) {
|
||||
await waitForBackpressureProgress();
|
||||
}
|
||||
if (context.cancelled || decodeError) break;
|
||||
if (context.decoder!.state !== "configured") {
|
||||
recordFirstDecodeError(
|
||||
new DOMException(
|
||||
"Decoder closed before the next video chunk was submitted.",
|
||||
"InvalidStateError",
|
||||
),
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
lastSubmittedChunk = chunk;
|
||||
lastSubmittedChunkIndex = submittedChunkCount;
|
||||
context.decoder!.decode(chunk);
|
||||
submittedChunkCount++;
|
||||
}
|
||||
|
||||
if (!context.cancelled && context.decoder!.state === "configured") {
|
||||
await context.decoder!.flush();
|
||||
}
|
||||
} catch (e) {
|
||||
recordFirstDecodeError(e);
|
||||
} finally {
|
||||
decodeDone = true;
|
||||
if (frameResolve) {
|
||||
const resolve = frameResolve;
|
||||
frameResolve = null;
|
||||
resolve(null);
|
||||
}
|
||||
notifyBackpressureProgress();
|
||||
}
|
||||
})();
|
||||
|
||||
// Route decoded frames into segments by timestamp, then deliver with VFR→CFR resampling
|
||||
let segmentIdx = 0;
|
||||
let segmentFrameIndex = 0;
|
||||
let lastDecodedFrameSec: number | null = null;
|
||||
let heldFrame: VideoFrame | null = null;
|
||||
let heldFrameSec = 0;
|
||||
|
||||
const emitHeldFrameForTarget = async (segment: {
|
||||
startSec: number;
|
||||
endSec: number;
|
||||
speed: number;
|
||||
}) => {
|
||||
if (!heldFrame) return false;
|
||||
const segmentFrameCount = segmentOutputFrameCounts[segmentIdx];
|
||||
if (segmentFrameIndex >= segmentFrameCount) return false;
|
||||
|
||||
const segmentDurationSec = segment.endSec - segment.startSec;
|
||||
const sourceTimeSec =
|
||||
segment.startSec + (segmentFrameIndex / segmentFrameCount) * segmentDurationSec;
|
||||
if (sourceTimeSec >= segment.endSec - epsilonSec) return false;
|
||||
|
||||
const sourceTimestampMs = sourceTimeSec * 1000;
|
||||
await onFrame(
|
||||
heldFrame,
|
||||
exportFrameIndex * frameDurationUs,
|
||||
sourceTimestampMs,
|
||||
sourceTimestampMs,
|
||||
);
|
||||
segmentFrameIndex++;
|
||||
exportFrameIndex++;
|
||||
return true;
|
||||
};
|
||||
|
||||
while (!context.cancelled && segmentIdx < segments.length) {
|
||||
const frame = await getNextFrame();
|
||||
if (!frame) break;
|
||||
|
||||
if (firstDecodedFrameTimestampUs === null) {
|
||||
firstDecodedFrameTimestampUs = frame.timestamp;
|
||||
decodedFrameTimelineOffsetUs = getDecodedFrameTimelineOffsetUs(
|
||||
firstDecodedFrameTimestampUs,
|
||||
context.metadata,
|
||||
);
|
||||
}
|
||||
|
||||
const normalizedFrameTimeSec = Math.max(
|
||||
0,
|
||||
(frame.timestamp - firstDecodedFrameTimestampUs + decodedFrameTimelineOffsetUs) /
|
||||
1_000_000,
|
||||
);
|
||||
const frameTimeSec: number =
|
||||
lastDecodedFrameSec === null
|
||||
? normalizedFrameTimeSec
|
||||
: Math.max(lastDecodedFrameSec, normalizedFrameTimeSec);
|
||||
lastDecodedFrameSec = frameTimeSec;
|
||||
|
||||
// Finalize completed segments before handling this frame.
|
||||
while (
|
||||
segmentIdx < segments.length &&
|
||||
frameTimeSec >= segments[segmentIdx].endSec - epsilonSec
|
||||
) {
|
||||
const segment = segments[segmentIdx];
|
||||
while (!context.cancelled && (await emitHeldFrameForTarget(segment))) {
|
||||
// Keep emitting remaining output frames for this segment from the last known frame.
|
||||
}
|
||||
|
||||
segmentIdx++;
|
||||
segmentFrameIndex = 0;
|
||||
if (
|
||||
heldFrame &&
|
||||
segmentIdx < segments.length &&
|
||||
heldFrameSec < segments[segmentIdx].startSec - epsilonSec
|
||||
) {
|
||||
heldFrame.close();
|
||||
heldFrame = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (segmentIdx >= segments.length) {
|
||||
frame.close();
|
||||
continue;
|
||||
}
|
||||
|
||||
const currentSegment = segments[segmentIdx];
|
||||
|
||||
// Before current segment (trimmed region or pre-roll).
|
||||
if (frameTimeSec < currentSegment.startSec - epsilonSec) {
|
||||
frame.close();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!heldFrame) {
|
||||
heldFrame = frame;
|
||||
heldFrameSec = frameTimeSec;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Any target timestamp before this midpoint is closer to heldFrame than current frame.
|
||||
const handoffBoundarySec = (heldFrameSec + frameTimeSec) / 2;
|
||||
while (!context.cancelled) {
|
||||
const segmentFrameCount = segmentOutputFrameCounts[segmentIdx];
|
||||
if (segmentFrameIndex >= segmentFrameCount) {
|
||||
break;
|
||||
}
|
||||
|
||||
const segmentDurationSec = currentSegment.endSec - currentSegment.startSec;
|
||||
const sourceTimeSec =
|
||||
currentSegment.startSec +
|
||||
(segmentFrameIndex / segmentFrameCount) * segmentDurationSec;
|
||||
if (sourceTimeSec >= currentSegment.endSec - epsilonSec) {
|
||||
break;
|
||||
}
|
||||
if (sourceTimeSec > handoffBoundarySec) {
|
||||
break;
|
||||
}
|
||||
|
||||
const sourceTimestampMs = sourceTimeSec * 1000;
|
||||
await onFrame(
|
||||
heldFrame,
|
||||
exportFrameIndex * frameDurationUs,
|
||||
sourceTimestampMs,
|
||||
sourceTimestampMs,
|
||||
);
|
||||
segmentFrameIndex++;
|
||||
exportFrameIndex++;
|
||||
}
|
||||
|
||||
heldFrame.close();
|
||||
heldFrame = frame;
|
||||
heldFrameSec = frameTimeSec;
|
||||
}
|
||||
|
||||
// Flush remaining output frames for the last decoded frame.
|
||||
if (!decodeError && heldFrame && segmentIdx < segments.length) {
|
||||
while (!context.cancelled && segmentIdx < segments.length) {
|
||||
const segment = segments[segmentIdx];
|
||||
if (heldFrameSec < segment.startSec - epsilonSec) {
|
||||
break;
|
||||
}
|
||||
|
||||
while (!context.cancelled && (await emitHeldFrameForTarget(segment))) {
|
||||
// Keep emitting output frames for the active segment.
|
||||
}
|
||||
|
||||
segmentIdx++;
|
||||
segmentFrameIndex = 0;
|
||||
if (
|
||||
segmentIdx < segments.length &&
|
||||
heldFrameSec < segments[segmentIdx].startSec - epsilonSec
|
||||
) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
heldFrame.close();
|
||||
heldFrame = null;
|
||||
} else if (heldFrame) {
|
||||
heldFrame.close();
|
||||
heldFrame = null;
|
||||
}
|
||||
|
||||
// Drain leftover decoded frames
|
||||
while (!decodeDone && !decodeError) {
|
||||
const frame = await getNextFrame();
|
||||
if (!frame) break;
|
||||
frame.close();
|
||||
}
|
||||
|
||||
try {
|
||||
reader.cancel();
|
||||
} catch {
|
||||
/* already closed */
|
||||
}
|
||||
await feedPromise;
|
||||
for (const f of pendingFrames) f.close();
|
||||
pendingFrames.length = 0;
|
||||
|
||||
if (context.decoder?.state === "configured") {
|
||||
context.decoder.close();
|
||||
}
|
||||
context.decoder = null;
|
||||
|
||||
if (decodeError) {
|
||||
throw decodeError;
|
||||
}
|
||||
|
||||
const requiredEndSec = segments.length > 0 ? segments[segments.length - 1].endSec : 0;
|
||||
if (
|
||||
!context.cancelled &&
|
||||
lastDecodedFrameSec !== null &&
|
||||
requiredEndSec - lastDecodedFrameSec > 1 &&
|
||||
exportFrameIndex < expectedOutputFrames
|
||||
) {
|
||||
throw new Error(
|
||||
`Video decode ended early at ${lastDecodedFrameSec.toFixed(3)}s (needed ${requiredEndSec.toFixed(3)}s; rendered ${exportFrameIndex}/${expectedOutputFrames} frames).`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -2,120 +2,27 @@ import { WebDemuxer } from "web-demuxer";
|
||||
import type { SpeedRegion, TrimRegion } from "@/components/video-editor/types";
|
||||
import { getEffectiveVideoStreamDurationSeconds } from "@/lib/mediaTiming";
|
||||
import { createFallbackDemuxerSource, resolveMediaResourceUrl } from "./localMediaSource";
|
||||
import { decodeVideoStream } from "./streamingDecodePipeline";
|
||||
import { computeVideoSegments, splitVideoSegmentsBySpeed } from "./videoTimelineSegments";
|
||||
|
||||
const DEFAULT_MAX_DECODE_QUEUE = 12;
|
||||
const DEFAULT_MAX_PENDING_FRAMES = 32;
|
||||
const STARTUP_STABILIZATION_SECONDS = 1.25;
|
||||
const STARTUP_MAX_DECODE_QUEUE = 12;
|
||||
const STARTUP_MAX_PENDING_FRAMES = 28;
|
||||
|
||||
export interface DecodedVideoInfo {
|
||||
width: number;
|
||||
height: number;
|
||||
duration: number; // seconds
|
||||
mediaStartTime?: number; // seconds
|
||||
streamStartTime?: number; // seconds
|
||||
streamDuration?: number; // seconds
|
||||
frameRate: number;
|
||||
codec: string;
|
||||
hasAudio: boolean;
|
||||
audioCodec?: string;
|
||||
audioSampleRate?: number;
|
||||
}
|
||||
import type { DecodedVideoInfo } from "./streamingDecoderSupport";
|
||||
export {
|
||||
buildVideoDecodeFailure,
|
||||
getDecodedFrameStartupOffsetUs,
|
||||
getDecodedFrameTimelineOffsetUs,
|
||||
getVideoDecodeFailureCode,
|
||||
preserveFirstVideoDecodeFailure,
|
||||
type DecodedVideoInfo,
|
||||
type VideoDecodeFailureContext,
|
||||
} from "./streamingDecoderSupport";
|
||||
|
||||
interface StreamingVideoDecoderLoadOptions {
|
||||
useFallbackMediaSource?: boolean;
|
||||
}
|
||||
|
||||
interface VideoDecodeFailureContext {
|
||||
decoderConfig: VideoDecoderConfig;
|
||||
sourceMetadata?: DecodedVideoInfo;
|
||||
chunkIndex?: number;
|
||||
chunk?: EncodedVideoChunk;
|
||||
decoderState?: CodecState;
|
||||
decodeQueueSize?: number;
|
||||
}
|
||||
|
||||
/** Maps WebCodecs failures to stable support-facing identifiers. */
|
||||
export function getVideoDecodeFailureCode(error: unknown): string {
|
||||
const name = error instanceof DOMException ? error.name : "";
|
||||
switch (name) {
|
||||
case "EncodingError":
|
||||
return "VIDEO_DECODE_ENCODING_ERROR";
|
||||
case "NotSupportedError":
|
||||
return "VIDEO_CODEC_UNSUPPORTED";
|
||||
case "QuotaExceededError":
|
||||
return "VIDEO_DECODER_RESOURCE_EXHAUSTED";
|
||||
case "InvalidStateError":
|
||||
return "VIDEO_DECODER_INVALID_STATE";
|
||||
default:
|
||||
return "VIDEO_DECODE_FAILED";
|
||||
}
|
||||
}
|
||||
|
||||
function describeUnknownError(error: unknown): string {
|
||||
if (error instanceof DOMException) {
|
||||
return `${error.name}: ${error.message}`;
|
||||
}
|
||||
|
||||
if (error instanceof Error) {
|
||||
return error.message;
|
||||
}
|
||||
|
||||
return String(error);
|
||||
}
|
||||
|
||||
/** Builds a decode error with codec, source, chunk, and decoder-state context. */
|
||||
export function buildVideoDecodeFailure(error: unknown, context: VideoDecodeFailureContext): Error {
|
||||
const details = [`codec=${context.decoderConfig.codec}`];
|
||||
const failureCode = getVideoDecodeFailureCode(error);
|
||||
const width = context.decoderConfig.codedWidth;
|
||||
const height = context.decoderConfig.codedHeight;
|
||||
if (width && height) {
|
||||
details.push(`codedSize=${width}x${height}`);
|
||||
}
|
||||
if (context.decoderConfig.hardwareAcceleration) {
|
||||
details.push(`hardwareAcceleration=${context.decoderConfig.hardwareAcceleration}`);
|
||||
}
|
||||
if (context.sourceMetadata) {
|
||||
details.push(`sourceFps=${context.sourceMetadata.frameRate}`);
|
||||
details.push(`sourceDurationSec=${context.sourceMetadata.duration}`);
|
||||
}
|
||||
if (context.chunkIndex !== undefined) {
|
||||
details.push(`chunkIndex=${context.chunkIndex}`);
|
||||
}
|
||||
if (context.chunk) {
|
||||
details.push(`chunkType=${context.chunk.type}`);
|
||||
details.push(`chunkTimestampUs=${context.chunk.timestamp}`);
|
||||
details.push(`sourceTimeSec=${(context.chunk.timestamp / 1_000_000).toFixed(3)}`);
|
||||
if (typeof context.chunk.duration === "number") {
|
||||
details.push(`chunkDurationUs=${context.chunk.duration}`);
|
||||
}
|
||||
details.push(`chunkBytes=${context.chunk.byteLength}`);
|
||||
}
|
||||
if (context.decoderState) {
|
||||
details.push(`decoderState=${context.decoderState}`);
|
||||
}
|
||||
if (context.decodeQueueSize !== undefined) {
|
||||
details.push(`decodeQueueSize=${context.decodeQueueSize}`);
|
||||
}
|
||||
|
||||
const failure = new Error(
|
||||
`[${failureCode}] VideoDecoder failure: ${describeUnknownError(error)} (${details.join(", ")})`,
|
||||
);
|
||||
(failure as Error & { cause?: unknown }).cause = error;
|
||||
return failure;
|
||||
}
|
||||
|
||||
/** Keeps the original decoder failure when cleanup triggers secondary errors. */
|
||||
export function preserveFirstVideoDecodeFailure(
|
||||
existingError: Error | null,
|
||||
error: unknown,
|
||||
context: VideoDecodeFailureContext,
|
||||
): Error {
|
||||
return existingError ?? buildVideoDecodeFailure(error, context);
|
||||
}
|
||||
|
||||
/** Decoder retains ownership of the VideoFrame and closes it after use. */
|
||||
type OnFrameCallback = (
|
||||
frame: VideoFrame,
|
||||
@@ -124,32 +31,6 @@ type OnFrameCallback = (
|
||||
cursorTimestampMs: number,
|
||||
) => Promise<void>;
|
||||
|
||||
export function getDecodedFrameStartupOffsetUs(
|
||||
firstDecodedFrameTimestampUs: number,
|
||||
metadata: Pick<DecodedVideoInfo, "mediaStartTime" | "streamStartTime">,
|
||||
): number {
|
||||
const streamStartTimeUs = Math.round(
|
||||
(metadata.streamStartTime ?? metadata.mediaStartTime ?? 0) * 1_000_000,
|
||||
);
|
||||
|
||||
return Math.max(0, firstDecodedFrameTimestampUs - streamStartTimeUs);
|
||||
}
|
||||
|
||||
export function getDecodedFrameTimelineOffsetUs(
|
||||
firstDecodedFrameTimestampUs: number,
|
||||
metadata: Pick<DecodedVideoInfo, "mediaStartTime" | "streamStartTime">,
|
||||
): number {
|
||||
const mediaStartTimeUs = Math.round((metadata.mediaStartTime ?? 0) * 1_000_000);
|
||||
const streamStartTimeUs = Math.round(
|
||||
(metadata.streamStartTime ?? metadata.mediaStartTime ?? 0) * 1_000_000,
|
||||
);
|
||||
|
||||
return (
|
||||
Math.max(0, streamStartTimeUs - mediaStartTimeUs) +
|
||||
getDecodedFrameStartupOffsetUs(firstDecodedFrameTimestampUs, metadata)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Decodes video frames via web-demuxer + VideoDecoder in a single forward pass.
|
||||
* Way faster than seeking an HTMLVideoElement per frame.
|
||||
@@ -291,498 +172,44 @@ export class StreamingVideoDecoder {
|
||||
throw new Error("Must call loadMetadata() before decodeAll()");
|
||||
}
|
||||
|
||||
const decoderConfig = await this.demuxer.getDecoderConfig("video");
|
||||
const codec = this.metadata.codec.toLowerCase();
|
||||
const shouldPreferSoftwareDecode = codec.includes("av01") || codec.includes("av1");
|
||||
const effectiveVideoDuration = getEffectiveVideoStreamDurationSeconds({
|
||||
duration: this.metadata.duration,
|
||||
streamDuration: this.metadata.streamDuration,
|
||||
});
|
||||
const segments = this.splitBySpeed(
|
||||
this.computeSegments(effectiveVideoDuration, trimRegions),
|
||||
const owner = this;
|
||||
await decodeVideoStream(
|
||||
{
|
||||
demuxer: this.demuxer,
|
||||
metadata: this.metadata,
|
||||
pendingFrames: this.pendingFrames,
|
||||
maxDecodeQueue: this.maxDecodeQueue,
|
||||
maxPendingFrames: this.maxPendingFrames,
|
||||
get cancelled() {
|
||||
return owner.cancelled;
|
||||
},
|
||||
get decoder() {
|
||||
return owner.decoder;
|
||||
},
|
||||
set decoder(value) {
|
||||
owner.decoder = value;
|
||||
},
|
||||
},
|
||||
targetFrameRate,
|
||||
trimRegions,
|
||||
speedRegions,
|
||||
onFrame,
|
||||
);
|
||||
const segmentOutputFrameCounts = segments.map((segment) =>
|
||||
Math.ceil(((segment.endSec - segment.startSec) / segment.speed) * targetFrameRate),
|
||||
);
|
||||
const expectedOutputFrames = segmentOutputFrameCounts.reduce(
|
||||
(sum, count) => sum + count,
|
||||
0,
|
||||
);
|
||||
const frameDurationUs = 1_000_000 / targetFrameRate;
|
||||
const epsilonSec = 0.001;
|
||||
const startupStabilizationSeconds = STARTUP_STABILIZATION_SECONDS;
|
||||
const startupFrameBudget = Math.max(
|
||||
1,
|
||||
Math.round(targetFrameRate * startupStabilizationSeconds),
|
||||
);
|
||||
let exportFrameIndex = 0;
|
||||
let loggedSteadyStateBackpressure = false;
|
||||
const backpressureWaiters = new Set<() => void>();
|
||||
|
||||
const notifyBackpressureProgress = () => {
|
||||
if (backpressureWaiters.size === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const waiters = [...backpressureWaiters];
|
||||
backpressureWaiters.clear();
|
||||
for (const resolve of waiters) {
|
||||
resolve();
|
||||
}
|
||||
};
|
||||
|
||||
const waitForBackpressureProgress = () =>
|
||||
new Promise<void>((resolve) => {
|
||||
backpressureWaiters.add(resolve);
|
||||
});
|
||||
|
||||
console.log(
|
||||
`[StreamingVideoDecoder] Startup-safe decode backpressure active for first ${startupStabilizationSeconds}s (${startupFrameBudget} frames)`,
|
||||
);
|
||||
|
||||
// Async frame queue — decoder pushes, consumer pulls
|
||||
this.pendingFrames.length = 0;
|
||||
const pendingFrames = this.pendingFrames;
|
||||
let frameResolve: ((frame: VideoFrame | null) => void) | null = null;
|
||||
let decodeError: Error | null = null;
|
||||
let decodeDone = false;
|
||||
let firstDecodedFrameTimestampUs: number | null = null;
|
||||
let decodedFrameTimelineOffsetUs = 0;
|
||||
let submittedChunkCount = 0;
|
||||
let lastSubmittedChunk: EncodedVideoChunk | undefined;
|
||||
let lastSubmittedChunkIndex: number | undefined;
|
||||
const preferredDecoderConfig = shouldPreferSoftwareDecode
|
||||
? {
|
||||
...decoderConfig,
|
||||
hardwareAcceleration: "prefer-software" as const,
|
||||
}
|
||||
: decoderConfig;
|
||||
let activeDecoderConfig = preferredDecoderConfig;
|
||||
const getDecoderFailureContext = (): VideoDecodeFailureContext => ({
|
||||
decoderConfig: activeDecoderConfig,
|
||||
sourceMetadata: this.metadata ?? undefined,
|
||||
chunkIndex: lastSubmittedChunkIndex,
|
||||
chunk: lastSubmittedChunk,
|
||||
decoderState: this.decoder?.state,
|
||||
decodeQueueSize: this.decoder?.decodeQueueSize,
|
||||
});
|
||||
const recordFirstDecodeError = (error: unknown) => {
|
||||
decodeError = preserveFirstVideoDecodeFailure(
|
||||
decodeError,
|
||||
error,
|
||||
getDecoderFailureContext(),
|
||||
);
|
||||
};
|
||||
|
||||
this.decoder = new VideoDecoder({
|
||||
output: (frame: VideoFrame) => {
|
||||
if (frameResolve) {
|
||||
const resolve = frameResolve;
|
||||
frameResolve = null;
|
||||
resolve(frame);
|
||||
} else {
|
||||
pendingFrames.push(frame);
|
||||
}
|
||||
notifyBackpressureProgress();
|
||||
},
|
||||
error: (e: DOMException) => {
|
||||
recordFirstDecodeError(e);
|
||||
if (frameResolve) {
|
||||
const resolve = frameResolve;
|
||||
frameResolve = null;
|
||||
resolve(null);
|
||||
}
|
||||
notifyBackpressureProgress();
|
||||
},
|
||||
});
|
||||
try {
|
||||
this.decoder.configure(preferredDecoderConfig);
|
||||
} catch (error) {
|
||||
if (!shouldPreferSoftwareDecode) {
|
||||
throw buildVideoDecodeFailure(error, getDecoderFailureContext());
|
||||
}
|
||||
// Fall back to default decoder config if software preference is unsupported.
|
||||
activeDecoderConfig = decoderConfig;
|
||||
try {
|
||||
this.decoder.configure(decoderConfig);
|
||||
} catch (fallbackError) {
|
||||
throw buildVideoDecodeFailure(fallbackError, getDecoderFailureContext());
|
||||
}
|
||||
}
|
||||
|
||||
const getNextFrame = (): Promise<VideoFrame | null> => {
|
||||
if (decodeError) return Promise.resolve(null);
|
||||
if (pendingFrames.length > 0) {
|
||||
const frame = pendingFrames.shift()!;
|
||||
notifyBackpressureProgress();
|
||||
return Promise.resolve(frame);
|
||||
}
|
||||
if (decodeDone) return Promise.resolve(null);
|
||||
return new Promise((resolve) => {
|
||||
frameResolve = resolve;
|
||||
});
|
||||
};
|
||||
|
||||
// One forward stream through the whole file.
|
||||
// Pass explicit range because some containers are truncated when no end is provided.
|
||||
const readEndSec =
|
||||
Math.max(
|
||||
this.metadata.duration + (this.metadata.mediaStartTime ?? 0),
|
||||
(this.metadata.streamDuration ?? this.metadata.duration) +
|
||||
(this.metadata.streamStartTime ?? this.metadata.mediaStartTime ?? 0),
|
||||
) + 0.5;
|
||||
const reader = this.demuxer.read("video", 0, readEndSec).getReader();
|
||||
|
||||
// Feed chunks to decoder in background with backpressure
|
||||
const feedPromise = (async () => {
|
||||
try {
|
||||
while (!this.cancelled && !decodeError) {
|
||||
const { done, value: chunk } = await reader.read();
|
||||
if (done || !chunk) break;
|
||||
|
||||
if (!loggedSteadyStateBackpressure && exportFrameIndex >= startupFrameBudget) {
|
||||
loggedSteadyStateBackpressure = true;
|
||||
console.log(
|
||||
"[StreamingVideoDecoder] Switched to steady-state decode backpressure",
|
||||
);
|
||||
}
|
||||
|
||||
const decodeQueueLimit =
|
||||
exportFrameIndex < startupFrameBudget
|
||||
? Math.min(this.maxDecodeQueue, STARTUP_MAX_DECODE_QUEUE)
|
||||
: this.maxDecodeQueue;
|
||||
const pendingFrameLimit =
|
||||
exportFrameIndex < startupFrameBudget
|
||||
? Math.min(this.maxPendingFrames, STARTUP_MAX_PENDING_FRAMES)
|
||||
: this.maxPendingFrames;
|
||||
|
||||
// Backpressure on both decode queue and decoded frame backlog.
|
||||
while (
|
||||
!decodeError &&
|
||||
this.decoder!.state === "configured" &&
|
||||
(this.decoder!.decodeQueueSize > decodeQueueLimit ||
|
||||
pendingFrames.length > pendingFrameLimit) &&
|
||||
!this.cancelled
|
||||
) {
|
||||
await waitForBackpressureProgress();
|
||||
}
|
||||
if (this.cancelled || decodeError) break;
|
||||
if (this.decoder!.state !== "configured") {
|
||||
recordFirstDecodeError(
|
||||
new DOMException(
|
||||
"Decoder closed before the next video chunk was submitted.",
|
||||
"InvalidStateError",
|
||||
),
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
lastSubmittedChunk = chunk;
|
||||
lastSubmittedChunkIndex = submittedChunkCount;
|
||||
this.decoder!.decode(chunk);
|
||||
submittedChunkCount++;
|
||||
}
|
||||
|
||||
if (!this.cancelled && this.decoder!.state === "configured") {
|
||||
await this.decoder!.flush();
|
||||
}
|
||||
} catch (e) {
|
||||
recordFirstDecodeError(e);
|
||||
} finally {
|
||||
decodeDone = true;
|
||||
if (frameResolve) {
|
||||
const resolve = frameResolve;
|
||||
frameResolve = null;
|
||||
resolve(null);
|
||||
}
|
||||
notifyBackpressureProgress();
|
||||
}
|
||||
})();
|
||||
|
||||
// Route decoded frames into segments by timestamp, then deliver with VFR→CFR resampling
|
||||
let segmentIdx = 0;
|
||||
let segmentFrameIndex = 0;
|
||||
let lastDecodedFrameSec: number | null = null;
|
||||
let heldFrame: VideoFrame | null = null;
|
||||
let heldFrameSec = 0;
|
||||
|
||||
const emitHeldFrameForTarget = async (segment: {
|
||||
startSec: number;
|
||||
endSec: number;
|
||||
speed: number;
|
||||
}) => {
|
||||
if (!heldFrame) return false;
|
||||
const segmentFrameCount = segmentOutputFrameCounts[segmentIdx];
|
||||
if (segmentFrameIndex >= segmentFrameCount) return false;
|
||||
|
||||
const segmentDurationSec = segment.endSec - segment.startSec;
|
||||
const sourceTimeSec =
|
||||
segment.startSec + (segmentFrameIndex / segmentFrameCount) * segmentDurationSec;
|
||||
if (sourceTimeSec >= segment.endSec - epsilonSec) return false;
|
||||
|
||||
const sourceTimestampMs = sourceTimeSec * 1000;
|
||||
await onFrame(
|
||||
heldFrame,
|
||||
exportFrameIndex * frameDurationUs,
|
||||
sourceTimestampMs,
|
||||
sourceTimestampMs,
|
||||
);
|
||||
segmentFrameIndex++;
|
||||
exportFrameIndex++;
|
||||
return true;
|
||||
};
|
||||
|
||||
while (!this.cancelled && segmentIdx < segments.length) {
|
||||
const frame = await getNextFrame();
|
||||
if (!frame) break;
|
||||
|
||||
if (firstDecodedFrameTimestampUs === null) {
|
||||
firstDecodedFrameTimestampUs = frame.timestamp;
|
||||
decodedFrameTimelineOffsetUs = getDecodedFrameTimelineOffsetUs(
|
||||
firstDecodedFrameTimestampUs,
|
||||
this.metadata,
|
||||
);
|
||||
}
|
||||
|
||||
const normalizedFrameTimeSec = Math.max(
|
||||
0,
|
||||
(frame.timestamp - firstDecodedFrameTimestampUs + decodedFrameTimelineOffsetUs) /
|
||||
1_000_000,
|
||||
);
|
||||
const frameTimeSec: number =
|
||||
lastDecodedFrameSec === null
|
||||
? normalizedFrameTimeSec
|
||||
: Math.max(lastDecodedFrameSec, normalizedFrameTimeSec);
|
||||
lastDecodedFrameSec = frameTimeSec;
|
||||
|
||||
// Finalize completed segments before handling this frame.
|
||||
while (
|
||||
segmentIdx < segments.length &&
|
||||
frameTimeSec >= segments[segmentIdx].endSec - epsilonSec
|
||||
) {
|
||||
const segment = segments[segmentIdx];
|
||||
while (!this.cancelled && (await emitHeldFrameForTarget(segment))) {
|
||||
// Keep emitting remaining output frames for this segment from the last known frame.
|
||||
}
|
||||
|
||||
segmentIdx++;
|
||||
segmentFrameIndex = 0;
|
||||
if (
|
||||
heldFrame &&
|
||||
segmentIdx < segments.length &&
|
||||
heldFrameSec < segments[segmentIdx].startSec - epsilonSec
|
||||
) {
|
||||
heldFrame.close();
|
||||
heldFrame = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (segmentIdx >= segments.length) {
|
||||
frame.close();
|
||||
continue;
|
||||
}
|
||||
|
||||
const currentSegment = segments[segmentIdx];
|
||||
|
||||
// Before current segment (trimmed region or pre-roll).
|
||||
if (frameTimeSec < currentSegment.startSec - epsilonSec) {
|
||||
frame.close();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!heldFrame) {
|
||||
heldFrame = frame;
|
||||
heldFrameSec = frameTimeSec;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Any target timestamp before this midpoint is closer to heldFrame than current frame.
|
||||
const handoffBoundarySec = (heldFrameSec + frameTimeSec) / 2;
|
||||
while (!this.cancelled) {
|
||||
const segmentFrameCount = segmentOutputFrameCounts[segmentIdx];
|
||||
if (segmentFrameIndex >= segmentFrameCount) {
|
||||
break;
|
||||
}
|
||||
|
||||
const segmentDurationSec = currentSegment.endSec - currentSegment.startSec;
|
||||
const sourceTimeSec =
|
||||
currentSegment.startSec +
|
||||
(segmentFrameIndex / segmentFrameCount) * segmentDurationSec;
|
||||
if (sourceTimeSec >= currentSegment.endSec - epsilonSec) {
|
||||
break;
|
||||
}
|
||||
if (sourceTimeSec > handoffBoundarySec) {
|
||||
break;
|
||||
}
|
||||
|
||||
const sourceTimestampMs = sourceTimeSec * 1000;
|
||||
await onFrame(
|
||||
heldFrame,
|
||||
exportFrameIndex * frameDurationUs,
|
||||
sourceTimestampMs,
|
||||
sourceTimestampMs,
|
||||
);
|
||||
segmentFrameIndex++;
|
||||
exportFrameIndex++;
|
||||
}
|
||||
|
||||
heldFrame.close();
|
||||
heldFrame = frame;
|
||||
heldFrameSec = frameTimeSec;
|
||||
}
|
||||
|
||||
// Flush remaining output frames for the last decoded frame.
|
||||
if (!decodeError && heldFrame && segmentIdx < segments.length) {
|
||||
while (!this.cancelled && segmentIdx < segments.length) {
|
||||
const segment = segments[segmentIdx];
|
||||
if (heldFrameSec < segment.startSec - epsilonSec) {
|
||||
break;
|
||||
}
|
||||
|
||||
while (!this.cancelled && (await emitHeldFrameForTarget(segment))) {
|
||||
// Keep emitting output frames for the active segment.
|
||||
}
|
||||
|
||||
segmentIdx++;
|
||||
segmentFrameIndex = 0;
|
||||
if (
|
||||
segmentIdx < segments.length &&
|
||||
heldFrameSec < segments[segmentIdx].startSec - epsilonSec
|
||||
) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
heldFrame.close();
|
||||
heldFrame = null;
|
||||
} else if (heldFrame) {
|
||||
heldFrame.close();
|
||||
heldFrame = null;
|
||||
}
|
||||
|
||||
// Drain leftover decoded frames
|
||||
while (!decodeDone && !decodeError) {
|
||||
const frame = await getNextFrame();
|
||||
if (!frame) break;
|
||||
frame.close();
|
||||
}
|
||||
|
||||
try {
|
||||
reader.cancel();
|
||||
} catch {
|
||||
/* already closed */
|
||||
}
|
||||
await feedPromise;
|
||||
for (const f of pendingFrames) f.close();
|
||||
pendingFrames.length = 0;
|
||||
|
||||
if (this.decoder?.state === "configured") {
|
||||
this.decoder.close();
|
||||
}
|
||||
this.decoder = null;
|
||||
|
||||
if (decodeError) {
|
||||
throw decodeError;
|
||||
}
|
||||
|
||||
const requiredEndSec = segments.length > 0 ? segments[segments.length - 1].endSec : 0;
|
||||
if (
|
||||
!this.cancelled &&
|
||||
lastDecodedFrameSec !== null &&
|
||||
requiredEndSec - lastDecodedFrameSec > 1 &&
|
||||
exportFrameIndex < expectedOutputFrames
|
||||
) {
|
||||
throw new Error(
|
||||
`Video decode ended early at ${lastDecodedFrameSec.toFixed(3)}s (needed ${requiredEndSec.toFixed(3)}s; rendered ${exportFrameIndex}/${expectedOutputFrames} frames).`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private computeSegments(
|
||||
totalDuration: number,
|
||||
trimRegions?: TrimRegion[],
|
||||
): Array<{ startSec: number; endSec: number }> {
|
||||
if (!trimRegions || trimRegions.length === 0) {
|
||||
return [{ startSec: 0, endSec: totalDuration }];
|
||||
}
|
||||
|
||||
const sorted = [...trimRegions].sort((a, b) => a.startMs - b.startMs);
|
||||
const segments: Array<{ startSec: number; endSec: number }> = [];
|
||||
let cursor = 0;
|
||||
|
||||
for (const trim of sorted) {
|
||||
const trimStart = trim.startMs / 1000;
|
||||
const trimEnd = trim.endMs / 1000;
|
||||
if (cursor < trimStart) {
|
||||
segments.push({ startSec: cursor, endSec: trimStart });
|
||||
}
|
||||
cursor = Math.max(cursor, trimEnd);
|
||||
}
|
||||
|
||||
if (cursor < totalDuration) {
|
||||
segments.push({ startSec: cursor, endSec: totalDuration });
|
||||
}
|
||||
|
||||
return segments;
|
||||
}
|
||||
|
||||
getEffectiveDuration(trimRegions?: TrimRegion[], speedRegions?: SpeedRegion[]): number {
|
||||
if (!this.metadata) throw new Error("Must call loadMetadata() first");
|
||||
const trimSegments = this.computeSegments(
|
||||
const trimSegments = computeVideoSegments(
|
||||
getEffectiveVideoStreamDurationSeconds({
|
||||
duration: this.metadata.duration,
|
||||
streamDuration: this.metadata.streamDuration,
|
||||
}),
|
||||
trimRegions,
|
||||
);
|
||||
const speedSegments = this.splitBySpeed(trimSegments, speedRegions);
|
||||
const speedSegments = splitVideoSegmentsBySpeed(trimSegments, speedRegions);
|
||||
return speedSegments.reduce((sum, seg) => sum + (seg.endSec - seg.startSec) / seg.speed, 0);
|
||||
}
|
||||
|
||||
private splitBySpeed(
|
||||
segments: Array<{ startSec: number; endSec: number }>,
|
||||
speedRegions?: SpeedRegion[],
|
||||
): Array<{ startSec: number; endSec: number; speed: number }> {
|
||||
if (!speedRegions || speedRegions.length === 0)
|
||||
return segments.map((s) => ({ ...s, speed: 1 }));
|
||||
|
||||
const result: Array<{ startSec: number; endSec: number; speed: number }> = [];
|
||||
for (const segment of segments) {
|
||||
const overlapping = speedRegions
|
||||
.filter(
|
||||
(sr) =>
|
||||
sr.startMs / 1000 < segment.endSec && sr.endMs / 1000 > segment.startSec,
|
||||
)
|
||||
.sort((a, b) => a.startMs - b.startMs);
|
||||
|
||||
if (overlapping.length === 0) {
|
||||
result.push({ ...segment, speed: 1 });
|
||||
continue;
|
||||
}
|
||||
|
||||
let cursor = segment.startSec;
|
||||
for (const sr of overlapping) {
|
||||
const srStart = Math.max(sr.startMs / 1000, segment.startSec);
|
||||
const srEnd = Math.min(sr.endMs / 1000, segment.endSec);
|
||||
if (cursor < srStart) {
|
||||
result.push({ startSec: cursor, endSec: srStart, speed: 1 });
|
||||
}
|
||||
const effectiveStart = Math.max(cursor, srStart);
|
||||
if (srEnd > effectiveStart) {
|
||||
result.push({
|
||||
startSec: effectiveStart,
|
||||
endSec: srEnd,
|
||||
speed: sr.speed,
|
||||
});
|
||||
}
|
||||
cursor = Math.max(cursor, srEnd);
|
||||
}
|
||||
if (cursor < segment.endSec)
|
||||
result.push({ startSec: cursor, endSec: segment.endSec, speed: 1 });
|
||||
}
|
||||
return result.filter((s) => s.endSec - s.startSec > 0.0001);
|
||||
}
|
||||
|
||||
cancel(): void {
|
||||
this.cancelled = true;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
export interface DecodedVideoInfo {
|
||||
width: number;
|
||||
height: number;
|
||||
duration: number; // seconds
|
||||
mediaStartTime?: number; // seconds
|
||||
streamStartTime?: number; // seconds
|
||||
streamDuration?: number; // seconds
|
||||
frameRate: number;
|
||||
codec: string;
|
||||
hasAudio: boolean;
|
||||
audioCodec?: string;
|
||||
audioSampleRate?: number;
|
||||
}
|
||||
|
||||
export interface VideoDecodeFailureContext {
|
||||
decoderConfig: VideoDecoderConfig;
|
||||
sourceMetadata?: DecodedVideoInfo;
|
||||
chunkIndex?: number;
|
||||
chunk?: EncodedVideoChunk;
|
||||
decoderState?: CodecState;
|
||||
decodeQueueSize?: number;
|
||||
}
|
||||
|
||||
/** Maps WebCodecs failures to stable support-facing identifiers. */
|
||||
export function getVideoDecodeFailureCode(error: unknown): string {
|
||||
const name = error instanceof DOMException ? error.name : "";
|
||||
switch (name) {
|
||||
case "EncodingError":
|
||||
return "VIDEO_DECODE_ENCODING_ERROR";
|
||||
case "NotSupportedError":
|
||||
return "VIDEO_CODEC_UNSUPPORTED";
|
||||
case "QuotaExceededError":
|
||||
return "VIDEO_DECODER_RESOURCE_EXHAUSTED";
|
||||
case "InvalidStateError":
|
||||
return "VIDEO_DECODER_INVALID_STATE";
|
||||
default:
|
||||
return "VIDEO_DECODE_FAILED";
|
||||
}
|
||||
}
|
||||
|
||||
function describeUnknownError(error: unknown): string {
|
||||
if (error instanceof DOMException) {
|
||||
return `${error.name}: ${error.message}`;
|
||||
}
|
||||
|
||||
if (error instanceof Error) {
|
||||
return error.message;
|
||||
}
|
||||
|
||||
return String(error);
|
||||
}
|
||||
|
||||
/** Builds a decode error with codec, source, chunk, and decoder-state context. */
|
||||
export function buildVideoDecodeFailure(error: unknown, context: VideoDecodeFailureContext): Error {
|
||||
const details = [`codec=${context.decoderConfig.codec}`];
|
||||
const failureCode = getVideoDecodeFailureCode(error);
|
||||
const width = context.decoderConfig.codedWidth;
|
||||
const height = context.decoderConfig.codedHeight;
|
||||
if (width && height) {
|
||||
details.push(`codedSize=${width}x${height}`);
|
||||
}
|
||||
if (context.decoderConfig.hardwareAcceleration) {
|
||||
details.push(`hardwareAcceleration=${context.decoderConfig.hardwareAcceleration}`);
|
||||
}
|
||||
if (context.sourceMetadata) {
|
||||
details.push(`sourceFps=${context.sourceMetadata.frameRate}`);
|
||||
details.push(`sourceDurationSec=${context.sourceMetadata.duration}`);
|
||||
}
|
||||
if (context.chunkIndex !== undefined) {
|
||||
details.push(`chunkIndex=${context.chunkIndex}`);
|
||||
}
|
||||
if (context.chunk) {
|
||||
details.push(`chunkType=${context.chunk.type}`);
|
||||
details.push(`chunkTimestampUs=${context.chunk.timestamp}`);
|
||||
details.push(`sourceTimeSec=${(context.chunk.timestamp / 1_000_000).toFixed(3)}`);
|
||||
if (typeof context.chunk.duration === "number") {
|
||||
details.push(`chunkDurationUs=${context.chunk.duration}`);
|
||||
}
|
||||
details.push(`chunkBytes=${context.chunk.byteLength}`);
|
||||
}
|
||||
if (context.decoderState) {
|
||||
details.push(`decoderState=${context.decoderState}`);
|
||||
}
|
||||
if (context.decodeQueueSize !== undefined) {
|
||||
details.push(`decodeQueueSize=${context.decodeQueueSize}`);
|
||||
}
|
||||
|
||||
const failure = new Error(
|
||||
`[${failureCode}] VideoDecoder failure: ${describeUnknownError(error)} (${details.join(", ")})`,
|
||||
);
|
||||
(failure as Error & { cause?: unknown }).cause = error;
|
||||
return failure;
|
||||
}
|
||||
|
||||
/** Keeps the original decoder failure when cleanup triggers secondary errors. */
|
||||
export function preserveFirstVideoDecodeFailure(
|
||||
existingError: Error | null,
|
||||
error: unknown,
|
||||
context: VideoDecodeFailureContext,
|
||||
): Error {
|
||||
return existingError ?? buildVideoDecodeFailure(error, context);
|
||||
}
|
||||
|
||||
export function getDecodedFrameStartupOffsetUs(
|
||||
firstDecodedFrameTimestampUs: number,
|
||||
metadata: Pick<DecodedVideoInfo, "mediaStartTime" | "streamStartTime">,
|
||||
): number {
|
||||
const streamStartTimeUs = Math.round(
|
||||
(metadata.streamStartTime ?? metadata.mediaStartTime ?? 0) * 1_000_000,
|
||||
);
|
||||
|
||||
return Math.max(0, firstDecodedFrameTimestampUs - streamStartTimeUs);
|
||||
}
|
||||
|
||||
export function getDecodedFrameTimelineOffsetUs(
|
||||
firstDecodedFrameTimestampUs: number,
|
||||
metadata: Pick<DecodedVideoInfo, "mediaStartTime" | "streamStartTime">,
|
||||
): number {
|
||||
const mediaStartTimeUs = Math.round((metadata.mediaStartTime ?? 0) * 1_000_000);
|
||||
const streamStartTimeUs = Math.round(
|
||||
(metadata.streamStartTime ?? metadata.mediaStartTime ?? 0) * 1_000_000,
|
||||
);
|
||||
|
||||
return (
|
||||
Math.max(0, streamStartTimeUs - mediaStartTimeUs) +
|
||||
getDecodedFrameStartupOffsetUs(firstDecodedFrameTimestampUs, metadata)
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
import type { SpeedRegion, TrimRegion } from "@/components/video-editor/types";
|
||||
|
||||
export function computeVideoSegments(
|
||||
totalDuration: number,
|
||||
trimRegions?: TrimRegion[],
|
||||
): Array<{ startSec: number; endSec: number }> {
|
||||
if (!trimRegions || trimRegions.length === 0) {
|
||||
return [{ startSec: 0, endSec: totalDuration }];
|
||||
}
|
||||
|
||||
const sorted = [...trimRegions].sort((a, b) => a.startMs - b.startMs);
|
||||
const segments: Array<{ startSec: number; endSec: number }> = [];
|
||||
let cursor = 0;
|
||||
|
||||
for (const trim of sorted) {
|
||||
const trimStart = trim.startMs / 1000;
|
||||
const trimEnd = trim.endMs / 1000;
|
||||
if (cursor < trimStart) {
|
||||
segments.push({ startSec: cursor, endSec: trimStart });
|
||||
}
|
||||
cursor = Math.max(cursor, trimEnd);
|
||||
}
|
||||
|
||||
if (cursor < totalDuration) {
|
||||
segments.push({ startSec: cursor, endSec: totalDuration });
|
||||
}
|
||||
|
||||
return segments;
|
||||
}
|
||||
|
||||
export function splitVideoSegmentsBySpeed(
|
||||
segments: Array<{ startSec: number; endSec: number }>,
|
||||
speedRegions?: SpeedRegion[],
|
||||
): Array<{ startSec: number; endSec: number; speed: number }> {
|
||||
if (!speedRegions || speedRegions.length === 0)
|
||||
return segments.map((s) => ({ ...s, speed: 1 }));
|
||||
|
||||
const result: Array<{ startSec: number; endSec: number; speed: number }> = [];
|
||||
for (const segment of segments) {
|
||||
const overlapping = speedRegions
|
||||
.filter(
|
||||
(sr) => sr.startMs / 1000 < segment.endSec && sr.endMs / 1000 > segment.startSec,
|
||||
)
|
||||
.sort((a, b) => a.startMs - b.startMs);
|
||||
|
||||
if (overlapping.length === 0) {
|
||||
result.push({ ...segment, speed: 1 });
|
||||
continue;
|
||||
}
|
||||
|
||||
let cursor = segment.startSec;
|
||||
for (const sr of overlapping) {
|
||||
const srStart = Math.max(sr.startMs / 1000, segment.startSec);
|
||||
const srEnd = Math.min(sr.endMs / 1000, segment.endSec);
|
||||
if (cursor < srStart) {
|
||||
result.push({ startSec: cursor, endSec: srStart, speed: 1 });
|
||||
}
|
||||
const effectiveStart = Math.max(cursor, srStart);
|
||||
if (srEnd > effectiveStart) {
|
||||
result.push({
|
||||
startSec: effectiveStart,
|
||||
endSec: srEnd,
|
||||
speed: sr.speed,
|
||||
});
|
||||
}
|
||||
cursor = Math.max(cursor, srEnd);
|
||||
}
|
||||
if (cursor < segment.endSec)
|
||||
result.push({ startSec: cursor, endSec: segment.endSec, speed: 1 });
|
||||
}
|
||||
return result.filter((s) => s.endSec - s.startSec > 0.0001);
|
||||
}
|
||||
Reference in New Issue
Block a user