lee/Recordly
1
0
Fork 0
mirror of https://github.com/webadderallorg/Recordly.git synced 2026-09-28 08:45:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

add: TimelineEditor waveformgenerator as a worker. to be extra efficient

Browse Source
This commit is contained in:
Alan Trebugeais
2026-05-09 13:35:41 +02:00
parent eff3bbf2f7
commit 29e10b9ec1
8 changed files with 222 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/components/video-editor/timeline/TimelineEditor.tsx
+3 -1
View File
@@ -229,7 +229,9 @@ const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
return { previewSpans, hiddenZoomIds };
}, [clipRegions, liveSpanPreviewById, zoomRegions]);
const { shortcuts: keyShortcuts, isMac } = useShortcuts();
const audioPeaks = useTimelineAudioPeaks(videoPath);
const audioPeaks = useTimelineAudioPeaks(videoPath, {
enableSourceSidecarFallback: true,
});
useEffect(() => {
if (aspectRatio === "native") {
src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx
+38 -8
View File
@@ -33,6 +33,7 @@ import {
import TimelineAxis from "../axis/TimelineAxis";
import ClipMarkerOverlay from "../overlays/ClipMarkerOverlay";
import PlaybackCursor from "../playhead/PlaybackCursor";
import { useTimelineAudioPeaks } from "../../hooks/useTimelineAudioPeaks";
const HINT_CLIP = "Press C to split clip";
const HINT_ANNOTATION = "Press A to add annotation";
@@ -234,6 +235,38 @@ interface TimelineCanvasRowsProps {
onZoomRowClick: MouseEventHandler<HTMLDivElement>;
}
interface AudioItemWithWaveformProps {
item: TimelineRenderItem;
span: { start: number; end: number };
waveformSpan: { start: number; end: number };
isSelected: boolean;
onSelectAudio?: (id: string | null) => void;
}
function AudioItemWithWaveform({
item,
span,
waveformSpan,
isSelected,
onSelectAudio,
}: AudioItemWithWaveformProps) {
const peaks = useTimelineAudioPeaks(item.audioPath ?? null);
return (
<Item
id={item.id}
rowId={item.rowId}
span={span}
isSelected={isSelected}
onSelectId={onSelectAudio}
variant="audio"
waveformPeaks={peaks}
waveformSegmentSpan={waveformSpan}
>
{item.label}
</Item>
);
}
const TimelineCanvasRows = memo(function TimelineCanvasRows({
items,
videoDurationMs,
@@ -422,17 +455,14 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({
{audioRows.map(({ rowId, items: rowItems }, index) => (
<Row key={rowId} id={rowId} isEmpty={rowItems.length === 0} hint={index === 0 ? HINT_AUDIO : undefined}>
{rowItems.map((item) => (
<Item
id={item.id}
<AudioItemWithWaveform
key={item.id}
rowId={item.rowId}
item={item}
span={item.span}
waveformSpan={liveSpanPreviewById?.[item.id] ?? item.span}
isSelected={selectAllBlocksActive || item.id === selectedAudioId}
onSelectId={onSelectAudio}
variant="audio"
>
{item.label}
</Item>
onSelectAudio={onSelectAudio}
/>
))}
</Row>
))}
src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx
+14 -6
View File
@@ -61,8 +61,13 @@ function AudioWaveformComponent({
const { peaks: peakData, durationMs } = peaks;
if (durationMs <= 0 || peakData.length === 0) return;
const visibleStartMs = segmentStartMs ?? range.start;
const visibleEndMs = segmentEndMs ?? range.end;
const rawVisibleStartMs = segmentStartMs ?? range.start;
const rawVisibleEndMs = segmentEndMs ?? range.end;
const msPerBin = durationMs / peakData.length;
const visibleStartMs =
msPerBin > 0 ? Math.round(rawVisibleStartMs / msPerBin) * msPerBin : rawVisibleStartMs;
const visibleEndMs =
msPerBin > 0 ? Math.round(rawVisibleEndMs / msPerBin) * msPerBin : rawVisibleEndMs;
const visibleDurationMs = visibleEndMs - visibleStartMs;
if (visibleDurationMs <= 0) return;
@@ -71,11 +76,14 @@ function AudioWaveformComponent({
ctx.beginPath();
for (let px = 0; px < width; px++) {
const t = visibleStartMs + (px / width) * visibleDurationMs;
const binIndex = Math.min(
peakData.length - 1,
Math.max(0, Math.floor((t / durationMs) * peakData.length)),
const exactIndex = Math.max(
0,
Math.min(peakData.length - 1, (t / durationMs) * (peakData.length - 1)),
);
const amplitude = peakData[binIndex];
const leftIndex = Math.floor(exactIndex);
const rightIndex = Math.min(peakData.length - 1, leftIndex + 1);
const mix = exactIndex - leftIndex;
const amplitude = peakData[leftIndex] * (1 - mix) + peakData[rightIndex] * mix;
const barHeight = amplitude * midY * 0.85;
ctx.moveTo(px, midY - barHeight);
src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts
+88
View File
@@ -0,0 +1,88 @@
import WorkerConstructor from "./waveform.worker?worker";
import type { AudioPeaksData } from "../../core/timelineTypes";
const DEFAULT_PEAK_COUNT = 2048;
export class WaveformGenerator {
private audioContext: AudioContext;
private worker: Worker;
private peaksCache = new Map<string, AudioPeaksData>();
private pending = new Map<string, Promise<AudioPeaksData>>();
constructor() {
this.audioContext = new (window.AudioContext || (window as typeof window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext)();
this.worker = new WorkerConstructor();
}
private computePeaksWithWorker(channelData: Float32Array, samples: number): Promise<Float32Array> {
return new Promise((resolve, reject) => {
const onMessage = (event: MessageEvent<Float32Array>) => {
this.worker.removeEventListener("message", onMessage);
this.worker.removeEventListener("error", onError);
resolve(event.data);
};
const onError = (error: ErrorEvent) => {
this.worker.removeEventListener("message", onMessage);
this.worker.removeEventListener("error", onError);
reject(error.error ?? new Error(error.message));
};
this.worker.addEventListener("message", onMessage);
this.worker.addEventListener("error", onError);
this.worker.postMessage(
{
channelData,
samples,
},
[channelData.buffer],
);
});
}
public async generate(url: string, peakCount = DEFAULT_PEAK_COUNT): Promise<AudioPeaksData> {
const cacheKey = `${url}::${peakCount}`;
const cached = this.peaksCache.get(cacheKey);
if (cached) return cached;
const inflight = this.pending.get(cacheKey);
if (inflight) return inflight;
const request = (async () => {
const response = await fetch(url);
if (!response.ok) {
throw new Error(`Failed to load media: ${response.status}`);
}
const arrayBuffer = await response.arrayBuffer();
const decoded = await this.audioContext.decodeAudioData(arrayBuffer);
const channelData = decoded.getChannelData(0).slice();
const peaks = await this.computePeaksWithWorker(channelData, peakCount);
let max = 0;
for (let i = 0; i < peaks.length; i++) {
if (peaks[i] > max) max = peaks[i];
}
if (max > 0) {
for (let i = 0; i < peaks.length; i++) {
peaks[i] /= max;
}
}
const result: AudioPeaksData = {
peaks,
durationMs: decoded.duration * 1000,
};
this.peaksCache.set(cacheKey, result);
this.pending.delete(cacheKey);
return result;
})().catch((error) => {
this.pending.delete(cacheKey);
throw error;
});
this.pending.set(cacheKey, request);
return request;
}
}
export const waveformGenerator = new WaveformGenerator();
src/components/video-editor/timeline/components/waveform/waveform.worker.ts
+31
View File
@@ -0,0 +1,31 @@
self.onmessage = (e: MessageEvent) => {
const { channelData, samples } = e.data as {
channelData: Float32Array;
samples: number;
};
if (!channelData || samples <= 0) {
self.postMessage(new Float32Array(0));
return;
}
try {
const step = Math.max(1, Math.floor(channelData.length / samples));
const result = new Float32Array(samples);
for (let i = 0; i < samples; i++) {
const start = i * step;
const end = Math.min(start + step, channelData.length);
let max = 0;
for (let j = start; j < end; j++) {
const val = Math.abs(channelData[j]);
if (val > max) max = val;
}
result[i] = max;
}
self.postMessage(result);
} catch {
self.postMessage(new Float32Array(0));
}
};
src/components/video-editor/timeline/core/timelineTypes.ts
+1
View File
@@ -32,6 +32,7 @@ export interface TimelineRenderItem {
rowId: string;
span: Span;
label: string;
audioPath?: string;
zoomDepth?: number;
zoomMode?: ZoomMode;
speedValue?: number;
src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts
+46 -171
View File
@@ -1,10 +1,9 @@
import { useEffect, useRef, useState } from "react";
import { fromFileUrl, resolveVideoUrl } from "../../projectPersistence";
import { resolveMediaResourceUrl } from "@/lib/exporter/localMediaSource";
import { fromFileUrl } from "../../projectPersistence";
import { waveformGenerator } from "../components/waveform/WaveformGenerator";
import type { AudioPeaksData } from "../core/timelineTypes";
/** Number of peak bins to produce — enough for smooth display at any zoom. */
const TARGET_PEAK_COUNT = 2048;
function buildSidecarAudioCandidates(sourcePath: string): string[] {
const normalized = sourcePath.replace(/\\/g, "/");
const lastSlash = normalized.lastIndexOf("/");
@@ -29,193 +28,69 @@ function extractLocalPathFromMediaServerUrl(input: string): string | null {
(url.hostname === "127.0.0.1" || url.hostname === "localhost") &&
url.pathname === "/video";
if (!isLocalMediaServer) return null;
const rawPath = url.searchParams.get("path");
if (!rawPath) return null;
return rawPath;
return url.searchParams.get("path");
} catch {
return null;
}
}
async function decodePeaksFromMediaUrl(
mediaUrl: string,
cancelled: () => boolean,
): Promise<AudioPeaksData> {
const response = await fetch(mediaUrl);
if (!response.ok) {
throw new Error(`Failed to load media for waveform: ${response.status}`);
}
if (cancelled()) throw new Error("cancelled");
console.debug("[timeline-audio-peaks] fetch ok", {
url: mediaUrl,
status: response.status,
});
const arrayBuffer = await response.arrayBuffer();
if (cancelled()) throw new Error("cancelled");
console.debug("[timeline-audio-peaks] bytes loaded", {
url: mediaUrl,
byteLength: arrayBuffer.byteLength,
});
const audioCtx = new OfflineAudioContext(1, 1, 44100);
const decoded = await audioCtx.decodeAudioData(arrayBuffer);
if (cancelled()) throw new Error("cancelled");
console.debug("[timeline-audio-peaks] decode ok", {
url: mediaUrl,
durationSec: decoded.duration,
channelCount: decoded.numberOfChannels,
sampleRate: decoded.sampleRate,
sampleFrames: decoded.length,
});
const channelData = decoded.getChannelData(0);
const durationMs = decoded.duration * 1000;
const binSize = Math.max(1, Math.floor(channelData.length / TARGET_PEAK_COUNT));
const peakCount = Math.ceil(channelData.length / binSize);
const peaks = new Float32Array(peakCount);
for (let i = 0; i < peakCount; i++) {
const start = i * binSize;
const end = Math.min(start + binSize, channelData.length);
let max = 0;
for (let j = start; j < end; j++) {
const abs = Math.abs(channelData[j]);
if (abs > max) max = abs;
}
peaks[i] = max;
}
let globalMax = 0;
for (let i = 0; i < peaks.length; i++) {
if (peaks[i] > globalMax) globalMax = peaks[i];
}
if (globalMax > 0) {
for (let i = 0; i < peaks.length; i++) {
peaks[i] /= globalMax;
}
}
return { peaks, durationMs };
interface TimelineAudioPeaksOptions {
enableSourceSidecarFallback?: boolean;
}
/**
* Decode audio from a media file URL and produce a fixed-length array of peak
* amplitudes suitable for waveform visualisation.
*
* Returns `null` while loading or if the file has no decodeable audio.
*/
export function useTimelineAudioPeaks(fileUrl: string | null | undefined): AudioPeaksData | null {
export function useTimelineAudioPeaks(
mediaResource: string | null | undefined,
options: TimelineAudioPeaksOptions = {},
): AudioPeaksData | null {
const [data, setData] = useState<AudioPeaksData | null>(null);
const urlRef = useRef(fileUrl);
const sourceRef = useRef(mediaResource);
const enableSourceSidecarFallback = options.enableSourceSidecarFallback ?? false;
useEffect(() => {
urlRef.current = fileUrl;
sourceRef.current = mediaResource;
setData(null);
console.debug("[timeline-audio-peaks] reset", { input: fileUrl ?? null });
if (!fileUrl) {
console.debug("[timeline-audio-peaks] no input source, skipping");
return;
}
if (!mediaResource) return;
let cancelled = false;
(async () => {
const run = async () => {
const localPathFromServer = extractLocalPathFromMediaServerUrl(mediaResource);
const localSourcePath =
localPathFromServer ||
(/^file:\/\//i.test(mediaResource) ? fromFileUrl(mediaResource) : mediaResource);
const tryGenerate = async (resource: string): Promise<AudioPeaksData> => {
const resolvedUrl = await resolveMediaResourceUrl(resource);
return waveformGenerator.generate(resolvedUrl);
};
try {
const extractedLocalPath = extractLocalPathFromMediaServerUrl(fileUrl);
const isRemoteLike = /^(https?:|blob:|data:)/i.test(fileUrl) && !extractedLocalPath;
const isFileUrl = /^file:\/\//i.test(fileUrl);
const sourceLocalPath = isRemoteLike
? null
: extractedLocalPath
? extractedLocalPath
: isFileUrl
? fromFileUrl(fileUrl)
: fileUrl;
console.debug("[timeline-audio-peaks] resolving source", {
input: fileUrl,
isRemoteLike,
isFileUrl,
sourceLocalPath,
extractedLocalPath,
});
const mediaUrl = isRemoteLike
? fileUrl
: isFileUrl
? await resolveVideoUrl(fromFileUrl(fileUrl))
: await resolveVideoUrl(fileUrl);
if (cancelled) return;
console.debug("[timeline-audio-peaks] resolved source", {
input: fileUrl,
resolved: mediaUrl,
});
let decodedPeaks: AudioPeaksData | null = null;
try {
decodedPeaks = await decodePeaksFromMediaUrl(mediaUrl, () => cancelled);
} catch (primaryError) {
if (primaryError instanceof Error && primaryError.message === "cancelled") {
return;
}
console.warn("[timeline-audio-peaks] primary decode failed", {
input: fileUrl,
url: mediaUrl,
error: primaryError,
});
if (sourceLocalPath) {
const candidates = buildSidecarAudioCandidates(sourceLocalPath);
console.debug("[timeline-audio-peaks] trying sidecar candidates", {
sourceLocalPath,
candidates,
});
for (const candidatePath of candidates) {
try {
const candidateUrl = await resolveVideoUrl(candidatePath);
console.debug("[timeline-audio-peaks] trying sidecar", {
candidatePath,
candidateUrl,
});
decodedPeaks = await decodePeaksFromMediaUrl(candidateUrl, () => cancelled);
console.info("[timeline-audio-peaks] sidecar decode succeeded", {
candidatePath,
candidateUrl,
});
break;
} catch (sidecarError) {
if (sidecarError instanceof Error && sidecarError.message === "cancelled") {
return;
}
console.debug("[timeline-audio-peaks] sidecar decode failed", {
candidatePath,
error: sidecarError,
});
}
}
}
}
if (!cancelled && urlRef.current === fileUrl && decodedPeaks) {
console.debug("[timeline-audio-peaks] peaks generated", {
input: fileUrl,
durationMs: Math.round(decodedPeaks.durationMs),
peakCount: decodedPeaks.peaks.length,
});
setData(decodedPeaks);
}
} catch (error) {
// File has no audio or decoding failed — leave as null.
console.warn("[timeline-audio-peaks] failed", {
input: fileUrl,
error,
});
const result = await tryGenerate(mediaResource);
if (!cancelled && sourceRef.current === mediaResource) setData(result);
return;
} catch {
// fallthrough
}
})();
if (!enableSourceSidecarFallback || !localSourcePath) return;
for (const candidate of buildSidecarAudioCandidates(localSourcePath)) {
try {
const result = await tryGenerate(candidate);
if (!cancelled && sourceRef.current === mediaResource) setData(result);
return;
} catch {
// try next
}
}
};
void run();
return () => {
cancelled = true;
};
}, [fileUrl]);
}, [mediaResource, enableSourceSidecarFallback]);
return data;
}
src/components/video-editor/timeline/model/timelineModel.ts
+1
View File
@@ -68,6 +68,7 @@ export function buildTimelineItems(params: {
rowId: getAudioTrackRowId(region.trackIndex ?? 0),
span: { start: region.startMs, end: region.endMs },
label: getAudioLabel(region),
audioPath: region.audioPath,
variant: "audio",
}));