diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx index 29b0f25c..cf7005e4 100644 --- a/src/components/video-editor/timeline/TimelineEditor.tsx +++ b/src/components/video-editor/timeline/TimelineEditor.tsx @@ -229,7 +229,9 @@ const TimelineEditor = forwardRef( return { previewSpans, hiddenZoomIds }; }, [clipRegions, liveSpanPreviewById, zoomRegions]); const { shortcuts: keyShortcuts, isMac } = useShortcuts(); - const audioPeaks = useTimelineAudioPeaks(videoPath); + const audioPeaks = useTimelineAudioPeaks(videoPath, { + enableSourceSidecarFallback: true, + }); useEffect(() => { if (aspectRatio === "native") { diff --git a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx index d6d13ed1..c7d65898 100644 --- a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx +++ b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx @@ -33,6 +33,7 @@ import { import TimelineAxis from "../axis/TimelineAxis"; import ClipMarkerOverlay from "../overlays/ClipMarkerOverlay"; import PlaybackCursor from "../playhead/PlaybackCursor"; +import { useTimelineAudioPeaks } from "../../hooks/useTimelineAudioPeaks"; const HINT_CLIP = "Press C to split clip"; const HINT_ANNOTATION = "Press A to add annotation"; @@ -234,6 +235,38 @@ interface TimelineCanvasRowsProps { onZoomRowClick: MouseEventHandler; } +interface AudioItemWithWaveformProps { + item: TimelineRenderItem; + span: { start: number; end: number }; + waveformSpan: { start: number; end: number }; + isSelected: boolean; + onSelectAudio?: (id: string | null) => void; +} + +function AudioItemWithWaveform({ + item, + span, + waveformSpan, + isSelected, + onSelectAudio, +}: AudioItemWithWaveformProps) { + const peaks = useTimelineAudioPeaks(item.audioPath ?? null); + return ( + + {item.label} + + ); +} + const TimelineCanvasRows = memo(function TimelineCanvasRows({ items, videoDurationMs, @@ -422,17 +455,14 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({ {audioRows.map(({ rowId, items: rowItems }, index) => ( {rowItems.map((item) => ( - - {item.label} - + onSelectAudio={onSelectAudio} + /> ))} ))} diff --git a/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx b/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx index 367ed1ed..eff26d09 100644 --- a/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx +++ b/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx @@ -61,8 +61,13 @@ function AudioWaveformComponent({ const { peaks: peakData, durationMs } = peaks; if (durationMs <= 0 || peakData.length === 0) return; - const visibleStartMs = segmentStartMs ?? range.start; - const visibleEndMs = segmentEndMs ?? range.end; + const rawVisibleStartMs = segmentStartMs ?? range.start; + const rawVisibleEndMs = segmentEndMs ?? range.end; + const msPerBin = durationMs / peakData.length; + const visibleStartMs = + msPerBin > 0 ? Math.round(rawVisibleStartMs / msPerBin) * msPerBin : rawVisibleStartMs; + const visibleEndMs = + msPerBin > 0 ? Math.round(rawVisibleEndMs / msPerBin) * msPerBin : rawVisibleEndMs; const visibleDurationMs = visibleEndMs - visibleStartMs; if (visibleDurationMs <= 0) return; @@ -71,11 +76,14 @@ function AudioWaveformComponent({ ctx.beginPath(); for (let px = 0; px < width; px++) { const t = visibleStartMs + (px / width) * visibleDurationMs; - const binIndex = Math.min( - peakData.length - 1, - Math.max(0, Math.floor((t / durationMs) * peakData.length)), + const exactIndex = Math.max( + 0, + Math.min(peakData.length - 1, (t / durationMs) * (peakData.length - 1)), ); - const amplitude = peakData[binIndex]; + const leftIndex = Math.floor(exactIndex); + const rightIndex = Math.min(peakData.length - 1, leftIndex + 1); + const mix = exactIndex - leftIndex; + const amplitude = peakData[leftIndex] * (1 - mix) + peakData[rightIndex] * mix; const barHeight = amplitude * midY * 0.85; ctx.moveTo(px, midY - barHeight); diff --git a/src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts b/src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts new file mode 100644 index 00000000..7ff7f26c --- /dev/null +++ b/src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts @@ -0,0 +1,88 @@ +import WorkerConstructor from "./waveform.worker?worker"; +import type { AudioPeaksData } from "../../core/timelineTypes"; + +const DEFAULT_PEAK_COUNT = 2048; + +export class WaveformGenerator { + private audioContext: AudioContext; + private worker: Worker; + private peaksCache = new Map(); + private pending = new Map>(); + + constructor() { + this.audioContext = new (window.AudioContext || (window as typeof window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext)(); + this.worker = new WorkerConstructor(); + } + + private computePeaksWithWorker(channelData: Float32Array, samples: number): Promise { + return new Promise((resolve, reject) => { + const onMessage = (event: MessageEvent) => { + this.worker.removeEventListener("message", onMessage); + this.worker.removeEventListener("error", onError); + resolve(event.data); + }; + const onError = (error: ErrorEvent) => { + this.worker.removeEventListener("message", onMessage); + this.worker.removeEventListener("error", onError); + reject(error.error ?? new Error(error.message)); + }; + + this.worker.addEventListener("message", onMessage); + this.worker.addEventListener("error", onError); + this.worker.postMessage( + { + channelData, + samples, + }, + [channelData.buffer], + ); + }); + } + + public async generate(url: string, peakCount = DEFAULT_PEAK_COUNT): Promise { + const cacheKey = `${url}::${peakCount}`; + const cached = this.peaksCache.get(cacheKey); + if (cached) return cached; + + const inflight = this.pending.get(cacheKey); + if (inflight) return inflight; + + const request = (async () => { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`Failed to load media: ${response.status}`); + } + + const arrayBuffer = await response.arrayBuffer(); + const decoded = await this.audioContext.decodeAudioData(arrayBuffer); + const channelData = decoded.getChannelData(0).slice(); + const peaks = await this.computePeaksWithWorker(channelData, peakCount); + + let max = 0; + for (let i = 0; i < peaks.length; i++) { + if (peaks[i] > max) max = peaks[i]; + } + if (max > 0) { + for (let i = 0; i < peaks.length; i++) { + peaks[i] /= max; + } + } + + const result: AudioPeaksData = { + peaks, + durationMs: decoded.duration * 1000, + }; + this.peaksCache.set(cacheKey, result); + this.pending.delete(cacheKey); + return result; + })().catch((error) => { + this.pending.delete(cacheKey); + throw error; + }); + + this.pending.set(cacheKey, request); + return request; + } +} + +export const waveformGenerator = new WaveformGenerator(); diff --git a/src/components/video-editor/timeline/components/waveform/waveform.worker.ts b/src/components/video-editor/timeline/components/waveform/waveform.worker.ts new file mode 100644 index 00000000..c8a7f8c5 --- /dev/null +++ b/src/components/video-editor/timeline/components/waveform/waveform.worker.ts @@ -0,0 +1,31 @@ +self.onmessage = (e: MessageEvent) => { + const { channelData, samples } = e.data as { + channelData: Float32Array; + samples: number; + }; + + if (!channelData || samples <= 0) { + self.postMessage(new Float32Array(0)); + return; + } + + try { + const step = Math.max(1, Math.floor(channelData.length / samples)); + const result = new Float32Array(samples); + + for (let i = 0; i < samples; i++) { + const start = i * step; + const end = Math.min(start + step, channelData.length); + let max = 0; + for (let j = start; j < end; j++) { + const val = Math.abs(channelData[j]); + if (val > max) max = val; + } + result[i] = max; + } + + self.postMessage(result); + } catch { + self.postMessage(new Float32Array(0)); + } +}; diff --git a/src/components/video-editor/timeline/core/timelineTypes.ts b/src/components/video-editor/timeline/core/timelineTypes.ts index 2222f7cc..4a58c969 100644 --- a/src/components/video-editor/timeline/core/timelineTypes.ts +++ b/src/components/video-editor/timeline/core/timelineTypes.ts @@ -32,6 +32,7 @@ export interface TimelineRenderItem { rowId: string; span: Span; label: string; + audioPath?: string; zoomDepth?: number; zoomMode?: ZoomMode; speedValue?: number; diff --git a/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts b/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts index 9af79a80..4ca2a8f4 100644 --- a/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts +++ b/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts @@ -1,10 +1,9 @@ import { useEffect, useRef, useState } from "react"; -import { fromFileUrl, resolveVideoUrl } from "../../projectPersistence"; +import { resolveMediaResourceUrl } from "@/lib/exporter/localMediaSource"; +import { fromFileUrl } from "../../projectPersistence"; +import { waveformGenerator } from "../components/waveform/WaveformGenerator"; import type { AudioPeaksData } from "../core/timelineTypes"; -/** Number of peak bins to produce — enough for smooth display at any zoom. */ -const TARGET_PEAK_COUNT = 2048; - function buildSidecarAudioCandidates(sourcePath: string): string[] { const normalized = sourcePath.replace(/\\/g, "/"); const lastSlash = normalized.lastIndexOf("/"); @@ -29,193 +28,69 @@ function extractLocalPathFromMediaServerUrl(input: string): string | null { (url.hostname === "127.0.0.1" || url.hostname === "localhost") && url.pathname === "/video"; if (!isLocalMediaServer) return null; - const rawPath = url.searchParams.get("path"); - if (!rawPath) return null; - return rawPath; + return url.searchParams.get("path"); } catch { return null; } } -async function decodePeaksFromMediaUrl( - mediaUrl: string, - cancelled: () => boolean, -): Promise { - const response = await fetch(mediaUrl); - if (!response.ok) { - throw new Error(`Failed to load media for waveform: ${response.status}`); - } - if (cancelled()) throw new Error("cancelled"); - console.debug("[timeline-audio-peaks] fetch ok", { - url: mediaUrl, - status: response.status, - }); - - const arrayBuffer = await response.arrayBuffer(); - if (cancelled()) throw new Error("cancelled"); - console.debug("[timeline-audio-peaks] bytes loaded", { - url: mediaUrl, - byteLength: arrayBuffer.byteLength, - }); - - const audioCtx = new OfflineAudioContext(1, 1, 44100); - const decoded = await audioCtx.decodeAudioData(arrayBuffer); - if (cancelled()) throw new Error("cancelled"); - console.debug("[timeline-audio-peaks] decode ok", { - url: mediaUrl, - durationSec: decoded.duration, - channelCount: decoded.numberOfChannels, - sampleRate: decoded.sampleRate, - sampleFrames: decoded.length, - }); - - const channelData = decoded.getChannelData(0); - const durationMs = decoded.duration * 1000; - const binSize = Math.max(1, Math.floor(channelData.length / TARGET_PEAK_COUNT)); - const peakCount = Math.ceil(channelData.length / binSize); - const peaks = new Float32Array(peakCount); - - for (let i = 0; i < peakCount; i++) { - const start = i * binSize; - const end = Math.min(start + binSize, channelData.length); - let max = 0; - for (let j = start; j < end; j++) { - const abs = Math.abs(channelData[j]); - if (abs > max) max = abs; - } - peaks[i] = max; - } - - let globalMax = 0; - for (let i = 0; i < peaks.length; i++) { - if (peaks[i] > globalMax) globalMax = peaks[i]; - } - if (globalMax > 0) { - for (let i = 0; i < peaks.length; i++) { - peaks[i] /= globalMax; - } - } - - return { peaks, durationMs }; +interface TimelineAudioPeaksOptions { + enableSourceSidecarFallback?: boolean; } -/** - * Decode audio from a media file URL and produce a fixed-length array of peak - * amplitudes suitable for waveform visualisation. - * - * Returns `null` while loading or if the file has no decodeable audio. - */ -export function useTimelineAudioPeaks(fileUrl: string | null | undefined): AudioPeaksData | null { +export function useTimelineAudioPeaks( + mediaResource: string | null | undefined, + options: TimelineAudioPeaksOptions = {}, +): AudioPeaksData | null { const [data, setData] = useState(null); - const urlRef = useRef(fileUrl); + const sourceRef = useRef(mediaResource); + const enableSourceSidecarFallback = options.enableSourceSidecarFallback ?? false; useEffect(() => { - urlRef.current = fileUrl; + sourceRef.current = mediaResource; setData(null); - console.debug("[timeline-audio-peaks] reset", { input: fileUrl ?? null }); - - if (!fileUrl) { - console.debug("[timeline-audio-peaks] no input source, skipping"); - return; - } + if (!mediaResource) return; let cancelled = false; - (async () => { + const run = async () => { + const localPathFromServer = extractLocalPathFromMediaServerUrl(mediaResource); + const localSourcePath = + localPathFromServer || + (/^file:\/\//i.test(mediaResource) ? fromFileUrl(mediaResource) : mediaResource); + + const tryGenerate = async (resource: string): Promise => { + const resolvedUrl = await resolveMediaResourceUrl(resource); + return waveformGenerator.generate(resolvedUrl); + }; + try { - const extractedLocalPath = extractLocalPathFromMediaServerUrl(fileUrl); - const isRemoteLike = /^(https?:|blob:|data:)/i.test(fileUrl) && !extractedLocalPath; - const isFileUrl = /^file:\/\//i.test(fileUrl); - const sourceLocalPath = isRemoteLike - ? null - : extractedLocalPath - ? extractedLocalPath - : isFileUrl - ? fromFileUrl(fileUrl) - : fileUrl; - console.debug("[timeline-audio-peaks] resolving source", { - input: fileUrl, - isRemoteLike, - isFileUrl, - sourceLocalPath, - extractedLocalPath, - }); - const mediaUrl = isRemoteLike - ? fileUrl - : isFileUrl - ? await resolveVideoUrl(fromFileUrl(fileUrl)) - : await resolveVideoUrl(fileUrl); - if (cancelled) return; - console.debug("[timeline-audio-peaks] resolved source", { - input: fileUrl, - resolved: mediaUrl, - }); - - let decodedPeaks: AudioPeaksData | null = null; - try { - decodedPeaks = await decodePeaksFromMediaUrl(mediaUrl, () => cancelled); - } catch (primaryError) { - if (primaryError instanceof Error && primaryError.message === "cancelled") { - return; - } - console.warn("[timeline-audio-peaks] primary decode failed", { - input: fileUrl, - url: mediaUrl, - error: primaryError, - }); - if (sourceLocalPath) { - const candidates = buildSidecarAudioCandidates(sourceLocalPath); - console.debug("[timeline-audio-peaks] trying sidecar candidates", { - sourceLocalPath, - candidates, - }); - for (const candidatePath of candidates) { - try { - const candidateUrl = await resolveVideoUrl(candidatePath); - console.debug("[timeline-audio-peaks] trying sidecar", { - candidatePath, - candidateUrl, - }); - decodedPeaks = await decodePeaksFromMediaUrl(candidateUrl, () => cancelled); - console.info("[timeline-audio-peaks] sidecar decode succeeded", { - candidatePath, - candidateUrl, - }); - break; - } catch (sidecarError) { - if (sidecarError instanceof Error && sidecarError.message === "cancelled") { - return; - } - console.debug("[timeline-audio-peaks] sidecar decode failed", { - candidatePath, - error: sidecarError, - }); - } - } - } - } - - if (!cancelled && urlRef.current === fileUrl && decodedPeaks) { - console.debug("[timeline-audio-peaks] peaks generated", { - input: fileUrl, - durationMs: Math.round(decodedPeaks.durationMs), - peakCount: decodedPeaks.peaks.length, - }); - setData(decodedPeaks); - } - } catch (error) { - // File has no audio or decoding failed — leave as null. - console.warn("[timeline-audio-peaks] failed", { - input: fileUrl, - error, - }); + const result = await tryGenerate(mediaResource); + if (!cancelled && sourceRef.current === mediaResource) setData(result); + return; + } catch { + // fallthrough } - })(); + + if (!enableSourceSidecarFallback || !localSourcePath) return; + + for (const candidate of buildSidecarAudioCandidates(localSourcePath)) { + try { + const result = await tryGenerate(candidate); + if (!cancelled && sourceRef.current === mediaResource) setData(result); + return; + } catch { + // try next + } + } + }; + + void run(); return () => { cancelled = true; }; - }, [fileUrl]); + }, [mediaResource, enableSourceSidecarFallback]); return data; } diff --git a/src/components/video-editor/timeline/model/timelineModel.ts b/src/components/video-editor/timeline/model/timelineModel.ts index 0981b85c..b36aabc9 100644 --- a/src/components/video-editor/timeline/model/timelineModel.ts +++ b/src/components/video-editor/timeline/model/timelineModel.ts @@ -68,6 +68,7 @@ export function buildTimelineItems(params: { rowId: getAudioTrackRowId(region.trackIndex ?? 0), span: { start: region.startMs, end: region.endMs }, label: getAudioLabel(region), + audioPath: region.audioPath, variant: "audio", }));