diff --git a/src/components/video-editor/timeline/Item.tsx b/src/components/video-editor/timeline/Item.tsx index 333b8855..33703a82 100644 --- a/src/components/video-editor/timeline/Item.tsx +++ b/src/components/video-editor/timeline/Item.tsx @@ -11,6 +11,8 @@ import type { Span } from "dnd-timeline"; import { useItem } from "dnd-timeline"; import { useMemo } from "react"; import { cn } from "@/lib/utils"; +import AudioWaveform from "./components/waveform/AudioWaveform"; +import type { AudioPeaksData } from "./core/timelineTypes"; import glassStyles from "./ItemGlass.module.css"; interface ItemProps { @@ -24,6 +26,8 @@ interface ItemProps { zoomDepth?: number; zoomMode?: "auto" | "manual"; speedValue?: number; + waveformPeaks?: AudioPeaksData | null; + waveformSegmentSpan?: Span; variant?: "zoom" | "trim" | "clip" | "annotation" | "speed" | "audio"; } @@ -57,6 +61,8 @@ export default function Item({ zoomDepth = 1, zoomMode = "auto", speedValue, + waveformPeaks = null, + waveformSegmentSpan, variant = "zoom", children, }: ItemProps) { @@ -71,6 +77,7 @@ export default function Item({ const isClip = variant === "clip"; const isSpeed = variant === "speed"; const isAudio = variant === "audio"; + const showAudioWaveform = isAudio && Boolean(waveformPeaks); const glassClass = isZoom ? glassStyles.glassPurple @@ -146,6 +153,14 @@ export default function Item({ style={{ cursor: "col-resize", pointerEvents: "auto" }} title="Resize right" /> + {showAudioWaveform && waveformPeaks && ( + + )} {/* Content */}
diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx index 818caa42..29b0f25c 100644 --- a/src/components/video-editor/timeline/TimelineEditor.tsx +++ b/src/components/video-editor/timeline/TimelineEditor.tsx @@ -178,6 +178,56 @@ const TimelineEditor = forwardRef( pan: "Shift + Ctrl + Scroll", zoom: "Ctrl + Scroll", }); + const [liveSpanPreviewById, setLiveSpanPreviewById] = useState>({}); + const liveZoomPreview = useMemo(() => { + const previewSpans: Record = { ...liveSpanPreviewById }; + const hiddenZoomIds = new Set(); + + for (const [previewId, previewSpan] of Object.entries(liveSpanPreviewById)) { + const oldClip = clipRegions.find((clip) => clip.id === previewId); + if (!oldClip) continue; + + const newStart = Math.round(previewSpan.start); + const newEnd = Math.round(previewSpan.end); + const removedSegments = [ + ...(newStart > oldClip.startMs + ? [{ startMs: oldClip.startMs, endMs: newStart }] + : []), + ...(newEnd < oldClip.endMs + ? [{ startMs: newEnd, endMs: oldClip.endMs }] + : []), + ]; + + const startDelta = newStart - oldClip.startMs; + const endDelta = newEnd - oldClip.endMs; + const isMove = Math.abs(startDelta - endDelta) < 1 && Math.abs(startDelta) > 0; + + if (isMove) { + const delta = startDelta; + for (const zoom of zoomRegions) { + const overlaps = + zoom.startMs < oldClip.endMs && zoom.endMs > oldClip.startMs; + if (!overlaps) continue; + previewSpans[zoom.id] = { + start: zoom.startMs + delta, + end: zoom.endMs + delta, + }; + } + } + + if (removedSegments.length > 0) { + for (const zoom of zoomRegions) { + const removed = removedSegments.some( + (segment) => + zoom.startMs < segment.endMs && zoom.endMs > segment.startMs, + ); + if (removed) hiddenZoomIds.add(zoom.id); + } + } + } + + return { previewSpans, hiddenZoomIds }; + }, [clipRegions, liveSpanPreviewById, zoomRegions]); const { shortcuts: keyShortcuts, isMac } = useShortcuts(); const audioPeaks = useTimelineAudioPeaks(videoPath); @@ -376,6 +426,25 @@ const TimelineEditor = forwardRef( onItemSpanChange={handleItemSpanChange} resolveTargetRowId={getResolvedDropRowId} allRegionSpans={allRegionSpans} + onLiveSpanPreviewChange={(id, span) => { + setLiveSpanPreviewById((prev) => { + if (!span) { + if (!(id in prev)) return prev; + const next = { ...prev }; + delete next[id]; + return next; + } + const current = prev[id]; + if ( + current && + current.start === span.start && + current.end === span.end + ) { + return prev; + } + return { ...prev, [id]: span }; + }); + }} > ( onClearBlockSelection={clearSelectedBlocks} keyframes={keyframes} audioPeaks={audioPeaks} + liveSpanPreviewById={liveZoomPreview.previewSpans} + liveHiddenItemIds={Array.from(liveZoomPreview.hiddenZoomIds)} />
diff --git a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx index 48e29b6f..d6d13ed1 100644 --- a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx +++ b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx @@ -17,7 +17,6 @@ import { getTimelineViewportStretchFactor, TIMELINE_AXIS_HEIGHT_PX, } from "../../timelineLayout"; -import AudioWaveform from "../waveform/AudioWaveform"; import glassStyles from "../../ItemGlass.module.css"; import Item from "../../Item"; import Row from "../../Row"; @@ -38,6 +37,7 @@ import PlaybackCursor from "../playhead/PlaybackCursor"; const HINT_CLIP = "Press C to split clip"; const HINT_ANNOTATION = "Press A to add annotation"; const HINT_AUDIO = "Click music icon to add audio"; +const SOURCE_AUDIO_ROW_ID = "row-source-audio"; interface TimelineCanvasProps { items: TimelineRenderItem[]; @@ -58,6 +58,8 @@ interface TimelineCanvasProps { onClearBlockSelection?: () => void; keyframes?: { id: string; time: number }[]; audioPeaks?: AudioPeaksData | null; + liveSpanPreviewById?: Record; + liveHiddenItemIds?: string[]; } interface TimelineHoverParams { @@ -219,6 +221,8 @@ interface TimelineCanvasRowsProps { onSelectAnnotation?: (id: string | null) => void; onSelectAudio?: (id: string | null) => void; audioPeaks?: AudioPeaksData | null; + liveSpanPreviewById?: Record; + liveHiddenItemIds?: string[]; direction: string; canShowGhostZoom: boolean; ghostStartMs: number | null; @@ -243,6 +247,8 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({ onSelectAnnotation, onSelectAudio, audioPeaks, + liveSpanPreviewById, + liveHiddenItemIds, direction, canShowGhostZoom, ghostStartMs, @@ -253,6 +259,7 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({ onZoomRowMouseLeave, onZoomRowClick, }: TimelineCanvasRowsProps) { + const hiddenIds = useMemo(() => new Set(liveHiddenItemIds ?? []), [liveHiddenItemIds]); const { clipItems, zoomItems, annotationRows, audioRows } = useMemo(() => { const nextClipItems: TimelineRenderItem[] = []; const nextZoomItems: TimelineRenderItem[] = []; @@ -307,7 +314,6 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({ return ( <> - {audioPeaks && } {clipItems.map((item) => ( ))} + {audioPeaks && ( + + {clipItems.map((item) => ( + onSelectClip?.(item.id)} + variant="audio" + waveformPeaks={audioPeaks} + waveformSegmentSpan={liveSpanPreviewById?.[item.id] ?? item.span} + > + Source + + ))} + + )}
)} - {zoomItems.map((item) => ( + {zoomItems + .filter((item) => !hiddenIds.has(item.id)) + .map((item) => ( (null); const { range } = useTimelineContext(); const [resizeKey, setResizeKey] = useState(0); @@ -53,8 +61,8 @@ function AudioWaveformComponent({ peaks }: AudioWaveformProps) { const { peaks: peakData, durationMs } = peaks; if (durationMs <= 0 || peakData.length === 0) return; - const visibleStartMs = range.start; - const visibleEndMs = range.end; + const visibleStartMs = segmentStartMs ?? range.start; + const visibleEndMs = segmentEndMs ?? range.end; const visibleDurationMs = visibleEndMs - visibleStartMs; if (visibleDurationMs <= 0) return; @@ -77,12 +85,12 @@ function AudioWaveformComponent({ peaks }: AudioWaveformProps) { ctx.strokeStyle = "rgba(255, 255, 255, 0.55)"; ctx.lineWidth = dpr; ctx.stroke(); - }, [peaks, range.start, range.end, resizeKey]); + }, [peaks, range.start, range.end, resizeKey, segmentStartMs, segmentEndMs]); return ( ); diff --git a/src/components/video-editor/timeline/components/wrapper/TimelineWrapper.tsx b/src/components/video-editor/timeline/components/wrapper/TimelineWrapper.tsx index 5e777e70..917b171a 100644 --- a/src/components/video-editor/timeline/components/wrapper/TimelineWrapper.tsx +++ b/src/components/video-editor/timeline/components/wrapper/TimelineWrapper.tsx @@ -29,6 +29,7 @@ interface TimelineWrapperProps { onItemSpanChange: (id: string, span: Span, rowId?: string) => void; resolveTargetRowId?: (id: string, proposedRowId: string) => string; allRegionSpans?: TimelineRegionSpan[]; + onLiveSpanPreviewChange?: (id: string, span: Span | null) => void; } export default function TimelineWrapper({ @@ -43,6 +44,7 @@ export default function TimelineWrapper({ onItemSpanChange, resolveTargetRowId, allRegionSpans = [], + onLiveSpanPreviewChange, }: TimelineWrapperProps) { const totalMs = Math.max(0, Math.round(videoDuration * 1000)); @@ -132,8 +134,9 @@ export default function TimelineWrapper({ (event: DragStartEvent) => { const span = event.active.data.current.getSpanFromDragEvent?.(event); if (span) showTooltip(span); + onLiveSpanPreviewChange?.(event.active.id as string, span ?? null); }, - [showTooltip], + [onLiveSpanPreviewChange, showTooltip], ); const onDragMove = useCallback( @@ -144,8 +147,9 @@ export default function TimelineWrapper({ ? (event.activatorEvent as PointerEvent).clientX + (event.delta?.x ?? 0) : undefined; if (span) showTooltip(span, screenX); + onLiveSpanPreviewChange?.(event.active.id as string, span ?? null); }, - [showTooltip], + [onLiveSpanPreviewChange, showTooltip], ); const onResizeMove = useCallback( @@ -156,8 +160,9 @@ export default function TimelineWrapper({ ? (event.activatorEvent as PointerEvent).clientX + (event.delta?.x ?? 0) : undefined; if (span) showTooltip(span, screenX); + onLiveSpanPreviewChange?.(event.active.id as string, span ?? null); }, - [showTooltip], + [onLiveSpanPreviewChange, showTooltip], ); const hideTooltip = useCallback(() => showTooltip(null), [showTooltip]); @@ -166,16 +171,18 @@ export default function TimelineWrapper({ (event: ResizeEndEvent) => { hideTooltip(); onResizeEnd(event); + onLiveSpanPreviewChange?.(event.active.id as string, null); }, - [hideTooltip, onResizeEnd], + [hideTooltip, onLiveSpanPreviewChange, onResizeEnd], ); const onDragEndWithTooltip = useCallback( (event: DragEndEvent) => { hideTooltip(); onDragEnd(event); + onLiveSpanPreviewChange?.(event.active.id as string, null); }, - [hideTooltip, onDragEnd], + [hideTooltip, onDragEnd, onLiveSpanPreviewChange], ); const handleRangeChange = useCallback( diff --git a/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts b/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts index ed377663..9af79a80 100644 --- a/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts +++ b/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts @@ -1,9 +1,104 @@ import { useEffect, useRef, useState } from "react"; +import { fromFileUrl, resolveVideoUrl } from "../../projectPersistence"; import type { AudioPeaksData } from "../core/timelineTypes"; /** Number of peak bins to produce — enough for smooth display at any zoom. */ const TARGET_PEAK_COUNT = 2048; +function buildSidecarAudioCandidates(sourcePath: string): string[] { + const normalized = sourcePath.replace(/\\/g, "/"); + const lastSlash = normalized.lastIndexOf("/"); + const dir = lastSlash >= 0 ? normalized.slice(0, lastSlash + 1) : ""; + const fileName = lastSlash >= 0 ? normalized.slice(lastSlash + 1) : normalized; + const dotIndex = fileName.lastIndexOf("."); + const baseName = dotIndex > 0 ? fileName.slice(0, dotIndex) : fileName; + + return [ + `${dir}${baseName}.system.wav`, + `${dir}${baseName}.mic.wav`, + `${dir}${baseName}.system.m4a`, + `${dir}${baseName}.mic.m4a`, + ]; +} + +function extractLocalPathFromMediaServerUrl(input: string): string | null { + try { + const url = new URL(input); + const isLocalMediaServer = + (url.protocol === "http:" || url.protocol === "https:") && + (url.hostname === "127.0.0.1" || url.hostname === "localhost") && + url.pathname === "/video"; + if (!isLocalMediaServer) return null; + const rawPath = url.searchParams.get("path"); + if (!rawPath) return null; + return rawPath; + } catch { + return null; + } +} + +async function decodePeaksFromMediaUrl( + mediaUrl: string, + cancelled: () => boolean, +): Promise { + const response = await fetch(mediaUrl); + if (!response.ok) { + throw new Error(`Failed to load media for waveform: ${response.status}`); + } + if (cancelled()) throw new Error("cancelled"); + console.debug("[timeline-audio-peaks] fetch ok", { + url: mediaUrl, + status: response.status, + }); + + const arrayBuffer = await response.arrayBuffer(); + if (cancelled()) throw new Error("cancelled"); + console.debug("[timeline-audio-peaks] bytes loaded", { + url: mediaUrl, + byteLength: arrayBuffer.byteLength, + }); + + const audioCtx = new OfflineAudioContext(1, 1, 44100); + const decoded = await audioCtx.decodeAudioData(arrayBuffer); + if (cancelled()) throw new Error("cancelled"); + console.debug("[timeline-audio-peaks] decode ok", { + url: mediaUrl, + durationSec: decoded.duration, + channelCount: decoded.numberOfChannels, + sampleRate: decoded.sampleRate, + sampleFrames: decoded.length, + }); + + const channelData = decoded.getChannelData(0); + const durationMs = decoded.duration * 1000; + const binSize = Math.max(1, Math.floor(channelData.length / TARGET_PEAK_COUNT)); + const peakCount = Math.ceil(channelData.length / binSize); + const peaks = new Float32Array(peakCount); + + for (let i = 0; i < peakCount; i++) { + const start = i * binSize; + const end = Math.min(start + binSize, channelData.length); + let max = 0; + for (let j = start; j < end; j++) { + const abs = Math.abs(channelData[j]); + if (abs > max) max = abs; + } + peaks[i] = max; + } + + let globalMax = 0; + for (let i = 0; i < peaks.length; i++) { + if (peaks[i] > globalMax) globalMax = peaks[i]; + } + if (globalMax > 0) { + for (let i = 0; i < peaks.length; i++) { + peaks[i] /= globalMax; + } + } + + return { peaks, durationMs }; +} + /** * Decode audio from a media file URL and produce a fixed-length array of peak * amplitudes suitable for waveform visualisation. @@ -17,8 +112,10 @@ export function useTimelineAudioPeaks(fileUrl: string | null | undefined): Audio useEffect(() => { urlRef.current = fileUrl; setData(null); + console.debug("[timeline-audio-peaks] reset", { input: fileUrl ?? null }); if (!fileUrl) { + console.debug("[timeline-audio-peaks] no input source, skipping"); return; } @@ -26,49 +123,92 @@ export function useTimelineAudioPeaks(fileUrl: string | null | undefined): Audio (async () => { try { - const response = await fetch(fileUrl); + const extractedLocalPath = extractLocalPathFromMediaServerUrl(fileUrl); + const isRemoteLike = /^(https?:|blob:|data:)/i.test(fileUrl) && !extractedLocalPath; + const isFileUrl = /^file:\/\//i.test(fileUrl); + const sourceLocalPath = isRemoteLike + ? null + : extractedLocalPath + ? extractedLocalPath + : isFileUrl + ? fromFileUrl(fileUrl) + : fileUrl; + console.debug("[timeline-audio-peaks] resolving source", { + input: fileUrl, + isRemoteLike, + isFileUrl, + sourceLocalPath, + extractedLocalPath, + }); + const mediaUrl = isRemoteLike + ? fileUrl + : isFileUrl + ? await resolveVideoUrl(fromFileUrl(fileUrl)) + : await resolveVideoUrl(fileUrl); if (cancelled) return; + console.debug("[timeline-audio-peaks] resolved source", { + input: fileUrl, + resolved: mediaUrl, + }); - const arrayBuffer = await response.arrayBuffer(); - if (cancelled) return; - - const audioCtx = new OfflineAudioContext(1, 1, 44100); - const decoded = await audioCtx.decodeAudioData(arrayBuffer); - if (cancelled) return; - - const channelData = decoded.getChannelData(0); - const durationMs = decoded.duration * 1000; - const binSize = Math.max(1, Math.floor(channelData.length / TARGET_PEAK_COUNT)); - const peakCount = Math.ceil(channelData.length / binSize); - const peaks = new Float32Array(peakCount); - - for (let i = 0; i < peakCount; i++) { - const start = i * binSize; - const end = Math.min(start + binSize, channelData.length); - let max = 0; - for (let j = start; j < end; j++) { - const abs = Math.abs(channelData[j]); - if (abs > max) max = abs; + let decodedPeaks: AudioPeaksData | null = null; + try { + decodedPeaks = await decodePeaksFromMediaUrl(mediaUrl, () => cancelled); + } catch (primaryError) { + if (primaryError instanceof Error && primaryError.message === "cancelled") { + return; } - peaks[i] = max; - } - - // Normalise to 0–1 range. - let globalMax = 0; - for (let i = 0; i < peaks.length; i++) { - if (peaks[i] > globalMax) globalMax = peaks[i]; - } - if (globalMax > 0) { - for (let i = 0; i < peaks.length; i++) { - peaks[i] /= globalMax; + console.warn("[timeline-audio-peaks] primary decode failed", { + input: fileUrl, + url: mediaUrl, + error: primaryError, + }); + if (sourceLocalPath) { + const candidates = buildSidecarAudioCandidates(sourceLocalPath); + console.debug("[timeline-audio-peaks] trying sidecar candidates", { + sourceLocalPath, + candidates, + }); + for (const candidatePath of candidates) { + try { + const candidateUrl = await resolveVideoUrl(candidatePath); + console.debug("[timeline-audio-peaks] trying sidecar", { + candidatePath, + candidateUrl, + }); + decodedPeaks = await decodePeaksFromMediaUrl(candidateUrl, () => cancelled); + console.info("[timeline-audio-peaks] sidecar decode succeeded", { + candidatePath, + candidateUrl, + }); + break; + } catch (sidecarError) { + if (sidecarError instanceof Error && sidecarError.message === "cancelled") { + return; + } + console.debug("[timeline-audio-peaks] sidecar decode failed", { + candidatePath, + error: sidecarError, + }); + } + } } } - if (!cancelled && urlRef.current === fileUrl) { - setData({ peaks, durationMs }); + if (!cancelled && urlRef.current === fileUrl && decodedPeaks) { + console.debug("[timeline-audio-peaks] peaks generated", { + input: fileUrl, + durationMs: Math.round(decodedPeaks.durationMs), + peakCount: decodedPeaks.peaks.length, + }); + setData(decodedPeaks); } - } catch { + } catch (error) { // File has no audio or decoding failed — leave as null. + console.warn("[timeline-audio-peaks] failed", { + input: fileUrl, + error, + }); } })();