fix: audio waveform on the wrong layer and not affected by clip size/region

This commit is contained in:
Alan Trebugeais
2026-05-09 13:29:06 +02:00
parent de4546999f
commit eff3bbf2f7
6 changed files with 323 additions and 50 deletions
@@ -11,6 +11,8 @@ import type { Span } from "dnd-timeline";
import { useItem } from "dnd-timeline";
import { useMemo } from "react";
import { cn } from "@/lib/utils";
import AudioWaveform from "./components/waveform/AudioWaveform";
import type { AudioPeaksData } from "./core/timelineTypes";
import glassStyles from "./ItemGlass.module.css";
interface ItemProps {
@@ -24,6 +26,8 @@ interface ItemProps {
zoomDepth?: number;
zoomMode?: "auto" | "manual";
speedValue?: number;
waveformPeaks?: AudioPeaksData | null;
waveformSegmentSpan?: Span;
variant?: "zoom" | "trim" | "clip" | "annotation" | "speed" | "audio";
}
@@ -57,6 +61,8 @@ export default function Item({
zoomDepth = 1,
zoomMode = "auto",
speedValue,
waveformPeaks = null,
waveformSegmentSpan,
variant = "zoom",
children,
}: ItemProps) {
@@ -71,6 +77,7 @@ export default function Item({
const isClip = variant === "clip";
const isSpeed = variant === "speed";
const isAudio = variant === "audio";
const showAudioWaveform = isAudio && Boolean(waveformPeaks);
const glassClass = isZoom
? glassStyles.glassPurple
@@ -146,6 +153,14 @@ export default function Item({
style={{ cursor: "col-resize", pointerEvents: "auto" }}
title="Resize right"
/>
{showAudioWaveform && waveformPeaks && (
<AudioWaveform
peaks={waveformPeaks}
segmentStartMs={waveformSegmentSpan?.start ?? span.start}
segmentEndMs={waveformSegmentSpan?.end ?? span.end}
className="absolute inset-0 w-full h-full pointer-events-none opacity-45"
/>
)}
{/* Content */}
<div className="relative z-10 flex flex-col items-center justify-center text-black/70 dark:text-white/90 opacity-80 group-hover:opacity-100 transition-opacity select-none overflow-hidden">
<div className="flex items-center gap-1.5">
@@ -178,6 +178,56 @@ const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
pan: "Shift + Ctrl + Scroll",
zoom: "Ctrl + Scroll",
});
const [liveSpanPreviewById, setLiveSpanPreviewById] = useState<Record<string, Span>>({});
const liveZoomPreview = useMemo(() => {
const previewSpans: Record<string, Span> = { ...liveSpanPreviewById };
const hiddenZoomIds = new Set<string>();
for (const [previewId, previewSpan] of Object.entries(liveSpanPreviewById)) {
const oldClip = clipRegions.find((clip) => clip.id === previewId);
if (!oldClip) continue;
const newStart = Math.round(previewSpan.start);
const newEnd = Math.round(previewSpan.end);
const removedSegments = [
...(newStart > oldClip.startMs
? [{ startMs: oldClip.startMs, endMs: newStart }]
: []),
...(newEnd < oldClip.endMs
? [{ startMs: newEnd, endMs: oldClip.endMs }]
: []),
];
const startDelta = newStart - oldClip.startMs;
const endDelta = newEnd - oldClip.endMs;
const isMove = Math.abs(startDelta - endDelta) < 1 && Math.abs(startDelta) > 0;
if (isMove) {
const delta = startDelta;
for (const zoom of zoomRegions) {
const overlaps =
zoom.startMs < oldClip.endMs && zoom.endMs > oldClip.startMs;
if (!overlaps) continue;
previewSpans[zoom.id] = {
start: zoom.startMs + delta,
end: zoom.endMs + delta,
};
}
}
if (removedSegments.length > 0) {
for (const zoom of zoomRegions) {
const removed = removedSegments.some(
(segment) =>
zoom.startMs < segment.endMs && zoom.endMs > segment.startMs,
);
if (removed) hiddenZoomIds.add(zoom.id);
}
}
}
return { previewSpans, hiddenZoomIds };
}, [clipRegions, liveSpanPreviewById, zoomRegions]);
const { shortcuts: keyShortcuts, isMac } = useShortcuts();
const audioPeaks = useTimelineAudioPeaks(videoPath);
@@ -376,6 +426,25 @@ const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
onItemSpanChange={handleItemSpanChange}
resolveTargetRowId={getResolvedDropRowId}
allRegionSpans={allRegionSpans}
onLiveSpanPreviewChange={(id, span) => {
setLiveSpanPreviewById((prev) => {
if (!span) {
if (!(id in prev)) return prev;
const next = { ...prev };
delete next[id];
return next;
}
const current = prev[id];
if (
current &&
current.start === span.start &&
current.end === span.end
) {
return prev;
}
return { ...prev, [id]: span };
});
}}
>
<KeyframeMarkers
keyframes={keyframes}
@@ -404,6 +473,8 @@ const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
onClearBlockSelection={clearSelectedBlocks}
keyframes={keyframes}
audioPeaks={audioPeaks}
liveSpanPreviewById={liveZoomPreview.previewSpans}
liveHiddenItemIds={Array.from(liveZoomPreview.hiddenZoomIds)}
/>
</TimelineWrapper>
</div>
@@ -17,7 +17,6 @@ import {
getTimelineViewportStretchFactor,
TIMELINE_AXIS_HEIGHT_PX,
} from "../../timelineLayout";
import AudioWaveform from "../waveform/AudioWaveform";
import glassStyles from "../../ItemGlass.module.css";
import Item from "../../Item";
import Row from "../../Row";
@@ -38,6 +37,7 @@ import PlaybackCursor from "../playhead/PlaybackCursor";
const HINT_CLIP = "Press C to split clip";
const HINT_ANNOTATION = "Press A to add annotation";
const HINT_AUDIO = "Click music icon to add audio";
const SOURCE_AUDIO_ROW_ID = "row-source-audio";
interface TimelineCanvasProps {
items: TimelineRenderItem[];
@@ -58,6 +58,8 @@ interface TimelineCanvasProps {
onClearBlockSelection?: () => void;
keyframes?: { id: string; time: number }[];
audioPeaks?: AudioPeaksData | null;
liveSpanPreviewById?: Record<string, { start: number; end: number }>;
liveHiddenItemIds?: string[];
}
interface TimelineHoverParams {
@@ -219,6 +221,8 @@ interface TimelineCanvasRowsProps {
onSelectAnnotation?: (id: string | null) => void;
onSelectAudio?: (id: string | null) => void;
audioPeaks?: AudioPeaksData | null;
liveSpanPreviewById?: Record<string, { start: number; end: number }>;
liveHiddenItemIds?: string[];
direction: string;
canShowGhostZoom: boolean;
ghostStartMs: number | null;
@@ -243,6 +247,8 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({
onSelectAnnotation,
onSelectAudio,
audioPeaks,
liveSpanPreviewById,
liveHiddenItemIds,
direction,
canShowGhostZoom,
ghostStartMs,
@@ -253,6 +259,7 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({
onZoomRowMouseLeave,
onZoomRowClick,
}: TimelineCanvasRowsProps) {
const hiddenIds = useMemo(() => new Set(liveHiddenItemIds ?? []), [liveHiddenItemIds]);
const { clipItems, zoomItems, annotationRows, audioRows } = useMemo(() => {
const nextClipItems: TimelineRenderItem[] = [];
const nextZoomItems: TimelineRenderItem[] = [];
@@ -307,7 +314,6 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({
return (
<>
<Row id={CLIP_ROW_ID} isEmpty={clipItems.length === 0} hint={HINT_CLIP}>
{audioPeaks && <AudioWaveform peaks={audioPeaks} />}
<ClipMarkerOverlay videoDurationMs={videoDurationMs} />
{clipItems.map((item) => (
<Item
@@ -323,6 +329,25 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({
</Item>
))}
</Row>
{audioPeaks && (
<Row id={SOURCE_AUDIO_ROW_ID}>
{clipItems.map((item) => (
<Item
key={`source-audio-${item.id}`}
id={`source-audio-${item.id}`}
rowId={SOURCE_AUDIO_ROW_ID}
span={liveSpanPreviewById?.[item.id] ?? item.span}
isSelected={selectAllBlocksActive || item.id === selectedClipId}
onSelect={() => onSelectClip?.(item.id)}
variant="audio"
waveformPeaks={audioPeaks}
waveformSegmentSpan={liveSpanPreviewById?.[item.id] ?? item.span}
>
Source
</Item>
))}
</Row>
)}
<Row
id={ZOOM_ROW_ID}
@@ -357,7 +382,9 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({
</div>
</div>
)}
{zoomItems.map((item) => (
{zoomItems
.filter((item) => !hiddenIds.has(item.id))
.map((item) => (
<Item
id={item.id}
key={item.id}
@@ -432,6 +459,8 @@ export default function TimelineCanvas({
onClearBlockSelection,
keyframes = [],
audioPeaks,
liveSpanPreviewById,
liveHiddenItemIds,
}: TimelineCanvasProps) {
const { setTimelineRef, style, sidebarWidth, direction, range, valueToPixels, pixelsToValue } =
useTimelineContext();
@@ -583,8 +612,9 @@ export default function TimelineCanvas({
if (isAnnotationTrackRowId(item.rowId)) annotationRowIds.add(item.rowId);
if (isAudioTrackRowId(item.rowId)) audioRowIds.add(item.rowId);
}
return 2 + annotationRowIds.size + audioRowIds.size;
}, [items]);
const sourceAudioRows = audioPeaks ? 1 : 0;
return 2 + sourceAudioRows + annotationRowIds.size + audioRowIds.size;
}, [items, audioPeaks]);
const timelineRowsMinHeightPx = getTimelineRowsMinHeightPx(timelineRowCount);
const timelineContentMinHeightPx = getTimelineContentMinHeightPx(timelineRowCount);
const timelineViewportStretchFactor = getTimelineViewportStretchFactor(timelineRowCount);
@@ -661,6 +691,8 @@ export default function TimelineCanvas({
onSelectAnnotation={onSelectAnnotation}
onSelectAudio={onSelectAudio}
audioPeaks={audioPeaks}
liveSpanPreviewById={liveSpanPreviewById}
liveHiddenItemIds={liveHiddenItemIds}
direction={direction}
canShowGhostZoom={canShowGhostZoom}
ghostStartMs={ghostStartMs}
@@ -4,6 +4,9 @@ import type { AudioPeaksData } from "../../core/timelineTypes";
interface AudioWaveformProps {
peaks: AudioPeaksData;
segmentStartMs?: number;
segmentEndMs?: number;
className?: string;
}
/**
@@ -11,7 +14,12 @@ interface AudioWaveformProps {
* Automatically syncs with the timeline's visible range so the waveform
* scrolls and zooms together with the clip items above it.
*/
function AudioWaveformComponent({ peaks }: AudioWaveformProps) {
function AudioWaveformComponent({
peaks,
segmentStartMs,
segmentEndMs,
className,
}: AudioWaveformProps) {
const canvasRef = useRef<HTMLCanvasElement>(null);
const { range } = useTimelineContext();
const [resizeKey, setResizeKey] = useState(0);
@@ -53,8 +61,8 @@ function AudioWaveformComponent({ peaks }: AudioWaveformProps) {
const { peaks: peakData, durationMs } = peaks;
if (durationMs <= 0 || peakData.length === 0) return;
const visibleStartMs = range.start;
const visibleEndMs = range.end;
const visibleStartMs = segmentStartMs ?? range.start;
const visibleEndMs = segmentEndMs ?? range.end;
const visibleDurationMs = visibleEndMs - visibleStartMs;
if (visibleDurationMs <= 0) return;
@@ -77,12 +85,12 @@ function AudioWaveformComponent({ peaks }: AudioWaveformProps) {
ctx.strokeStyle = "rgba(255, 255, 255, 0.55)";
ctx.lineWidth = dpr;
ctx.stroke();
}, [peaks, range.start, range.end, resizeKey]);
}, [peaks, range.start, range.end, resizeKey, segmentStartMs, segmentEndMs]);
return (
<canvas
ref={setCanvasRef}
className="absolute inset-0 w-full h-full pointer-events-none"
className={className ?? "absolute inset-0 w-full h-full pointer-events-none"}
style={{ display: "block" }}
/>
);
@@ -29,6 +29,7 @@ interface TimelineWrapperProps {
onItemSpanChange: (id: string, span: Span, rowId?: string) => void;
resolveTargetRowId?: (id: string, proposedRowId: string) => string;
allRegionSpans?: TimelineRegionSpan[];
onLiveSpanPreviewChange?: (id: string, span: Span | null) => void;
}
export default function TimelineWrapper({
@@ -43,6 +44,7 @@ export default function TimelineWrapper({
onItemSpanChange,
resolveTargetRowId,
allRegionSpans = [],
onLiveSpanPreviewChange,
}: TimelineWrapperProps) {
const totalMs = Math.max(0, Math.round(videoDuration * 1000));
@@ -132,8 +134,9 @@ export default function TimelineWrapper({
(event: DragStartEvent) => {
const span = event.active.data.current.getSpanFromDragEvent?.(event);
if (span) showTooltip(span);
onLiveSpanPreviewChange?.(event.active.id as string, span ?? null);
},
[showTooltip],
[onLiveSpanPreviewChange, showTooltip],
);
const onDragMove = useCallback(
@@ -144,8 +147,9 @@ export default function TimelineWrapper({
? (event.activatorEvent as PointerEvent).clientX + (event.delta?.x ?? 0)
: undefined;
if (span) showTooltip(span, screenX);
onLiveSpanPreviewChange?.(event.active.id as string, span ?? null);
},
[showTooltip],
[onLiveSpanPreviewChange, showTooltip],
);
const onResizeMove = useCallback(
@@ -156,8 +160,9 @@ export default function TimelineWrapper({
? (event.activatorEvent as PointerEvent).clientX + (event.delta?.x ?? 0)
: undefined;
if (span) showTooltip(span, screenX);
onLiveSpanPreviewChange?.(event.active.id as string, span ?? null);
},
[showTooltip],
[onLiveSpanPreviewChange, showTooltip],
);
const hideTooltip = useCallback(() => showTooltip(null), [showTooltip]);
@@ -166,16 +171,18 @@ export default function TimelineWrapper({
(event: ResizeEndEvent) => {
hideTooltip();
onResizeEnd(event);
onLiveSpanPreviewChange?.(event.active.id as string, null);
},
[hideTooltip, onResizeEnd],
[hideTooltip, onLiveSpanPreviewChange, onResizeEnd],
);
const onDragEndWithTooltip = useCallback(
(event: DragEndEvent) => {
hideTooltip();
onDragEnd(event);
onLiveSpanPreviewChange?.(event.active.id as string, null);
},
[hideTooltip, onDragEnd],
[hideTooltip, onDragEnd, onLiveSpanPreviewChange],
);
const handleRangeChange = useCallback(
@@ -1,9 +1,104 @@
import { useEffect, useRef, useState } from "react";
import { fromFileUrl, resolveVideoUrl } from "../../projectPersistence";
import type { AudioPeaksData } from "../core/timelineTypes";
/** Number of peak bins to produce — enough for smooth display at any zoom. */
const TARGET_PEAK_COUNT = 2048;
function buildSidecarAudioCandidates(sourcePath: string): string[] {
const normalized = sourcePath.replace(/\\/g, "/");
const lastSlash = normalized.lastIndexOf("/");
const dir = lastSlash >= 0 ? normalized.slice(0, lastSlash + 1) : "";
const fileName = lastSlash >= 0 ? normalized.slice(lastSlash + 1) : normalized;
const dotIndex = fileName.lastIndexOf(".");
const baseName = dotIndex > 0 ? fileName.slice(0, dotIndex) : fileName;
return [
`${dir}${baseName}.system.wav`,
`${dir}${baseName}.mic.wav`,
`${dir}${baseName}.system.m4a`,
`${dir}${baseName}.mic.m4a`,
];
}
function extractLocalPathFromMediaServerUrl(input: string): string | null {
try {
const url = new URL(input);
const isLocalMediaServer =
(url.protocol === "http:" || url.protocol === "https:") &&
(url.hostname === "127.0.0.1" || url.hostname === "localhost") &&
url.pathname === "/video";
if (!isLocalMediaServer) return null;
const rawPath = url.searchParams.get("path");
if (!rawPath) return null;
return rawPath;
} catch {
return null;
}
}
async function decodePeaksFromMediaUrl(
mediaUrl: string,
cancelled: () => boolean,
): Promise<AudioPeaksData> {
const response = await fetch(mediaUrl);
if (!response.ok) {
throw new Error(`Failed to load media for waveform: ${response.status}`);
}
if (cancelled()) throw new Error("cancelled");
console.debug("[timeline-audio-peaks] fetch ok", {
url: mediaUrl,
status: response.status,
});
const arrayBuffer = await response.arrayBuffer();
if (cancelled()) throw new Error("cancelled");
console.debug("[timeline-audio-peaks] bytes loaded", {
url: mediaUrl,
byteLength: arrayBuffer.byteLength,
});
const audioCtx = new OfflineAudioContext(1, 1, 44100);
const decoded = await audioCtx.decodeAudioData(arrayBuffer);
if (cancelled()) throw new Error("cancelled");
console.debug("[timeline-audio-peaks] decode ok", {
url: mediaUrl,
durationSec: decoded.duration,
channelCount: decoded.numberOfChannels,
sampleRate: decoded.sampleRate,
sampleFrames: decoded.length,
});
const channelData = decoded.getChannelData(0);
const durationMs = decoded.duration * 1000;
const binSize = Math.max(1, Math.floor(channelData.length / TARGET_PEAK_COUNT));
const peakCount = Math.ceil(channelData.length / binSize);
const peaks = new Float32Array(peakCount);
for (let i = 0; i < peakCount; i++) {
const start = i * binSize;
const end = Math.min(start + binSize, channelData.length);
let max = 0;
for (let j = start; j < end; j++) {
const abs = Math.abs(channelData[j]);
if (abs > max) max = abs;
}
peaks[i] = max;
}
let globalMax = 0;
for (let i = 0; i < peaks.length; i++) {
if (peaks[i] > globalMax) globalMax = peaks[i];
}
if (globalMax > 0) {
for (let i = 0; i < peaks.length; i++) {
peaks[i] /= globalMax;
}
}
return { peaks, durationMs };
}
/**
* Decode audio from a media file URL and produce a fixed-length array of peak
* amplitudes suitable for waveform visualisation.
@@ -17,8 +112,10 @@ export function useTimelineAudioPeaks(fileUrl: string | null | undefined): Audio
useEffect(() => {
urlRef.current = fileUrl;
setData(null);
console.debug("[timeline-audio-peaks] reset", { input: fileUrl ?? null });
if (!fileUrl) {
console.debug("[timeline-audio-peaks] no input source, skipping");
return;
}
@@ -26,49 +123,92 @@ export function useTimelineAudioPeaks(fileUrl: string | null | undefined): Audio
(async () => {
try {
const response = await fetch(fileUrl);
const extractedLocalPath = extractLocalPathFromMediaServerUrl(fileUrl);
const isRemoteLike = /^(https?:|blob:|data:)/i.test(fileUrl) && !extractedLocalPath;
const isFileUrl = /^file:\/\//i.test(fileUrl);
const sourceLocalPath = isRemoteLike
? null
: extractedLocalPath
? extractedLocalPath
: isFileUrl
? fromFileUrl(fileUrl)
: fileUrl;
console.debug("[timeline-audio-peaks] resolving source", {
input: fileUrl,
isRemoteLike,
isFileUrl,
sourceLocalPath,
extractedLocalPath,
});
const mediaUrl = isRemoteLike
? fileUrl
: isFileUrl
? await resolveVideoUrl(fromFileUrl(fileUrl))
: await resolveVideoUrl(fileUrl);
if (cancelled) return;
console.debug("[timeline-audio-peaks] resolved source", {
input: fileUrl,
resolved: mediaUrl,
});
const arrayBuffer = await response.arrayBuffer();
if (cancelled) return;
const audioCtx = new OfflineAudioContext(1, 1, 44100);
const decoded = await audioCtx.decodeAudioData(arrayBuffer);
if (cancelled) return;
const channelData = decoded.getChannelData(0);
const durationMs = decoded.duration * 1000;
const binSize = Math.max(1, Math.floor(channelData.length / TARGET_PEAK_COUNT));
const peakCount = Math.ceil(channelData.length / binSize);
const peaks = new Float32Array(peakCount);
for (let i = 0; i < peakCount; i++) {
const start = i * binSize;
const end = Math.min(start + binSize, channelData.length);
let max = 0;
for (let j = start; j < end; j++) {
const abs = Math.abs(channelData[j]);
if (abs > max) max = abs;
let decodedPeaks: AudioPeaksData | null = null;
try {
decodedPeaks = await decodePeaksFromMediaUrl(mediaUrl, () => cancelled);
} catch (primaryError) {
if (primaryError instanceof Error && primaryError.message === "cancelled") {
return;
}
peaks[i] = max;
}
// Normalise to 0–1 range.
let globalMax = 0;
for (let i = 0; i < peaks.length; i++) {
if (peaks[i] > globalMax) globalMax = peaks[i];
}
if (globalMax > 0) {
for (let i = 0; i < peaks.length; i++) {
peaks[i] /= globalMax;
console.warn("[timeline-audio-peaks] primary decode failed", {
input: fileUrl,
url: mediaUrl,
error: primaryError,
});
if (sourceLocalPath) {
const candidates = buildSidecarAudioCandidates(sourceLocalPath);
console.debug("[timeline-audio-peaks] trying sidecar candidates", {
sourceLocalPath,
candidates,
});
for (const candidatePath of candidates) {
try {
const candidateUrl = await resolveVideoUrl(candidatePath);
console.debug("[timeline-audio-peaks] trying sidecar", {
candidatePath,
candidateUrl,
});
decodedPeaks = await decodePeaksFromMediaUrl(candidateUrl, () => cancelled);
console.info("[timeline-audio-peaks] sidecar decode succeeded", {
candidatePath,
candidateUrl,
});
break;
} catch (sidecarError) {
if (sidecarError instanceof Error && sidecarError.message === "cancelled") {
return;
}
console.debug("[timeline-audio-peaks] sidecar decode failed", {
candidatePath,
error: sidecarError,
});
}
}
}
}
if (!cancelled && urlRef.current === fileUrl) {
setData({ peaks, durationMs });
if (!cancelled && urlRef.current === fileUrl && decodedPeaks) {
console.debug("[timeline-audio-peaks] peaks generated", {
input: fileUrl,
durationMs: Math.round(decodedPeaks.durationMs),
peakCount: decodedPeaks.peaks.length,
});
setData(decodedPeaks);
}
} catch {
} catch (error) {
// File has no audio or decoding failed — leave as null.
console.warn("[timeline-audio-peaks] failed", {
input: fileUrl,
error,
});
}
})();