From 62f015cb0b0b7e4d3c77a15a35dbf7e166f5f3a0 Mon Sep 17 00:00:00 2001 From: webadderall Date: Sat, 2 May 2026 15:31:35 +1000 Subject: [PATCH] Use click clusters for auto zoom suggestions --- .../video-editor/timeline/TimelineEditor.tsx | 8 +- .../timeline/zoomSuggestionUtils.test.ts | 117 ++++++++++++++++++ .../timeline/zoomSuggestionUtils.ts | 94 ++++++++++---- 3 files changed, 194 insertions(+), 25 deletions(-) create mode 100644 src/components/video-editor/timeline/zoomSuggestionUtils.test.ts diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx index 32643bec..603820ce 100644 --- a/src/components/video-editor/timeline/TimelineEditor.tsx +++ b/src/components/video-editor/timeline/TimelineEditor.tsx @@ -1580,7 +1580,7 @@ const TimelineEditor = forwardRef( return; } - if (cursorTelemetry.length < 2) { + if (cursorTelemetry.length === 0) { toast.info("No cursor telemetry available", { description: "Record a screencast first to generate cursor-based suggestions.", }); @@ -1610,14 +1610,14 @@ const TimelineEditor = forwardRef( if (result.status === "no-interactions") { toast.info("No clear interaction moments found", { - description: "Try a recording with pauses or clicks around important actions.", + description: "Try a recording with clicks around important actions.", }); return; } if (result.status === "no-slots" || result.suggestions.length === 0) { toast.info("No auto-zoom slots available", { - description: "Detected dwell points overlap existing zoom regions.", + description: "Detected click groups overlap existing zoom regions.", }); return; } @@ -1627,7 +1627,7 @@ const TimelineEditor = forwardRef( } toast.success( - `Added ${result.suggestions.length} interaction-based zoom suggestion${result.suggestions.length === 1 ? "" : "s"}`, + `Added ${result.suggestions.length} click-based zoom suggestion${result.suggestions.length === 1 ? "" : "s"}`, ); }, [ videoDuration, diff --git a/src/components/video-editor/timeline/zoomSuggestionUtils.test.ts b/src/components/video-editor/timeline/zoomSuggestionUtils.test.ts new file mode 100644 index 00000000..17d706a3 --- /dev/null +++ b/src/components/video-editor/timeline/zoomSuggestionUtils.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, it } from "vitest"; + +import type { CursorTelemetryPoint } from "../types"; +import { buildInteractionZoomSuggestions } from "./zoomSuggestionUtils"; + +function createSample( + timeMs: number, + interactionType: CursorTelemetryPoint["interactionType"] = "move", + cx = 0.5, + cy = 0.5, +): CursorTelemetryPoint { + return { timeMs, interactionType, cx, cy }; +} + +describe("buildInteractionZoomSuggestions", () => { + it("creates a zoom from 500ms before the first click through 500ms after the last click", () => { + const result = buildInteractionZoomSuggestions({ + cursorTelemetry: [ + createSample(100, "move"), + createSample(2000, "click", 0.25, 0.4), + createSample(2400, "move", 0.3, 0.45), + ], + totalMs: 6000, + defaultDurationMs: 1500, + }); + + expect(result.status).toBe("ok"); + expect(result.suggestions).toEqual([ + { + start: 1500, + end: 2500, + focus: { cx: 0.25, cy: 0.4 }, + }, + ]); + }); + + it("groups clicks within 2500ms into the same zoom region", () => { + const result = buildInteractionZoomSuggestions({ + cursorTelemetry: [ + createSample(0, "move"), + createSample(1000, "click", 0.2, 0.2), + createSample(3200, "double-click", 0.8, 0.6), + createSample(4500, "move", 0.75, 0.5), + ], + totalMs: 8000, + defaultDurationMs: 1200, + }); + + expect(result.status).toBe("ok"); + expect(result.suggestions).toEqual([ + { + start: 500, + end: 3700, + focus: { cx: 0.5, cy: 0.4 }, + }, + ]); + }); + + it("splits clicks that are more than 2500ms apart into separate zooms", () => { + const result = buildInteractionZoomSuggestions({ + cursorTelemetry: [ + createSample(0, "move"), + createSample(1000, "click", 0.1, 0.2), + createSample(4000, "right-click", 0.9, 0.7), + createSample(5000, "move", 0.9, 0.7), + ], + totalMs: 7000, + defaultDurationMs: 1500, + }); + + expect(result.status).toBe("ok"); + expect(result.suggestions).toEqual([ + { + start: 500, + end: 1500, + focus: { cx: 0.1, cy: 0.2 }, + }, + { + start: 3500, + end: 4500, + focus: { cx: 0.9, cy: 0.7 }, + }, + ]); + }); + + it("skips click-cluster suggestions that overlap reserved spans", () => { + const result = buildInteractionZoomSuggestions({ + cursorTelemetry: [ + createSample(0, "move"), + createSample(1000, "click", 0.2, 0.3), + createSample(5000, "middle-click", 0.7, 0.6), + ], + totalMs: 7000, + defaultDurationMs: 1500, + reservedSpans: [{ start: 450, end: 1800 }], + }); + + expect(result.status).toBe("ok"); + expect(result.suggestions).toEqual([ + { + start: 4500, + end: 5500, + focus: { cx: 0.7, cy: 0.6 }, + }, + ]); + }); + + it("returns no-interactions when telemetry has no click events", () => { + const result = buildInteractionZoomSuggestions({ + cursorTelemetry: [createSample(0), createSample(1200), createSample(2400)], + totalMs: 3000, + defaultDurationMs: 1500, + }); + + expect(result).toEqual({ status: "no-interactions", suggestions: [] }); + }); +}); \ No newline at end of file diff --git a/src/components/video-editor/timeline/zoomSuggestionUtils.ts b/src/components/video-editor/timeline/zoomSuggestionUtils.ts index 2df2db85..1fa4f39c 100644 --- a/src/components/video-editor/timeline/zoomSuggestionUtils.ts +++ b/src/components/video-editor/timeline/zoomSuggestionUtils.ts @@ -38,9 +38,17 @@ export interface InteractionZoomSuggestionResult { suggestions: SuggestedZoomRegion[]; } -const DEFAULT_SUGGESTION_SPACING_MS = 1800; +const ZOOM_CLICK_LEAD_IN_MS = 500; +const ZOOM_CLICK_TAIL_OUT_MS = 500; +const ZOOM_CLICK_GROUP_GAP_MS = 2500; const DEFAULT_MERGE_NEARBY_GAP_MS = 1500; +interface ClickClusterCandidate { + start: number; + end: number; + focus: ZoomFocus; +} + function normalizeTelemetrySample( sample: CursorTelemetryPoint, totalMs: number, @@ -259,7 +267,6 @@ export function buildInteractionZoomSuggestions(params: { totalMs: number; defaultDurationMs: number; reservedSpans?: Array<{ start: number; end: number }>; - spacingMs?: number; mergeGapMs?: number; }): InteractionZoomSuggestionResult { const { @@ -267,7 +274,6 @@ export function buildInteractionZoomSuggestions(params: { totalMs, defaultDurationMs, reservedSpans = [], - spacingMs = DEFAULT_SUGGESTION_SPACING_MS, mergeGapMs = DEFAULT_MERGE_NEARBY_GAP_MS, } = params; @@ -277,32 +283,21 @@ export function buildInteractionZoomSuggestions(params: { } const normalizedSamples = normalizeCursorTelemetry(cursorTelemetry, totalMs); - if (normalizedSamples.length < 2) { + if (normalizedSamples.length === 0) { return { status: "no-telemetry", suggestions: [] }; } - const interactionCandidates = detectInteractionCandidates(normalizedSamples); - if (interactionCandidates.length === 0) { + const clickClusters = buildClickClusterCandidates(normalizedSamples, totalMs); + if (clickClusters.length === 0) { return { status: "no-interactions", suggestions: [] }; } - const sortedCandidates = [...interactionCandidates].sort((a, b) => b.strength - a.strength); - const acceptedCenters: number[] = []; const accepted: SuggestedZoomRegion[] = []; const reserved = [...reservedSpans].sort((a, b) => a.start - b.start); - sortedCandidates.forEach((candidate) => { - const tooCloseToAccepted = acceptedCenters.some( - (center) => Math.abs(center - candidate.centerTimeMs) < spacingMs, - ); - - if (tooCloseToAccepted) { - return; - } - - const centeredStart = Math.round(candidate.centerTimeMs - defaultDuration / 2); - const candidateStart = Math.max(0, Math.min(centeredStart, totalMs - defaultDuration)); - const candidateEnd = candidateStart + defaultDuration; + clickClusters.forEach((candidate) => { + const candidateStart = candidate.start; + const candidateEnd = candidate.end; const hasOverlap = reserved.some( (span) => candidateEnd > span.start && candidateStart < span.end, ); @@ -312,7 +307,6 @@ export function buildInteractionZoomSuggestions(params: { } reserved.push({ start: candidateStart, end: candidateEnd }); - acceptedCenters.push(candidate.centerTimeMs); accepted.push({ start: candidateStart, end: candidateEnd, @@ -339,6 +333,64 @@ export function buildInteractionZoomSuggestions(params: { return { status: "ok", suggestions: merged }; } +function buildClickClusterCandidates( + samples: CursorTelemetryPoint[], + totalMs: number, +): ClickClusterCandidate[] { + const clickSamples = samples.filter((sample) => isZoomTriggerClick(sample.interactionType)); + if (clickSamples.length === 0) { + return []; + } + + const clusters: CursorTelemetryPoint[][] = []; + let currentCluster: CursorTelemetryPoint[] = []; + + for (const sample of clickSamples) { + const previous = currentCluster[currentCluster.length - 1]; + if (!previous || sample.timeMs - previous.timeMs <= ZOOM_CLICK_GROUP_GAP_MS) { + currentCluster.push(sample); + continue; + } + + clusters.push(currentCluster); + currentCluster = [sample]; + } + + if (currentCluster.length > 0) { + clusters.push(currentCluster); + } + + return clusters.map((cluster) => { + const firstClick = cluster[0]; + const lastClick = cluster[cluster.length - 1]; + const focus = cluster.reduce( + (accumulator, sample) => ({ + cx: accumulator.cx + sample.cx, + cy: accumulator.cy + sample.cy, + }), + { cx: 0, cy: 0 }, + ); + + return { + start: Math.max(0, Math.round(firstClick.timeMs - ZOOM_CLICK_LEAD_IN_MS)), + end: Math.min(totalMs, Math.round(lastClick.timeMs + ZOOM_CLICK_TAIL_OUT_MS)), + focus: { + cx: focus.cx / cluster.length, + cy: focus.cy / cluster.length, + }, + }; + }); +} + +function isZoomTriggerClick(interactionType: CursorTelemetryPoint["interactionType"]): boolean { + return ( + interactionType === "click" || + interactionType === "double-click" || + interactionType === "right-click" || + interactionType === "middle-click" + ); +} + /** * Analyzes cursor movement after a click to classify the interaction pattern. *