diff --git a/src/components/video-editor/timeline/zoomSuggestionUtils.test.ts b/src/components/video-editor/timeline/zoomSuggestionUtils.test.ts new file mode 100644 index 00000000..77080bfc --- /dev/null +++ b/src/components/video-editor/timeline/zoomSuggestionUtils.test.ts @@ -0,0 +1,207 @@ +import { describe, expect, it } from "vitest"; +import { + CLICK_CLUSTER_MERGE_GAP_MS, + CLICK_CLUSTER_PAD_MS, + buildInteractionZoomSuggestions, +} from "./zoomSuggestionUtils"; +import type { CursorTelemetryPoint } from "../types"; + +function makeClick( + timeMs: number, + cx = 0.5, + cy = 0.5, + interactionType: CursorTelemetryPoint["interactionType"] = "click", +): CursorTelemetryPoint { + return { timeMs, cx, cy, interactionType }; +} + +function makeMove(timeMs: number, cx = 0.5, cy = 0.5): CursorTelemetryPoint { + return { timeMs, cx, cy, interactionType: "move" }; +} + +/** Wraps click samples with surrounding move events to mimic real mixed telemetry. */ +function withMoves( + clicks: CursorTelemetryPoint[], + totalMs: number, +): CursorTelemetryPoint[] { + return [ + makeMove(0), + ...clicks, + makeMove(totalMs), + ]; +} + +const TOTAL_MS = 30_000; + +describe("buildInteractionZoomSuggestions (click-cluster logic)", () => { + it("creates one zoom track for a single isolated click with 500ms padding", () => { + const telemetry = withMoves([makeClick(5_000)], TOTAL_MS); + + const result = buildInteractionZoomSuggestions({ + cursorTelemetry: telemetry, + totalMs: TOTAL_MS, + defaultDurationMs: 3_000, + }); + + expect(result.status).toBe("ok"); + expect(result.suggestions).toHaveLength(1); + + const [s] = result.suggestions; + expect(s.start).toBe(5_000 - CLICK_CLUSTER_PAD_MS); + expect(s.end).toBe(5_000 + CLICK_CLUSTER_PAD_MS); + }); + + it("accepts a single explicit click sample without needing surrounding moves", () => { + const result = buildInteractionZoomSuggestions({ + cursorTelemetry: [makeClick(5_000)], + totalMs: TOTAL_MS, + defaultDurationMs: 3_000, + }); + + expect(result.status).toBe("ok"); + expect(result.suggestions).toHaveLength(1); + }); + + it.each(["right-click", "middle-click"] as const)( + "accepts %s telemetry like a standard click", + (interactionType) => { + const result = buildInteractionZoomSuggestions({ + cursorTelemetry: withMoves([makeClick(5_000, 0.5, 0.5, interactionType)], TOTAL_MS), + totalMs: TOTAL_MS, + defaultDurationMs: 3_000, + }); + + expect(result.status).toBe("ok"); + expect(result.suggestions).toHaveLength(1); + + const [suggestion] = result.suggestions; + expect(suggestion.start).toBe(5_000 - CLICK_CLUSTER_PAD_MS); + expect(suggestion.end).toBe(5_000 + CLICK_CLUSTER_PAD_MS); + }, + ); + + it("merges two clicks within 2500ms into one zoom track", () => { + const telemetry = withMoves( + [makeClick(4_000), makeClick(4_000 + CLICK_CLUSTER_MERGE_GAP_MS - 1)], + TOTAL_MS, + ); + + const result = buildInteractionZoomSuggestions({ + cursorTelemetry: telemetry, + totalMs: TOTAL_MS, + defaultDurationMs: 3_000, + }); + + expect(result.status).toBe("ok"); + expect(result.suggestions).toHaveLength(1); + + const [s] = result.suggestions; + const lastClickMs = 4_000 + CLICK_CLUSTER_MERGE_GAP_MS - 1; + expect(s.start).toBe(4_000 - CLICK_CLUSTER_PAD_MS); + expect(s.end).toBe(lastClickMs + CLICK_CLUSTER_PAD_MS); + }); + + it("splits two clicks more than 2500ms apart into separate zoom tracks", () => { + const click1 = 3_000; + const click2 = 3_000 + CLICK_CLUSTER_MERGE_GAP_MS + 1; // just outside the merge gap + + const telemetry = withMoves([makeClick(click1), makeClick(click2)], TOTAL_MS); + + const result = buildInteractionZoomSuggestions({ + cursorTelemetry: telemetry, + totalMs: TOTAL_MS, + defaultDurationMs: 3_000, + }); + + expect(result.status).toBe("ok"); + expect(result.suggestions).toHaveLength(2); + + const [a, b] = result.suggestions; + expect(a.start).toBe(click1 - CLICK_CLUSTER_PAD_MS); + expect(a.end).toBe(click1 + CLICK_CLUSTER_PAD_MS); + expect(b.start).toBe(click2 - CLICK_CLUSTER_PAD_MS); + expect(b.end).toBe(click2 + CLICK_CLUSTER_PAD_MS); + }); + + it("chains multiple clicks: 3 in a row within 2500ms each become one track", () => { + // click at 0, 2000, 4000 — each gap is 2000ms < 2500ms + const telemetry = withMoves([makeClick(0), makeClick(2_000), makeClick(4_000)], TOTAL_MS); + + const result = buildInteractionZoomSuggestions({ + cursorTelemetry: telemetry, + totalMs: TOTAL_MS, + defaultDurationMs: 3_000, + }); + + expect(result.status).toBe("ok"); + expect(result.suggestions).toHaveLength(1); + + const [s] = result.suggestions; + expect(s.start).toBe(0); // clamped to 0 (would be -500) + expect(s.end).toBe(4_000 + CLICK_CLUSTER_PAD_MS); + }); + + it("returns no-interactions when there are no click telemetry points", () => { + // Move events only — no clicks + const telemetry: CursorTelemetryPoint[] = [ + { timeMs: 0, cx: 0.5, cy: 0.5, interactionType: "move" }, + { timeMs: 1_000, cx: 0.5, cy: 0.5, interactionType: "move" }, + { timeMs: 2_000, cx: 0.6, cy: 0.6, interactionType: "move" }, + { timeMs: TOTAL_MS, cx: 0.6, cy: 0.6, interactionType: "move" }, + ]; + + const result = buildInteractionZoomSuggestions({ + cursorTelemetry: telemetry, + totalMs: TOTAL_MS, + defaultDurationMs: 3_000, + }); + + expect(result.status).toBe("no-interactions"); + expect(result.suggestions).toHaveLength(0); + }); + + it("ignores dwell-derived click-like heuristics when there are no explicit clicks", () => { + const telemetry: CursorTelemetryPoint[] = [ + makeMove(0, 0.5, 0.5), + makeMove(200, 0.5005, 0.5005), + makeMove(400, 0.5008, 0.5008), + makeMove(600, 0.501, 0.501), + ]; + + const result = buildInteractionZoomSuggestions({ + cursorTelemetry: telemetry, + totalMs: TOTAL_MS, + defaultDurationMs: 3_000, + }); + + expect(result.status).toBe("no-interactions"); + expect(result.suggestions).toHaveLength(0); + }); + + it("skips clusters that overlap reserved spans", () => { + const click = 5_000; + + const result = buildInteractionZoomSuggestions({ + cursorTelemetry: withMoves([makeClick(click)], TOTAL_MS), + totalMs: TOTAL_MS, + defaultDurationMs: 3_000, + reservedSpans: [{ start: 4_000, end: 6_000 }], // overlaps the cluster window + }); + + expect(result.status).toBe("no-slots"); + expect(result.suggestions).toHaveLength(0); + }); + + it("clamps start to 0 and end to totalMs at video boundaries", () => { + const result = buildInteractionZoomSuggestions({ + cursorTelemetry: withMoves([makeClick(200)], 1_000), + totalMs: 1_000, + defaultDurationMs: 3_000, + }); + + expect(result.status).toBe("ok"); + const [s] = result.suggestions; + expect(s.start).toBeGreaterThanOrEqual(0); + expect(s.end).toBeLessThanOrEqual(1_000); + }); +}); diff --git a/src/components/video-editor/timeline/zoomSuggestionUtils.ts b/src/components/video-editor/timeline/zoomSuggestionUtils.ts index 2df2db85..189c7604 100644 --- a/src/components/video-editor/timeline/zoomSuggestionUtils.ts +++ b/src/components/video-editor/timeline/zoomSuggestionUtils.ts @@ -19,6 +19,7 @@ export interface CursorInteractionCandidate extends ZoomDwellCandidate { | "dropdown-open" | "text-selection" | "text-field-click"; + source: "explicit" | "heuristic"; } export interface SuggestedZoomRegion { @@ -38,8 +39,22 @@ export interface InteractionZoomSuggestionResult { suggestions: SuggestedZoomRegion[]; } -const DEFAULT_SUGGESTION_SPACING_MS = 1800; -const DEFAULT_MERGE_NEARBY_GAP_MS = 1500; +/** Max gap between consecutive clicks before they are split into separate zoom clusters. */ +export const CLICK_CLUSTER_MERGE_GAP_MS = 2500; +/** Padding added before the first click and after the last click in a cluster. */ +export const CLICK_CLUSTER_PAD_MS = 500; +const EXPLICIT_CLICK_TYPES = new Set>([ + "click", + "double-click", + "right-click", + "middle-click", +]); + +function isExplicitClickType( + interactionType: CursorTelemetryPoint["interactionType"], +): interactionType is NonNullable { + return typeof interactionType === "string" && EXPLICIT_CLICK_TYPES.has(interactionType); +} function normalizeTelemetrySample( sample: CursorTelemetryPoint, @@ -185,9 +200,7 @@ export function detectInteractionCandidates( samples: CursorTelemetryPoint[], ): CursorInteractionCandidate[] { // --- Phase 1: Explicit interaction events (from uiohook telemetry) --- - const clickEvents = samples.filter( - (s) => s.interactionType && s.interactionType !== "move" && s.interactionType !== "mouseup", - ); + const clickEvents = samples.filter((sample) => isExplicitClickType(sample.interactionType)); const explicitInteractionCandidates: CursorInteractionCandidate[] = []; @@ -211,6 +224,7 @@ export function detectInteractionCandidates( focus: { cx: clickSample.cx, cy: clickSample.cy }, strength: baseStrength, kind, + source: "explicit", }); } @@ -218,12 +232,12 @@ export function detectInteractionCandidates( const dwellCandidates = detectZoomDwellCandidates(samples).map( (candidate) => { if (candidate.strength >= 1100) { - return { ...candidate, kind: "text-focus-like" }; + return { ...candidate, kind: "text-focus-like", source: "heuristic" }; } if (candidate.strength <= 800) { - return { ...candidate, kind: "click-like" }; + return { ...candidate, kind: "click-like", source: "heuristic" }; } - return { ...candidate, kind: "dwell" }; + return { ...candidate, kind: "dwell", source: "heuristic" }; }, ); @@ -247,6 +261,7 @@ export function detectInteractionCandidates( }, strength: prev.strength + curr.strength + 500, kind: "double-click-like", + source: "heuristic", }); } } @@ -254,6 +269,73 @@ export function detectInteractionCandidates( return [...explicitInteractionCandidates, ...dwellCandidates, ...doubleClickCandidates]; } +/** + * Groups a sorted list of click timestamps into clusters where consecutive + * clicks are no more than `mergeGapMs` apart. Returns an array of + * `{ firstMs, lastMs, focus }` objects, one per cluster. The focus is taken + * from the click with the highest interaction strength, falling back to the + * centroid of all clicks in the cluster. + */ +function buildClickClusters( + clicks: CursorInteractionCandidate[], + mergeGapMs: number, +): Array<{ firstMs: number; lastMs: number; focus: ZoomFocus }> { + if (clicks.length === 0) { + return []; + } + + const sorted = [...clicks].sort((a, b) => a.centerTimeMs - b.centerTimeMs); + const clusters: Array<{ firstMs: number; lastMs: number; focus: ZoomFocus }> = []; + + let clusterStart = sorted[0].centerTimeMs; + let clusterEnd = sorted[0].centerTimeMs; + let bestStrength = sorted[0].strength; + let bestFocus = sorted[0].focus; + let sumCx = sorted[0].focus.cx; + let sumCy = sorted[0].focus.cy; + let count = 1; + + for (let i = 1; i < sorted.length; i++) { + const click = sorted[i]; + const gap = click.centerTimeMs - clusterEnd; + + if (gap <= mergeGapMs) { + // Extend current cluster + clusterEnd = Math.max(clusterEnd, click.centerTimeMs); + if (click.strength > bestStrength) { + bestStrength = click.strength; + bestFocus = click.focus; + } + sumCx += click.focus.cx; + sumCy += click.focus.cy; + count += 1; + } else { + // Flush current cluster and start a new one + clusters.push({ + firstMs: clusterStart, + lastMs: clusterEnd, + focus: bestFocus ?? { cx: sumCx / count, cy: sumCy / count }, + }); + clusterStart = click.centerTimeMs; + clusterEnd = click.centerTimeMs; + bestStrength = click.strength; + bestFocus = click.focus; + sumCx = click.focus.cx; + sumCy = click.focus.cy; + count = 1; + } + } + + // Flush last cluster + clusters.push({ + firstMs: clusterStart, + lastMs: clusterEnd, + focus: bestFocus ?? { cx: sumCx / count, cy: sumCy / count }, + }); + + return clusters; +} + export function buildInteractionZoomSuggestions(params: { cursorTelemetry: CursorTelemetryPoint[]; totalMs: number; @@ -261,82 +343,79 @@ export function buildInteractionZoomSuggestions(params: { reservedSpans?: Array<{ start: number; end: number }>; spacingMs?: number; mergeGapMs?: number; + padMs?: number; }): InteractionZoomSuggestionResult { const { cursorTelemetry, totalMs, - defaultDurationMs, reservedSpans = [], - spacingMs = DEFAULT_SUGGESTION_SPACING_MS, - mergeGapMs = DEFAULT_MERGE_NEARBY_GAP_MS, + mergeGapMs = CLICK_CLUSTER_MERGE_GAP_MS, + padMs = CLICK_CLUSTER_PAD_MS, } = params; - const defaultDuration = Math.min(defaultDurationMs, totalMs); - if (defaultDuration <= 0) { + if (totalMs <= 0) { return { status: "no-slots", suggestions: [] }; } const normalizedSamples = normalizeCursorTelemetry(cursorTelemetry, totalMs); - if (normalizedSamples.length < 2) { + if (normalizedSamples.length === 0) { return { status: "no-telemetry", suggestions: [] }; } - const interactionCandidates = detectInteractionCandidates(normalizedSamples); - if (interactionCandidates.length === 0) { + if ( + normalizedSamples.length === 1 && + !isExplicitClickType(normalizedSamples[0].interactionType) + ) { + return { status: "no-telemetry", suggestions: [] }; + } + + // Only use explicit click events (uiohook telemetry) – ignore dwell heuristics + const clickCandidates = detectInteractionCandidates(normalizedSamples).filter( + (candidate) => candidate.source === "explicit", + ); + + if (clickCandidates.length === 0) { return { status: "no-interactions", suggestions: [] }; } - const sortedCandidates = [...interactionCandidates].sort((a, b) => b.strength - a.strength); - const acceptedCenters: number[] = []; - const accepted: SuggestedZoomRegion[] = []; + // Group nearby clicks into clusters, then derive zoom windows from those clusters + const clusters = buildClickClusters(clickCandidates, mergeGapMs); + const reserved = [...reservedSpans].sort((a, b) => a.start - b.start); + const suggestions: SuggestedZoomRegion[] = []; - sortedCandidates.forEach((candidate) => { - const tooCloseToAccepted = acceptedCenters.some( - (center) => Math.abs(center - candidate.centerTimeMs) < spacingMs, - ); + for (const cluster of clusters) { + const regionStart = Math.max(0, cluster.firstMs - padMs); + const regionEnd = Math.min(totalMs, cluster.lastMs + padMs); - if (tooCloseToAccepted) { - return; - } - - const centeredStart = Math.round(candidate.centerTimeMs - defaultDuration / 2); - const candidateStart = Math.max(0, Math.min(centeredStart, totalMs - defaultDuration)); - const candidateEnd = candidateStart + defaultDuration; - const hasOverlap = reserved.some( - (span) => candidateEnd > span.start && candidateStart < span.end, - ); - - if (hasOverlap) { - return; - } - - reserved.push({ start: candidateStart, end: candidateEnd }); - acceptedCenters.push(candidate.centerTimeMs); - accepted.push({ - start: candidateStart, - end: candidateEnd, - focus: candidate.focus, - }); - }); - - const sortedAccepted = [...accepted].sort((a, b) => a.start - b.start); - const merged: SuggestedZoomRegion[] = []; - for (const region of sortedAccepted) { - const previous = merged[merged.length - 1]; - if (previous && region.start - previous.end <= mergeGapMs) { - previous.end = Math.max(previous.end, region.end); + if (regionEnd <= regionStart) { continue; } - merged.push({ ...region }); + const hasOverlap = reserved.some( + (span) => regionEnd > span.start && regionStart < span.end, + ); + + if (hasOverlap) { + continue; + } + + reserved.push({ start: regionStart, end: regionEnd }); + suggestions.push({ + start: regionStart, + end: regionEnd, + focus: cluster.focus, + }); } - if (merged.length === 0) { + if (suggestions.length === 0) { return { status: "no-slots", suggestions: [] }; } - return { status: "ok", suggestions: merged }; + // Sort chronologically + suggestions.sort((a, b) => a.start - b.start); + + return { status: "ok", suggestions }; } /**