Merge pull request #405 from webadderallorg/copilot/click-cluster-zoom

timeline: suggest zoom tracks from click clusters instead of dwell heuristics

src/components/video-editor/timeline/zoomSuggestionUtils.test.ts

@@ -0,0 +1,207 @@
+import { describe, expect, it } from "vitest";
+import {
+	CLICK_CLUSTER_MERGE_GAP_MS,
+	CLICK_CLUSTER_PAD_MS,
+	buildInteractionZoomSuggestions,
+} from "./zoomSuggestionUtils";
+import type { CursorTelemetryPoint } from "../types";
+
+function makeClick(
+	timeMs: number,
+	cx = 0.5,
+	cy = 0.5,
+	interactionType: CursorTelemetryPoint["interactionType"] = "click",
+): CursorTelemetryPoint {
+	return { timeMs, cx, cy, interactionType };
+}
+
+function makeMove(timeMs: number, cx = 0.5, cy = 0.5): CursorTelemetryPoint {
+	return { timeMs, cx, cy, interactionType: "move" };
+}
+
+/** Wraps click samples with surrounding move events to mimic real mixed telemetry. */
+function withMoves(
+	clicks: CursorTelemetryPoint[],
+	totalMs: number,
+): CursorTelemetryPoint[] {
+	return [
+		makeMove(0),
+		...clicks,
+		makeMove(totalMs),
+	];
+}
+
+const TOTAL_MS = 30_000;
+
+describe("buildInteractionZoomSuggestions (click-cluster logic)", () => {
+	it("creates one zoom track for a single isolated click with 500ms padding", () => {
+		const telemetry = withMoves([makeClick(5_000)], TOTAL_MS);
+
+		const result = buildInteractionZoomSuggestions({
+			cursorTelemetry: telemetry,
+			totalMs: TOTAL_MS,
+			defaultDurationMs: 3_000,
+		});
+
+		expect(result.status).toBe("ok");
+		expect(result.suggestions).toHaveLength(1);
+
+		const [s] = result.suggestions;
+		expect(s.start).toBe(5_000 - CLICK_CLUSTER_PAD_MS);
+		expect(s.end).toBe(5_000 + CLICK_CLUSTER_PAD_MS);
+	});
+
+	it("accepts a single explicit click sample without needing surrounding moves", () => {
+		const result = buildInteractionZoomSuggestions({
+			cursorTelemetry: [makeClick(5_000)],
+			totalMs: TOTAL_MS,
+			defaultDurationMs: 3_000,
+		});
+
+		expect(result.status).toBe("ok");
+		expect(result.suggestions).toHaveLength(1);
+	});
+
+	it.each(["right-click", "middle-click"] as const)(
+		"accepts %s telemetry like a standard click",
+		(interactionType) => {
+			const result = buildInteractionZoomSuggestions({
+				cursorTelemetry: withMoves([makeClick(5_000, 0.5, 0.5, interactionType)], TOTAL_MS),
+				totalMs: TOTAL_MS,
+				defaultDurationMs: 3_000,
+			});
+
+			expect(result.status).toBe("ok");
+			expect(result.suggestions).toHaveLength(1);
+
+			const [suggestion] = result.suggestions;
+			expect(suggestion.start).toBe(5_000 - CLICK_CLUSTER_PAD_MS);
+			expect(suggestion.end).toBe(5_000 + CLICK_CLUSTER_PAD_MS);
+		},
+	);
+
+	it("merges two clicks within 2500ms into one zoom track", () => {
+		const telemetry = withMoves(
+			[makeClick(4_000), makeClick(4_000 + CLICK_CLUSTER_MERGE_GAP_MS - 1)],
+			TOTAL_MS,
+		);
+
+		const result = buildInteractionZoomSuggestions({
+			cursorTelemetry: telemetry,
+			totalMs: TOTAL_MS,
+			defaultDurationMs: 3_000,
+		});
+
+		expect(result.status).toBe("ok");
+		expect(result.suggestions).toHaveLength(1);
+
+		const [s] = result.suggestions;
+		const lastClickMs = 4_000 + CLICK_CLUSTER_MERGE_GAP_MS - 1;
+		expect(s.start).toBe(4_000 - CLICK_CLUSTER_PAD_MS);
+		expect(s.end).toBe(lastClickMs + CLICK_CLUSTER_PAD_MS);
+	});
+
+	it("splits two clicks more than 2500ms apart into separate zoom tracks", () => {
+		const click1 = 3_000;
+		const click2 = 3_000 + CLICK_CLUSTER_MERGE_GAP_MS + 1; // just outside the merge gap
+
+		const telemetry = withMoves([makeClick(click1), makeClick(click2)], TOTAL_MS);
+
+		const result = buildInteractionZoomSuggestions({
+			cursorTelemetry: telemetry,
+			totalMs: TOTAL_MS,
+			defaultDurationMs: 3_000,
+		});
+
+		expect(result.status).toBe("ok");
+		expect(result.suggestions).toHaveLength(2);
+
+		const [a, b] = result.suggestions;
+		expect(a.start).toBe(click1 - CLICK_CLUSTER_PAD_MS);
+		expect(a.end).toBe(click1 + CLICK_CLUSTER_PAD_MS);
+		expect(b.start).toBe(click2 - CLICK_CLUSTER_PAD_MS);
+		expect(b.end).toBe(click2 + CLICK_CLUSTER_PAD_MS);
+	});
+
+	it("chains multiple clicks: 3 in a row within 2500ms each become one track", () => {
+		// click at 0, 2000, 4000 — each gap is 2000ms < 2500ms
+		const telemetry = withMoves([makeClick(0), makeClick(2_000), makeClick(4_000)], TOTAL_MS);
+
+		const result = buildInteractionZoomSuggestions({
+			cursorTelemetry: telemetry,
+			totalMs: TOTAL_MS,
+			defaultDurationMs: 3_000,
+		});
+
+		expect(result.status).toBe("ok");
+		expect(result.suggestions).toHaveLength(1);
+
+		const [s] = result.suggestions;
+		expect(s.start).toBe(0); // clamped to 0 (would be -500)
+		expect(s.end).toBe(4_000 + CLICK_CLUSTER_PAD_MS);
+	});
+
+	it("returns no-interactions when there are no click telemetry points", () => {
+		// Move events only — no clicks
+		const telemetry: CursorTelemetryPoint[] = [
+			{ timeMs: 0, cx: 0.5, cy: 0.5, interactionType: "move" },
+			{ timeMs: 1_000, cx: 0.5, cy: 0.5, interactionType: "move" },
+			{ timeMs: 2_000, cx: 0.6, cy: 0.6, interactionType: "move" },
+			{ timeMs: TOTAL_MS, cx: 0.6, cy: 0.6, interactionType: "move" },
+		];
+
+		const result = buildInteractionZoomSuggestions({
+			cursorTelemetry: telemetry,
+			totalMs: TOTAL_MS,
+			defaultDurationMs: 3_000,
+		});
+
+		expect(result.status).toBe("no-interactions");
+		expect(result.suggestions).toHaveLength(0);
+	});
+
+	it("ignores dwell-derived click-like heuristics when there are no explicit clicks", () => {
+		const telemetry: CursorTelemetryPoint[] = [
+			makeMove(0, 0.5, 0.5),
+			makeMove(200, 0.5005, 0.5005),
+			makeMove(400, 0.5008, 0.5008),
+			makeMove(600, 0.501, 0.501),
+		];
+
+		const result = buildInteractionZoomSuggestions({
+			cursorTelemetry: telemetry,
+			totalMs: TOTAL_MS,
+			defaultDurationMs: 3_000,
+		});
+
+		expect(result.status).toBe("no-interactions");
+		expect(result.suggestions).toHaveLength(0);
+	});
+
+	it("skips clusters that overlap reserved spans", () => {
+		const click = 5_000;
+
+		const result = buildInteractionZoomSuggestions({
+			cursorTelemetry: withMoves([makeClick(click)], TOTAL_MS),
+			totalMs: TOTAL_MS,
+			defaultDurationMs: 3_000,
+			reservedSpans: [{ start: 4_000, end: 6_000 }], // overlaps the cluster window
+		});
+
+		expect(result.status).toBe("no-slots");
+		expect(result.suggestions).toHaveLength(0);
+	});
+
+	it("clamps start to 0 and end to totalMs at video boundaries", () => {
+		const result = buildInteractionZoomSuggestions({
+			cursorTelemetry: withMoves([makeClick(200)], 1_000),
+			totalMs: 1_000,
+			defaultDurationMs: 3_000,
+		});
+
+		expect(result.status).toBe("ok");
+		const [s] = result.suggestions;
+		expect(s.start).toBeGreaterThanOrEqual(0);
+		expect(s.end).toBeLessThanOrEqual(1_000);
+	});
+});

src/components/video-editor/timeline/zoomSuggestionUtils.ts

@@ -19,6 +19,7 @@ export interface CursorInteractionCandidate extends ZoomDwellCandidate {
 		| "dropdown-open"
 		| "text-selection"
 		| "text-field-click";
+	source: "explicit" | "heuristic";
 }
 
 export interface SuggestedZoomRegion {
@@ -38,8 +39,22 @@ export interface InteractionZoomSuggestionResult {
 	suggestions: SuggestedZoomRegion[];
 }
 
-const DEFAULT_SUGGESTION_SPACING_MS = 1800;
-const DEFAULT_MERGE_NEARBY_GAP_MS = 1500;
+/** Max gap between consecutive clicks before they are split into separate zoom clusters. */
+export const CLICK_CLUSTER_MERGE_GAP_MS = 2500;
+/** Padding added before the first click and after the last click in a cluster. */
+export const CLICK_CLUSTER_PAD_MS = 500;
+const EXPLICIT_CLICK_TYPES = new Set<NonNullable<CursorTelemetryPoint["interactionType"]>>([
+	"click",
+	"double-click",
+	"right-click",
+	"middle-click",
+]);
+
+function isExplicitClickType(
+	interactionType: CursorTelemetryPoint["interactionType"],
+): interactionType is NonNullable<CursorTelemetryPoint["interactionType"]> {
+	return typeof interactionType === "string" && EXPLICIT_CLICK_TYPES.has(interactionType);
+}
 
 function normalizeTelemetrySample(
 	sample: CursorTelemetryPoint,
@@ -185,9 +200,7 @@ export function detectInteractionCandidates(
 	samples: CursorTelemetryPoint[],
 ): CursorInteractionCandidate[] {
 	// --- Phase 1: Explicit interaction events (from uiohook telemetry) ---
-	const clickEvents = samples.filter(
-		(s) => s.interactionType && s.interactionType !== "move" && s.interactionType !== "mouseup",
-	);
+	const clickEvents = samples.filter((sample) => isExplicitClickType(sample.interactionType));
 
 	const explicitInteractionCandidates: CursorInteractionCandidate[] = [];
 
@@ -211,6 +224,7 @@ export function detectInteractionCandidates(
 			focus: { cx: clickSample.cx, cy: clickSample.cy },
 			strength: baseStrength,
 			kind,
+			source: "explicit",
 		});
 	}
 
@@ -218,12 +232,12 @@ export function detectInteractionCandidates(
 	const dwellCandidates = detectZoomDwellCandidates(samples).map<CursorInteractionCandidate>(
 		(candidate) => {
 			if (candidate.strength >= 1100) {
-				return { ...candidate, kind: "text-focus-like" };
+				return { ...candidate, kind: "text-focus-like", source: "heuristic" };
 			}
 			if (candidate.strength <= 800) {
-				return { ...candidate, kind: "click-like" };
+				return { ...candidate, kind: "click-like", source: "heuristic" };
 			}
-			return { ...candidate, kind: "dwell" };
+			return { ...candidate, kind: "dwell", source: "heuristic" };
 		},
 	);
 
@@ -247,6 +261,7 @@ export function detectInteractionCandidates(
 				},
 				strength: prev.strength + curr.strength + 500,
 				kind: "double-click-like",
+				source: "heuristic",
 			});
 		}
 	}
@@ -254,6 +269,73 @@ export function detectInteractionCandidates(
 	return [...explicitInteractionCandidates, ...dwellCandidates, ...doubleClickCandidates];
 }
 
+/**
+ * Groups a sorted list of click timestamps into clusters where consecutive
+ * clicks are no more than `mergeGapMs` apart. Returns an array of
+ * `{ firstMs, lastMs, focus }` objects, one per cluster.  The focus is taken
+ * from the click with the highest interaction strength, falling back to the
+ * centroid of all clicks in the cluster.
+ */
+function buildClickClusters(
+	clicks: CursorInteractionCandidate[],
+	mergeGapMs: number,
+): Array<{ firstMs: number; lastMs: number; focus: ZoomFocus }> {
+	if (clicks.length === 0) {
+		return [];
+	}
+
+	const sorted = [...clicks].sort((a, b) => a.centerTimeMs - b.centerTimeMs);
+	const clusters: Array<{ firstMs: number; lastMs: number; focus: ZoomFocus }> = [];
+
+	let clusterStart = sorted[0].centerTimeMs;
+	let clusterEnd = sorted[0].centerTimeMs;
+	let bestStrength = sorted[0].strength;
+	let bestFocus = sorted[0].focus;
+	let sumCx = sorted[0].focus.cx;
+	let sumCy = sorted[0].focus.cy;
+	let count = 1;
+
+	for (let i = 1; i < sorted.length; i++) {
+		const click = sorted[i];
+		const gap = click.centerTimeMs - clusterEnd;
+
+		if (gap <= mergeGapMs) {
+			// Extend current cluster
+			clusterEnd = Math.max(clusterEnd, click.centerTimeMs);
+			if (click.strength > bestStrength) {
+				bestStrength = click.strength;
+				bestFocus = click.focus;
+			}
+			sumCx += click.focus.cx;
+			sumCy += click.focus.cy;
+			count += 1;
+		} else {
+			// Flush current cluster and start a new one
+			clusters.push({
+				firstMs: clusterStart,
+				lastMs: clusterEnd,
+				focus: bestFocus ?? { cx: sumCx / count, cy: sumCy / count },
+			});
+			clusterStart = click.centerTimeMs;
+			clusterEnd = click.centerTimeMs;
+			bestStrength = click.strength;
+			bestFocus = click.focus;
+			sumCx = click.focus.cx;
+			sumCy = click.focus.cy;
+			count = 1;
+		}
+	}
+
+	// Flush last cluster
+	clusters.push({
+		firstMs: clusterStart,
+		lastMs: clusterEnd,
+		focus: bestFocus ?? { cx: sumCx / count, cy: sumCy / count },
+	});
+
+	return clusters;
+}
+
 export function buildInteractionZoomSuggestions(params: {
 	cursorTelemetry: CursorTelemetryPoint[];
 	totalMs: number;
@@ -261,82 +343,79 @@ export function buildInteractionZoomSuggestions(params: {
 	reservedSpans?: Array<{ start: number; end: number }>;
 	spacingMs?: number;
 	mergeGapMs?: number;
+	padMs?: number;
 }): InteractionZoomSuggestionResult {
 	const {
 		cursorTelemetry,
 		totalMs,
-		defaultDurationMs,
 		reservedSpans = [],
-		spacingMs = DEFAULT_SUGGESTION_SPACING_MS,
-		mergeGapMs = DEFAULT_MERGE_NEARBY_GAP_MS,
+		mergeGapMs = CLICK_CLUSTER_MERGE_GAP_MS,
+		padMs = CLICK_CLUSTER_PAD_MS,
 	} = params;
 
-	const defaultDuration = Math.min(defaultDurationMs, totalMs);
-	if (defaultDuration <= 0) {
+	if (totalMs <= 0) {
 		return { status: "no-slots", suggestions: [] };
 	}
 
 	const normalizedSamples = normalizeCursorTelemetry(cursorTelemetry, totalMs);
-	if (normalizedSamples.length < 2) {
+	if (normalizedSamples.length === 0) {
 		return { status: "no-telemetry", suggestions: [] };
 	}
 
-	const interactionCandidates = detectInteractionCandidates(normalizedSamples);
-	if (interactionCandidates.length === 0) {
+	if (
+		normalizedSamples.length === 1 &&
+		!isExplicitClickType(normalizedSamples[0].interactionType)
+	) {
+		return { status: "no-telemetry", suggestions: [] };
+	}
+
+	// Only use explicit click events (uiohook telemetry) – ignore dwell heuristics
+	const clickCandidates = detectInteractionCandidates(normalizedSamples).filter(
+		(candidate) => candidate.source === "explicit",
+	);
+
+	if (clickCandidates.length === 0) {
 		return { status: "no-interactions", suggestions: [] };
 	}
 
-	const sortedCandidates = [...interactionCandidates].sort((a, b) => b.strength - a.strength);
-	const acceptedCenters: number[] = [];
-	const accepted: SuggestedZoomRegion[] = [];
+	// Group nearby clicks into clusters, then derive zoom windows from those clusters
+	const clusters = buildClickClusters(clickCandidates, mergeGapMs);
+
 	const reserved = [...reservedSpans].sort((a, b) => a.start - b.start);
+	const suggestions: SuggestedZoomRegion[] = [];
 
-	sortedCandidates.forEach((candidate) => {
-		const tooCloseToAccepted = acceptedCenters.some(
-			(center) => Math.abs(center - candidate.centerTimeMs) < spacingMs,
-		);
+	for (const cluster of clusters) {
+		const regionStart = Math.max(0, cluster.firstMs - padMs);
+		const regionEnd = Math.min(totalMs, cluster.lastMs + padMs);
 
-		if (tooCloseToAccepted) {
-			return;
-		}
-
-		const centeredStart = Math.round(candidate.centerTimeMs - defaultDuration / 2);
-		const candidateStart = Math.max(0, Math.min(centeredStart, totalMs - defaultDuration));
-		const candidateEnd = candidateStart + defaultDuration;
-		const hasOverlap = reserved.some(
-			(span) => candidateEnd > span.start && candidateStart < span.end,
-		);
-
-		if (hasOverlap) {
-			return;
-		}
-
-		reserved.push({ start: candidateStart, end: candidateEnd });
-		acceptedCenters.push(candidate.centerTimeMs);
-		accepted.push({
-			start: candidateStart,
-			end: candidateEnd,
-			focus: candidate.focus,
-		});
-	});
-
-	const sortedAccepted = [...accepted].sort((a, b) => a.start - b.start);
-	const merged: SuggestedZoomRegion[] = [];
-	for (const region of sortedAccepted) {
-		const previous = merged[merged.length - 1];
-		if (previous && region.start - previous.end <= mergeGapMs) {
-			previous.end = Math.max(previous.end, region.end);
+		if (regionEnd <= regionStart) {
 			continue;
 		}
 
-		merged.push({ ...region });
+		const hasOverlap = reserved.some(
+			(span) => regionEnd > span.start && regionStart < span.end,
+		);
+
+		if (hasOverlap) {
+			continue;
+		}
+
+		reserved.push({ start: regionStart, end: regionEnd });
+		suggestions.push({
+			start: regionStart,
+			end: regionEnd,
+			focus: cluster.focus,
+		});
 	}
 
-	if (merged.length === 0) {
+	if (suggestions.length === 0) {
 		return { status: "no-slots", suggestions: [] };
 	}
 
-	return { status: "ok", suggestions: merged };
+	// Sort chronologically
+	suggestions.sort((a, b) => a.start - b.start);
+
+	return { status: "ok", suggestions };
 }
 
 /**