diff --git a/graphify/cli.py b/graphify/cli.py index 3c1ae0e5..6d037332 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -96,6 +96,153 @@ def _stamped_manifest_files( ] for ftype, flist in files_by_type.items() } + + +def _stale_graph_sources( + graph_path: Path, + scan_root: Path, + seen_files: set[str], +) -> list[str]: + """Source files graph.json still references but the current scan no longer + contains (#1909). + + Incremental extract's prune set was historically derived from the manifest + alone (``manifest - corpus``), so a file that became EXCLUDED + (.graphifyignore/.gitignore/--exclude changed) without being listed in the + manifest kept its stale nodes in graph.json forever. Derive prune + candidates from the graph's own node ``source_file``s instead: anything + the graph references that the post-exclude detect corpus no longer + contains is stale, whether the file was deleted or newly excluded. + + Only IN-ROOT paths are candidates: out-of-root/absolute entries + (--include sources, symlinked external corpora) are never walked by + detect, so their absence from the corpus is not staleness evidence. + Relative entries are re-anchored against both the scan root and the + graph's own output root (``--out`` extracts store source_files relative + to the OUT root, e.g. ``../project/x.py``, #555/#1899); only anchors + that land inside the scan root count. + ``seen_files`` must be the FULL detect output including unclassified + files, so nodes from walked-but-unsupported sources (e.g. introspected + Cargo.toml manifests) are not misread as stale. + """ + try: + data = json.loads(graph_path.read_text(encoding="utf-8")) + except Exception: + return [] + if not isinstance(data, dict): + return [] + try: + root_res = scan_root.resolve() + except (OSError, RuntimeError): + root_res = scan_root + # /graphify-out/graph.json — relative source_files may be anchored here. + out_base = graph_path.parent.parent + try: + out_base = out_base.resolve() + except (OSError, RuntimeError): + pass + + def _within_root(p: Path) -> bool: + try: + p.relative_to(root_res) + return True + except ValueError: + pass + try: + p.resolve().relative_to(root_res) + return True + except (ValueError, OSError, RuntimeError): + return False + + def _in_seen(p: Path) -> bool: + if str(p) in seen_files: + return True + try: + return str(p.resolve()) in seen_files + except (OSError, RuntimeError): + return False + + stale: list[str] = [] + checked: set[str] = set() + for n in data.get("nodes", []): + if not isinstance(n, dict): + continue + sf = n.get("source_file") + if not sf or not isinstance(sf, str) or sf in checked: + continue + checked.add(sf) + if "://" in sf: + continue # remote/virtual source (e.g. Google Workspace), not a scanned path + p = Path(sf) + if p.is_absolute(): + candidates = [p] + else: + rel = sf.replace("\\", "/") + bases = [root_res] + if out_base != root_res: + bases.append(out_base) + candidates = [ + Path(os.path.normpath(str(base / rel))) for base in bases + ] + in_root = [c for c in candidates if _within_root(c)] + if not in_root: + continue # out-of-root under every anchor: never prune + if any(_in_seen(c) for c in in_root): + continue # still part of the scan corpus + stale.append(sf) + return stale + + +def _prune_graph_json_sources(graph_path: Path, stale_sources: list[str]) -> int: + """Drop nodes/edges/hyperedges owned by ``stale_sources`` from graph.json + in place. Returns the number of nodes removed. + + Used by the ``--no-cluster`` incremental early-exit: that path never runs + ``build_merge`` (it would raw-dump only the new chunks), so an + exclusion-only change must prune the existing raw graph directly or the + newly-excluded file's nodes survive forever (#1909). + ``stale_sources`` comes from :func:`_stale_graph_sources`, i.e. the + graph's own ``source_file`` spellings, so exact string matching is enough. + """ + try: + data = json.loads(graph_path.read_text(encoding="utf-8")) + except Exception: + return 0 + if not isinstance(data, dict): + return 0 + stale = set(stale_sources) + links_key = "links" if "links" in data else "edges" + nodes = [n for n in data.get("nodes", []) if isinstance(n, dict)] + kept_nodes = [n for n in nodes if n.get("source_file") not in stale] + removed_ids = { + n.get("id") for n in nodes if n.get("source_file") in stale + } + n_removed = len(nodes) - len(kept_nodes) + kept_edges = [ + e for e in data.get(links_key, []) + if isinstance(e, dict) + and e.get("source_file") not in stale + and e.get("source") not in removed_ids + and e.get("target") not in removed_ids + ] + kept_hyper = [ + h for h in data.get("hyperedges", []) + if isinstance(h, dict) and h.get("source_file") not in stale + ] + if n_removed == 0 and len(kept_edges) == len(data.get(links_key, [])) and ( + len(kept_hyper) == len(data.get("hyperedges", [])) + ): + return 0 + data["nodes"] = kept_nodes + data[links_key] = kept_edges + if "hyperedges" in data: + data["hyperedges"] = kept_hyper + from graphify.export import backup_if_protected as _backup + _backup(graph_path.parent) + graph_path.write_text(json.dumps(data, indent=2), encoding="utf-8") + return n_removed + + class _StageTimer: """Print per-stage wall-clock timings to stderr when --timing is set (#1490). @@ -2137,6 +2284,8 @@ def dispatch_command(cmd: str) -> None: paper_files = [] image_files = [] deleted_files = [] + excluded_files = [] + graph_stale_sources = [] unchanged_total = 0 files_by_type = {} elif incremental_mode: @@ -2154,7 +2303,18 @@ def dispatch_command(cmd: str) -> None: paper_files = [Path(p) for p in new_by_type.get("paper", [])] image_files = [Path(p) for p in new_by_type.get("image", [])] deleted_files = list(detection.get("deleted_files", [])) + excluded_files = list(detection.get("excluded_files", [])) unchanged_total = sum(len(v) for v in detection.get("unchanged_files", {}).values()) + # #1909: derive the prune set from the existing graph itself, not + # just the manifest. A file that became excluded without ever + # being manifest-listed (every pre-#1897 graph is in this state) + # still has stale nodes carried forward by build_merge unless the + # graph's own sources are reconciled against the current corpus. + _seen_files = {f for _fl in files_by_type.values() for f in _fl} + _seen_files.update(detection.get("unclassified", [])) + graph_stale_sources = _stale_graph_sources( + existing_graph_path, target, _seen_files + ) else: print(f"[graphify extract] scanning {target}") detection = _detect(target, google_workspace=google_workspace or None, extra_excludes=cli_excludes or None, cache_root=out_root) @@ -2164,6 +2324,8 @@ def dispatch_command(cmd: str) -> None: paper_files = [Path(p) for p in files_by_type.get("paper", [])] image_files = [Path(p) for p in files_by_type.get("image", [])] deleted_files = [] + excluded_files = [] + graph_stale_sources = [] unchanged_total = 0 semantic_files = doc_files + paper_files + image_files @@ -2182,10 +2344,15 @@ def dispatch_command(cmd: str) -> None: paper_files = [] image_files = [] if incremental_mode: + # Excluded-but-alive files are reported separately from deletions + # (#1908): they still exist on disk, the scan just stopped + # covering them (ignore rules / --exclude changed). + _excl_note = f"; {len(excluded_files)} excluded" if excluded_files else "" print( f"[graphify extract] {len(code_files)} code, {len(doc_files)} docs, " f"{len(paper_files)} papers, {len(image_files)} images changed; " f"{unchanged_total} unchanged; {len(deleted_files)} deleted" + f"{_excl_note}" ) else: print( @@ -2487,6 +2654,16 @@ def dispatch_command(cmd: str) -> None: # absolute file lists. _manifest_files = _stamped_manifest_files(files_by_type, sem_result, target) + # Full-scan manifest saves prune rows for in-root files that left the + # scan corpus but still exist on disk (#1908). The corpus must be the + # RAW detect output (files_by_type), NOT the #933-stamp-filtered + # _manifest_files above — pruning to the filtered set would erase + # failed-chunk/omitted-doc rows and every doc row on --code-only runs. + _scan_corpus = ( + {f for _fl in files_by_type.values() for f in _fl} + if has_path else None + ) + if no_cluster: # --no-cluster: dump the raw merged extraction as graph.json. # No NetworkX, no community detection, no analysis sidecar. @@ -2506,12 +2683,26 @@ def dispatch_command(cmd: str) -> None: and not cargo_result.get("nodes") and not cargo_result.get("edges") ): + # An exclusion-only change reaches this gate (excluded files + # are deliberately NOT in deleted_files, #1908) but must still + # scrub the newly-excluded sources from the raw graph (#1909). + # This path never runs build_merge, so prune in place. + if graph_stale_sources: + _n_pruned = _prune_graph_json_sources( + existing_graph_path, graph_stale_sources + ) + if _n_pruned: + print( + f"[graphify extract] pruned {_n_pruned} node(s) from " + f"{len(graph_stale_sources)} source file(s) no longer " + "in the scan (deleted or excluded)." + ) print( "[graphify extract] no incremental changes detected " "(--no-cluster); outputs left untouched." ) try: - _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target) + _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target, scan_corpus=_scan_corpus) except Exception as exc: print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) stages.total() @@ -2548,7 +2739,7 @@ def dispatch_command(cmd: str) -> None: f"est. cost: ${cost:.4f}" ) try: - _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target) + _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target, scan_corpus=_scan_corpus) except Exception as exc: print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) if global_merge: @@ -2577,10 +2768,18 @@ def dispatch_command(cmd: str) -> None: from graphify.analyze import god_nodes as _god_nodes, surprising_connections as _surprising dedup_backend = backend if dedup_llm else None if incremental_mode: + # Prune everything the current scan no longer covers: genuinely + # deleted manifest rows, excluded-but-alive manifest rows (#1908), + # and the graph's own stale sources — which catches files that + # became excluded without ever being manifest-listed (#1909). + _prune_sources: list[str] = list(deleted_files) + for _src in list(excluded_files) + graph_stale_sources: + if _src not in _prune_sources: + _prune_sources.append(_src) G = _build_merge( [merged], graph_path=existing_graph_path, - prune_sources=deleted_files or None, + prune_sources=_prune_sources or None, dedup=True, dedup_llm_backend=dedup_backend, root=target, @@ -2642,7 +2841,7 @@ def dispatch_command(cmd: str) -> None: } analysis_path.write_text(json.dumps(analysis, indent=2), encoding="utf-8") try: - _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target) + _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target, scan_corpus=_scan_corpus) except Exception as exc: print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) @@ -2654,11 +2853,12 @@ def dispatch_command(cmd: str) -> None: ) print(f"[graphify extract] wrote {analysis_path}") if incremental_mode: + _excl_note = f", {len(excluded_files)} excluded" if excluded_files else "" print( f"[graphify extract] incremental summary: " f"{sem_cache_hits + unchanged_total} files cached/unchanged, " f"{len(code_files) + sem_cache_misses} re-extracted, " - f"{len(deleted_files)} deleted" + f"{len(deleted_files)} deleted{_excl_note}" ) elif sem_cache_hits: print(f"[graphify extract] semantic cache: {sem_cache_hits} cached, {sem_cache_misses} re-extracted") diff --git a/graphify/detect.py b/graphify/detect.py index bc73fbde..195b5e54 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -1466,6 +1466,7 @@ def save_manifest( *, kind: str = "both", root: Path | None = None, + scan_corpus: set[str] | list[str] | None = None, ) -> None: """Save current file mtimes + content hashes for change detection. @@ -1481,9 +1482,51 @@ def save_manifest( machines and checkout locations (#777). Out-of-root entries are written as absolute so they continue to round-trip on the saving machine. When ``root`` is None the legacy absolute-keyed format is preserved. + + ``scan_corpus`` (#1908): full-scan callers pass the COMPLETE detect + corpus (absolute paths) so seeded rows for in-root files that are still + alive on disk but no longer part of the scan (newly excluded via + .graphifyignore/.gitignore/--exclude) are dropped instead of surviving + forever and masquerading as deletions in detect_incremental. It must be + the RAW detect output, not a stamp-filtered subset — pruning to a + filtered set would erase rows the filter merely omitted (failed chunks, + --code-only doc rows). Out-of-root entries are never pruned. Callers + saving a SUBSET of files (changed_paths hooks, skill runbooks, #917) + must leave this None so their untouched rows are preserved. """ existing = load_manifest(manifest_path, root=root) + scan_set: set[str] | None = set(scan_corpus) if scan_corpus is not None else None + try: + root_res: Path | None = Path(root).resolve() if root is not None else None + except (OSError, RuntimeError): + root_res = Path(root) if root is not None else None + + def _in_scan(path_str: str) -> bool: + if path_str in scan_set: + return True + try: + return str(Path(path_str).resolve()) in scan_set + except (OSError, RuntimeError): + return False + + def _in_root(path_str: str) -> bool: + # Without a root we cannot tell in-root from out-of-root; fail open + # (keep the row) so out-of-root corpora are never pruned by accident. + if root_res is None: + return False + p = Path(path_str) + try: + p.relative_to(root_res) + return True + except ValueError: + pass + try: + p.resolve().relative_to(root_res) + return True + except (ValueError, OSError, RuntimeError): + return False + def _normalise_entry(entry): if isinstance(entry, (int, float)): return {"mtime": entry, "ast_hash": "", "semantic_hash": ""} @@ -1496,17 +1539,23 @@ def save_manifest( # Seed from the existing manifest so incremental callers passing a subset # of files don't silently erase entries for untouched files (#917). # Prune entries whose file no longer exists on disk — those are genuine - # deletions that detect_incremental() should treat as gone. + # deletions that detect_incremental() should treat as gone. When the + # caller supplied the full scan corpus, additionally prune in-root rows + # the scan no longer covers: those files were excluded, not deleted, and + # keeping the row makes them look deleted on every future run (#1908). manifest: dict[str, dict] = {} for f, entry in existing.items(): normalised = _normalise_entry(entry) if normalised is None: continue try: - if Path(f).exists(): - manifest[f] = normalised + if not Path(f).exists(): + continue except OSError: continue + if scan_set is not None and not _in_scan(f) and _in_root(f): + continue # excluded-but-alive: drop the stale row (#1908) + manifest[f] = normalised all_files = [f for file_list in files.values() for f in file_list] with ThreadPoolExecutor() as pool: @@ -1584,6 +1633,8 @@ def detect_incremental( full["new_files"] = full["files"] full["unchanged_files"] = {k: [] for k in full["files"]} full["new_total"] = full["total_files"] + full["deleted_files"] = [] + full["excluded_files"] = [] return full new_files: dict[str, list[str]] = {k: [] for k in full["files"]} @@ -1636,9 +1687,23 @@ def detect_incremental( else: unchanged_files[ftype].append(f) - # Files in manifest that no longer exist - their cached nodes are now ghost nodes + # Manifest rows that left the corpus, split by disk existence (#1908): + # a row whose file is gone from DISK is a genuine deletion (its cached + # nodes are ghosts); a row whose file still exists but is out of the + # current scan was EXCLUDED (ignore rules / --exclude changed) and must + # not be reported as deleted. Mirrors the watch-side excluded-vs-deleted + # distinction (#1795). current_files = {f for flist in full["files"].values() for f in flist} - deleted_files = [f for f in manifest if f not in current_files] + deleted_files: list[str] = [] + excluded_files: list[str] = [] + for f in manifest: + if f in current_files: + continue + try: + alive = Path(f).exists() + except OSError: + alive = False + (excluded_files if alive else deleted_files).append(f) new_total = sum(len(v) for v in new_files.values()) full["incremental"] = True @@ -1646,4 +1711,5 @@ def detect_incremental( full["unchanged_files"] = unchanged_files full["new_total"] = new_total full["deleted_files"] = deleted_files + full["excluded_files"] = excluded_files return full diff --git a/graphify/watch.py b/graphify/watch.py index df911ef8..57364228 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -1021,7 +1021,14 @@ def _rebuild_code( try: from graphify.detect import save_manifest - save_manifest(detected["files"], kind="ast", root=project_root) + # detected["files"] is a FULL detect of the watched root, so + # pass it as the scan corpus too: rows for files that left the + # scan but still exist on disk (newly excluded) are pruned + # instead of surviving as phantom "deleted" entries (#1908). + save_manifest( + detected["files"], kind="ast", root=project_root, + scan_corpus={f for _fl in detected["files"].values() for f in _fl}, + ) except Exception: pass @@ -1060,7 +1067,11 @@ def _rebuild_code( if same_topology: try: from graphify.detect import save_manifest - save_manifest(detected["files"], kind="ast", root=project_root) + # Full-scan save: prune excluded-but-alive rows (#1908). + save_manifest( + detected["files"], kind="ast", root=project_root, + scan_corpus={f for _fl in detected["files"].values() for f in _fl}, + ) except Exception: pass flag = out / "needs_update" @@ -1138,7 +1149,11 @@ def _rebuild_code( try: from graphify.detect import save_manifest - save_manifest(detected["files"], kind="ast", root=project_root) + # Full-scan save: prune excluded-but-alive rows (#1908). + save_manifest( + detected["files"], kind="ast", root=project_root, + scan_corpus={f for _fl in detected["files"].values() for f in _fl}, + ) except Exception: pass diff --git a/tests/test_detect.py b/tests/test_detect.py index 374b2552..a783eaf6 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -1851,3 +1851,162 @@ def test_nested_gitignore_patterns_still_apply_inside_their_dir(tmp_path): result = detect(tmp_path) assert result["total_files"] == 2 # main.py + sub/keep.py; sub/noise.log ignored + + +# --------------------------------------------------------------------------- +# #1908: manifest must not retain scan-excluded files as permanent +# "deleted" entries. Full-scan saves prune excluded-but-alive rows; subset +# saves keep preserving untouched rows (#917); out-of-root rows never prune. +# --------------------------------------------------------------------------- + +def test_save_manifest_full_scan_prunes_excluded_but_alive_row(tmp_path): + """A row for a file that still exists on disk but left the scan corpus + (newly excluded) is dropped when the caller passes the full corpus.""" + import json + a = tmp_path / "a.py" + b = tmp_path / "b.py" + a.write_text("x = 1\n") + b.write_text("y = 2\n") + manifest_path = str(tmp_path / "graphify-out" / "manifest.json") + + save_manifest({"code": [str(a), str(b)]}, manifest_path, root=tmp_path) + raw = json.loads(Path(manifest_path).read_text(encoding="utf-8")) + assert set(raw) == {"a.py", "b.py"} + + # Second full scan no longer covers b.py (excluded), yet b.py is alive. + save_manifest( + {"code": [str(a)]}, manifest_path, root=tmp_path, + scan_corpus={str(a)}, + ) + raw = json.loads(Path(manifest_path).read_text(encoding="utf-8")) + assert set(raw) == {"a.py"}, ( + f"excluded-but-alive row must be pruned on a full-scan save, got {set(raw)}" + ) + + +def test_save_manifest_full_scan_still_prunes_missing_file(tmp_path): + """Genuine deletions keep being pruned when scan_corpus is passed.""" + import json + a = tmp_path / "a.py" + gone = tmp_path / "gone.py" + a.write_text("x = 1\n") + gone.write_text("y = 2\n") + manifest_path = str(tmp_path / "graphify-out" / "manifest.json") + save_manifest({"code": [str(a), str(gone)]}, manifest_path, root=tmp_path) + + gone.unlink() + save_manifest( + {"code": [str(a)]}, manifest_path, root=tmp_path, + scan_corpus={str(a)}, + ) + raw = json.loads(Path(manifest_path).read_text(encoding="utf-8")) + assert set(raw) == {"a.py"} + + +def test_save_manifest_subset_save_preserves_untouched_rows(tmp_path): + """Without scan_corpus (changed_paths hooks, skill runbooks, #917) a + subset save must keep seeding rows for files it wasn't given.""" + import json + a = tmp_path / "a.py" + b = tmp_path / "b.py" + a.write_text("x = 1\n") + b.write_text("y = 2\n") + manifest_path = str(tmp_path / "graphify-out" / "manifest.json") + save_manifest({"code": [str(a), str(b)]}, manifest_path, root=tmp_path) + + # Incremental hook re-stamps only a.py; b.py's row must survive. + save_manifest({"code": [str(a)]}, manifest_path, root=tmp_path) + raw = json.loads(Path(manifest_path).read_text(encoding="utf-8")) + assert set(raw) == {"a.py", "b.py"}, ( + f"subset saves must preserve untouched rows (#917), got {set(raw)}" + ) + + +def test_save_manifest_full_scan_keeps_out_of_root_rows(tmp_path): + """Out-of-root entries (--include sources, symlinked corpora) are never + walked by detect, so their absence from the corpus is not exclusion + evidence — a full-scan save must keep them.""" + import json + a = tmp_path / "a.py" + a.write_text("x = 1\n") + outside = tmp_path.parent / f"{tmp_path.name}-extern.py" + outside.write_text("z = 3\n") + try: + manifest_path = str(tmp_path / "graphify-out" / "manifest.json") + save_manifest( + {"code": [str(a), str(outside)]}, manifest_path, root=tmp_path + ) + save_manifest( + {"code": [str(a)]}, manifest_path, root=tmp_path, + scan_corpus={str(a)}, + ) + raw = json.loads(Path(manifest_path).read_text(encoding="utf-8")) + assert "a.py" in raw + assert str(outside.resolve()) in raw, ( + f"out-of-root rows must never be pruned to the scan, got {set(raw)}" + ) + finally: + outside.unlink(missing_ok=True) + + +def test_detect_incremental_reports_excluded_not_deleted(tmp_path): + """A previously-indexed file that becomes excluded (still on disk) must + land in excluded_files, not deleted_files (#1908).""" + a = tmp_path / "a.py" + b = tmp_path / "b.py" + a.write_text("x = 1\n") + b.write_text("y = 2\n") + manifest_path = str(tmp_path / "graphify-out" / "manifest.json") + full = detect(tmp_path) + save_manifest(full["files"], manifest_path, root=tmp_path) + + inc = detect_incremental( + tmp_path, manifest_path, extra_excludes=["b.py"] + ) + assert inc["deleted_files"] == [], ( + f"excluded-but-alive file misreported as deleted: {inc['deleted_files']}" + ) + assert [Path(f).name for f in inc["excluded_files"]] == ["b.py"] + + +def test_detect_incremental_still_reports_real_deletions(tmp_path): + """Counterpart: a manifest row whose file is gone from disk stays in + deleted_files.""" + a = tmp_path / "a.py" + b = tmp_path / "b.py" + a.write_text("x = 1\n") + b.write_text("y = 2\n") + manifest_path = str(tmp_path / "graphify-out" / "manifest.json") + full = detect(tmp_path) + save_manifest(full["files"], manifest_path, root=tmp_path) + + b.unlink() + inc = detect_incremental(tmp_path, manifest_path) + assert [Path(f).name for f in inc["deleted_files"]] == ["b.py"] + assert inc["excluded_files"] == [] + + +def test_detect_incremental_exclusion_stable_across_runs(tmp_path): + """After a full-scan save prunes the excluded row, later incremental runs + report the file neither as deleted nor as excluded — the exclusion has + fully settled instead of resurfacing forever.""" + a = tmp_path / "a.py" + b = tmp_path / "b.py" + a.write_text("x = 1\n") + b.write_text("y = 2\n") + manifest_path = str(tmp_path / "graphify-out" / "manifest.json") + full = detect(tmp_path) + save_manifest(full["files"], manifest_path, root=tmp_path) + + # Run 1: b.py newly excluded — reported as excluded, then the full-scan + # save (what extract does at the end of the run) prunes its row. + inc1 = detect_incremental(tmp_path, manifest_path, extra_excludes=["b.py"]) + assert [Path(f).name for f in inc1["excluded_files"]] == ["b.py"] + assert inc1["deleted_files"] == [] + corpus = {f for flist in inc1["files"].values() for f in flist} + save_manifest(inc1["files"], manifest_path, root=tmp_path, scan_corpus=corpus) + + # Run 2 (and beyond): steady state — nothing deleted, nothing excluded. + inc2 = detect_incremental(tmp_path, manifest_path, extra_excludes=["b.py"]) + assert inc2["deleted_files"] == [] + assert inc2["excluded_files"] == [] diff --git a/tests/test_extract_cli.py b/tests/test_extract_cli.py index f56de047..3442a487 100644 --- a/tests/test_extract_cli.py +++ b/tests/test_extract_cli.py @@ -364,3 +364,170 @@ def test_extract_timing_flag_emits_stage_timings(monkeypatch, tmp_path, capsys): mainmod.main() assert exc2.value.code == 0 assert "graphify timing" not in capsys.readouterr().err + + +# --------------------------------------------------------------------------- +# #1909: a newly-excluded file's nodes must be pruned from graph.json on the +# next incremental extract even when the manifest never listed the file (the +# pre-#1897 state every 0.9.16 graph is in), so the manifest-diff prune set +# (`manifest - corpus`) can never see it. +# --------------------------------------------------------------------------- + +def _two_file_corpus(tmp_path): + project = tmp_path / "project" + project.mkdir() + (project / "x.py").write_text( + "def secret_helper():\n return 42\n\n" + "def secret_caller():\n return secret_helper()\n" + ) + (project / "keep.py").write_text( + "def kept():\n return still_here()\n\n" + "def still_here():\n return 1\n" + ) + return project + + +def _node_sources(graph_path): + import json + data = json.loads(graph_path.read_text(encoding="utf-8")) + return {n.get("source_file", "") for n in data.get("nodes", [])} + + +def _run_extract(monkeypatch, argv): + monkeypatch.setattr(mainmod.sys, "argv", argv) + try: + mainmod.main() + except SystemExit as exc: + assert exc.code in (None, 0), f"unexpected exit code {exc.code}" + + +def test_incremental_extract_prunes_newly_excluded_file_not_in_manifest( + monkeypatch, tmp_path +): + """Seed a graph with nodes for x.py, drop x.py from the manifest (pre-#1897 + manifests never listed excluded/omitted files), exclude x.py via + .graphifyignore, re-run extract: x.py's nodes must be gone even though it + was never on the deleted list.""" + import json + project = _two_file_corpus(tmp_path) + out_dir = tmp_path / "out" + _clear_backend_keys(monkeypatch) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + + _run_extract( + monkeypatch, + ["graphify", "extract", str(project), "--out", str(out_dir)], + ) + graph_path = out_dir / "graphify-out" / "graph.json" + manifest_path = out_dir / "graphify-out" / "manifest.json" + assert any("x.py" in s for s in _node_sources(graph_path)), ( + "seed extract must produce nodes for x.py" + ) + + # Simulate the pre-#1897 manifest state: x.py was never manifest-listed, + # so `manifest - corpus` can never flag it. + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest = {k: v for k, v in manifest.items() if "x.py" not in k} + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + (project / ".graphifyignore").write_text("x.py\n") + _run_extract( + monkeypatch, + ["graphify", "extract", str(project), "--out", str(out_dir)], + ) + + sources = _node_sources(graph_path) + assert not any("x.py" in s for s in sources), ( + f"newly-excluded x.py must be pruned from graph.json, still see {sources}" + ) + assert any("keep.py" in s for s in sources), ( + "unchanged keep.py nodes must survive the incremental merge" + ) + # x.py exists on disk, is excluded, and must not creep into the manifest. + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + assert not any("x.py" in k for k in manifest), ( + f"excluded x.py must not be (re)listed in the manifest: {set(manifest)}" + ) + + +def test_incremental_extract_prunes_excluded_file_listed_in_manifest( + monkeypatch, tmp_path +): + """Post-#1897 state: the excluded file IS manifest-listed. It must be + pruned from graph.json AND dropped from the manifest (#1908), and stay + settled on a further run.""" + import json + project = _two_file_corpus(tmp_path) + out_dir = tmp_path / "out" + _clear_backend_keys(monkeypatch) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + + _run_extract( + monkeypatch, + ["graphify", "extract", str(project), "--out", str(out_dir)], + ) + graph_path = out_dir / "graphify-out" / "graph.json" + manifest_path = out_dir / "graphify-out" / "manifest.json" + assert any("x.py" in k for k in json.loads(manifest_path.read_text())) + + (project / ".graphifyignore").write_text("x.py\n") + _run_extract( + monkeypatch, + ["graphify", "extract", str(project), "--out", str(out_dir)], + ) + + sources = _node_sources(graph_path) + assert not any("x.py" in s for s in sources) + assert any("keep.py" in s for s in sources) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + assert not any("x.py" in k for k in manifest), ( + "excluded-but-alive manifest row must be pruned (#1908)" + ) + + # Steady state: a third run neither resurrects x.py nor loses keep.py. + _run_extract( + monkeypatch, + ["graphify", "extract", str(project), "--out", str(out_dir)], + ) + sources = _node_sources(graph_path) + assert not any("x.py" in s for s in sources) + assert any("keep.py" in s for s in sources) + + +def test_no_cluster_incremental_prunes_newly_excluded_file( + monkeypatch, tmp_path, capsys +): + """--no-cluster's exclusion-only early exit must still scrub the excluded + file's nodes from the raw graph.json (that path never runs build_merge), + and must not report the alive file as deleted.""" + import json + project = _two_file_corpus(tmp_path) + out_dir = tmp_path / "out" + _clear_backend_keys(monkeypatch) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + + monkeypatch.setattr( + mainmod.sys, "argv", + ["graphify", "extract", str(project), "--no-cluster", "--out", str(out_dir)], + ) + with pytest.raises(SystemExit) as exc: + mainmod.main() + assert exc.value.code == 0 + graph_path = out_dir / "graphify-out" / "graph.json" + assert any("x.py" in s for s in _node_sources(graph_path)) + capsys.readouterr() + + (project / ".graphifyignore").write_text("x.py\n") + with pytest.raises(SystemExit) as exc: + mainmod.main() + assert exc.value.code == 0 + out_text = capsys.readouterr().out + assert "1 deleted" not in out_text, ( + "excluded-but-alive file must not be reported as deleted" + ) + + sources = _node_sources(graph_path) + assert not any("x.py" in s for s in sources), ( + f"--no-cluster early exit must prune excluded sources, still see {sources}" + ) + assert any("keep.py" in s for s in sources)