diff --git a/README.md b/README.md index c4fd30da7..b42bb7c86 100644 --- a/README.md +++ b/README.md @@ -396,6 +396,8 @@ Create a `.graphifyignore` in your project root — same syntax as `.gitignore`, **`.gitignore` is respected automatically.** graphify reads the `.gitignore` in each directory. If a `.graphifyignore` is also present, the two are **merged** — `.graphifyignore` patterns are evaluated last, so they win on conflicts (including `!` negations). Adding a `.graphifyignore` only ever excludes more; it never re-includes a file your `.gitignore` already excluded. Subdirectory scoping works the same way as git — an ignore file only affects its own subtree. +Pass `--no-gitignore` to `graphify extract` when git-ignored generated or transpiled code belongs in the graph. This disables `.gitignore` and `.git/info/exclude`; `.graphifyignore` still applies. + ``` # .graphifyignore node_modules/ @@ -714,6 +716,7 @@ graphify extract ./docs --token-budget 30000 # smaller semantic chunks for loc graphify extract ./docs --max-concurrency 2 # fewer parallel LLM calls (useful for local inference) graphify extract ./docs --api-timeout 900 # longer HTTP timeout for slow local models (default 600s) graphify extract ./docs --google-workspace # export .gdoc/.gsheet/.gslides via gws before extraction +graphify extract ./src --no-gitignore # include git-ignored source; still honor .graphifyignore graphify extract ./docs --mode deep # richer semantic extraction via extended system prompt graphify extract ./docs --no-cluster # raw extraction only, skip clustering graphify extract ./docs --timing # print per-stage wall-clock timings to stderr (also works on cluster-only) diff --git a/graphify/__main__.py b/graphify/__main__.py index b4243275a..d97d48fda 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -605,6 +605,7 @@ def _run_cli() -> None: print(" --api-timeout S per-request timeout in seconds for the LLM client (default: 600)") print(" --out DIR output dir (default: ); writes /graphify-out/") print(" --google-workspace export .gdoc/.gsheet/.gslides shortcuts via gws before extraction") + print(" --no-gitignore ignore .gitignore and .git/info/exclude (prioritizes .graphifyignore)") print(" --no-cluster skip clustering, write raw extraction only") print(" --code-only index code (local AST, no API key) and skip doc/paper/image files") print(" --postgres DSN extract schema from a live PostgreSQL database") diff --git a/graphify/cli.py b/graphify/cli.py index 8bca710ec..174810857 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -2325,6 +2325,7 @@ def dispatch_command(cmd: str) -> None: print( "Usage: graphify extract [--backend gemini|kimi|claude|openai|deepseek|ollama] " "[--model M] [--mode deep] [--out DIR] [--google-workspace] [--no-cluster] " + "[--no-gitignore] " "[--max-workers N] [--token-budget N] [--max-concurrency N] " "[--api-timeout S] [--postgres DSN] [--cargo] [--allow-partial] [--timing]", file=sys.stderr, @@ -2353,6 +2354,7 @@ def dispatch_command(cmd: str) -> None: google_workspace = False global_merge = False code_only = False + no_gitignore = False global_repo_tag: str | None = None # Performance/tuning knobs (issue #792). None means "use library default". cli_max_workers: int | None = None @@ -2418,6 +2420,8 @@ def dispatch_command(cmd: str) -> None: code_only = True; i += 1 elif a == "--google-workspace": google_workspace = True; i += 1 + elif a == "--no-gitignore": + no_gitignore = True; i += 1 elif a == "--global": global_merge = True; i += 1 elif a == "--as" and i + 1 < len(args): @@ -2495,10 +2499,14 @@ def dispatch_command(cmd: str) -> None: out_root = (out_dir.resolve() if out_dir else target) graphify_out = out_root / _GRAPHIFY_OUT graphify_out.mkdir(parents=True, exist_ok=True) - # Persist --exclude so later update/watch/hook rebuilds re-apply it - # instead of silently re-including the excluded paths (#1886). + # Persist corpus-shaping options so later update/watch/hook rebuilds + # use the same file set as the initial extraction (#1886). from graphify.watch import _write_build_config as _write_build_cfg - _write_build_cfg(graphify_out, excludes=cli_excludes or None) + _write_build_cfg( + graphify_out, + excludes=cli_excludes or None, + gitignore=not no_gitignore, + ) stages = _StageTimer(cli_timing) @@ -2547,6 +2555,7 @@ def dispatch_command(cmd: str) -> None: manifest_path=str(manifest_path), google_workspace=google_workspace or None, extra_excludes=cli_excludes or None, + gitignore=not no_gitignore, ) files_by_type = detection.get("files", {}) new_by_type = detection.get("new_files", {}) @@ -2569,7 +2578,13 @@ def dispatch_command(cmd: str) -> None: ) else: print(f"[graphify extract] scanning {target}") - detection = _detect(target, google_workspace=google_workspace or None, extra_excludes=cli_excludes or None, cache_root=out_root) + detection = _detect( + target, + google_workspace=google_workspace or None, + extra_excludes=cli_excludes or None, + cache_root=out_root, + gitignore=not no_gitignore, + ) files_by_type = detection.get("files", {}) code_files = [Path(p) for p in files_by_type.get("code", [])] doc_files = [Path(p) for p in files_by_type.get("document", [])] diff --git a/graphify/detect.py b/graphify/detect.py index 1c9227114..c2638a0ef 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -863,7 +863,7 @@ def _git_info_exclude(vcs_root: Path) -> Path | None: return exclude if exclude.is_file() else None -def _load_dir_own_ignore(d: Path) -> list[tuple[Path, str]]: +def _load_dir_own_ignore(d: Path, *, gitignore: bool = True) -> list[tuple[Path, str]]: """Read .gitignore/.graphifyignore directly inside *d* (not its ancestors). Merges .gitignore and .graphifyignore for this one directory (#1363): @@ -880,7 +880,7 @@ def _load_dir_own_ignore(d: Path) -> list[tuple[Path, str]]: were read, so e.g. `vendor/sub/.gitignore` was silently ignored (#1206). """ patterns: list[tuple[Path, str]] = [] - for fname in (".gitignore", ".graphifyignore"): + for fname in ((".gitignore", ".graphifyignore") if gitignore else (".graphifyignore",)): ignore_file = d / fname if ignore_file.exists(): for raw in ignore_file.read_text(encoding="utf-8", errors="ignore").splitlines(): @@ -890,7 +890,7 @@ def _load_dir_own_ignore(d: Path) -> list[tuple[Path, str]]: return patterns -def _load_graphifyignore(root: Path) -> list[tuple[Path, str]]: +def _load_graphifyignore(root: Path, *, gitignore: bool = True) -> list[tuple[Path, str]]: """Read .graphifyignore files and return (anchor_dir, pattern) pairs. Patterns are returned outer-first so that inner (closer) rules are @@ -923,7 +923,7 @@ def _load_graphifyignore(root: Path) -> list[tuple[Path, str]]: # per-directory .gitignore/.graphifyignore — so load it first (lowest priority # under last-match-wins) anchored at the VCS root, letting a nearer `!` # re-include still override it (#1810). - info_exclude = _git_info_exclude(ceiling) + info_exclude = _git_info_exclude(ceiling) if gitignore else None if info_exclude is not None: for raw in info_exclude.read_text(encoding="utf-8", errors="ignore").splitlines(): line = _parse_gitignore_line(raw) @@ -931,7 +931,7 @@ def _load_graphifyignore(root: Path) -> list[tuple[Path, str]]: patterns.append((ceiling, line)) for d in dirs: - patterns.extend(_load_dir_own_ignore(d)) + patterns.extend(_load_dir_own_ignore(d, gitignore=gitignore)) return patterns @@ -1189,7 +1189,7 @@ def _resolves_under_root(path: Path, root: Path) -> bool: return True -def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: bool | None = None, extra_excludes: list[str] | None = None, cache_root: Path | None = None) -> dict: +def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: bool | None = None, extra_excludes: list[str] | None = None, cache_root: Path | None = None, gitignore: bool = True) -> dict: root = root.resolve() if follow_symlinks is None: follow_symlinks = False @@ -1218,7 +1218,7 @@ def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: # of silently vanishing from the graph (#1922). Directory-level entries keep # this bounded — a pruned `data/` is one entry, not one per contained file. ignored: list[str] = [] - ignore_patterns = _load_graphifyignore(root) + ignore_patterns = _load_graphifyignore(root, gitignore=gitignore) ignore_cache: dict[Path, bool] = {} # shared across all _is_ignored calls in this scan # CLI --exclude patterns are anchored at the scan root and appended last # so they win over any .graphifyignore/.gitignore rules (#947). @@ -1276,7 +1276,7 @@ def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: # Load it now, before pruning dp's children, so a nested ignore # file governs its own subtree the same way git honors it (#1206). if dp != root: - ignore_patterns.extend(_load_dir_own_ignore(dp)) + ignore_patterns.extend(_load_dir_own_ignore(dp, gitignore=gitignore)) # Prune noise dirs in-place so os.walk never descends into them. # Dot dirs are allowed — users often want .github/, .claude/, etc. # Framework caches (.next, .nuxt, …) are caught by _is_noise_dir. @@ -1695,6 +1695,7 @@ def detect_incremental( google_workspace: bool | None = None, kind: str = "semantic", extra_excludes: list[str] | None = None, + gitignore: bool = True, ) -> dict: """Like detect(), but returns only new or modified files since the last run. @@ -1718,7 +1719,13 @@ def detect_incremental( runs. ``None`` (default) does not follow symlinked directories; callers must opt in explicitly, and resolved targets outside the scan root are skipped. """ - full = detect(root, follow_symlinks=follow_symlinks, google_workspace=google_workspace, extra_excludes=extra_excludes) + full = detect( + root, + follow_symlinks=follow_symlinks, + google_workspace=google_workspace, + extra_excludes=extra_excludes, + gitignore=gitignore, + ) # Pass ``root`` so a manifest written with relative keys (post-#777) is # re-anchored to the absolute form the rest of this function compares # against. Legacy absolute-keyed manifests pass through unchanged. diff --git a/graphify/watch.py b/graphify/watch.py index 79006ba40..37edc9cb1 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -76,19 +76,32 @@ def _drain_pending(out_dir: Path) -> list[Path]: _BUILD_CONFIG_FILENAME = ".graphify_build.json" -def _write_build_config(out_dir: Path, *, excludes: "list[str] | None") -> None: - """Persist build options (currently ``--exclude`` patterns) under ``out_dir``. +def _write_build_config( + out_dir: Path, + *, + excludes: "list[str] | None", + gitignore: bool | None = None, +) -> None: + """Persist corpus-shaping options under ``out_dir``. - Best-effort and non-clobbering: with no excludes it leaves any existing file - untouched, so a plain rebuild never erases patterns a prior extract recorded. + Best effort and non clobbering: omitted options retain their existing values. """ - if not excludes: + if not excludes and gitignore is None: return try: out_dir.mkdir(parents=True, exist_ok=True) - (out_dir / _BUILD_CONFIG_FILENAME).write_text( - json.dumps({"excludes": list(excludes)}), encoding="utf-8" - ) + path = out_dir / _BUILD_CONFIG_FILENAME + try: + config = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {} + except (OSError, json.JSONDecodeError): + config = {} + if not isinstance(config, dict): + config = {} + if excludes: + config["excludes"] = list(excludes) + if gitignore is not None: + config["gitignore"] = gitignore + path.write_text(json.dumps(config), encoding="utf-8") except OSError: pass @@ -107,6 +120,19 @@ def _read_build_excludes(out_dir: Path) -> list[str]: return [] +def _read_build_gitignore(out_dir: Path) -> bool: + """Return whether rebuilds should honor VCS ignore files (default True).""" + try: + path = out_dir / _BUILD_CONFIG_FILENAME + if path.is_file(): + cfg = json.loads(path.read_text(encoding="utf-8")) + if isinstance(cfg, dict) and isinstance(cfg.get("gitignore"), bool): + return cfg["gitignore"] + except (OSError, json.JSONDecodeError): + pass + return True + + def _merge_changed_paths(*sources: "list[Path] | None") -> list[Path]: """Concatenate path lists, preserving order and dropping duplicates. @@ -872,6 +898,7 @@ def _rebuild_code( detected = detect( watch_path, follow_symlinks=follow_symlinks, extra_excludes=_persisted_excludes or None, + gitignore=_read_build_gitignore(out), ) code_files = [Path(f) for f in detected['files']['code']] @@ -1335,7 +1362,10 @@ def watch(watch_path: Path, debounce: float = 3.0) -> None: # without this short-circuit a busy volume can saturate a CPU core # discarding events one extension at a time. (gh-928) watch_root_for_ignore = watch_path.resolve() - ignore_patterns = _load_graphifyignore(watch_root_for_ignore) + ignore_patterns = _load_graphifyignore( + watch_root_for_ignore, + gitignore=_read_build_gitignore(watch_path / _GRAPHIFY_OUT), + ) class Handler(FileSystemEventHandler): def on_any_event(self, event): diff --git a/tests/test_extract_code_only_cli.py b/tests/test_extract_code_only_cli.py index 2fcc8a7e3..1c27d9b49 100644 --- a/tests/test_extract_code_only_cli.py +++ b/tests/test_extract_code_only_cli.py @@ -7,6 +7,7 @@ still builds, and the no-key error now points users at the flag. from __future__ import annotations import os +import json import subprocess import sys from pathlib import Path @@ -53,3 +54,29 @@ def test_mixed_repo_without_key_errors_and_points_at_code_only(tmp_path): r = _run(repo) # no --code-only, no key assert r.returncode != 0, "mixed repo with no key should still error without the flag" assert "--code-only" in r.stderr, "the no-key error must point users at --code-only" + + +def test_no_gitignore_indexes_vcs_ignored_code_but_keeps_graphifyignore(tmp_path): + repo = tmp_path / "repo" + generated = repo / "proj" / "deep" / "generated" + generated.mkdir(parents=True) + (repo / ".git" / "info").mkdir(parents=True) + (repo / ".git" / "info" / "exclude").write_text("local/\n") + (repo / "proj" / ".gitignore").write_text("generated/\n") + (repo / "proj" / ".graphifyignore").write_text("hidden/\n") + (generated / "Gen.cs").write_text("namespace N { public class Gen {} }\n") + local = repo / "local" + local.mkdir() + (local / "Local.cs").write_text("namespace N { public class Local {} }\n") + hidden = repo / "proj" / "hidden" + hidden.mkdir() + (hidden / "Hidden.cs").write_text("namespace N { public class Hidden {} }\n") + + result = _run(repo, "--no-gitignore", "--no-cluster") + + assert result.returncode == 0, result.stderr + graph = json.loads((repo / "graphify-out" / "graph.json").read_text()) + sources = {Path(str(node.get("source_file", ""))).as_posix() for node in graph["nodes"]} + assert any(source.endswith("proj/deep/generated/Gen.cs") for source in sources) + assert any(source.endswith("local/Local.cs") for source in sources) + assert not any(source.endswith("proj/hidden/Hidden.cs") for source in sources) diff --git a/tests/test_watch.py b/tests/test_watch.py index 54dcdf98d..4b16d1e03 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -277,6 +277,26 @@ def test_rebuild_honors_persisted_excludes(tmp_path): ) +def test_rebuild_honors_persisted_no_gitignore(tmp_path): + import json + from graphify.watch import _rebuild_code, _write_build_config + + corpus = tmp_path / "corpus" + generated = corpus / "generated" + generated.mkdir(parents=True) + (corpus / ".gitignore").write_text("generated/\n") + (generated / "gen.py").write_text("def generated(): return 1\n") + _write_build_config( + corpus / "graphify-out", excludes=None, gitignore=False + ) + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + + graph = json.loads((corpus / "graphify-out" / "graph.json").read_text()) + sources = {Path(str(node.get("source_file", ""))).as_posix() for node in graph["nodes"]} + assert any(source.endswith("generated/gen.py") for source in sources) + + def test_graphify_root_preserves_absolute_when_user_supplied(tmp_path): """When the caller supplies an absolute path, ``.graphify_root`` stores that absolute form verbatim — preserving explicit-absolute intent."""