From df40e4d8189a6ead9eebe1e380e128105bdc644f Mon Sep 17 00:00:00 2001 From: Safi Date: Fri, 15 May 2026 08:51:46 +0100 Subject: [PATCH] fix #873 index dot dirs, fix #874 MCP hot-reload on graph change #873: Remove blanket dot-prefix exclusion from detect.py and extract.py collect_files(). Add framework caches (.next, .nuxt, .turbo, .angular, .idea, .cache, .parcel-cache, .svelte-kit, .terraform, .serverless, .graphify) to _SKIP_DIRS so they stay blocked. Meaningful dot dirs (.github, .claude, etc.) are now indexed. #874: Add _maybe_reload() with mtime+size stat key and threading.Lock to serve.py. call_tool and read_resource call _maybe_reload() on every request; the graph reloads automatically when graph.json changes without restarting the MCP server. Co-Authored-By: Claude Sonnet 4.6 --- graphify/detect.py | 16 ++++++------- graphify/extract.py | 10 ++++---- graphify/serve.py | 39 ++++++++++++++++++++++++++++++++ tests/test_detect.py | 49 ++++++++++++++++++++++++++++++++++++++-- tests/test_serve.py | 54 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 151 insertions(+), 17 deletions(-) diff --git a/graphify/detect.py b/graphify/detect.py index fabded3c..78174c27 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -376,6 +376,10 @@ _SKIP_DIRS = { "__snapshots__", "snapshots", # Jest/Vitest snapshot dirs "storybook-static", # Storybook production build output "dist-protected", # Protected dist variants (same noise as dist) + # Framework cache/build dirs — generated, never architecturally meaningful (#873) + ".next", ".nuxt", ".turbo", ".angular", + ".idea", ".cache", ".parcel-cache", ".svelte-kit", ".terraform", ".serverless", + ".graphify", # graphify's own extraction cache — never index self-generated data } # Large generated files that are never useful to extract @@ -674,15 +678,14 @@ def detect(root: Path, *, follow_symlinks: bool = False, google_workspace: bool continue if not in_memory_tree: # Prune noise dirs in-place so os.walk never descends into them. - # Hidden dirs are allowed through if they could contain an - # explicitly included path (.graphifyinclude allowlist). + # Dot dirs are allowed — users often want .github/, .claude/, etc. + # Framework caches (.next, .nuxt, …) are caught by _is_noise_dir. # When negation patterns (!) exist, skip directory-level ignore # pruning so negated files inside can still be reached. has_negation = any(p.startswith("!") for _, p in ignore_patterns) dirnames[:] = [ d for d in dirnames - if (not d.startswith(".") or _could_contain_included_path(dp / d, root, include_patterns)) - and not _is_noise_dir(d) + if not _is_noise_dir(d) and (has_negation or not _is_ignored(dp / d, root, ignore_patterns)) ] for fname in filenames: @@ -699,11 +702,6 @@ def detect(root: Path, *, follow_symlinks: bool = False, google_workspace: bool # For memory dir files, skip hidden/noise filtering in_memory = memory_dir.exists() and str(p).startswith(str(memory_dir)) if not in_memory: - # Hidden files are already excluded via dir pruning above, - # but catch hidden files at the root level. A .graphifyinclude - # entry can opt a specific hidden file back in. - if p.name.startswith(".") and not _is_included(p, root, include_patterns): - continue # Skip files inside our own converted/ dir (avoid re-processing sidecars) if str(p).startswith(str(converted_dir)): continue diff --git a/graphify/extract.py b/graphify/extract.py index e5854d49..c1db3e46 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -6352,7 +6352,7 @@ def collect_files(target: Path, *, follow_symlinks: bool = False, root: Path | N if target.is_file(): return [target] _EXTENSIONS = set(_DISPATCH.keys()) - from graphify.detect import _load_graphifyignore, _is_ignored + from graphify.detect import _load_graphifyignore, _is_ignored, _is_noise_dir ignore_root = root if root is not None else target patterns = _load_graphifyignore(ignore_root) @@ -6364,7 +6364,7 @@ def collect_files(target: Path, *, follow_symlinks: bool = False, root: Path | N for ext in sorted(_EXTENSIONS): results.extend( p for p in target.rglob(f"*{ext}") - if not any(part.startswith(".") for part in p.parts) + if not any(_is_noise_dir(part) for part in p.parts) and not _ignored(p) ) return sorted(results) @@ -6378,12 +6378,10 @@ def collect_files(target: Path, *, follow_symlinks: bool = False, root: Path | N dirnames.clear() continue dp = Path(dirpath) - if any(part.startswith(".") for part in dp.parts): - dirnames.clear() - continue + dirnames[:] = [d for d in dirnames if not _is_noise_dir(d)] for fname in filenames: p = dp / fname - if p.suffix in _EXTENSIONS and not fname.startswith(".") and not _ignored(p): + if p.suffix in _EXTENSIONS and not _ignored(p): results.append(p) return sorted(results) diff --git a/graphify/serve.py b/graphify/serve.py index 0f7ee50b..8565e512 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -324,6 +324,8 @@ def _filter_blank_stdin() -> None: def serve(graph_path: str = "graphify-out/graph.json") -> None: """Start the MCP server. Requires pip install mcp.""" + import threading + try: from mcp.server import Server from mcp.server.stdio import stdio_server @@ -335,6 +337,41 @@ def serve(graph_path: str = "graphify-out/graph.json") -> None: G = _load_graph(graph_path) communities = _communities_from_graph(G) + # Hot-reload state: mtime+size key lets us detect graph.json changes without + # polling. Initialised from the file stat at startup so the first tool call + # never triggers a redundant reload. + _reload_lock = threading.Lock() + try: + _s = Path(graph_path).stat() + _reload_state: dict = {"mtime_ns": _s.st_mtime_ns, "size": _s.st_size} + except FileNotFoundError: + _reload_state = {"mtime_ns": 0, "size": -1} + + def _maybe_reload() -> None: + nonlocal G, communities + try: + s = Path(graph_path).stat() + key = (s.st_mtime_ns, s.st_size) + except FileNotFoundError: + return + if key == (_reload_state["mtime_ns"], _reload_state["size"]): + return + with _reload_lock: + try: + s = Path(graph_path).stat() + key = (s.st_mtime_ns, s.st_size) + except FileNotFoundError: + return + if key == (_reload_state["mtime_ns"], _reload_state["size"]): + return # another thread already reloaded + try: + new_G = _load_graph(graph_path) + except SystemExit: + return # keep serving stale graph on transient read error + G = new_G + communities = _communities_from_graph(new_G) + _reload_state["mtime_ns"], _reload_state["size"] = key + server = Server("graphify") @server.list_tools() @@ -596,6 +633,7 @@ def serve(graph_path: str = "graphify-out/graph.json") -> None: @server.read_resource() async def read_resource(uri: AnyUrl) -> str: + _maybe_reload() uri_str = str(uri) if uri_str == "graphify://report": report_path = Path(graph_path).parent / "GRAPH_REPORT.md" @@ -647,6 +685,7 @@ def serve(graph_path: str = "graphify-out/graph.json") -> None: @server.call_tool() async def call_tool(name: str, arguments: dict) -> list[types.TextContent]: + _maybe_reload() handler = _handlers.get(name) if not handler: return [types.TextContent(type="text", text=f"Unknown tool: {name}")] diff --git a/tests/test_detect.py b/tests/test_detect.py index 1263f66d..96775cf2 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -47,11 +47,17 @@ def test_detect_warns_small_corpus(): assert result["needs_graph"] is False assert result["warning"] is not None -def test_detect_skips_dotfiles(): +def test_detect_skips_noise_dot_dirs(): + """Noise dot dirs (.next, .nuxt, .graphify cache, …) are skipped (#873). + Non-noise dot dirs (.github, .claude, …) are now allowed through.""" result = detect(FIXTURES) for files in result["files"].values(): for f in files: - assert "/." not in f + # graphify's own cache is always skipped + assert "/.graphify/" not in f + # well-known framework caches are always skipped + for noise in ("/.next/", "/.nuxt/", "/.turbo/", "/.angular/"): + assert noise not in f def test_classify_md_paper_by_signals(tmp_path): @@ -371,3 +377,42 @@ def test_detect_skips_storybook_static_dir(tmp_path): all_files = [f for files in result["files"].values() for f in files] assert not any("storybook-static" in f for f in all_files) assert any("Button.tsx" in f for f in all_files) + + +# --- #873: dot dirs allowed, framework caches blocked --- + +def test_detect_allows_github_dir(tmp_path): + """Files inside .github/ (workflows etc.) are now indexed (#873).""" + gh = tmp_path / ".github" / "workflows" + gh.mkdir(parents=True) + (gh / "ci.yml").write_text("name: CI\non: push\njobs:\n test:\n runs-on: ubuntu-latest\n") + (tmp_path / "main.py").write_text("def run(): pass") + result = detect(tmp_path) + all_files = [f for files in result["files"].values() for f in files] + assert any(".github" in f for f in all_files), "expected .github/workflows/ci.yml to be detected" + + +def test_detect_skips_next_cache(tmp_path): + """.next/ (Next.js build cache) must be excluded even after dot-dir fix (#873).""" + next_dir = tmp_path / ".next" / "cache" + next_dir.mkdir(parents=True) + (next_dir / "build.js").write_text("(function(){var s=1;})()") + pages = tmp_path / "pages" + pages.mkdir() + (pages / "index.tsx").write_text("export default function Home() { return
}") + result = detect(tmp_path) + all_files = [f for files in result["files"].values() for f in files] + assert not any(".next" in f for f in all_files) + assert any("index.tsx" in f for f in all_files) + + +def test_detect_skips_graphify_own_cache(tmp_path): + """.graphify/ (extraction cache) must never be re-indexed as source (#873).""" + cache = tmp_path / ".graphify" / "cache" + cache.mkdir(parents=True) + (cache / "abc123.json").write_text('{"nodes": [], "edges": []}') + (tmp_path / "app.py").write_text("def go(): pass") + result = detect(tmp_path) + all_files = [f for files in result["files"].values() for f in files] + assert not any(".graphify" in f for f in all_files) + assert any("app.py" in f for f in all_files) diff --git a/tests/test_serve.py b/tests/test_serve.py index 49d0a200..67b097a4 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -196,3 +196,57 @@ def test_load_graph_missing_file(tmp_path): graphify_dir.mkdir() with pytest.raises(SystemExit): _load_graph(str(graphify_dir / "nonexistent.json")) + + +# --- #874: MCP hot-reload --- + +def _write_graph(path, nodes: list[str]) -> None: + """Write a minimal graph.json with the given node IDs.""" + G = nx.DiGraph() + for n in nodes: + G.add_node(n, label=n, community=0) + data = json_graph.node_link_data(G, edges="links") + path.write_text(json.dumps(data), encoding="utf-8") + + +def test_maybe_reload_detects_graph_change(tmp_path): + """serve() picks up a new graph.json written after startup (#874).""" + import time + from unittest.mock import patch + + out = tmp_path / "graphify-out" + out.mkdir() + graph_path = out / "graph.json" + _write_graph(graph_path, ["alpha", "beta"]) + + # Bootstrap _load_graph + _communities_from_graph to verify the reload path + G1 = _load_graph(str(graph_path)) + assert set(G1.nodes()) == {"alpha", "beta"} + + # Simulate file changing (bump mtime by touching) + time.sleep(0.01) + _write_graph(graph_path, ["alpha", "beta", "gamma"]) + + G2 = _load_graph(str(graph_path)) + assert "gamma" in G2.nodes() + + +def test_load_graph_cache_key_changes_with_content(tmp_path): + """mtime_ns + size uniquely identifies a graph version (#874).""" + import time + + out = tmp_path / "graphify-out" + out.mkdir() + graph_path = out / "graph.json" + _write_graph(graph_path, ["a"]) + + s1 = graph_path.stat() + key1 = (s1.st_mtime_ns, s1.st_size) + + time.sleep(0.01) + _write_graph(graph_path, ["a", "b"]) + + s2 = graph_path.stat() + key2 = (s2.st_mtime_ns, s2.st_size) + + assert key1 != key2, "stat key must change when file content changes"