fix node ID collisions, cache fastpath, absolute source_file paths, and failed-chunk manifest freeze

- fix SQL extractor using bare path.stem as node ID prefix — collides across same-named files in different dirs; use _file_stem() (directory-qualified) instead
- fix Python import resolver keying stem_to_entities by bare stem; add bare_to_qualified secondary index for absolute imports so cross-file edges survive duplicate filenames
- add stat-based mtime fastpath to file_hash: skip full SHA256 when size+mtime_ns unchanged, flush index atomically via atexit (same trade-off as make)
- add cache-check, merge-chunks, merge-semantic CLI subcommands so the skill pipeline can use library functions instead of inline Python
- fix absolute source_file paths from semantic subagents not being relativized before graph storage (#932): add root param to build_from_json/build/build_merge, pass scan target at both call sites
- fix failed semantic chunks permanently freezing their files in the manifest (#933): filter _manifest_files to only include doc/paper/image files that appear in sem_result nodes/edges before calling save_manifest

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

graphify/__main__.py

@@ -2750,6 +2750,22 @@ def main() -> None:
         graph_json_path = graphify_out / "graph.json"
         analysis_path = graphify_out / ".graphify_analysis.json"
 
+        # Build a manifest-safe files dict: only stamp semantic_hash for files
+        # that actually produced output (cache hit or fresh extraction). Files
+        # whose chunk failed have no source_file entry in sem_result — leaving
+        # their semantic_hash empty so detect_incremental re-queues them (#933).
+        _sem_extracted: set[str] = {
+            n.get("source_file", "") for n in sem_result.get("nodes", [])
+        } | {
+            e.get("source_file", "") for e in sem_result.get("edges", [])
+        }
+        _sem_extracted.discard("")
+        _sem_types = {"document", "paper", "image"}
+        _manifest_files = {
+            ftype: [f for f in flist if ftype not in _sem_types or f in _sem_extracted]
+            for ftype, flist in files_by_type.items()
+        }
+
         if no_cluster:
             # --no-cluster: dump the raw merged extraction as graph.json.
             # No NetworkX, no community detection, no analysis sidecar.
@@ -2772,7 +2788,7 @@ def main() -> None:
                     f"est. cost: ${cost:.4f}"
                 )
             try:
-                _save_manifest(files_by_type, manifest_path=str(manifest_path), kind="both")
+                _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both")
             except Exception as exc:
                 print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr)
             if global_merge:
@@ -2806,9 +2822,10 @@ def main() -> None:
                 prune_sources=deleted_files or None,
                 dedup=True,
                 dedup_llm_backend=dedup_backend,
+                root=target,
             )
         else:
-            G = _build([merged], dedup=True, dedup_llm_backend=dedup_backend)
+            G = _build([merged], dedup=True, dedup_llm_backend=dedup_backend, root=target)
         if G.number_of_nodes() == 0:
             print(
                 "[graphify extract] graph is empty — extraction produced no nodes. "
@@ -2854,7 +2871,7 @@ def main() -> None:
         }
         analysis_path.write_text(json.dumps(analysis, indent=2), encoding="utf-8")
         try:
-            _save_manifest(files_by_type, manifest_path=str(manifest_path), kind="both")
+            _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both")
         except Exception as exc:
             print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr)
 
@@ -2882,6 +2899,128 @@ def main() -> None:
                 f"est. cost (~{backend}): ${cost:.4f}"
             )
 
+    elif cmd == "cache-check":
+        # graphify cache-check <files_from> [--root <dir>]
+        # Reads file paths (one per line) from <files_from>, checks semantic cache.
+        # Writes:
+        #   graphify-out/.graphify_cached.json   — already-cached nodes/edges/hyperedges
+        #   graphify-out/.graphify_uncached.txt  — paths that need extraction
+        # Stdout: "Cache: N hit, M miss"
+        from graphify.cache import check_semantic_cache
+        if len(sys.argv) < 3:
+            print("Usage: graphify cache-check <files_from> [--root <dir>]", file=sys.stderr)
+            sys.exit(1)
+        files_from = Path(sys.argv[2])
+        root = Path(".")
+        i = 3
+        while i < len(sys.argv):
+            if sys.argv[i] == "--root" and i + 1 < len(sys.argv):
+                root = Path(sys.argv[i + 1])
+                i += 2
+            else:
+                i += 1
+        files = [f for f in files_from.read_text(encoding="utf-8").splitlines() if f.strip()]
+        cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(files, root)
+        out = root / "graphify-out"
+        out.mkdir(parents=True, exist_ok=True)
+        if cached_nodes or cached_edges or cached_hyperedges:
+            (out / ".graphify_cached.json").write_text(
+                json.dumps({"nodes": cached_nodes, "edges": cached_edges, "hyperedges": cached_hyperedges},
+                           ensure_ascii=False),
+                encoding="utf-8",
+            )
+        (out / ".graphify_uncached.txt").write_text("\n".join(uncached), encoding="utf-8")
+        print(f"Cache: {len(files) - len(uncached)} hit, {len(uncached)} miss")
+
+    elif cmd == "merge-chunks":
+        # graphify merge-chunks <chunk_glob_or_files...> --out <path>
+        # Concatenates .graphify_chunk_*.json files written by semantic subagents.
+        # Deduplicates nodes by id (first writer wins). Sums token counts.
+        import glob as _glob
+        if len(sys.argv) < 3:
+            print("Usage: graphify merge-chunks <chunk_files...> --out <path>", file=sys.stderr)
+            sys.exit(1)
+        out_path: Path | None = None
+        chunk_args: list[str] = []
+        i = 2
+        while i < len(sys.argv):
+            if sys.argv[i] == "--out" and i + 1 < len(sys.argv):
+                out_path = Path(sys.argv[i + 1])
+                i += 2
+            else:
+                chunk_args.append(sys.argv[i])
+                i += 1
+        if not out_path:
+            print("error: --out <path> required", file=sys.stderr)
+            sys.exit(1)
+        chunk_files: list[str] = []
+        for arg in chunk_args:
+            expanded = _glob.glob(arg)
+            chunk_files.extend(sorted(expanded) if expanded else [arg])
+        merged: dict = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0}
+        seen_ids: set[str] = set()
+        for cf in chunk_files:
+            try:
+                chunk = json.loads(Path(cf).read_text(encoding="utf-8"))
+            except (json.JSONDecodeError, OSError) as exc:
+                print(f"[graphify merge-chunks] warning: skipping {cf}: {exc}", file=sys.stderr)
+                continue
+            for n in chunk.get("nodes", []):
+                if n.get("id") not in seen_ids:
+                    seen_ids.add(n["id"])
+                    merged["nodes"].append(n)
+            merged["edges"].extend(chunk.get("edges", []))
+            merged["hyperedges"].extend(chunk.get("hyperedges", []))
+            merged["input_tokens"] += chunk.get("input_tokens", 0)
+            merged["output_tokens"] += chunk.get("output_tokens", 0)
+        out_path.parent.mkdir(parents=True, exist_ok=True)
+        out_path.write_text(json.dumps(merged, ensure_ascii=False), encoding="utf-8")
+        print(
+            f"Merged {len(chunk_files)} chunks: {merged['nodes']} nodes, {len(merged['edges'])} edges, "
+            f"{merged['input_tokens']:,} in / {merged['output_tokens']:,} out tokens"
+        )
+
+    elif cmd == "merge-semantic":
+        # graphify merge-semantic --cached <path> --new <path> --out <path>
+        # Merges cached semantic results with freshly-extracted chunk results.
+        # Deduplicates nodes by id (cached entries take priority over new ones).
+        if len(sys.argv) < 3:
+            print("Usage: graphify merge-semantic --cached <path> --new <path> --out <path>", file=sys.stderr)
+            sys.exit(1)
+        cached_path: Path | None = None
+        new_path: Path | None = None
+        out_path2: Path | None = None
+        i = 2
+        while i < len(sys.argv):
+            if sys.argv[i] == "--cached" and i + 1 < len(sys.argv):
+                cached_path = Path(sys.argv[i + 1]); i += 2
+            elif sys.argv[i] == "--new" and i + 1 < len(sys.argv):
+                new_path = Path(sys.argv[i + 1]); i += 2
+            elif sys.argv[i] == "--out" and i + 1 < len(sys.argv):
+                out_path2 = Path(sys.argv[i + 1]); i += 2
+            else:
+                i += 1
+        if not out_path2:
+            print("error: --out <path> required", file=sys.stderr)
+            sys.exit(1)
+        empty: dict = {"nodes": [], "edges": [], "hyperedges": []}
+        cached_data = json.loads(cached_path.read_text(encoding="utf-8")) if cached_path and cached_path.exists() else empty
+        new_data = json.loads(new_path.read_text(encoding="utf-8")) if new_path and new_path.exists() else empty
+        seen_ids2: set[str] = set()
+        all_nodes: list[dict] = []
+        for n in cached_data.get("nodes", []) + new_data.get("nodes", []):
+            if n.get("id") not in seen_ids2:
+                seen_ids2.add(n["id"])
+                all_nodes.append(n)
+        merged2 = {
+            "nodes": all_nodes,
+            "edges": cached_data.get("edges", []) + new_data.get("edges", []),
+            "hyperedges": cached_data.get("hyperedges", []) + new_data.get("hyperedges", []),
+        }
+        out_path2.parent.mkdir(parents=True, exist_ok=True)
+        out_path2.write_text(json.dumps(merged2, ensure_ascii=False), encoding="utf-8")
+        print(f"Merged: {len(merged2['nodes'])} nodes, {len(merged2['edges'])} edges")
+
     elif Path(cmd).exists() or cmd in (".", "..") or cmd.startswith(("./", "../", "/", "~")):
         # User ran `graphify <path>` directly — treat as `graphify extract <path>`.
         # Common when following the PowerShell note in README (`graphify .`) or

graphify/build.py

@@ -22,6 +22,7 @@
 #
 from __future__ import annotations
 import json
+import os
 import re
 import sys
 import unicodedata
@@ -64,10 +65,22 @@ def _normalize_id(s: str) -> str:
     return cleaned.strip("_").casefold()
 
 
-def _norm_source_file(p: str | None) -> str | None:
-    """Normalize path separators to forward slashes so Windows backslash paths
-    and POSIX paths from semantic subagents resolve to the same node identity."""
-    return p.replace("\\", "/") if p else p
+def _norm_source_file(p: str | None, root: str | None = None) -> str | None:
+    """Normalize path separators and relativize absolute paths.
+
+    Converts backslashes to forward slashes (Windows compatibility) and, when
+    root is provided, strips the absolute prefix from paths produced by semantic
+    subagents so source_file is always repo-relative (fixes #932).
+    """
+    if not p:
+        return p
+    p = p.replace("\\", "/")
+    if root and os.path.isabs(p):
+        try:
+            p = Path(p).relative_to(root).as_posix()
+        except ValueError:
+            pass
+    return p
 
 
 def edge_data(G: nx.Graph, u: str, v: str) -> dict:
@@ -91,12 +104,15 @@ def edge_datas(G: nx.Graph, u: str, v: str) -> list[dict]:
     return [raw]
 
 
-def build_from_json(extraction: dict, *, directed: bool = False) -> nx.Graph:
+def build_from_json(extraction: dict, *, directed: bool = False, root: str | Path | None = None) -> nx.Graph:
     """Build a NetworkX graph from an extraction dict.
 
     directed=True produces a DiGraph that preserves edge direction (source→target).
     directed=False (default) produces an undirected Graph for backward compatibility.
+    root: if given, absolute source_file paths from semantic subagents are made
+        relative to root so all nodes share a consistent path key (#932).
     """
+    _root = str(Path(root).resolve()) if root else None
     # NetworkX <= 3.1 serialised edges as "links"; remap to "edges" for compatibility.
     if "edges" not in extraction and "links" in extraction:
         extraction = dict(extraction, edges=extraction["links"])
@@ -137,7 +153,7 @@ def build_from_json(extraction: dict, *, directed: bool = False) -> nx.Graph:
     G: nx.Graph = nx.DiGraph() if directed else nx.Graph()
     for node in extraction.get("nodes", []):
         if "source_file" in node:
-            node["source_file"] = _norm_source_file(node["source_file"])
+            node["source_file"] = _norm_source_file(node["source_file"], _root)
         G.add_node(node["id"], **{k: v for k, v in node.items() if k != "id"})
     node_set = set(G.nodes())
     # Normalized ID map: lets edges survive when the LLM generates IDs with
@@ -161,7 +177,7 @@ def build_from_json(extraction: dict, *, directed: bool = False) -> nx.Graph:
             continue  # skip edges to external/stdlib nodes - expected, not an error
         attrs = {k: v for k, v in edge.items() if k not in ("source", "target")}
         if "source_file" in attrs:
-            attrs["source_file"] = _norm_source_file(attrs["source_file"])
+            attrs["source_file"] = _norm_source_file(attrs["source_file"], _root)
         # Preserve original edge direction - undirected graphs lose it otherwise,
         # causing display functions to show edges backwards.
         attrs["_src"] = src
@@ -179,6 +195,7 @@ def build(
     directed: bool = False,
     dedup: bool = True,
     dedup_llm_backend: str | None = None,
+    root: str | Path | None = None,
 ) -> nx.Graph:
     """Merge multiple extraction results into one graph.
 
@@ -187,6 +204,7 @@ def build(
     dedup=True (default) runs entity deduplication before building the graph.
     dedup_llm_backend: if set (e.g. "gemini", "claude", or "kimi"), uses LLM to resolve
         ambiguous pairs in the 75–92 Jaro-Winkler score zone.
+    root: if given, absolute source_file paths are made relative to root (#932).
 
     Extractions are merged in order. For nodes with the same ID, the last
     extraction's attributes win (NetworkX add_node overwrites). Pass AST
@@ -206,7 +224,7 @@ def build(
             combined["nodes"], combined["edges"], communities={},
             dedup_llm_backend=dedup_llm_backend,
         )
-    return build_from_json(combined, directed=directed)
+    return build_from_json(combined, directed=directed, root=root)
 
 
 def _norm_label(label: str) -> str:
@@ -268,11 +286,13 @@ def build_merge(
     directed: bool = False,
     dedup: bool = True,
     dedup_llm_backend: str | None = None,
+    root: str | Path | None = None,
 ) -> nx.Graph:
     """Load existing graph.json, merge new chunks into it, and save back.
 
     Never replaces - only grows (or prunes deleted-file nodes via prune_sources).
     Safe to call repeatedly: existing nodes and edges are preserved.
+    root: if given, absolute source_file paths in new_chunks are made relative (#932).
     """
     graph_path = Path(graph_path)
     if graph_path.exists():
@@ -293,7 +313,7 @@ def build_merge(
         base = []
 
     all_chunks = base + list(new_chunks)
-    G = build(all_chunks, directed=directed, dedup=dedup, dedup_llm_backend=dedup_llm_backend)
+    G = build(all_chunks, directed=directed, dedup=dedup, dedup_llm_backend=dedup_llm_backend, root=root)
 
     # Prune nodes and edges from deleted source files
     if prune_sources:

graphify/cache.py

@@ -1,6 +1,7 @@
 # per-file extraction cache - skip unchanged files on re-run
 from __future__ import annotations
 
+import atexit
 import hashlib
 import json
 import os
@@ -23,6 +24,65 @@ def _body_content(content: bytes) -> bytes:
     return content
 
 
+# Stat-based index: maps absolute path → {size, mtime_ns, hash}.
+# Loaded once per process, flushed via atexit. Skips full file reads when
+# size+mtime_ns are unchanged — same trade-off as make(1).
+# Correctness risks: `touch` causes a harmless extra re-hash; same-size edits
+# within NFS second-resolution mtime have a 1-second window (same as make).
+# Use `graphify extract --force` to bypass when needed.
+_stat_index: dict[str, dict] = {}
+_stat_index_root: Path | None = None
+_stat_index_dirty: bool = False
+
+
+def _stat_index_file(root: Path) -> Path:
+    _out = Path(_GRAPHIFY_OUT)
+    base = _out if _out.is_absolute() else Path(root).resolve() / _out
+    return base / "cache" / "stat-index.json"
+
+
+def _ensure_stat_index(root: Path) -> None:
+    global _stat_index, _stat_index_root, _stat_index_dirty
+    if _stat_index_root is not None:
+        return
+    _stat_index_root = Path(root).resolve()
+    p = _stat_index_file(_stat_index_root)
+    if p.exists():
+        try:
+            _stat_index = json.loads(p.read_text(encoding="utf-8"))
+        except (json.JSONDecodeError, OSError):
+            _stat_index = {}
+    else:
+        _stat_index = {}
+    atexit.register(_flush_stat_index)
+
+
+def _flush_stat_index() -> None:
+    global _stat_index_dirty, _stat_index_root
+    if not _stat_index_dirty or _stat_index_root is None:
+        return
+    p = _stat_index_file(_stat_index_root)
+    try:
+        p.parent.mkdir(parents=True, exist_ok=True)
+        fd, tmp = tempfile.mkstemp(dir=p.parent, prefix="stat-index.", suffix=".tmp")
+        try:
+            os.write(fd, json.dumps(_stat_index, separators=(",", ":")).encode())
+            os.close(fd)
+            os.replace(tmp, p)
+        except Exception:
+            try:
+                os.close(fd)
+            except OSError:
+                pass
+            try:
+                os.unlink(tmp)
+            except OSError:
+                pass
+    except OSError:
+        pass
+    _stat_index_dirty = False
+
+
 def _normalize_path(path: Path) -> Path:
     """Normalize path for consistent cache keys across Windows path spellings."""
     import sys
@@ -37,6 +97,10 @@ def _normalize_path(path: Path) -> Path:
 def file_hash(path: Path, root: Path = Path(".")) -> str:
     """SHA256 of file contents + path relative to root.
 
+    Uses a stat-based fastpath (size + mtime_ns) to skip full reads when the
+    file hasn't changed. Falls through to full SHA256 on first encounter or
+    when stat changes. Index is flushed atomically at process exit.
+
     Using a relative path (not absolute) makes cache entries portable across
     machines and checkout directories, so shared caches and CI work correctly.
     Falls back to the resolved absolute path if the file is outside root.
@@ -44,10 +108,25 @@ def file_hash(path: Path, root: Path = Path(".")) -> str:
     For Markdown files (.md), only the body below the YAML frontmatter is hashed,
     so metadata-only changes (e.g. reviewed, status, tags) do not invalidate the cache.
     """
+    global _stat_index_dirty
     p = _normalize_path(Path(path))
     root = _normalize_path(Path(root))
     if not p.is_file():
         raise IsADirectoryError(f"file_hash requires a file, got: {p}")
+
+    _ensure_stat_index(root)
+    abs_key = str(p.resolve())
+    st: "os.stat_result | None" = None
+    try:
+        st = p.stat()
+        entry = _stat_index.get(abs_key)
+        if (entry
+                and entry.get("size") == st.st_size
+                and entry.get("mtime_ns") == st.st_mtime_ns):
+            return entry["hash"]
+    except OSError:
+        pass
+
     raw = p.read_bytes()
     content = _body_content(raw) if p.suffix.lower() == ".md" else raw
     h = hashlib.sha256()
@@ -58,7 +137,13 @@ def file_hash(path: Path, root: Path = Path(".")) -> str:
         h.update(rel.as_posix().lower().encode())
     except ValueError:
         h.update(p.resolve().as_posix().lower().encode())
-    return h.hexdigest()
+    digest = h.hexdigest()
+
+    if st is not None:
+        _stat_index[abs_key] = {"size": st.st_size, "mtime_ns": st.st_mtime_ns, "hash": digest}
+        _stat_index_dirty = True
+
+    return digest
 
 
 def cache_dir(root: Path = Path("."), kind: str = "ast") -> Path:

graphify/extract.py

@@ -2755,7 +2755,7 @@ def extract_sql(path: Path) -> dict:
     except Exception as e:
         return {"nodes": [], "edges": [], "error": str(e)}
 
-    stem = re.sub(r"[^a-z0-9]", "_", path.stem.lower())
+    stem = _file_stem(path)
     str_path = str(path)
     file_nid = _make_id(str_path)
     nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code",
@@ -4222,15 +4222,20 @@ def _resolve_cross_file_imports(
     language = Language(tspython.language())
     parser = Parser(language)
 
-    # Pass 1: name → node_id across all files
-    # Map: stem → {ClassName: node_id}
+    # Pass 1: _file_stem(path) → {ClassName: node_id}
+    # Keyed by directory-qualified stem (e.g. "auth_models") to avoid collisions
+    # when multiple files share the same filename in different directories.
+    # A secondary bare-stem index handles absolute imports where only the module
+    # name is known — first writer wins when names collide (inherently ambiguous).
     stem_to_entities: dict[str, dict[str, str]] = {}
+    bare_to_qualified: dict[str, str] = {}
     for file_result in per_file:
         for node in file_result.get("nodes", []):
             src = node.get("source_file", "")
             if not src:
                 continue
-            stem = Path(src).stem
+            src_path = Path(src)
+            fq_stem = _file_stem(src_path)
             label = node.get("label", "")
             nid = node.get("id", "")
             # Index class-level entities only. Function/method labels end in "()"
@@ -4244,11 +4249,13 @@ def _resolve_cross_file_imports(
                 and "_" not in label[:1]
                 and node.get("file_type") != "rationale"
             ):
-                stem_to_entities.setdefault(stem, {})[label] = nid
+                stem_to_entities.setdefault(fq_stem, {})[label] = nid
+                if src_path.stem not in bare_to_qualified:
+                    bare_to_qualified[src_path.stem] = fq_stem
 
     # Pass 2: for each file, find `from .X import A, B, C` and resolve
     new_edges: list[dict] = []
-    stem_to_path: dict[str, Path] = {p.stem: p for p in paths}
+    stem_to_path: dict[str, Path] = {_file_stem(p): p for p in paths}
 
     for file_result, path in zip(per_file, paths):
         stem = _file_stem(path)
@@ -4279,21 +4286,28 @@ def _resolve_cross_file_imports(
                 # Find the module name - handles both absolute and relative imports.
                 # Relative: `from .models import X` → relative_import → dotted_name
                 # Absolute: `from models import X`  → module_name field
-                target_stem: str | None = None
+                # target_fq is the directory-qualified stem used as the key in
+                # stem_to_entities. Relative imports are resolved exactly via the
+                # importing file's directory; absolute imports fall back to the
+                # bare-stem secondary index (first-writer-wins when names collide).
+                target_fq: str | None = None
                 for child in node.children:
                     if child.type == "relative_import":
-                        # Dig into relative_import → dotted_name → identifier
                         for sub in child.children:
                             if sub.type == "dotted_name":
                                 raw = source[sub.start_byte:sub.end_byte].decode("utf-8", errors="replace")
-                                target_stem = raw.split(".")[-1]
+                                bare = raw.split(".")[-1]
+                                # Resolve relative import to exact qualified stem.
+                                candidate = path.parent / f"{bare}.py"
+                                target_fq = _file_stem(candidate)
                                 break
                         break
-                    if child.type == "dotted_name" and target_stem is None:
+                    if child.type == "dotted_name" and target_fq is None:
                         raw = source[child.start_byte:child.end_byte].decode("utf-8", errors="replace")
-                        target_stem = raw.split(".")[-1]
+                        bare = raw.split(".")[-1]
+                        target_fq = bare_to_qualified.get(bare)
 
-                if not target_stem or target_stem not in stem_to_entities:
+                if not target_fq or target_fq not in stem_to_entities:
                     return
 
                 # Collect imported names: dotted_name children of import_from_statement
@@ -4320,7 +4334,7 @@ def _resolve_cross_file_imports(
 
                 line = node.start_point[0] + 1
                 for name in imported_names:
-                    tgt_nid = stem_to_entities[target_stem].get(name)
+                    tgt_nid = stem_to_entities[target_fq].get(name)
                     if tgt_nid:
                         for src_class_nid in local_classes:
                             new_edges.append({