diff --git a/graphify/__main__.py b/graphify/__main__.py index 9e7b9c15..2520d6b0 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -949,11 +949,15 @@ def _clone_repo(url: str, branch: str | None = None, out_dir: Path | None = None else: dest = Path.home() / ".graphify" / "repos" / owner / repo + if branch and branch.startswith("-"): + print(f"error: invalid branch name: {branch!r}", file=sys.stderr) + sys.exit(1) + if dest.exists(): print(f"Repo already cloned at {dest} — pulling latest...", flush=True) cmd = ["git", "-C", str(dest), "pull"] if branch: - cmd += ["origin", branch] + cmd += ["origin", "--", branch] result = _sp.run(cmd, capture_output=True, text=True) if result.returncode != 0: print(f"warning: git pull failed:\n{result.stderr}", file=sys.stderr) @@ -963,7 +967,7 @@ def _clone_repo(url: str, branch: str | None = None, out_dir: Path | None = None cmd = ["git", "clone", "--depth", "1"] if branch: cmd += ["--branch", branch] - cmd += [git_url, str(dest)] + cmd += ["--", git_url, str(dest)] result = _sp.run(cmd, capture_output=True, text=True) if result.returncode != 0: print(f"error: git clone failed:\n{result.stderr}", file=sys.stderr) diff --git a/graphify/cache.py b/graphify/cache.py index 3de993cf..1dedcac2 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -4,6 +4,7 @@ from __future__ import annotations import hashlib import json import os +import tempfile from pathlib import Path @@ -111,20 +112,29 @@ def save_cached(path: Path, result: dict, root: Path = Path("."), kind: str = "a if not p.is_file(): return h = file_hash(p, root) - entry = cache_dir(root, kind) / f"{h}.json" - tmp = entry.with_suffix(".tmp") + target_dir = cache_dir(root, kind) + entry = target_dir / f"{h}.json" + fd, tmp_path = tempfile.mkstemp(dir=target_dir, prefix=f"{h}.", suffix=".tmp") try: - tmp.write_text(json.dumps(result), encoding="utf-8") + os.write(fd, json.dumps(result).encode()) + os.close(fd) try: - os.replace(tmp, entry) + os.replace(tmp_path, entry) except PermissionError: # Windows: os.replace can fail with WinError 5 if the target is # briefly locked. Fall back to copy-then-delete. import shutil - shutil.copy2(tmp, entry) - tmp.unlink(missing_ok=True) + shutil.copy2(tmp_path, entry) + os.unlink(tmp_path) except Exception: - tmp.unlink(missing_ok=True) + try: + os.close(fd) + except OSError: + pass + try: + os.unlink(tmp_path) + except OSError: + pass raise diff --git a/graphify/detect.py b/graphify/detect.py index 93e0adec..ba1595e6 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -385,16 +385,23 @@ def _parse_gitignore_line(raw: str) -> str: """Parse one raw line from a .graphifyignore file per gitignore spec. - Strip newline chars + - Strip inline comments (whitespace + # suffix), but only when # is + preceded by whitespace — so path#with#hash.py is preserved + - Unescape \\# to literal # - Remove trailing spaces unless escaped with backslash - Strip leading whitespace - - Return empty string for blank lines and comments + - Return empty string for blank lines and full-line comments """ line = raw.rstrip("\n\r") - # Remove unescaped trailing spaces (per gitignore spec) - line = re.sub(r"(? dict: } -def load_manifest(manifest_path: str = _MANIFEST_PATH) -> dict[str, float]: - """Load the file modification time manifest from a previous run.""" +def _md5_file(path: Path) -> str: + """MD5 of file contents streamed in 64KB chunks — for change detection only.""" + import hashlib as _hl + h = _hl.md5() + try: + with path.open("rb") as f: + for chunk in iter(lambda: f.read(65536), b""): + h.update(chunk) + except OSError: + return "" + return h.hexdigest() + + +def load_manifest(manifest_path: str = _MANIFEST_PATH) -> dict: + """Load the manifest from a previous run. Returns {} on any error.""" try: return json.loads(Path(manifest_path).read_text(encoding="utf-8")) except Exception: @@ -622,12 +642,13 @@ def load_manifest(manifest_path: str = _MANIFEST_PATH) -> dict[str, float]: def save_manifest(files: dict[str, list[str]], manifest_path: str = _MANIFEST_PATH) -> None: - """Save current file mtimes so the next --update run can diff against them.""" - manifest: dict[str, float] = {} + """Save current file mtimes + content hashes for change detection on --update.""" + manifest: dict[str, dict] = {} for file_list in files.values(): for f in file_list: try: - manifest[f] = Path(f).stat().st_mtime + p = Path(f) + manifest[f] = {"mtime": p.stat().st_mtime, "hash": _md5_file(p)} except OSError: pass # file deleted between detect() and manifest write - skip it Path(manifest_path).parent.mkdir(parents=True, exist_ok=True) @@ -637,8 +658,11 @@ def save_manifest(files: dict[str, list[str]], manifest_path: str = _MANIFEST_PA def detect_incremental(root: Path, manifest_path: str = _MANIFEST_PATH) -> dict: """Like detect(), but returns only new or modified files since the last run. - Compares current file mtimes against the stored manifest. - Use for --update mode: re-extract only what changed, merge into existing graph. + Fast path: mtime unchanged → unchanged (free, no hash). + Slow path: mtime bumped → compare MD5. Same hash = sync tool touched mtime, + treat as unchanged. Different hash = actually changed, re-extract. + + Backwards compatible with legacy manifests storing plain float mtime values. """ full = detect(root) manifest = load_manifest(manifest_path) @@ -656,12 +680,26 @@ def detect_incremental(root: Path, manifest_path: str = _MANIFEST_PATH) -> dict: for ftype, file_list in full["files"].items(): for f in file_list: - stored_mtime = manifest.get(f) + stored = manifest.get(f) try: current_mtime = Path(f).stat().st_mtime except Exception: current_mtime = 0 - if stored_mtime is None or current_mtime > stored_mtime: + + # Legacy manifest: plain float value + if isinstance(stored, (int, float)): + changed = stored is None or current_mtime > stored + elif isinstance(stored, dict): + stored_mtime = stored.get("mtime") + if stored_mtime is None or current_mtime != stored_mtime: + # mtime bumped — verify with content hash before re-extracting + changed = _md5_file(Path(f)) != stored.get("hash", "") + else: + changed = False + else: + changed = True # unknown format, re-extract to be safe + + if changed: new_files[ftype].append(f) else: unchanged_files[ftype].append(f) diff --git a/graphify/ingest.py b/graphify/ingest.py index 62d8386b..d811bfcd 100644 --- a/graphify/ingest.py +++ b/graphify/ingest.py @@ -49,19 +49,16 @@ def _fetch_html(url: str) -> str: def _html_to_markdown(html: str, url: str) -> str: - """Convert HTML to clean markdown. Uses html2text if available, else basic strip.""" + """Convert HTML to clean markdown. Uses markdownify if available, else basic strip.""" + # Always pre-strip script/style so their text content never leaks into output + html = re.sub(r"]*>.*?", "", html, flags=re.DOTALL | re.IGNORECASE) + html = re.sub(r"]*>.*?", "", html, flags=re.DOTALL | re.IGNORECASE) try: - import html2text - h = html2text.HTML2Text() - h.ignore_links = False - h.ignore_images = True - h.body_width = 0 - return h.handle(html) + from markdownify import markdownify + return markdownify(html, heading_style="ATX", bullets="-", strip=["img"]) except ImportError: - # Fallback: strip tags - text = re.sub(r"]*>.*?", "", html, flags=re.DOTALL | re.IGNORECASE) - text = re.sub(r"]*>.*?", "", text, flags=re.DOTALL | re.IGNORECASE) - text = re.sub(r"<[^>]+>", " ", text) + # Fallback: basic tag strip + text = re.sub(r"<[^>]+>", " ", html) text = re.sub(r"\s+", " ", text).strip() return text[:8000] diff --git a/graphify/llm.py b/graphify/llm.py index f07f8ec0..0f47d4ee 100644 --- a/graphify/llm.py +++ b/graphify/llm.py @@ -102,6 +102,9 @@ def _call_openai_compat( } if temperature is not None: kwargs["temperature"] = temperature + # Kimi-k2.6 is a reasoning model — disable thinking so content isn't empty + if "moonshot" in base_url: + kwargs["extra_body"] = {"thinking": {"type": "disabled"}} resp = client.chat.completions.create(**kwargs) result = _parse_llm_json(resp.choices[0].message.content or "{}") result["input_tokens"] = resp.usage.prompt_tokens if resp.usage else 0 diff --git a/graphify/serve.py b/graphify/serve.py index 361dec3c..dd82bcb6 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -45,6 +45,9 @@ def _strip_diacritics(text: str) -> str: return "".join(c for c in nfkd if not unicodedata.combining(c)) +_EXACT_MATCH_BONUS = 100.0 + + def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]: scored = [] norm_terms = [_strip_diacritics(t).lower() for t in terms] @@ -52,6 +55,9 @@ def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]: norm_label = data.get("norm_label") or _strip_diacritics(data.get("label") or "").lower() source = (data.get("source_file") or "").lower() score = sum(1 for t in norm_terms if t in norm_label) + sum(0.5 for t in norm_terms if t in source) + # Exact match: single term equals the full label (strip trailing () for functions) + if any(t == norm_label or t == norm_label.rstrip("()") for t in norm_terms): + score += _EXACT_MATCH_BONUS if score > 0: scored.append((score, nid)) return sorted(scored, reverse=True) @@ -89,11 +95,18 @@ def _dfs(G: nx.Graph, start_nodes: list[str], depth: int) -> tuple[set[str], lis return visited, edges_seen -def _subgraph_to_text(G: nx.Graph, nodes: set[str], edges: list[tuple], token_budget: int = 2000) -> str: - """Render subgraph as text, cutting at token_budget (approx 3 chars/token).""" +def _subgraph_to_text(G: nx.Graph, nodes: set[str], edges: list[tuple], token_budget: int = 2000, *, seeds: list[str] | None = None) -> str: + """Render subgraph as text, cutting at token_budget (approx 3 chars/token). + + seeds: exact-match nodes rendered first before the degree-sorted expansion, + so the queried symbol always appears at the top of the output. + """ char_budget = token_budget * 3 lines = [] - for nid in sorted(nodes, key=lambda n: G.degree(n), reverse=True): + seed_set = set(seeds or []) + ordered = [n for n in (seeds or []) if n in nodes] + \ + sorted(nodes - seed_set, key=lambda n: G.degree(n), reverse=True) + for nid in ordered: d = G.nodes[nid] line = f"NODE {sanitize_label(d.get('label', nid))} [src={d.get('source_file', '')} loc={d.get('source_location', '')} community={d.get('community', '')}]" lines.append(line) @@ -246,7 +259,7 @@ def serve(graph_path: str = "graphify-out/graph.json") -> None: return "No matching nodes found." nodes, edges = _dfs(G, start_nodes, depth) if mode == "dfs" else _bfs(G, start_nodes, depth) header = f"Traversal: {mode.upper()} depth={depth} | Start: {[G.nodes[n].get('label', n) for n in start_nodes]} | {len(nodes)} nodes found\n\n" - return header + _subgraph_to_text(G, nodes, edges, budget) + return header + _subgraph_to_text(G, nodes, edges, budget, seeds=start_nodes) def _tool_get_node(arguments: dict) -> str: label = arguments["label"].lower() diff --git a/graphify/watch.py b/graphify/watch.py index 3902a5e3..1e07bd39 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -104,6 +104,12 @@ def _rebuild_code(watch_path: Path, *, follow_symlinks: bool = False) -> bool: if not json_written: return False + try: + from graphify.detect import save_manifest + save_manifest(detected["files"]) + except Exception: + pass + report = generate(G, communities, cohesion, labels, gods, surprises, detection, {"input": 0, "output": 0}, report_root, suggested_questions=questions) (out / "GRAPH_REPORT.md").write_text(report, encoding="utf-8") diff --git a/pyproject.toml b/pyproject.toml index 1109afa3..d954f416 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ Issues = "https://github.com/safishamsi/graphify/issues" [project.optional-dependencies] mcp = ["mcp"] neo4j = ["neo4j"] -pdf = ["pypdf", "html2text"] +pdf = ["pypdf", "markdownify"] watch = ["watchdog"] svg = ["matplotlib"] leiden = ["graspologic; python_version < '3.13'"] @@ -52,7 +52,7 @@ office = ["python-docx", "openpyxl"] video = ["faster-whisper", "yt-dlp"] kimi = ["openai"] sql = ["tree-sitter-sql"] -all = ["mcp", "neo4j", "pypdf", "html2text", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper", "yt-dlp", "matplotlib", "openai", "tree-sitter-sql"] +all = ["mcp", "neo4j", "pypdf", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper", "yt-dlp", "matplotlib", "openai", "tree-sitter-sql"] [project.scripts] graphify = "graphify.__main__:main"