From 64f38acf3e3ff0f73c2b17ff11ca99635ef3ba09 Mon Sep 17 00:00:00 2001 From: Safi Date: Wed, 22 Apr 2026 23:28:09 +0100 Subject: [PATCH] implement #488 #482 #472 #490: legacy schema canonicalization, Java inheritance, aggregated HTML viz, check-update subcommand --- graphify/__main__.py | 8 +++ graphify/build.py | 6 ++ graphify/export.py | 18 +++-- graphify/extract.py | 138 +++++++++++++++++++++++++++++++++++++ graphify/skill-opencode.md | 26 ++++++- graphify/skill.md | 26 ++++++- graphify/watch.py | 15 ++++ tests/test_build.py | 21 ++++++ tests/test_export.py | 11 +++ tests/test_watch.py | 28 ++++++++ 10 files changed, 289 insertions(+), 8 deletions(-) diff --git a/graphify/__main__.py b/graphify/__main__.py index dc899c40..9abaf5a5 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -940,6 +940,7 @@ def main() -> None: print(" --type T query type: query|path_query|explain (default: query)") print(" --nodes N1 N2 ... source node labels cited in the answer") print(" --memory-dir DIR memory directory (default: graphify-out/memory)") + print(" check-update check needs_update flag and notify if semantic re-extraction is pending (cron-safe)") print(" benchmark [graph.json] measure token reduction vs naive full-corpus approach") print(" hook install install post-commit/post-checkout git hooks (all platforms)") print(" hook uninstall remove git hooks") @@ -1351,6 +1352,13 @@ def main() -> None: print("Nothing to update or rebuild failed — check output above.", file=sys.stderr) sys.exit(1) + elif cmd == "check-update": + if len(sys.argv) < 3: + print("Usage: graphify check-update ", file=sys.stderr) + sys.exit(1) + from graphify.watch import check_update + check_update(Path(sys.argv[2]).resolve()) + sys.exit(0) elif cmd == "benchmark": from graphify.benchmark import run_benchmark, print_benchmark graph_path = sys.argv[2] if len(sys.argv) > 2 else "graphify-out/graph.json" diff --git a/graphify/build.py b/graphify/build.py index f00f9842..19b5b186 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -46,6 +46,12 @@ def build_from_json(extraction: dict, *, directed: bool = False) -> nx.Graph: # NetworkX <= 3.1 serialised edges as "links"; remap to "edges" for compatibility. if "edges" not in extraction and "links" in extraction: extraction = dict(extraction, edges=extraction["links"]) + + # Canonicalize legacy node/edge schema before validation. + for node in extraction.get("nodes", []): + if isinstance(node, dict) and "source" in node and "source_file" not in node: + node["source_file"] = node.pop("source") + errors = validate_extraction(extraction) # Dangling edges (stdlib/external imports) are expected - only warn about real schema errors. real_errors = [e for e in errors if "does not match any node id" not in e] diff --git a/graphify/export.py b/graphify/export.py index 0e96bdb7..12cafb87 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -344,12 +344,16 @@ def to_html( communities: dict[int, list[str]], output_path: str, community_labels: dict[int, str] | None = None, + member_counts: dict[int, int] | None = None, ) -> None: """Generate an interactive vis.js HTML visualization of the graph. Features: node size by degree, click-to-inspect panel, search box, community filter, physics clustering by community, confidence-styled edges. Raises ValueError if graph exceeds MAX_NODES_FOR_VIZ. + + If member_counts is provided (aggregated community view), node sizes are + based on community member counts rather than graph degree. """ if G.number_of_nodes() > MAX_NODES_FOR_VIZ: raise ValueError( @@ -360,6 +364,7 @@ def to_html( node_community = _node_community_map(communities) degree = dict(G.degree()) max_deg = max(degree.values(), default=1) or 1 + max_mc = (max(member_counts.values(), default=1) or 1) if member_counts else 1 # Build nodes list for vis.js vis_nodes = [] @@ -368,9 +373,14 @@ def to_html( color = COMMUNITY_COLORS[cid % len(COMMUNITY_COLORS)] label = sanitize_label(data.get("label", node_id)) deg = degree.get(node_id, 1) - size = 10 + 30 * (deg / max_deg) - # Only show label for high-degree nodes by default; others show on hover - font_size = 12 if deg >= max_deg * 0.15 else 0 + if member_counts: + mc = member_counts.get(cid, 1) + size = 10 + 30 * (mc / max_mc) + font_size = 12 + else: + size = 10 + 30 * (deg / max_deg) + # Only show label for high-degree nodes by default; others show on hover + font_size = 12 if deg >= max_deg * 0.15 else 0 vis_nodes.append({ "id": node_id, "label": label, @@ -406,7 +416,7 @@ def to_html( for cid in sorted((community_labels or {}).keys()): color = COMMUNITY_COLORS[cid % len(COMMUNITY_COLORS)] lbl = _html.escape(sanitize_label((community_labels or {}).get(cid, f"Community {cid}"))) - n = len(communities.get(cid, [])) + n = member_counts.get(cid, len(communities.get(cid, []))) if member_counts else len(communities.get(cid, [])) legend_data.append({"cid": cid, "color": color, "label": lbl, "count": n}) # Escape sequences so embedded JSON cannot break out of the script tag diff --git a/graphify/extract.py b/graphify/extract.py index e12ff80a..dbd441c6 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -803,6 +803,49 @@ def _extract_generic(path: Path, config: LanguageConfig) -> dict: seen_ids.add(base_nid) add_edge(class_nid, base_nid, "inherits", line) + # Java-specific: extends (superclass) / implements (interfaces) / interface-extends + if config.ts_module == "tree_sitter_java": + def _emit_java_parent(base_name: str, rel: str, at_line: int) -> None: + if not base_name: + return + base_nid = _make_id(stem, base_name) + if base_nid not in seen_ids: + base_nid = _make_id(base_name) + if base_nid not in seen_ids: + nodes.append({ + "id": base_nid, + "label": base_name, + "file_type": "code", + "source_file": "", + "source_location": "", + }) + seen_ids.add(base_nid) + add_edge(class_nid, base_nid, rel, at_line) + + sup = node.child_by_field_name("superclass") + if sup is not None: + for sub in sup.children: + if sub.type == "type_identifier": + _emit_java_parent(_read_text(sub, source), "extends", line) + break + + ifs = node.child_by_field_name("interfaces") + if ifs is not None: + for sub in ifs.children: + if sub.type == "type_list": + for tid in sub.children: + if tid.type == "type_identifier": + _emit_java_parent(_read_text(tid, source), "implements", line) + + if t == "interface_declaration": + for child in node.children: + if child.type == "extends_interfaces": + for sub in child.children: + if sub.type == "type_list": + for tid in sub.children: + if tid.type == "type_identifier": + _emit_java_parent(_read_text(tid, source), "extends", line) + # Find body and recurse body = _find_body(node, config) if body: @@ -2664,6 +2707,91 @@ def _resolve_cross_file_imports( return new_edges +def _resolve_cross_file_java_imports( + per_file: list[dict], + paths: list[Path], +) -> list[dict]: + """Two-pass Java import resolution. + + Pass 1: build a global index {ClassName: [node_id, ...]} across all Java nodes. + Pass 2: re-parse each Java file; for every `import a.b.C;`, resolve C against + the index. Wildcard and stdlib imports produce no edge. + """ + try: + import tree_sitter_java as tsjava + from tree_sitter import Language, Parser + except ImportError: + return [] + + language = Language(tsjava.language()) + parser = Parser(language) + + # Pass 1: class-name → node_id index (only internal, uppercase-starting names) + name_to_ids: dict[str, list[str]] = {} + for file_result in per_file: + for node in file_result.get("nodes", []): + label = node.get("label", "") + nid = node.get("id", "") + src = node.get("source_file", "") + if not label or not nid or not src: + continue + if label.endswith(")") or label.endswith(".java"): + continue + if not label[0].isalpha() or not label[0].isupper(): + continue + name_to_ids.setdefault(label, []).append(nid) + + # Pass 2: resolve imports to real node IDs + new_edges: list[dict] = [] + seen_pairs: set[tuple[str, str]] = set() + for path in paths: + file_nid = _make_id(path.stem) + try: + source = path.read_bytes() + tree = parser.parse(source) + except Exception: + continue + + def walk(n) -> None: + if n.type == "import_declaration": + raw = _read_text(n, source).strip() + body = raw[len("import"):].strip().rstrip(";").strip() + if body.startswith("static "): + body = body[len("static "):].strip() + if body.endswith(".*"): + return + parts = body.split(".") + if not parts: + return + last = parts[-1] + if last and last[0].islower() and len(parts) >= 2: + last = parts[-2] + at_line = n.start_point[0] + 1 + for tgt_nid in name_to_ids.get(last, []): + if tgt_nid == file_nid: + continue + key = (file_nid, tgt_nid) + if key in seen_pairs: + continue + seen_pairs.add(key) + new_edges.append({ + "source": file_nid, + "target": tgt_nid, + "relation": "imports", + "confidence": "EXTRACTED", + "confidence_score": 1.0, + "source_file": str(path), + "source_location": f"L{at_line}", + "weight": 1.0, + }) + for child in n.children: + walk(child) + + walk(tree.root_node) + + return new_edges + + def extract_objc(path: Path) -> dict: """Extract interfaces, implementations, protocols, methods, and imports from .m/.mm/.h files.""" try: @@ -3204,6 +3332,16 @@ def extract(paths: list[Path], cache_root: Path | None = None) -> dict: import logging logging.getLogger(__name__).warning("Cross-file import resolution failed, skipping: %s", exc) + # Cross-file Java import resolution + java_paths = [p for p in paths if p.suffix == ".java"] + if java_paths: + java_results = [r for r, p in zip(per_file, paths) if p.suffix == ".java"] + try: + all_edges.extend(_resolve_cross_file_java_imports(java_results, java_paths)) + except Exception as exc: + import logging + logging.getLogger(__name__).warning("Java cross-file import resolution failed, skipping: %s", exc) + # Cross-file call resolution for all languages # Each extractor saved unresolved calls in raw_calls. Now that we have all # nodes from all files, resolve any callee that exists in another file. diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index 2d4abc4a..479b677d 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -531,8 +531,30 @@ G = build_from_json(extraction) communities = {int(k): v for k, v in analysis['communities'].items()} labels = {int(k): v for k, v in labels_raw.items()} -if G.number_of_nodes() > 5000: - print(f'Graph has {G.number_of_nodes()} nodes - too large for HTML viz. Use Obsidian vault instead.') +NODE_LIMIT = 5000 +if G.number_of_nodes() > NODE_LIMIT: + from collections import Counter + print(f'Graph has {G.number_of_nodes()} nodes (above {NODE_LIMIT} limit). Building aggregated community view...') + node_to_community = {nid: cid for cid, members in communities.items() for nid in members} + import networkx as nx_meta + meta = nx_meta.Graph() + for cid, members in communities.items(): + meta.add_node(str(cid), label=labels.get(cid, f'Community {cid}')) + edge_counts = Counter() + for u, v in G.edges(): + cu, cv = node_to_community.get(u), node_to_community.get(v) + if cu is not None and cv is not None and cu != cv: + edge_counts[(min(cu, cv), max(cu, cv))] += 1 + for (cu, cv), w in edge_counts.items(): + meta.add_edge(str(cu), str(cv), weight=w, relation=f'{w} cross-community edges', confidence='AGGREGATED') + if meta.number_of_nodes() > 1: + meta_communities = {cid: [str(cid)] for cid in communities} + member_counts = {cid: len(members) for cid, members in communities.items()} + to_html(meta, meta_communities, 'graphify-out/graph.html', community_labels=labels or None, member_counts=member_counts) + print(f'graph.html written (aggregated: {meta.number_of_nodes()} community nodes, {meta.number_of_edges()} cross-community edges)') + print('Tip: run with --obsidian for full node-level detail.') + else: + print('Single community — aggregated view not useful. Skipping graph.html.') else: to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) print('graph.html written - open in any browser, no server needed') diff --git a/graphify/skill.md b/graphify/skill.md index a2c029ab..0674b0a7 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -550,8 +550,30 @@ G = build_from_json(extraction) communities = {int(k): v for k, v in analysis['communities'].items()} labels = {int(k): v for k, v in labels_raw.items()} -if G.number_of_nodes() > 5000: - print(f'Graph has {G.number_of_nodes()} nodes - too large for HTML viz. Use Obsidian vault instead.') +NODE_LIMIT = 5000 +if G.number_of_nodes() > NODE_LIMIT: + from collections import Counter + print(f'Graph has {G.number_of_nodes()} nodes (above {NODE_LIMIT} limit). Building aggregated community view...') + node_to_community = {nid: cid for cid, members in communities.items() for nid in members} + import networkx as nx_meta + meta = nx_meta.Graph() + for cid, members in communities.items(): + meta.add_node(str(cid), label=labels.get(cid, f'Community {cid}')) + edge_counts = Counter() + for u, v in G.edges(): + cu, cv = node_to_community.get(u), node_to_community.get(v) + if cu is not None and cv is not None and cu != cv: + edge_counts[(min(cu, cv), max(cu, cv))] += 1 + for (cu, cv), w in edge_counts.items(): + meta.add_edge(str(cu), str(cv), weight=w, relation=f'{w} cross-community edges', confidence='AGGREGATED') + if meta.number_of_nodes() > 1: + meta_communities = {cid: [str(cid)] for cid in communities} + member_counts = {cid: len(members) for cid, members in communities.items()} + to_html(meta, meta_communities, 'graphify-out/graph.html', community_labels=labels or None, member_counts=member_counts) + print(f'graph.html written (aggregated: {meta.number_of_nodes()} community nodes, {meta.number_of_edges()} cross-community edges)') + print('Tip: run with --obsidian for full node-level detail.') + else: + print('Single community — aggregated view not useful. Skipping graph.html.') else: to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) print('graph.html written - open in any browser, no server needed') diff --git a/graphify/watch.py b/graphify/watch.py index 5babd89f..a09dd51e 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -132,6 +132,21 @@ def _rebuild_code(watch_path: Path, *, follow_symlinks: bool = False) -> bool: return False +def check_update(watch_path: Path) -> bool: + """Check for pending semantic update flag and notify the user if set. + + Cron-safe: always returns True so cron jobs do not alarm. + Non-code file changes (docs, papers, images) require LLM-backed + re-extraction via `/graphify --update` — this function only signals + that the update is needed. + """ + flag = Path(watch_path) / "graphify-out" / "needs_update" + if flag.exists(): + print(f"[graphify check-update] Pending non-code changes in {watch_path}.") + print("[graphify check-update] Run `/graphify --update` to apply semantic re-extraction.") + return True + + def _notify_only(watch_path: Path) -> None: """Write a flag file and print a notification (fallback for non-code-only corpora).""" flag = watch_path / "graphify-out" / "needs_update" diff --git a/tests/test_build.py b/tests/test_build.py index 54a7fa45..2b47bc86 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -29,6 +29,27 @@ def test_ambiguous_edge_preserved(): data = G.edges["n_layernorm", "n_concept_attn"] assert data["confidence"] == "AMBIGUOUS" +def test_legacy_node_source_canonicalized(): + """Legacy 'source' key on nodes is renamed to 'source_file' before graph build.""" + ext = {"nodes": [{"id": "n1", "label": "A", "file_type": "code", "source": "a.py"}], + "edges": [], "input_tokens": 0, "output_tokens": 0} + G = build_from_json(ext) + assert "source_file" in G.nodes["n1"] + assert G.nodes["n1"]["source_file"] == "a.py" + assert "source" not in G.nodes["n1"] + + +def test_legacy_edge_from_to_canonicalized(): + """Legacy 'from'/'to' keys on edges are accepted alongside 'source'/'target'.""" + ext = {"nodes": [{"id": "n1", "label": "A", "file_type": "code", "source_file": "a.py"}, + {"id": "n2", "label": "B", "file_type": "code", "source_file": "b.py"}], + "edges": [{"from": "n1", "to": "n2", "relation": "calls", + "confidence": "EXTRACTED", "source_file": "a.py", "weight": 1.0}], + "input_tokens": 0, "output_tokens": 0} + G = build_from_json(ext) + assert G.number_of_edges() == 1 + + def test_build_merges_multiple_extractions(): ext1 = {"nodes": [{"id": "n1", "label": "A", "file_type": "code", "source_file": "a.py"}], "edges": [], "input_tokens": 0, "output_tokens": 0} diff --git a/tests/test_export.py b/tests/test_export.py index bb5e4ba5..e93ba815 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -127,6 +127,17 @@ def test_to_html_contains_nodes_and_edges(): assert "RAW_EDGES" in content +def test_to_html_member_counts_accepted(): + """to_html accepts member_counts without raising.""" + G = make_graph() + communities = cluster(G) + member_counts = {cid: len(members) for cid, members in communities.items()} + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "graph.html" + to_html(G, communities, str(out), member_counts=member_counts) + assert out.exists() + + def test_to_canvas_file_paths_relative_to_vault(): """Node file paths in canvas must be vault-root-relative (just fname.md), not hardcoded.""" G = make_graph() diff --git a/tests/test_watch.py b/tests/test_watch.py index 5f17892a..ac396aa6 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -52,6 +52,34 @@ def test_watched_extensions_excludes_noise(): # --- watch() import error without watchdog --- +def test_check_update_no_flag_returns_true(tmp_path): + """check_update returns True and is silent when needs_update flag is absent.""" + from graphify.watch import check_update + assert check_update(tmp_path) is True + + +def test_check_update_with_flag_returns_true_and_prints(tmp_path, capsys): + """check_update returns True and prints notification when flag exists.""" + from graphify.watch import check_update + flag = tmp_path / "graphify-out" / "needs_update" + flag.parent.mkdir(parents=True, exist_ok=True) + flag.write_text("1") + result = check_update(tmp_path) + assert result is True + out = capsys.readouterr().out + assert "graphify --update" in out + + +def test_check_update_does_not_clear_flag(tmp_path): + """check_update never removes the needs_update flag (clearing is LLM's job).""" + from graphify.watch import check_update + flag = tmp_path / "graphify-out" / "needs_update" + flag.parent.mkdir(parents=True, exist_ok=True) + flag.write_text("1") + check_update(tmp_path) + assert flag.exists() + + def test_watch_raises_without_watchdog(tmp_path, monkeypatch): import builtins real_import = builtins.__import__