diff --git a/graphify/export.py b/graphify/export.py index 14306328..e733221d 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -195,11 +195,18 @@ LEGEND.forEach(c => {{ """ +_CONFIDENCE_SCORE_DEFAULTS = {"EXTRACTED": 1.0, "INFERRED": 0.5, "AMBIGUOUS": 0.2} + + def to_json(G: nx.Graph, communities: dict[int, list[str]], output_path: str) -> None: node_community = _node_community_map(communities) data = json_graph.node_link_data(G, edges="links") for node in data["nodes"]: node["community"] = node_community.get(node["id"]) + for link in data["links"]: + if "confidence_score" not in link: + conf = link.get("confidence", "EXTRACTED") + link["confidence_score"] = _CONFIDENCE_SCORE_DEFAULTS.get(conf, 1.0) with open(output_path, "w") as f: json.dump(data, f, indent=2) diff --git a/graphify/report.py b/graphify/report.py index 1a67a52b..3cfe4ad1 100644 --- a/graphify/report.py +++ b/graphify/report.py @@ -24,6 +24,10 @@ def generate( inf_pct = round(confidences.count("INFERRED") / total * 100) amb_pct = round(confidences.count("AMBIGUOUS") / total * 100) + inf_edges = [(u, v, d) for u, v, d in G.edges(data=True) if d.get("confidence") == "INFERRED"] + inf_scores = [d.get("confidence_score", 0.5) for _, _, d in inf_edges] + inf_avg = round(sum(inf_scores) / len(inf_scores), 2) if inf_scores else None + lines = [ f"# Graph Report - {root} ({today})", "", @@ -41,7 +45,8 @@ def generate( "", "## Summary", f"- {G.number_of_nodes()} nodes · {G.number_of_edges()} edges · {len(communities)} communities detected", - f"- Extraction: {ext_pct}% EXTRACTED · {inf_pct}% INFERRED · {amb_pct}% AMBIGUOUS", + f"- Extraction: {ext_pct}% EXTRACTED · {inf_pct}% INFERRED · {amb_pct}% AMBIGUOUS" + + (f" · INFERRED: {len(inf_edges)} edges (avg confidence: {inf_avg})" if inf_avg is not None else ""), f"- Token cost: {token_cost.get('input', 0):,} input · {token_cost.get('output', 0):,} output", "", "## God Nodes (most connected - your core abstractions)", @@ -55,8 +60,14 @@ def generate( relation = s.get("relation", "related_to") note = s.get("note", "") files = s.get("source_files", ["", ""]) + conf = s.get("confidence", "EXTRACTED") + cscore = s.get("confidence_score") + if conf == "INFERRED" and cscore is not None: + conf_tag = f"INFERRED {cscore:.2f}" + else: + conf_tag = conf lines += [ - f"- `{s['source']}` --{relation}--> `{s['target']}` [{s['confidence']}]", + f"- `{s['source']}` --{relation}--> `{s['target']}` [{conf_tag}]", f" {files[0]} → {files[1]}" + (f" _{note}_" if note else ""), ] else: diff --git a/graphify/skill.md b/graphify/skill.md index a2aed2d1..5774d07b 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -210,8 +210,15 @@ DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indire If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. +confidence_score rules: +- EXTRACTED edges: confidence_score must be 1.0 +- INFERRED edges: score 0.4-0.9 based on how certain you are. + Strong structural inference (e.g. two classes clearly share data): 0.8-0.9. + Reasonable but not certain: 0.6-0.7. Weak inference: 0.4-0.5. +- AMBIGUOUS edges: score 0.1-0.3 + Output exactly this JSON (no other text): -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","source_file":"relative/path","source_location":null,"weight":1.0}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"input_tokens":0,"output_tokens":0} ``` **Step B3 - Collect, cache, and merge** diff --git a/pyproject.toml b/pyproject.toml index 6cbcfe69..2bf16951 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" -version = "0.1.11" +version = "0.1.12" description = "Claude Code skill - turn any folder of code, docs, papers, images, or tweets into a queryable knowledge graph" readme = "README.md" license = { text = "MIT" } diff --git a/skills/graphify/skill.md b/skills/graphify/skill.md index a2aed2d1..5774d07b 100644 --- a/skills/graphify/skill.md +++ b/skills/graphify/skill.md @@ -210,8 +210,15 @@ DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indire If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. +confidence_score rules: +- EXTRACTED edges: confidence_score must be 1.0 +- INFERRED edges: score 0.4-0.9 based on how certain you are. + Strong structural inference (e.g. two classes clearly share data): 0.8-0.9. + Reasonable but not certain: 0.6-0.7. Weak inference: 0.4-0.5. +- AMBIGUOUS edges: score 0.1-0.3 + Output exactly this JSON (no other text): -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","source_file":"relative/path","source_location":null,"weight":1.0}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"input_tokens":0,"output_tokens":0} ``` **Step B3 - Collect, cache, and merge** diff --git a/tests/test_confidence.py b/tests/test_confidence.py new file mode 100644 index 00000000..299548ac --- /dev/null +++ b/tests/test_confidence.py @@ -0,0 +1,192 @@ +"""Tests for confidence_score on edges.""" +import json +import tempfile +from pathlib import Path + +import networkx as nx + +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections +from graphify.export import to_json +from graphify.report import generate + +FIXTURES = Path(__file__).parent / "fixtures" + + +def _make_extraction(**edge_overrides): + """Return a minimal extraction dict with one edge of each confidence type.""" + base = { + "nodes": [ + {"id": "n_a", "label": "A", "file_type": "code", "source_file": "a.py"}, + {"id": "n_b", "label": "B", "file_type": "code", "source_file": "b.py"}, + {"id": "n_c", "label": "C", "file_type": "document", "source_file": "c.md"}, + {"id": "n_d", "label": "D", "file_type": "document", "source_file": "d.md"}, + ], + "edges": [ + {"source": "n_a", "target": "n_b", "relation": "calls", "confidence": "EXTRACTED", + "confidence_score": 1.0, "source_file": "a.py", "weight": 1.0}, + {"source": "n_b", "target": "n_c", "relation": "implements", "confidence": "INFERRED", + "confidence_score": 0.75, "source_file": "b.py", "weight": 0.8}, + {"source": "n_c", "target": "n_d", "relation": "references", "confidence": "AMBIGUOUS", + "confidence_score": 0.2, "source_file": "c.md", "weight": 0.5}, + ], + "input_tokens": 100, + "output_tokens": 50, + } + return base + + +def test_extracted_edges_have_score_1(): + """EXTRACTED edges must have confidence_score == 1.0.""" + G = build_from_json(_make_extraction()) + for u, v, d in G.edges(data=True): + if d.get("confidence") == "EXTRACTED": + assert d.get("confidence_score") == 1.0, ( + f"EXTRACTED edge ({u},{v}) should have confidence_score=1.0, got {d.get('confidence_score')}" + ) + + +def test_inferred_edges_score_in_range(): + """INFERRED edges must have confidence_score between 0.0 and 1.0.""" + G = build_from_json(_make_extraction()) + found = False + for u, v, d in G.edges(data=True): + if d.get("confidence") == "INFERRED": + found = True + score = d.get("confidence_score") + assert score is not None, f"INFERRED edge ({u},{v}) missing confidence_score" + assert 0.0 <= score <= 1.0, ( + f"INFERRED edge ({u},{v}) confidence_score={score} out of range [0,1]" + ) + assert found, "No INFERRED edges found in test fixture" + + +def test_ambiguous_edges_score_at_most_04(): + """AMBIGUOUS edges must have confidence_score <= 0.4.""" + G = build_from_json(_make_extraction()) + found = False + for u, v, d in G.edges(data=True): + if d.get("confidence") == "AMBIGUOUS": + found = True + score = d.get("confidence_score") + assert score is not None, f"AMBIGUOUS edge ({u},{v}) missing confidence_score" + assert score <= 0.4, ( + f"AMBIGUOUS edge ({u},{v}) confidence_score={score} should be <= 0.4" + ) + assert found, "No AMBIGUOUS edges found in test fixture" + + +def test_confidence_score_round_trip(): + """confidence_score survives build_from_json → to_json → JSON parse round-trip.""" + extraction = _make_extraction() + G = build_from_json(extraction) + communities = cluster(G) + + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "graph.json" + to_json(G, communities, str(out)) + data = json.loads(out.read_text()) + + # to_json uses node_link_data which puts edges in "links" + links = data.get("links", []) + assert links, "No links found in exported graph.json" + for link in links: + assert "confidence_score" in link, f"Link missing confidence_score: {link}" + score = link["confidence_score"] + assert isinstance(score, float), f"confidence_score should be float, got {type(score)}" + assert 0.0 <= score <= 1.0, f"confidence_score={score} out of range" + + +def test_to_json_defaults_missing_confidence_score(): + """Edges lacking confidence_score get sensible defaults in to_json.""" + extraction = { + "nodes": [ + {"id": "n_x", "label": "X", "file_type": "code", "source_file": "x.py"}, + {"id": "n_y", "label": "Y", "file_type": "code", "source_file": "y.py"}, + {"id": "n_z", "label": "Z", "file_type": "code", "source_file": "z.py"}, + ], + "edges": [ + # No confidence_score field on any of these + {"source": "n_x", "target": "n_y", "relation": "calls", + "confidence": "EXTRACTED", "source_file": "x.py", "weight": 1.0}, + {"source": "n_y", "target": "n_z", "relation": "depends_on", + "confidence": "INFERRED", "source_file": "y.py", "weight": 1.0}, + ], + "input_tokens": 0, + "output_tokens": 0, + } + G = build_from_json(extraction) + communities = cluster(G) + + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "graph.json" + to_json(G, communities, str(out)) + data = json.loads(out.read_text()) + + links_by_conf = {} + for link in data.get("links", []): + conf = link.get("confidence", "EXTRACTED") + links_by_conf[conf] = link.get("confidence_score") + + assert links_by_conf.get("EXTRACTED") == 1.0, "EXTRACTED default should be 1.0" + assert links_by_conf.get("INFERRED") == 0.5, "INFERRED default should be 0.5" + + +def test_report_shows_avg_confidence_for_inferred(): + """Report summary line should include avg confidence for INFERRED edges.""" + extraction = _make_extraction() + G = build_from_json(extraction) + communities = cluster(G) + cohesion = score_all(G, communities) + labels = {cid: f"Community {cid}" for cid in communities} + gods = god_nodes(G) + surprises = surprising_connections(G) + detection = {"total_files": 2, "total_words": 5000, "needs_graph": True, "warning": None} + tokens = {"input": 100, "output": 50} + + report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, ".") + assert "avg confidence" in report, "Report should show avg confidence for INFERRED edges" + # The fixture has one INFERRED edge with score 0.75, so avg should be 0.75 + assert "0.75" in report, f"Expected avg confidence 0.75 in report" + + +def test_report_inferred_tag_with_score(): + """Surprising connections section shows confidence score next to INFERRED edges.""" + # Build a graph where surprising_connections will find an INFERRED cross-file edge + extraction = { + "nodes": [ + {"id": "n_p", "label": "Parser", "file_type": "code", "source_file": "parser.py"}, + {"id": "n_q", "label": "Renderer", "file_type": "code", "source_file": "renderer.py"}, + ], + "edges": [ + {"source": "n_p", "target": "n_q", "relation": "feeds", + "confidence": "INFERRED", "confidence_score": 0.82, + "source_file": "parser.py", "weight": 1.0}, + ], + "input_tokens": 0, + "output_tokens": 0, + } + G = build_from_json(extraction) + + # Manually construct a surprise entry the way analyze.surprising_connections would + surprise = { + "source": "Parser", + "target": "Renderer", + "relation": "feeds", + "confidence": "INFERRED", + "confidence_score": 0.82, + "source_files": ["parser.py", "renderer.py"], + "note": "", + } + communities = cluster(G) + cohesion = score_all(G, communities) + labels = {cid: f"Community {cid}" for cid in communities} + gods = god_nodes(G) + detection = {"total_files": 2, "total_words": 1000, "needs_graph": True, "warning": None} + tokens = {"input": 0, "output": 0} + + report = generate(G, communities, cohesion, labels, gods, [surprise], detection, tokens, ".") + assert "INFERRED 0.82" in report, ( + f"Report should show 'INFERRED 0.82' in surprising connections section. Got:\n{report}" + )