diff --git a/graphify/build.py b/graphify/build.py index dd21f6f2..a0dbcae0 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -30,6 +30,26 @@ import networkx as nx from .validate import validate_extraction +# Synonym mapper for known invalid file_type values that LLM subagents commonly +# emit. Keeps semantic intent close (markdown→document, tool→code) and falls +# back to "concept" for any other invalid value (see #840). +_FILE_TYPE_SYNONYMS = { + "markdown": "document", + "text": "document", + "tool": "code", + "library": "code", + "pattern": "concept", + "principle": "concept", + "constraint": "concept", + "tech": "concept", + "technology": "concept", + "data-source": "concept", + "data_source": "concept", + "gotcha": "concept", + "framework": "concept", +} + + def _normalize_id(s: str) -> str: r"""Normalize an ID string the same way extract._make_id does. @@ -105,6 +125,9 @@ def build_from_json(extraction: dict, *, directed: bool = False) -> nx.Graph: # trigger spurious "invalid file_type 'None'" validator warnings (#660). if node.get("file_type") in (None, ""): node["file_type"] = "concept" + ft = node.get("file_type", "") + if ft and ft not in {"code", "document", "paper", "image", "rationale", "concept"}: + node["file_type"] = _FILE_TYPE_SYNONYMS.get(ft, "concept") errors = validate_extraction(extraction) # Dangling edges (stdlib/external imports) are expected - only warn about real schema errors. diff --git a/graphify/llm.py b/graphify/llm.py index 26787ed3..c1932697 100644 --- a/graphify/llm.py +++ b/graphify/llm.py @@ -122,7 +122,7 @@ Node ID format: lowercase, only [a-z0-9_], no dots or slashes. Format: {stem}_{entity} where stem = filename without extension, entity = symbol name (both normalised). Output exactly this schema: -{"nodes":[{"id":"stem_entity","label":"Human Readable Name","file_type":"code|document|paper|image|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"stem_entity","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[],"input_tokens":0,"output_tokens":0} """ diff --git a/graphify/skill.md b/graphify/skill.md index edee6975..9e7f5d26 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -315,7 +315,7 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. -Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). Do NOT invent file_types like `concept` — valid values are only `code|document|paper|image|rationale`. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. @@ -360,7 +360,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the filename without extension and entity is the symbol name, both normalized (lowercase, non-alphanumeric chars replaced with `_`). Example: `src/auth/session.py` + `ValidateToken` → `session_validatetoken`. This must match the ID the AST extractor generates so cross-references between code and semantic nodes connect correctly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH @@ -672,7 +672,9 @@ from graphify.detect import save_manifest # Save manifest for --update detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -save_manifest(detect['files']) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) # Update cumulative cost tracker extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) @@ -772,6 +774,24 @@ print(f'{new_total} new/changed file(s) to re-extract.') " ``` +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + If new files exist, first check whether all changed files are code files: ```bash diff --git a/tests/test_build.py b/tests/test_build.py index 95acc1d4..54497b46 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -116,8 +116,9 @@ def test_missing_file_type_defaults_to_concept(capsys): assert G.nodes["n1"]["file_type"] == "concept" -def test_real_invalid_file_type_still_warns(capsys): - """Truly invalid file_type values (not None, not empty) must still warn.""" +def test_real_invalid_file_type_coerced_to_concept(): + """Unknown file_type values are coerced through the synonym mapper, falling + back to 'concept' for anything that isn't a known LLM synonym (#840).""" ext = { "nodes": [ {"id": "n1", "label": "Bad", "file_type": "weird_type", "source_file": "a.py"}, @@ -126,10 +127,26 @@ def test_real_invalid_file_type_still_warns(capsys): "input_tokens": 0, "output_tokens": 0, } - build_from_json(ext) - err = capsys.readouterr().err - assert "invalid file_type" in err - assert "weird_type" in err + G = build_from_json(ext) + assert G.nodes["n1"]["file_type"] == "concept" + + +def test_file_type_synonym_mapping(): + """Known invalid file_type values map to their canonical equivalents.""" + ext = { + "nodes": [ + {"id": "n1", "label": "MD", "file_type": "markdown", "source_file": "a.md"}, + {"id": "n2", "label": "Tool", "file_type": "tool", "source_file": "b.py"}, + {"id": "n3", "label": "Pat", "file_type": "pattern", "source_file": "c.md"}, + ], + "edges": [], + "input_tokens": 0, + "output_tokens": 0, + } + G = build_from_json(ext) + assert G.nodes["n1"]["file_type"] == "document" + assert G.nodes["n2"]["file_type"] == "code" + assert G.nodes["n3"]["file_type"] == "concept" def test_build_merge_preserves_call_edge_direction(tmp_path):