diff --git a/graphify/__main__.py b/graphify/__main__.py index cdb40ea3..584b02b5 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -254,6 +254,8 @@ _AGENTS_MD_SECTION = """\ This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. +When the user types `/graphify`, invoke the `skill` tool with `skill: "graphify"` before doing anything else. + Rules: - ALWAYS read graphify-out/GRAPH_REPORT.md before reading any source files, running grep/glob searches, or answering codebase questions. The graph is your primary map of the codebase. - IF graphify-out/wiki/index.md EXISTS, navigate it instead of reading raw files @@ -1525,6 +1527,25 @@ def main() -> None: print(f"No node matching '{target_label}' found.", file=sys.stderr) sys.exit(1) src_nid, tgt_nid = src_scored[0][1], tgt_scored[0][1] + # Ambiguity guard: when both queries resolve to the same node, the + # shortest path is trivially zero hops, which is almost never what the + # caller wanted (see bug #828). + if src_nid == tgt_nid: + print( + f"'{source_label}' and '{target_label}' both resolved to the same " + f"node '{src_nid}'. Use a more specific label or the exact node ID.", + file=sys.stderr, + ) + sys.exit(1) + for _name, _scored in (("source", src_scored), ("target", tgt_scored)): + if len(_scored) >= 2: + _top, _runner = _scored[0][0], _scored[1][0] + if _top > 0 and (_top - _runner) / _top < 0.10: + print( + f"warning: {_name} match was ambiguous " + f"(top score {_top:g}, runner-up {_runner:g})", + file=sys.stderr, + ) try: path_nodes = _nx.shortest_path(G, src_nid, tgt_nid) except (_nx.NetworkXNoPath, _nx.NodeNotFound): diff --git a/graphify/cache.py b/graphify/cache.py index ef7582d0..bdaf1e77 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -55,9 +55,9 @@ def file_hash(path: Path, root: Path = Path(".")) -> str: h.update(b"\x00") try: rel = p.resolve().relative_to(Path(root).resolve()) - h.update(str(rel).encode()) + h.update(rel.as_posix().lower().encode()) except ValueError: - h.update(str(p.resolve()).encode()) + h.update(p.resolve().as_posix().lower().encode()) return h.hexdigest() @@ -190,7 +190,10 @@ def check_semantic_cache( uncached: list[str] = [] for fpath in files: - result = load_cached(Path(fpath), root, kind="semantic") + p = Path(fpath) + if not p.is_absolute(): + p = Path(root) / p + result = load_cached(p, root, kind="semantic") if result is not None: cached_nodes.extend(result.get("nodes", [])) cached_edges.extend(result.get("edges", [])) diff --git a/graphify/serve.py b/graphify/serve.py index 295d1e75..9ffd6b46 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -48,7 +48,10 @@ def _strip_diacritics(text: str) -> str: return "".join(c for c in nfkd if not unicodedata.combining(c)) -_EXACT_MATCH_BONUS = 100.0 +_EXACT_MATCH_BONUS = 1000.0 +_PREFIX_MATCH_BONUS = 100.0 +_SUBSTRING_MATCH_BONUS = 1.0 +_SOURCE_MATCH_BONUS = 0.5 def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]: @@ -56,11 +59,20 @@ def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]: norm_terms = [_strip_diacritics(t).lower() for t in terms] for nid, data in G.nodes(data=True): norm_label = data.get("norm_label") or _strip_diacritics(data.get("label") or "").lower() + bare_label = norm_label.rstrip("()") source = (data.get("source_file") or "").lower() - score = sum(1 for t in norm_terms if t in norm_label) + sum(0.5 for t in norm_terms if t in source) - # Exact match: single term equals the full label (strip trailing () for functions) - if any(t == norm_label or t == norm_label.rstrip("()") for t in norm_terms): - score += _EXACT_MATCH_BONUS + score = 0.0 + for t in norm_terms: + # Three-tier precedence: exact > prefix > substring (take the + # strongest tier per term so a single term cannot double-count). + if t == norm_label or t == bare_label: + score += _EXACT_MATCH_BONUS + elif norm_label.startswith(t) or bare_label.startswith(t): + score += _PREFIX_MATCH_BONUS + elif t in norm_label: + score += _SUBSTRING_MATCH_BONUS + if t in source: + score += _SOURCE_MATCH_BONUS if score > 0: scored.append((score, nid)) return sorted(scored, reverse=True) @@ -233,11 +245,26 @@ def _query_graph_text( def _find_node(G: nx.Graph, label: str) -> list[str]: - """Return node IDs whose label or ID matches the search term (diacritic-insensitive).""" + """Return node IDs whose label or ID matches the search term (diacritic-insensitive). + + Results are ordered by three-tier precedence: exact match, then prefix match, + then substring match. Node-ID exact matches are grouped with label exact matches. + """ term = _strip_diacritics(label).lower() - return [nid for nid, d in G.nodes(data=True) - if term in (d.get("norm_label") or _strip_diacritics(d.get("label") or "").lower()) - or term == nid.lower()] + exact: list[str] = [] + prefix: list[str] = [] + substring: list[str] = [] + for nid, d in G.nodes(data=True): + norm_label = d.get("norm_label") or _strip_diacritics(d.get("label") or "").lower() + bare_label = norm_label.rstrip("()") + nid_lower = nid.lower() + if term == norm_label or term == bare_label or term == nid_lower: + exact.append(nid) + elif norm_label.startswith(term) or bare_label.startswith(term) or nid_lower.startswith(term): + prefix.append(nid) + elif term in norm_label: + substring.append(nid) + return exact + prefix + substring def _filter_blank_stdin() -> None: @@ -456,6 +483,23 @@ def serve(graph_path: str = "graphify-out/graph.json") -> None: if not tgt_scored: return f"No node matching target '{arguments['target']}' found." src_nid, tgt_nid = src_scored[0][1], tgt_scored[0][1] + # Ambiguity guard: when both queries resolve to the same node, the + # shortest path is trivially zero hops, which is almost never what the + # caller wanted (see bug #828). + if src_nid == tgt_nid: + return ( + f"'{arguments['source']}' and '{arguments['target']}' both resolved to " + f"the same node '{src_nid}'. Use a more specific label or the exact node ID." + ) + warnings: list[str] = [] + for name, scored in (("source", src_scored), ("target", tgt_scored)): + if len(scored) >= 2: + top, runner = scored[0][0], scored[1][0] + if top > 0 and (top - runner) / top < 0.10: + warnings.append( + f"warning: {name} match was ambiguous " + f"(top score {top:g}, runner-up {runner:g})" + ) max_hops = int(arguments.get("max_hops", 8)) try: path_nodes = nx.shortest_path(G, src_nid, tgt_nid) @@ -474,7 +518,8 @@ def serve(graph_path: str = "graphify-out/graph.json") -> None: if i == 0: segments.append(G.nodes[u].get("label", u)) segments.append(f"--{rel}{conf_str}--> {G.nodes[v].get('label', v)}") - return f"Shortest path ({hops} hops):\n " + " ".join(segments) + prefix = ("\n".join(warnings) + "\n") if warnings else "" + return prefix + f"Shortest path ({hops} hops):\n " + " ".join(segments) _handlers = { "query_graph": _tool_query_graph, diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index 37bbdf95..2bbf9fd2 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -69,10 +69,16 @@ import graphify '@ | Out-File -FilePath .graphify_step_1_ensure_graphify_is_installed_1.py -Encoding utf8 python .graphify_step_1_ensure_graphify_is_installed_1.py 2>$null Remove-Item -ErrorAction SilentlyContinue .graphify_step_1_ensure_graphify_is_installed_1.py -if ($LASTEXITCODE -ne 0) { pip install graphifyy -q 2>&1 | Select-Object -Last 3 } +if ($LASTEXITCODE -ne 0) { + if (Get-Command uv -ErrorAction SilentlyContinue) { + uv tool install --upgrade graphifyy -q 2>&1 | Select-Object -Last 3 + } else { + pip install graphifyy -q 2>&1 | Select-Object -Last 3 + } +} # Write interpreter path for all subsequent steps @' -import sys; open('.graphify_python', 'w').write(sys.executable) +import sys; open('.graphify_python', 'w', encoding='utf-8').write(sys.executable) '@ | Out-File -FilePath .graphify_step_1_ensure_graphify_is_installed_2.py -Encoding utf8 python .graphify_step_1_ensure_graphify_is_installed_2.py Remove-Item -ErrorAction SilentlyContinue .graphify_step_1_ensure_graphify_is_installed_2.py @@ -88,7 +94,7 @@ import json from graphify.detect import detect from pathlib import Path result = detect(Path('INPUT_PATH')) -print(json.dumps(result)) +print(json.dumps(result, ensure_ascii=False)) '@ | Out-File -FilePath .graphify_step_2_detect_files_3.py -Encoding utf8 python .graphify_step_2_detect_files_3.py > .graphify_detect.json Remove-Item -ErrorAction SilentlyContinue .graphify_step_2_detect_files_3.py @@ -140,12 +146,12 @@ import json, os from pathlib import Path from graphify.transcribe import transcribe_all -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding="utf-8")) video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths)) +print(json.dumps(transcript_paths, ensure_ascii=False)) '@ | Out-File -FilePath .graphify_step_transcribe.py -Encoding utf8 & (Get-Content graphify-out\.graphify_python) .graphify_step_transcribe.py | Out-File -FilePath graphify-out\.graphify_transcripts.json -Encoding utf8 Remove-Item -ErrorAction SilentlyContinue .graphify_step_transcribe.py @@ -182,16 +188,16 @@ from pathlib import Path def main(): code_files = [] - detect = json.loads(Path('.graphify_detect.json').read_text()) + detect = json.loads(Path('.graphify_detect.json').read_text(encoding="utf-8")) for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: result = extract(code_files) - Path('.graphify_ast.json').write_text(json.dumps(result, indent=2)) + Path('.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8") print(f'AST: {len(result["nodes"])} nodes, {len(result["edges"])} edges') else: - Path('.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) + Path('.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding="utf-8") print('No code files - skipping AST extraction') @@ -229,14 +235,14 @@ import json from graphify.cache import check_semantic_cache from pathlib import Path -detect = json.loads(Path('.graphify_detect.json').read_text()) +detect = json.loads(Path('.graphify_detect.json').read_text(encoding="utf-8")) all_files = [f for files in detect['files'].values() for f in files] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) if cached_nodes or cached_edges or cached_hyperedges: - Path('.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) -Path('.graphify_uncached.txt').write_text('\n'.join(uncached)) + Path('.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding="utf-8") +Path('.graphify_uncached.txt').write_text('\n'.join(uncached), encoding="utf-8") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') '@ | Out-File -FilePath .graphify_step_3_extract_entities_and_relations_5.py -Encoding utf8 python .graphify_step_3_extract_entities_and_relations_5.py @@ -343,7 +349,7 @@ chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: - d = json.loads(Path(c).read_text()) + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) @@ -352,7 +358,7 @@ for c in chunks: Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2)) +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') " ``` @@ -364,7 +370,7 @@ import json from graphify.cache import save_semantic_cache from pathlib import Path -new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('.graphify_semantic_new.json').read_text(encoding="utf-8")) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) print(f'Cached {saved} files') '@ | Out-File -FilePath .graphify_step_3_extract_entities_and_relations_6.py -Encoding utf8 @@ -378,8 +384,8 @@ Merge cached + new results into `.graphify_semantic.json`: import json from pathlib import Path -cached = json.loads(Path('.graphify_cached.json').read_text()) if Path('.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +cached = json.loads(Path('.graphify_cached.json').read_text(encoding="utf-8")) if Path('.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('.graphify_semantic_new.json').read_text(encoding="utf-8")) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) @@ -398,7 +404,7 @@ merged = { 'input_tokens': new.get('input_tokens', 0), 'output_tokens': new.get('output_tokens', 0), } -Path('.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) +Path('.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding="utf-8") print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached["nodes"])} from cache, {len(new.get("nodes",[]))} new)') '@ | Out-File -FilePath .graphify_step_3_extract_entities_and_relations_7.py -Encoding utf8 python .graphify_step_3_extract_entities_and_relations_7.py @@ -413,8 +419,8 @@ Clean up temp files: `Remove-Item -ErrorAction SilentlyContinue .graphify_cached import sys, json from pathlib import Path -ast = json.loads(Path('.graphify_ast.json').read_text()) -sem = json.loads(Path('.graphify_semantic.json').read_text()) +ast = json.loads(Path('.graphify_ast.json').read_text(encoding="utf-8")) +sem = json.loads(Path('.graphify_semantic.json').read_text(encoding="utf-8")) # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} @@ -433,7 +439,7 @@ merged = { 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } -Path('.graphify_extract.json').write_text(json.dumps(merged, indent=2)) +Path('.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding="utf-8") total = len(merged_nodes) edges = len(merged_edges) print(f'Merged: {total} nodes, {edges} edges ({len(ast["nodes"])} AST + {len(sem["nodes"])} semantic)') @@ -455,8 +461,8 @@ from graphify.report import generate from graphify.export import to_json from pathlib import Path -extraction = json.loads(Path('.graphify_extract.json').read_text()) -detection = json.loads(Path('.graphify_detect.json').read_text()) +extraction = json.loads(Path('.graphify_extract.json').read_text(encoding="utf-8")) +detection = json.loads(Path('.graphify_detect.json').read_text(encoding="utf-8")) G = build_from_json(extraction) communities = cluster(G) @@ -469,7 +475,7 @@ labels = {cid: 'Community ' + str(cid) for cid in communities} questions = suggest_questions(G, communities, labels) report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding="utf-8") to_json(G, communities, 'graphify-out/graph.json') analysis = { @@ -479,7 +485,7 @@ analysis = { 'surprises': surprises, 'questions': questions, } -Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding="utf-8") if G.number_of_nodes() == 0: print('ERROR: Graph is empty - extraction produced no nodes.') print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') @@ -509,9 +515,9 @@ from graphify.analyze import god_nodes, surprising_connections, suggest_question from graphify.report import generate from pathlib import Path -extraction = json.loads(Path('.graphify_extract.json').read_text()) -detection = json.loads(Path('.graphify_detect.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) +extraction = json.loads(Path('.graphify_extract.json').read_text(encoding="utf-8")) +detection = json.loads(Path('.graphify_detect.json').read_text(encoding="utf-8")) +analysis = json.loads(Path('.graphify_analysis.json').read_text(encoding="utf-8")) G = build_from_json(extraction) communities = {int(k): v for k, v in analysis['communities'].items()} @@ -525,8 +531,8 @@ labels = LABELS_DICT questions = suggest_questions(G, communities, labels) report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding="utf-8") +Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding="utf-8") print('Report updated with community labels') '@ | Out-File -FilePath .graphify_step_5_label_communities_10.py -Encoding utf8 python .graphify_step_5_label_communities_10.py @@ -551,9 +557,9 @@ from graphify.build import build_from_json from graphify.export import to_obsidian, to_canvas from pathlib import Path -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} +extraction = json.loads(Path('.graphify_extract.json').read_text(encoding="utf-8")) +analysis = json.loads(Path('.graphify_analysis.json').read_text(encoding="utf-8")) +labels_raw = json.loads(Path('.graphify_labels.json').read_text(encoding="utf-8")) if Path('.graphify_labels.json').exists() else {} G = build_from_json(extraction) communities = {int(k): v for k, v in analysis['communities'].items()} @@ -586,9 +592,9 @@ from graphify.build import build_from_json from graphify.export import to_html from pathlib import Path -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} +extraction = json.loads(Path('.graphify_extract.json').read_text(encoding="utf-8")) +analysis = json.loads(Path('.graphify_analysis.json').read_text(encoding="utf-8")) +labels_raw = json.loads(Path('.graphify_labels.json').read_text(encoding="utf-8")) if Path('.graphify_labels.json').exists() else {} G = build_from_json(extraction) communities = {int(k): v for k, v in analysis['communities'].items()} @@ -615,7 +621,7 @@ from graphify.build import build_from_json from graphify.export import to_cypher from pathlib import Path -G = build_from_json(json.loads(Path('.graphify_extract.json').read_text())) +G = build_from_json(json.loads(Path('.graphify_extract.json').read_text(encoding="utf-8"))) to_cypher(G, 'graphify-out/cypher.txt') print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') '@ | Out-File -FilePath .graphify_step_7_neo4j_export_only_if_neo4j_or__13.py -Encoding utf8 @@ -633,8 +639,8 @@ from graphify.cluster import cluster from graphify.export import push_to_neo4j from pathlib import Path -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) +extraction = json.loads(Path('.graphify_extract.json').read_text(encoding="utf-8")) +analysis = json.loads(Path('.graphify_analysis.json').read_text(encoding="utf-8")) G = build_from_json(extraction) communities = {int(k): v for k, v in analysis['communities'].items()} @@ -656,9 +662,9 @@ from graphify.build import build_from_json from graphify.export import to_svg from pathlib import Path -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} +extraction = json.loads(Path('.graphify_extract.json').read_text(encoding="utf-8")) +analysis = json.loads(Path('.graphify_analysis.json').read_text(encoding="utf-8")) +labels_raw = json.loads(Path('.graphify_labels.json').read_text(encoding="utf-8")) if Path('.graphify_labels.json').exists() else {} G = build_from_json(extraction) communities = {int(k): v for k, v in analysis['communities'].items()} @@ -680,8 +686,8 @@ from graphify.build import build_from_json from graphify.export import to_graphml from pathlib import Path -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) +extraction = json.loads(Path('.graphify_extract.json').read_text(encoding="utf-8")) +analysis = json.loads(Path('.graphify_analysis.json').read_text(encoding="utf-8")) G = build_from_json(extraction) communities = {int(k): v for k, v in analysis['communities'].items()} @@ -723,7 +729,7 @@ import json from graphify.benchmark import run_benchmark, print_benchmark from pathlib import Path -detection = json.loads(Path('.graphify_detect.json').read_text()) +detection = json.loads(Path('.graphify_detect.json').read_text(encoding="utf-8")) result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) print_benchmark(result) '@ | Out-File -FilePath .graphify_step_8_token_reduction_benchmark_only_17.py -Encoding utf8 @@ -745,17 +751,17 @@ from datetime import datetime, timezone from graphify.detect import save_manifest # Save manifest for --update -detect = json.loads(Path('.graphify_detect.json').read_text()) +detect = json.loads(Path('.graphify_detect.json').read_text(encoding="utf-8")) save_manifest(detect['files']) # Update cumulative cost tracker -extract = json.loads(Path('.graphify_extract.json').read_text()) +extract = json.loads(Path('.graphify_extract.json').read_text(encoding="utf-8")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) cost_path = Path('graphify-out/cost.json') if cost_path.exists(): - cost = json.loads(cost_path.read_text()) + cost = json.loads(cost_path.read_text(encoding="utf-8")) else: cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} @@ -767,7 +773,7 @@ cost['runs'].append({ }) cost['total_input_tokens'] += input_tok cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2)) +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding="utf-8") print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost["total_input_tokens"]:,} input, {cost["total_output_tokens"]:,} output ({len(cost["runs"])} runs)') @@ -821,8 +827,8 @@ from pathlib import Path result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2)) -Path('.graphify_incremental.json').write_text(json.dumps(result)) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding="utf-8") if new_total == 0: print('No files changed since last run. Nothing to update.') raise SystemExit(0) @@ -839,7 +845,7 @@ If new files exist, first check whether all changed files are code files: import json from pathlib import Path -result = json.loads(open('.graphify_incremental.json').read()) if Path('.graphify_incremental.json').exists() else {} +result = json.loads(open('.graphify_incremental.json', encoding='utf-8').read()) if Path('.graphify_incremental.json').exists() else {} code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc'} new_files = result.get('new_files', {}) all_changed = [f for files in new_files.values() for f in files] @@ -866,15 +872,15 @@ import networkx as nx from pathlib import Path # Load existing graph -existing_data = json.loads(Path('graphify-out/graph.json').read_text()) +existing_data = json.loads(Path('graphify-out/graph.json').read_text(encoding="utf-8")) G_existing = json_graph.node_link_graph(existing_data, edges='links') # Load new extraction -new_extraction = json.loads(Path('.graphify_extract.json').read_text()) +new_extraction = json.loads(Path('.graphify_extract.json').read_text(encoding="utf-8")) G_new = build_from_json(new_extraction) # Prune nodes from deleted files -incremental = json.loads(Path('.graphify_incremental.json').read_text()) +incremental = json.loads(Path('.graphify_incremental.json').read_text(encoding="utf-8")) deleted = set(incremental.get('deleted_files', [])) if deleted: to_remove = [n for n, d in G_existing.nodes(data=True) if d.get('source_file') in deleted] @@ -914,8 +920,8 @@ import networkx as nx from pathlib import Path # Load old graph (before update) from backup written before merge -old_data = json.loads(Path('.graphify_old.json').read_text()) if Path('.graphify_old.json').exists() else None -new_extract = json.loads(Path('.graphify_extract.json').read_text()) +old_data = json.loads(Path('.graphify_old.json').read_text(encoding="utf-8")) if Path('.graphify_old.json').exists() else None +new_extract = json.loads(Path('.graphify_extract.json').read_text(encoding="utf-8")) G_new = build_from_json(new_extract) if old_data: @@ -951,7 +957,7 @@ from networkx.readwrite import json_graph import networkx as nx from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text()) +data = json.loads(Path('graphify-out/graph.json').read_text(encoding="utf-8")) G = json_graph.node_link_graph(data, edges='links') detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, @@ -965,7 +971,7 @@ surprises = surprising_connections(G, communities) labels = {cid: 'Community ' + str(cid) for cid in communities} report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') -Path('graphify-out/GRAPH_REPORT.md').write_text(report) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding="utf-8") to_json(G, communities, 'graphify-out/graph.json') analysis = { @@ -974,7 +980,7 @@ analysis = { 'gods': gods, 'surprises': surprises, } -Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding="utf-8") print(f'Re-clustered: {len(communities)} communities') '@ | Out-File -FilePath .graphify_step_for_cluster_only_23.py -Encoding utf8 python .graphify_step_for_cluster_only_23.py @@ -1022,7 +1028,7 @@ from networkx.readwrite import json_graph import networkx as nx from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text()) +data = json.loads(Path('graphify-out/graph.json').read_text(encoding="utf-8")) G = json_graph.node_link_graph(data, edges='links') question = 'QUESTION' @@ -1140,7 +1146,7 @@ import networkx as nx from networkx.readwrite import json_graph from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text()) +data = json.loads(Path('graphify-out/graph.json').read_text(encoding="utf-8")) G = json_graph.node_link_graph(data, edges='links') a_term = 'NODE_A' @@ -1217,7 +1223,7 @@ import networkx as nx from networkx.readwrite import json_graph from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text()) +data = json.loads(Path('graphify-out/graph.json').read_text(encoding="utf-8")) G = json_graph.node_link_graph(data, edges='links') term = 'NODE_NAME' diff --git a/graphify/skill.md b/graphify/skill.md index 67a48d14..edee6975 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -99,10 +99,19 @@ if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then fi # 3. Fall back to python3 if [ -z "$PYTHON" ]; then PYTHON="python3"; fi -"$PYTHON" -c "import graphify" 2>/dev/null || "$PYTHON" -m pip install graphifyy -q 2>/dev/null || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 +if ! "$PYTHON" -c "import graphify" 2>/dev/null; then + if command -v uv >/dev/null 2>&1; then + uv tool install --upgrade graphifyy -q 2>&1 | tail -3 + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi + else + "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ + || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 + fi +fi # Write interpreter path for all subsequent steps (persists across invocations) mkdir -p graphify-out -"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" # Save scan root so `graphify update` (no args) knows where to look next time echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` @@ -119,7 +128,7 @@ import json from graphify.detect import detect from pathlib import Path result = detect(Path('INPUT_PATH')) -print(json.dumps(result)) +print(json.dumps(result, ensure_ascii=False)) " > graphify-out/.graphify_detect.json ``` @@ -170,12 +179,12 @@ import json, os from pathlib import Path from graphify.transcribe import transcribe_all -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) video_files = detect.get('files', {}).get('video', []) prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths)) +print(json.dumps(transcript_paths, ensure_ascii=False)) " > graphify-out/.graphify_transcripts.json ``` @@ -214,16 +223,16 @@ from pathlib import Path import json code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: result = extract(code_files, cache_root=Path('.')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2)) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') " ``` @@ -250,14 +259,14 @@ import json from graphify.cache import check_semantic_cache from pathlib import Path -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) all_files = [f for files in detect['files'].values() for f in files] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached)) + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " ``` @@ -377,7 +386,7 @@ chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: - d = json.loads(Path(c).read_text()) + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) @@ -386,7 +395,7 @@ for c in chunks: Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2)) +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') " ``` @@ -398,7 +407,7 @@ import json from graphify.cache import save_semantic_cache from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) print(f'Cached {saved} files') " @@ -410,8 +419,8 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text()) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) @@ -430,7 +439,7 @@ merged = { 'input_tokens': new.get('input_tokens', 0), 'output_tokens': new.get('output_tokens', 0), } -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` @@ -443,8 +452,8 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from pathlib import Path -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text()) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text()) +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} @@ -463,7 +472,7 @@ merged = { 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2)) +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") total = len(merged_nodes) edges = len(merged_edges) print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') @@ -485,8 +494,8 @@ from graphify.report import generate from graphify.export import to_json from pathlib import Path -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) G = build_from_json(extraction) communities = cluster(G) @@ -499,7 +508,7 @@ labels = {cid: 'Community ' + str(cid) for cid in communities} questions = suggest_questions(G, communities, labels) report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") to_json(G, communities, 'graphify-out/graph.json') analysis = { @@ -509,7 +518,7 @@ analysis = { 'surprises': surprises, 'questions': questions, } -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") if G.number_of_nodes() == 0: print('ERROR: Graph is empty - extraction produced no nodes.') print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') @@ -537,9 +546,9 @@ from graphify.analyze import god_nodes, surprising_connections, suggest_question from graphify.report import generate from pathlib import Path -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) G = build_from_json(extraction) communities = {int(k): v for k, v in analysis['communities'].items()} @@ -553,8 +562,8 @@ labels = LABELS_DICT questions = suggest_questions(G, communities, labels) report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') " ``` @@ -662,17 +671,17 @@ from datetime import datetime, timezone from graphify.detect import save_manifest # Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) save_manifest(detect['files']) # Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) cost_path = Path('graphify-out/cost.json') if cost_path.exists(): - cost = json.loads(cost_path.read_text()) + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) else: cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} @@ -684,7 +693,7 @@ cost['runs'].append({ }) cost['total_input_tokens'] += input_tok cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2)) +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') @@ -738,7 +747,7 @@ if [ ! -f graphify-out/.graphify_python ]; then PYTHON="python3" fi mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" fi ``` @@ -754,8 +763,8 @@ from pathlib import Path result = detect_incremental(Path('INPUT_PATH')) new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result)) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") if new_total == 0: print('No files changed since last run. Nothing to update.') raise SystemExit(0) @@ -770,7 +779,7 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path -result = json.loads(open('graphify-out/.graphify_incremental.json').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} new_files = result.get('new_files', {}) all_changed = [f for files in new_files.values() for f in files] @@ -793,8 +802,8 @@ from graphify.build import build_merge from graphify.detect import save_manifest # Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text()) +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) # Use build_merge() — reads graph.json directly without NetworkX round-trip @@ -822,7 +831,7 @@ merged_out = { 'input_tokens': new_extraction.get('input_tokens', 0), 'output_tokens': new_extraction.get('output_tokens', 0), } -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out)) +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') # Save manifest so next --update diffs against today's state, not the @@ -846,8 +855,8 @@ import networkx as nx from pathlib import Path # Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text()) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) G_new = build_from_json(new_extract) if old_data: