mirror of
https://github.com/safishamsi/graphify.git
synced 2026-08-28 09:17:02 +00:00
- skill.md + skill-windows.md: add encoding="utf-8" to all read_text()/write_text() calls and ensure_ascii=False to json.dumps — bare calls defaulted to system codepage on Chinese-locale Windows, mojibaking non-ASCII content (#832) - skill.md + skill-windows.md: prefer uv tool install --upgrade graphifyy over pip in the Step 1 install fallback — pip installs to the wrong env when graphify was installed via uv tool (#831) - serve.py + __main__.py: replace flat substring scoring in _score_nodes with three-tier precedence (exact 1000 / prefix 100 / substring 1); _find_node returns results ordered exact→prefix→substring; both path CLI and MCP now emit a clear error when src and tgt resolve to the same node (#828) - cache.py: normalize path key via .as_posix().lower() in file_hash so Windows junction/case variants hash identically; mirror abs-path normalization from save_semantic_cache into check_semantic_cache so relative source_file paths resolve the same way on both sides (#826) - __main__.py: add /graphify skill trigger line to _AGENTS_MD_SECTION — affects all 7 AGENTS.md platforms (OpenCode, Codex, Aider, Trae, Hermes, Claw, Droid) so typing /graphify actually invokes the skill tool (#827) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Safi
co-authored by
Claude Sonnet 4.6
Claude Sonnet 4.6
parent
094d8ba731
commit
994b17be46
@@ -254,6 +254,8 @@ _AGENTS_MD_SECTION = """\
|
||||
|
||||
This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships.
|
||||
|
||||
When the user types `/graphify`, invoke the `skill` tool with `skill: "graphify"` before doing anything else.
|
||||
|
||||
Rules:
|
||||
- ALWAYS read graphify-out/GRAPH_REPORT.md before reading any source files, running grep/glob searches, or answering codebase questions. The graph is your primary map of the codebase.
|
||||
- IF graphify-out/wiki/index.md EXISTS, navigate it instead of reading raw files
|
||||
@@ -1525,6 +1527,25 @@ def main() -> None:
|
||||
print(f"No node matching '{target_label}' found.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
src_nid, tgt_nid = src_scored[0][1], tgt_scored[0][1]
|
||||
# Ambiguity guard: when both queries resolve to the same node, the
|
||||
# shortest path is trivially zero hops, which is almost never what the
|
||||
# caller wanted (see bug #828).
|
||||
if src_nid == tgt_nid:
|
||||
print(
|
||||
f"'{source_label}' and '{target_label}' both resolved to the same "
|
||||
f"node '{src_nid}'. Use a more specific label or the exact node ID.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
for _name, _scored in (("source", src_scored), ("target", tgt_scored)):
|
||||
if len(_scored) >= 2:
|
||||
_top, _runner = _scored[0][0], _scored[1][0]
|
||||
if _top > 0 and (_top - _runner) / _top < 0.10:
|
||||
print(
|
||||
f"warning: {_name} match was ambiguous "
|
||||
f"(top score {_top:g}, runner-up {_runner:g})",
|
||||
file=sys.stderr,
|
||||
)
|
||||
try:
|
||||
path_nodes = _nx.shortest_path(G, src_nid, tgt_nid)
|
||||
except (_nx.NetworkXNoPath, _nx.NodeNotFound):
|
||||
|
||||
+6
-3
@@ -55,9 +55,9 @@ def file_hash(path: Path, root: Path = Path(".")) -> str:
|
||||
h.update(b"\x00")
|
||||
try:
|
||||
rel = p.resolve().relative_to(Path(root).resolve())
|
||||
h.update(str(rel).encode())
|
||||
h.update(rel.as_posix().lower().encode())
|
||||
except ValueError:
|
||||
h.update(str(p.resolve()).encode())
|
||||
h.update(p.resolve().as_posix().lower().encode())
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
@@ -190,7 +190,10 @@ def check_semantic_cache(
|
||||
uncached: list[str] = []
|
||||
|
||||
for fpath in files:
|
||||
result = load_cached(Path(fpath), root, kind="semantic")
|
||||
p = Path(fpath)
|
||||
if not p.is_absolute():
|
||||
p = Path(root) / p
|
||||
result = load_cached(p, root, kind="semantic")
|
||||
if result is not None:
|
||||
cached_nodes.extend(result.get("nodes", []))
|
||||
cached_edges.extend(result.get("edges", []))
|
||||
|
||||
+55
-10
@@ -48,7 +48,10 @@ def _strip_diacritics(text: str) -> str:
|
||||
return "".join(c for c in nfkd if not unicodedata.combining(c))
|
||||
|
||||
|
||||
_EXACT_MATCH_BONUS = 100.0
|
||||
_EXACT_MATCH_BONUS = 1000.0
|
||||
_PREFIX_MATCH_BONUS = 100.0
|
||||
_SUBSTRING_MATCH_BONUS = 1.0
|
||||
_SOURCE_MATCH_BONUS = 0.5
|
||||
|
||||
|
||||
def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]:
|
||||
@@ -56,11 +59,20 @@ def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]:
|
||||
norm_terms = [_strip_diacritics(t).lower() for t in terms]
|
||||
for nid, data in G.nodes(data=True):
|
||||
norm_label = data.get("norm_label") or _strip_diacritics(data.get("label") or "").lower()
|
||||
bare_label = norm_label.rstrip("()")
|
||||
source = (data.get("source_file") or "").lower()
|
||||
score = sum(1 for t in norm_terms if t in norm_label) + sum(0.5 for t in norm_terms if t in source)
|
||||
# Exact match: single term equals the full label (strip trailing () for functions)
|
||||
if any(t == norm_label or t == norm_label.rstrip("()") for t in norm_terms):
|
||||
score += _EXACT_MATCH_BONUS
|
||||
score = 0.0
|
||||
for t in norm_terms:
|
||||
# Three-tier precedence: exact > prefix > substring (take the
|
||||
# strongest tier per term so a single term cannot double-count).
|
||||
if t == norm_label or t == bare_label:
|
||||
score += _EXACT_MATCH_BONUS
|
||||
elif norm_label.startswith(t) or bare_label.startswith(t):
|
||||
score += _PREFIX_MATCH_BONUS
|
||||
elif t in norm_label:
|
||||
score += _SUBSTRING_MATCH_BONUS
|
||||
if t in source:
|
||||
score += _SOURCE_MATCH_BONUS
|
||||
if score > 0:
|
||||
scored.append((score, nid))
|
||||
return sorted(scored, reverse=True)
|
||||
@@ -233,11 +245,26 @@ def _query_graph_text(
|
||||
|
||||
|
||||
def _find_node(G: nx.Graph, label: str) -> list[str]:
|
||||
"""Return node IDs whose label or ID matches the search term (diacritic-insensitive)."""
|
||||
"""Return node IDs whose label or ID matches the search term (diacritic-insensitive).
|
||||
|
||||
Results are ordered by three-tier precedence: exact match, then prefix match,
|
||||
then substring match. Node-ID exact matches are grouped with label exact matches.
|
||||
"""
|
||||
term = _strip_diacritics(label).lower()
|
||||
return [nid for nid, d in G.nodes(data=True)
|
||||
if term in (d.get("norm_label") or _strip_diacritics(d.get("label") or "").lower())
|
||||
or term == nid.lower()]
|
||||
exact: list[str] = []
|
||||
prefix: list[str] = []
|
||||
substring: list[str] = []
|
||||
for nid, d in G.nodes(data=True):
|
||||
norm_label = d.get("norm_label") or _strip_diacritics(d.get("label") or "").lower()
|
||||
bare_label = norm_label.rstrip("()")
|
||||
nid_lower = nid.lower()
|
||||
if term == norm_label or term == bare_label or term == nid_lower:
|
||||
exact.append(nid)
|
||||
elif norm_label.startswith(term) or bare_label.startswith(term) or nid_lower.startswith(term):
|
||||
prefix.append(nid)
|
||||
elif term in norm_label:
|
||||
substring.append(nid)
|
||||
return exact + prefix + substring
|
||||
|
||||
|
||||
def _filter_blank_stdin() -> None:
|
||||
@@ -456,6 +483,23 @@ def serve(graph_path: str = "graphify-out/graph.json") -> None:
|
||||
if not tgt_scored:
|
||||
return f"No node matching target '{arguments['target']}' found."
|
||||
src_nid, tgt_nid = src_scored[0][1], tgt_scored[0][1]
|
||||
# Ambiguity guard: when both queries resolve to the same node, the
|
||||
# shortest path is trivially zero hops, which is almost never what the
|
||||
# caller wanted (see bug #828).
|
||||
if src_nid == tgt_nid:
|
||||
return (
|
||||
f"'{arguments['source']}' and '{arguments['target']}' both resolved to "
|
||||
f"the same node '{src_nid}'. Use a more specific label or the exact node ID."
|
||||
)
|
||||
warnings: list[str] = []
|
||||
for name, scored in (("source", src_scored), ("target", tgt_scored)):
|
||||
if len(scored) >= 2:
|
||||
top, runner = scored[0][0], scored[1][0]
|
||||
if top > 0 and (top - runner) / top < 0.10:
|
||||
warnings.append(
|
||||
f"warning: {name} match was ambiguous "
|
||||
f"(top score {top:g}, runner-up {runner:g})"
|
||||
)
|
||||
max_hops = int(arguments.get("max_hops", 8))
|
||||
try:
|
||||
path_nodes = nx.shortest_path(G, src_nid, tgt_nid)
|
||||
@@ -474,7 +518,8 @@ def serve(graph_path: str = "graphify-out/graph.json") -> None:
|
||||
if i == 0:
|
||||
segments.append(G.nodes[u].get("label", u))
|
||||
segments.append(f"--{rel}{conf_str}--> {G.nodes[v].get('label', v)}")
|
||||
return f"Shortest path ({hops} hops):\n " + " ".join(segments)
|
||||
prefix = ("\n".join(warnings) + "\n") if warnings else ""
|
||||
return prefix + f"Shortest path ({hops} hops):\n " + " ".join(segments)
|
||||
|
||||
_handlers = {
|
||||
"query_graph": _tool_query_graph,
|
||||
|
||||
+68
-62
@@ -69,10 +69,16 @@ import graphify
|
||||
'@ | Out-File -FilePath .graphify_step_1_ensure_graphify_is_installed_1.py -Encoding utf8
|
||||
python .graphify_step_1_ensure_graphify_is_installed_1.py 2>$null
|
||||
Remove-Item -ErrorAction SilentlyContinue .graphify_step_1_ensure_graphify_is_installed_1.py
|
||||
if ($LASTEXITCODE -ne 0) { pip install graphifyy -q 2>&1 | Select-Object -Last 3 }
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
if (Get-Command uv -ErrorAction SilentlyContinue) {
|
||||
uv tool install --upgrade graphifyy -q 2>&1 | Select-Object -Last 3
|
||||
} else {
|
||||
pip install graphifyy -q 2>&1 | Select-Object -Last 3
|
||||
}
|
||||
}
|
||||
# Write interpreter path for all subsequent steps
|
||||
@'
|
||||
import sys; open('.graphify_python', 'w').write(sys.executable)
|
||||
import sys; open('.graphify_python', 'w', encoding='utf-8').write(sys.executable)
|
||||
'@ | Out-File -FilePath .graphify_step_1_ensure_graphify_is_installed_2.py -Encoding utf8
|
||||
python .graphify_step_1_ensure_graphify_is_installed_2.py
|
||||
Remove-Item -ErrorAction SilentlyContinue .graphify_step_1_ensure_graphify_is_installed_2.py
|
||||
@@ -88,7 +94,7 @@ import json
|
||||
from graphify.detect import detect
|
||||
from pathlib import Path
|
||||
result = detect(Path('INPUT_PATH'))
|
||||
print(json.dumps(result))
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
'@ | Out-File -FilePath .graphify_step_2_detect_files_3.py -Encoding utf8
|
||||
python .graphify_step_2_detect_files_3.py > .graphify_detect.json
|
||||
Remove-Item -ErrorAction SilentlyContinue .graphify_step_2_detect_files_3.py
|
||||
@@ -140,12 +146,12 @@ import json, os
|
||||
from pathlib import Path
|
||||
from graphify.transcribe import transcribe_all
|
||||
|
||||
detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text())
|
||||
detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding="utf-8"))
|
||||
video_files = detect.get('files', {}).get('video', [])
|
||||
prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.')
|
||||
|
||||
transcript_paths = transcribe_all(video_files, initial_prompt=prompt)
|
||||
print(json.dumps(transcript_paths))
|
||||
print(json.dumps(transcript_paths, ensure_ascii=False))
|
||||
'@ | Out-File -FilePath .graphify_step_transcribe.py -Encoding utf8
|
||||
& (Get-Content graphify-out\.graphify_python) .graphify_step_transcribe.py | Out-File -FilePath graphify-out\.graphify_transcripts.json -Encoding utf8
|
||||
Remove-Item -ErrorAction SilentlyContinue .graphify_step_transcribe.py
|
||||
@@ -182,16 +188,16 @@ from pathlib import Path
|
||||
|
||||
def main():
|
||||
code_files = []
|
||||
detect = json.loads(Path('.graphify_detect.json').read_text())
|
||||
detect = json.loads(Path('.graphify_detect.json').read_text(encoding="utf-8"))
|
||||
for f in detect.get('files', {}).get('code', []):
|
||||
code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)])
|
||||
|
||||
if code_files:
|
||||
result = extract(code_files)
|
||||
Path('.graphify_ast.json').write_text(json.dumps(result, indent=2))
|
||||
Path('.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
print(f'AST: {len(result["nodes"])} nodes, {len(result["edges"])} edges')
|
||||
else:
|
||||
Path('.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}))
|
||||
Path('.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding="utf-8")
|
||||
print('No code files - skipping AST extraction')
|
||||
|
||||
|
||||
@@ -229,14 +235,14 @@ import json
|
||||
from graphify.cache import check_semantic_cache
|
||||
from pathlib import Path
|
||||
|
||||
detect = json.loads(Path('.graphify_detect.json').read_text())
|
||||
detect = json.loads(Path('.graphify_detect.json').read_text(encoding="utf-8"))
|
||||
all_files = [f for files in detect['files'].values() for f in files]
|
||||
|
||||
cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files)
|
||||
|
||||
if cached_nodes or cached_edges or cached_hyperedges:
|
||||
Path('.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}))
|
||||
Path('.graphify_uncached.txt').write_text('\n'.join(uncached))
|
||||
Path('.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding="utf-8")
|
||||
Path('.graphify_uncached.txt').write_text('\n'.join(uncached), encoding="utf-8")
|
||||
print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction')
|
||||
'@ | Out-File -FilePath .graphify_step_3_extract_entities_and_relations_5.py -Encoding utf8
|
||||
python .graphify_step_3_extract_entities_and_relations_5.py
|
||||
@@ -343,7 +349,7 @@ chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json'))
|
||||
all_nodes, all_edges, all_hyperedges = [], [], []
|
||||
total_in, total_out = 0, 0
|
||||
for c in chunks:
|
||||
d = json.loads(Path(c).read_text())
|
||||
d = json.loads(Path(c).read_text(encoding=\"utf-8\"))
|
||||
all_nodes += d.get('nodes', [])
|
||||
all_edges += d.get('edges', [])
|
||||
all_hyperedges += d.get('hyperedges', [])
|
||||
@@ -352,7 +358,7 @@ for c in chunks:
|
||||
Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({
|
||||
'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges,
|
||||
'input_tokens': total_in, 'output_tokens': total_out,
|
||||
}, indent=2))
|
||||
}, indent=2, ensure_ascii=False), encoding=\"utf-8\")
|
||||
print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens')
|
||||
"
|
||||
```
|
||||
@@ -364,7 +370,7 @@ import json
|
||||
from graphify.cache import save_semantic_cache
|
||||
from pathlib import Path
|
||||
|
||||
new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
|
||||
new = json.loads(Path('.graphify_semantic_new.json').read_text(encoding="utf-8")) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
|
||||
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []))
|
||||
print(f'Cached {saved} files')
|
||||
'@ | Out-File -FilePath .graphify_step_3_extract_entities_and_relations_6.py -Encoding utf8
|
||||
@@ -378,8 +384,8 @@ Merge cached + new results into `.graphify_semantic.json`:
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
cached = json.loads(Path('.graphify_cached.json').read_text()) if Path('.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
|
||||
new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
|
||||
cached = json.loads(Path('.graphify_cached.json').read_text(encoding="utf-8")) if Path('.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
|
||||
new = json.loads(Path('.graphify_semantic_new.json').read_text(encoding="utf-8")) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
|
||||
|
||||
all_nodes = cached['nodes'] + new.get('nodes', [])
|
||||
all_edges = cached['edges'] + new.get('edges', [])
|
||||
@@ -398,7 +404,7 @@ merged = {
|
||||
'input_tokens': new.get('input_tokens', 0),
|
||||
'output_tokens': new.get('output_tokens', 0),
|
||||
}
|
||||
Path('.graphify_semantic.json').write_text(json.dumps(merged, indent=2))
|
||||
Path('.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached["nodes"])} from cache, {len(new.get("nodes",[]))} new)')
|
||||
'@ | Out-File -FilePath .graphify_step_3_extract_entities_and_relations_7.py -Encoding utf8
|
||||
python .graphify_step_3_extract_entities_and_relations_7.py
|
||||
@@ -413,8 +419,8 @@ Clean up temp files: `Remove-Item -ErrorAction SilentlyContinue .graphify_cached
|
||||
import sys, json
|
||||
from pathlib import Path
|
||||
|
||||
ast = json.loads(Path('.graphify_ast.json').read_text())
|
||||
sem = json.loads(Path('.graphify_semantic.json').read_text())
|
||||
ast = json.loads(Path('.graphify_ast.json').read_text(encoding="utf-8"))
|
||||
sem = json.loads(Path('.graphify_semantic.json').read_text(encoding="utf-8"))
|
||||
|
||||
# Merge: AST nodes first, semantic nodes deduplicated by id
|
||||
seen = {n['id'] for n in ast['nodes']}
|
||||
@@ -433,7 +439,7 @@ merged = {
|
||||
'input_tokens': sem.get('input_tokens', 0),
|
||||
'output_tokens': sem.get('output_tokens', 0),
|
||||
}
|
||||
Path('.graphify_extract.json').write_text(json.dumps(merged, indent=2))
|
||||
Path('.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
total = len(merged_nodes)
|
||||
edges = len(merged_edges)
|
||||
print(f'Merged: {total} nodes, {edges} edges ({len(ast["nodes"])} AST + {len(sem["nodes"])} semantic)')
|
||||
@@ -455,8 +461,8 @@ from graphify.report import generate
|
||||
from graphify.export import to_json
|
||||
from pathlib import Path
|
||||
|
||||
extraction = json.loads(Path('.graphify_extract.json').read_text())
|
||||
detection = json.loads(Path('.graphify_detect.json').read_text())
|
||||
extraction = json.loads(Path('.graphify_extract.json').read_text(encoding="utf-8"))
|
||||
detection = json.loads(Path('.graphify_detect.json').read_text(encoding="utf-8"))
|
||||
|
||||
G = build_from_json(extraction)
|
||||
communities = cluster(G)
|
||||
@@ -469,7 +475,7 @@ labels = {cid: 'Community ' + str(cid) for cid in communities}
|
||||
questions = suggest_questions(G, communities, labels)
|
||||
|
||||
report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions)
|
||||
Path('graphify-out/GRAPH_REPORT.md').write_text(report)
|
||||
Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding="utf-8")
|
||||
to_json(G, communities, 'graphify-out/graph.json')
|
||||
|
||||
analysis = {
|
||||
@@ -479,7 +485,7 @@ analysis = {
|
||||
'surprises': surprises,
|
||||
'questions': questions,
|
||||
}
|
||||
Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2))
|
||||
Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
if G.number_of_nodes() == 0:
|
||||
print('ERROR: Graph is empty - extraction produced no nodes.')
|
||||
print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.')
|
||||
@@ -509,9 +515,9 @@ from graphify.analyze import god_nodes, surprising_connections, suggest_question
|
||||
from graphify.report import generate
|
||||
from pathlib import Path
|
||||
|
||||
extraction = json.loads(Path('.graphify_extract.json').read_text())
|
||||
detection = json.loads(Path('.graphify_detect.json').read_text())
|
||||
analysis = json.loads(Path('.graphify_analysis.json').read_text())
|
||||
extraction = json.loads(Path('.graphify_extract.json').read_text(encoding="utf-8"))
|
||||
detection = json.loads(Path('.graphify_detect.json').read_text(encoding="utf-8"))
|
||||
analysis = json.loads(Path('.graphify_analysis.json').read_text(encoding="utf-8"))
|
||||
|
||||
G = build_from_json(extraction)
|
||||
communities = {int(k): v for k, v in analysis['communities'].items()}
|
||||
@@ -525,8 +531,8 @@ labels = LABELS_DICT
|
||||
questions = suggest_questions(G, communities, labels)
|
||||
|
||||
report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions)
|
||||
Path('graphify-out/GRAPH_REPORT.md').write_text(report)
|
||||
Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}))
|
||||
Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding="utf-8")
|
||||
Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding="utf-8")
|
||||
print('Report updated with community labels')
|
||||
'@ | Out-File -FilePath .graphify_step_5_label_communities_10.py -Encoding utf8
|
||||
python .graphify_step_5_label_communities_10.py
|
||||
@@ -551,9 +557,9 @@ from graphify.build import build_from_json
|
||||
from graphify.export import to_obsidian, to_canvas
|
||||
from pathlib import Path
|
||||
|
||||
extraction = json.loads(Path('.graphify_extract.json').read_text())
|
||||
analysis = json.loads(Path('.graphify_analysis.json').read_text())
|
||||
labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {}
|
||||
extraction = json.loads(Path('.graphify_extract.json').read_text(encoding="utf-8"))
|
||||
analysis = json.loads(Path('.graphify_analysis.json').read_text(encoding="utf-8"))
|
||||
labels_raw = json.loads(Path('.graphify_labels.json').read_text(encoding="utf-8")) if Path('.graphify_labels.json').exists() else {}
|
||||
|
||||
G = build_from_json(extraction)
|
||||
communities = {int(k): v for k, v in analysis['communities'].items()}
|
||||
@@ -586,9 +592,9 @@ from graphify.build import build_from_json
|
||||
from graphify.export import to_html
|
||||
from pathlib import Path
|
||||
|
||||
extraction = json.loads(Path('.graphify_extract.json').read_text())
|
||||
analysis = json.loads(Path('.graphify_analysis.json').read_text())
|
||||
labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {}
|
||||
extraction = json.loads(Path('.graphify_extract.json').read_text(encoding="utf-8"))
|
||||
analysis = json.loads(Path('.graphify_analysis.json').read_text(encoding="utf-8"))
|
||||
labels_raw = json.loads(Path('.graphify_labels.json').read_text(encoding="utf-8")) if Path('.graphify_labels.json').exists() else {}
|
||||
|
||||
G = build_from_json(extraction)
|
||||
communities = {int(k): v for k, v in analysis['communities'].items()}
|
||||
@@ -615,7 +621,7 @@ from graphify.build import build_from_json
|
||||
from graphify.export import to_cypher
|
||||
from pathlib import Path
|
||||
|
||||
G = build_from_json(json.loads(Path('.graphify_extract.json').read_text()))
|
||||
G = build_from_json(json.loads(Path('.graphify_extract.json').read_text(encoding="utf-8")))
|
||||
to_cypher(G, 'graphify-out/cypher.txt')
|
||||
print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt')
|
||||
'@ | Out-File -FilePath .graphify_step_7_neo4j_export_only_if_neo4j_or__13.py -Encoding utf8
|
||||
@@ -633,8 +639,8 @@ from graphify.cluster import cluster
|
||||
from graphify.export import push_to_neo4j
|
||||
from pathlib import Path
|
||||
|
||||
extraction = json.loads(Path('.graphify_extract.json').read_text())
|
||||
analysis = json.loads(Path('.graphify_analysis.json').read_text())
|
||||
extraction = json.loads(Path('.graphify_extract.json').read_text(encoding="utf-8"))
|
||||
analysis = json.loads(Path('.graphify_analysis.json').read_text(encoding="utf-8"))
|
||||
G = build_from_json(extraction)
|
||||
communities = {int(k): v for k, v in analysis['communities'].items()}
|
||||
|
||||
@@ -656,9 +662,9 @@ from graphify.build import build_from_json
|
||||
from graphify.export import to_svg
|
||||
from pathlib import Path
|
||||
|
||||
extraction = json.loads(Path('.graphify_extract.json').read_text())
|
||||
analysis = json.loads(Path('.graphify_analysis.json').read_text())
|
||||
labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {}
|
||||
extraction = json.loads(Path('.graphify_extract.json').read_text(encoding="utf-8"))
|
||||
analysis = json.loads(Path('.graphify_analysis.json').read_text(encoding="utf-8"))
|
||||
labels_raw = json.loads(Path('.graphify_labels.json').read_text(encoding="utf-8")) if Path('.graphify_labels.json').exists() else {}
|
||||
|
||||
G = build_from_json(extraction)
|
||||
communities = {int(k): v for k, v in analysis['communities'].items()}
|
||||
@@ -680,8 +686,8 @@ from graphify.build import build_from_json
|
||||
from graphify.export import to_graphml
|
||||
from pathlib import Path
|
||||
|
||||
extraction = json.loads(Path('.graphify_extract.json').read_text())
|
||||
analysis = json.loads(Path('.graphify_analysis.json').read_text())
|
||||
extraction = json.loads(Path('.graphify_extract.json').read_text(encoding="utf-8"))
|
||||
analysis = json.loads(Path('.graphify_analysis.json').read_text(encoding="utf-8"))
|
||||
|
||||
G = build_from_json(extraction)
|
||||
communities = {int(k): v for k, v in analysis['communities'].items()}
|
||||
@@ -723,7 +729,7 @@ import json
|
||||
from graphify.benchmark import run_benchmark, print_benchmark
|
||||
from pathlib import Path
|
||||
|
||||
detection = json.loads(Path('.graphify_detect.json').read_text())
|
||||
detection = json.loads(Path('.graphify_detect.json').read_text(encoding="utf-8"))
|
||||
result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words'])
|
||||
print_benchmark(result)
|
||||
'@ | Out-File -FilePath .graphify_step_8_token_reduction_benchmark_only_17.py -Encoding utf8
|
||||
@@ -745,17 +751,17 @@ from datetime import datetime, timezone
|
||||
from graphify.detect import save_manifest
|
||||
|
||||
# Save manifest for --update
|
||||
detect = json.loads(Path('.graphify_detect.json').read_text())
|
||||
detect = json.loads(Path('.graphify_detect.json').read_text(encoding="utf-8"))
|
||||
save_manifest(detect['files'])
|
||||
|
||||
# Update cumulative cost tracker
|
||||
extract = json.loads(Path('.graphify_extract.json').read_text())
|
||||
extract = json.loads(Path('.graphify_extract.json').read_text(encoding="utf-8"))
|
||||
input_tok = extract.get('input_tokens', 0)
|
||||
output_tok = extract.get('output_tokens', 0)
|
||||
|
||||
cost_path = Path('graphify-out/cost.json')
|
||||
if cost_path.exists():
|
||||
cost = json.loads(cost_path.read_text())
|
||||
cost = json.loads(cost_path.read_text(encoding="utf-8"))
|
||||
else:
|
||||
cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0}
|
||||
|
||||
@@ -767,7 +773,7 @@ cost['runs'].append({
|
||||
})
|
||||
cost['total_input_tokens'] += input_tok
|
||||
cost['total_output_tokens'] += output_tok
|
||||
cost_path.write_text(json.dumps(cost, indent=2))
|
||||
cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens')
|
||||
print(f'All time: {cost["total_input_tokens"]:,} input, {cost["total_output_tokens"]:,} output ({len(cost["runs"])} runs)')
|
||||
@@ -821,8 +827,8 @@ from pathlib import Path
|
||||
|
||||
result = detect_incremental(Path('INPUT_PATH'))
|
||||
new_total = result.get('new_total', 0)
|
||||
print(json.dumps(result, indent=2))
|
||||
Path('.graphify_incremental.json').write_text(json.dumps(result))
|
||||
print(json.dumps(result, indent=2, ensure_ascii=False))
|
||||
Path('.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding="utf-8")
|
||||
if new_total == 0:
|
||||
print('No files changed since last run. Nothing to update.')
|
||||
raise SystemExit(0)
|
||||
@@ -839,7 +845,7 @@ If new files exist, first check whether all changed files are code files:
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
result = json.loads(open('.graphify_incremental.json').read()) if Path('.graphify_incremental.json').exists() else {}
|
||||
result = json.loads(open('.graphify_incremental.json', encoding='utf-8').read()) if Path('.graphify_incremental.json').exists() else {}
|
||||
code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc'}
|
||||
new_files = result.get('new_files', {})
|
||||
all_changed = [f for files in new_files.values() for f in files]
|
||||
@@ -866,15 +872,15 @@ import networkx as nx
|
||||
from pathlib import Path
|
||||
|
||||
# Load existing graph
|
||||
existing_data = json.loads(Path('graphify-out/graph.json').read_text())
|
||||
existing_data = json.loads(Path('graphify-out/graph.json').read_text(encoding="utf-8"))
|
||||
G_existing = json_graph.node_link_graph(existing_data, edges='links')
|
||||
|
||||
# Load new extraction
|
||||
new_extraction = json.loads(Path('.graphify_extract.json').read_text())
|
||||
new_extraction = json.loads(Path('.graphify_extract.json').read_text(encoding="utf-8"))
|
||||
G_new = build_from_json(new_extraction)
|
||||
|
||||
# Prune nodes from deleted files
|
||||
incremental = json.loads(Path('.graphify_incremental.json').read_text())
|
||||
incremental = json.loads(Path('.graphify_incremental.json').read_text(encoding="utf-8"))
|
||||
deleted = set(incremental.get('deleted_files', []))
|
||||
if deleted:
|
||||
to_remove = [n for n, d in G_existing.nodes(data=True) if d.get('source_file') in deleted]
|
||||
@@ -914,8 +920,8 @@ import networkx as nx
|
||||
from pathlib import Path
|
||||
|
||||
# Load old graph (before update) from backup written before merge
|
||||
old_data = json.loads(Path('.graphify_old.json').read_text()) if Path('.graphify_old.json').exists() else None
|
||||
new_extract = json.loads(Path('.graphify_extract.json').read_text())
|
||||
old_data = json.loads(Path('.graphify_old.json').read_text(encoding="utf-8")) if Path('.graphify_old.json').exists() else None
|
||||
new_extract = json.loads(Path('.graphify_extract.json').read_text(encoding="utf-8"))
|
||||
G_new = build_from_json(new_extract)
|
||||
|
||||
if old_data:
|
||||
@@ -951,7 +957,7 @@ from networkx.readwrite import json_graph
|
||||
import networkx as nx
|
||||
from pathlib import Path
|
||||
|
||||
data = json.loads(Path('graphify-out/graph.json').read_text())
|
||||
data = json.loads(Path('graphify-out/graph.json').read_text(encoding="utf-8"))
|
||||
G = json_graph.node_link_graph(data, edges='links')
|
||||
|
||||
detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None,
|
||||
@@ -965,7 +971,7 @@ surprises = surprising_connections(G, communities)
|
||||
labels = {cid: 'Community ' + str(cid) for cid in communities}
|
||||
|
||||
report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.')
|
||||
Path('graphify-out/GRAPH_REPORT.md').write_text(report)
|
||||
Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding="utf-8")
|
||||
to_json(G, communities, 'graphify-out/graph.json')
|
||||
|
||||
analysis = {
|
||||
@@ -974,7 +980,7 @@ analysis = {
|
||||
'gods': gods,
|
||||
'surprises': surprises,
|
||||
}
|
||||
Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2))
|
||||
Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
print(f'Re-clustered: {len(communities)} communities')
|
||||
'@ | Out-File -FilePath .graphify_step_for_cluster_only_23.py -Encoding utf8
|
||||
python .graphify_step_for_cluster_only_23.py
|
||||
@@ -1022,7 +1028,7 @@ from networkx.readwrite import json_graph
|
||||
import networkx as nx
|
||||
from pathlib import Path
|
||||
|
||||
data = json.loads(Path('graphify-out/graph.json').read_text())
|
||||
data = json.loads(Path('graphify-out/graph.json').read_text(encoding="utf-8"))
|
||||
G = json_graph.node_link_graph(data, edges='links')
|
||||
|
||||
question = 'QUESTION'
|
||||
@@ -1140,7 +1146,7 @@ import networkx as nx
|
||||
from networkx.readwrite import json_graph
|
||||
from pathlib import Path
|
||||
|
||||
data = json.loads(Path('graphify-out/graph.json').read_text())
|
||||
data = json.loads(Path('graphify-out/graph.json').read_text(encoding="utf-8"))
|
||||
G = json_graph.node_link_graph(data, edges='links')
|
||||
|
||||
a_term = 'NODE_A'
|
||||
@@ -1217,7 +1223,7 @@ import networkx as nx
|
||||
from networkx.readwrite import json_graph
|
||||
from pathlib import Path
|
||||
|
||||
data = json.loads(Path('graphify-out/graph.json').read_text())
|
||||
data = json.loads(Path('graphify-out/graph.json').read_text(encoding="utf-8"))
|
||||
G = json_graph.node_link_graph(data, edges='links')
|
||||
|
||||
term = 'NODE_NAME'
|
||||
|
||||
+51
-42
@@ -99,10 +99,19 @@ if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then
|
||||
fi
|
||||
# 3. Fall back to python3
|
||||
if [ -z "$PYTHON" ]; then PYTHON="python3"; fi
|
||||
"$PYTHON" -c "import graphify" 2>/dev/null || "$PYTHON" -m pip install graphifyy -q 2>/dev/null || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3
|
||||
if ! "$PYTHON" -c "import graphify" 2>/dev/null; then
|
||||
if command -v uv >/dev/null 2>&1; then
|
||||
uv tool install --upgrade graphifyy -q 2>&1 | tail -3
|
||||
_UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null)
|
||||
if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi
|
||||
else
|
||||
"$PYTHON" -m pip install graphifyy -q 2>/dev/null \
|
||||
|| "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3
|
||||
fi
|
||||
fi
|
||||
# Write interpreter path for all subsequent steps (persists across invocations)
|
||||
mkdir -p graphify-out
|
||||
"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)"
|
||||
"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)"
|
||||
# Save scan root so `graphify update` (no args) knows where to look next time
|
||||
echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root
|
||||
```
|
||||
@@ -119,7 +128,7 @@ import json
|
||||
from graphify.detect import detect
|
||||
from pathlib import Path
|
||||
result = detect(Path('INPUT_PATH'))
|
||||
print(json.dumps(result))
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
" > graphify-out/.graphify_detect.json
|
||||
```
|
||||
|
||||
@@ -170,12 +179,12 @@ import json, os
|
||||
from pathlib import Path
|
||||
from graphify.transcribe import transcribe_all
|
||||
|
||||
detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text())
|
||||
detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\"))
|
||||
video_files = detect.get('files', {}).get('video', [])
|
||||
prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.')
|
||||
|
||||
transcript_paths = transcribe_all(video_files, initial_prompt=prompt)
|
||||
print(json.dumps(transcript_paths))
|
||||
print(json.dumps(transcript_paths, ensure_ascii=False))
|
||||
" > graphify-out/.graphify_transcripts.json
|
||||
```
|
||||
|
||||
@@ -214,16 +223,16 @@ from pathlib import Path
|
||||
import json
|
||||
|
||||
code_files = []
|
||||
detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text())
|
||||
detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\"))
|
||||
for f in detect.get('files', {}).get('code', []):
|
||||
code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)])
|
||||
|
||||
if code_files:
|
||||
result = extract(code_files, cache_root=Path('.'))
|
||||
Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2))
|
||||
Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\")
|
||||
print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges')
|
||||
else:
|
||||
Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}))
|
||||
Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\")
|
||||
print('No code files - skipping AST extraction')
|
||||
"
|
||||
```
|
||||
@@ -250,14 +259,14 @@ import json
|
||||
from graphify.cache import check_semantic_cache
|
||||
from pathlib import Path
|
||||
|
||||
detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text())
|
||||
detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\"))
|
||||
all_files = [f for files in detect['files'].values() for f in files]
|
||||
|
||||
cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files)
|
||||
|
||||
if cached_nodes or cached_edges or cached_hyperedges:
|
||||
Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}))
|
||||
Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached))
|
||||
Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\")
|
||||
Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\")
|
||||
print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction')
|
||||
"
|
||||
```
|
||||
@@ -377,7 +386,7 @@ chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json'))
|
||||
all_nodes, all_edges, all_hyperedges = [], [], []
|
||||
total_in, total_out = 0, 0
|
||||
for c in chunks:
|
||||
d = json.loads(Path(c).read_text())
|
||||
d = json.loads(Path(c).read_text(encoding=\"utf-8\"))
|
||||
all_nodes += d.get('nodes', [])
|
||||
all_edges += d.get('edges', [])
|
||||
all_hyperedges += d.get('hyperedges', [])
|
||||
@@ -386,7 +395,7 @@ for c in chunks:
|
||||
Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({
|
||||
'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges,
|
||||
'input_tokens': total_in, 'output_tokens': total_out,
|
||||
}, indent=2))
|
||||
}, indent=2, ensure_ascii=False), encoding=\"utf-8\")
|
||||
print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens')
|
||||
"
|
||||
```
|
||||
@@ -398,7 +407,7 @@ import json
|
||||
from graphify.cache import save_semantic_cache
|
||||
from pathlib import Path
|
||||
|
||||
new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
|
||||
new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
|
||||
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []))
|
||||
print(f'Cached {saved} files')
|
||||
"
|
||||
@@ -410,8 +419,8 @@ $(cat graphify-out/.graphify_python) -c "
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text()) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
|
||||
new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
|
||||
cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
|
||||
new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
|
||||
|
||||
all_nodes = cached['nodes'] + new.get('nodes', [])
|
||||
all_edges = cached['edges'] + new.get('edges', [])
|
||||
@@ -430,7 +439,7 @@ merged = {
|
||||
'input_tokens': new.get('input_tokens', 0),
|
||||
'output_tokens': new.get('output_tokens', 0),
|
||||
}
|
||||
Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2))
|
||||
Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\")
|
||||
print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)')
|
||||
"
|
||||
```
|
||||
@@ -443,8 +452,8 @@ $(cat graphify-out/.graphify_python) -c "
|
||||
import sys, json
|
||||
from pathlib import Path
|
||||
|
||||
ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text())
|
||||
sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text())
|
||||
ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\"))
|
||||
sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\"))
|
||||
|
||||
# Merge: AST nodes first, semantic nodes deduplicated by id
|
||||
seen = {n['id'] for n in ast['nodes']}
|
||||
@@ -463,7 +472,7 @@ merged = {
|
||||
'input_tokens': sem.get('input_tokens', 0),
|
||||
'output_tokens': sem.get('output_tokens', 0),
|
||||
}
|
||||
Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2))
|
||||
Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\")
|
||||
total = len(merged_nodes)
|
||||
edges = len(merged_edges)
|
||||
print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)')
|
||||
@@ -485,8 +494,8 @@ from graphify.report import generate
|
||||
from graphify.export import to_json
|
||||
from pathlib import Path
|
||||
|
||||
extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text())
|
||||
detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text())
|
||||
extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\"))
|
||||
detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\"))
|
||||
|
||||
G = build_from_json(extraction)
|
||||
communities = cluster(G)
|
||||
@@ -499,7 +508,7 @@ labels = {cid: 'Community ' + str(cid) for cid in communities}
|
||||
questions = suggest_questions(G, communities, labels)
|
||||
|
||||
report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions)
|
||||
Path('graphify-out/GRAPH_REPORT.md').write_text(report)
|
||||
Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\")
|
||||
to_json(G, communities, 'graphify-out/graph.json')
|
||||
|
||||
analysis = {
|
||||
@@ -509,7 +518,7 @@ analysis = {
|
||||
'surprises': surprises,
|
||||
'questions': questions,
|
||||
}
|
||||
Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2))
|
||||
Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\")
|
||||
if G.number_of_nodes() == 0:
|
||||
print('ERROR: Graph is empty - extraction produced no nodes.')
|
||||
print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.')
|
||||
@@ -537,9 +546,9 @@ from graphify.analyze import god_nodes, surprising_connections, suggest_question
|
||||
from graphify.report import generate
|
||||
from pathlib import Path
|
||||
|
||||
extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text())
|
||||
detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text())
|
||||
analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text())
|
||||
extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\"))
|
||||
detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\"))
|
||||
analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\"))
|
||||
|
||||
G = build_from_json(extraction)
|
||||
communities = {int(k): v for k, v in analysis['communities'].items()}
|
||||
@@ -553,8 +562,8 @@ labels = LABELS_DICT
|
||||
questions = suggest_questions(G, communities, labels)
|
||||
|
||||
report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions)
|
||||
Path('graphify-out/GRAPH_REPORT.md').write_text(report)
|
||||
Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}))
|
||||
Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\")
|
||||
Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\")
|
||||
print('Report updated with community labels')
|
||||
"
|
||||
```
|
||||
@@ -662,17 +671,17 @@ from datetime import datetime, timezone
|
||||
from graphify.detect import save_manifest
|
||||
|
||||
# Save manifest for --update
|
||||
detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text())
|
||||
detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\"))
|
||||
save_manifest(detect['files'])
|
||||
|
||||
# Update cumulative cost tracker
|
||||
extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text())
|
||||
extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\"))
|
||||
input_tok = extract.get('input_tokens', 0)
|
||||
output_tok = extract.get('output_tokens', 0)
|
||||
|
||||
cost_path = Path('graphify-out/cost.json')
|
||||
if cost_path.exists():
|
||||
cost = json.loads(cost_path.read_text())
|
||||
cost = json.loads(cost_path.read_text(encoding=\"utf-8\"))
|
||||
else:
|
||||
cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0}
|
||||
|
||||
@@ -684,7 +693,7 @@ cost['runs'].append({
|
||||
})
|
||||
cost['total_input_tokens'] += input_tok
|
||||
cost['total_output_tokens'] += output_tok
|
||||
cost_path.write_text(json.dumps(cost, indent=2))
|
||||
cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\")
|
||||
|
||||
print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens')
|
||||
print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)')
|
||||
@@ -738,7 +747,7 @@ if [ ! -f graphify-out/.graphify_python ]; then
|
||||
PYTHON="python3"
|
||||
fi
|
||||
mkdir -p graphify-out
|
||||
"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)"
|
||||
"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)"
|
||||
fi
|
||||
```
|
||||
|
||||
@@ -754,8 +763,8 @@ from pathlib import Path
|
||||
|
||||
result = detect_incremental(Path('INPUT_PATH'))
|
||||
new_total = result.get('new_total', 0)
|
||||
print(json.dumps(result, indent=2))
|
||||
Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result))
|
||||
print(json.dumps(result, indent=2, ensure_ascii=False))
|
||||
Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\")
|
||||
if new_total == 0:
|
||||
print('No files changed since last run. Nothing to update.')
|
||||
raise SystemExit(0)
|
||||
@@ -770,7 +779,7 @@ $(cat graphify-out/.graphify_python) -c "
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
result = json.loads(open('graphify-out/.graphify_incremental.json').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {}
|
||||
result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {}
|
||||
code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'}
|
||||
new_files = result.get('new_files', {})
|
||||
all_changed = [f for files in new_files.values() for f in files]
|
||||
@@ -793,8 +802,8 @@ from graphify.build import build_merge
|
||||
from graphify.detect import save_manifest
|
||||
|
||||
# Load new extraction and incremental state
|
||||
new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text())
|
||||
incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text())
|
||||
new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\"))
|
||||
incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\"))
|
||||
deleted = list(incremental.get('deleted_files', []))
|
||||
|
||||
# Use build_merge() — reads graph.json directly without NetworkX round-trip
|
||||
@@ -822,7 +831,7 @@ merged_out = {
|
||||
'input_tokens': new_extraction.get('input_tokens', 0),
|
||||
'output_tokens': new_extraction.get('output_tokens', 0),
|
||||
}
|
||||
Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out))
|
||||
Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\")
|
||||
print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)')
|
||||
|
||||
# Save manifest so next --update diffs against today's state, not the
|
||||
@@ -846,8 +855,8 @@ import networkx as nx
|
||||
from pathlib import Path
|
||||
|
||||
# Load old graph (before update) from backup written before merge
|
||||
old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text()) if Path('graphify-out/.graphify_old.json').exists() else None
|
||||
new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text())
|
||||
old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None
|
||||
new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\"))
|
||||
G_new = build_from_json(new_extract)
|
||||
|
||||
if old_data:
|
||||
|
||||
Reference in New Issue
Block a user