diff --git a/graphify/benchmark.py b/graphify/benchmark.py index f362d5e7..e57da1c1 100644 --- a/graphify/benchmark.py +++ b/graphify/benchmark.py @@ -7,6 +7,7 @@ import networkx as nx from networkx.readwrite import json_graph from graphify.build import edge_data +from graphify.serve import _query_terms _CHARS_PER_TOKEN = 4 # standard approximation @@ -37,7 +38,7 @@ def _estimate_tokens(text: str) -> int: def _query_subgraph_tokens(G: nx.Graph, question: str, depth: int = 3) -> int: """Run BFS from best-matching nodes and return estimated tokens in the subgraph context.""" - terms = [t.lower() for t in question.split() if len(t) > 2] + terms = _query_terms(question) scored = [] for nid, data in G.nodes(data=True): label = data.get("label", "").lower() diff --git a/graphify/serve.py b/graphify/serve.py index 605c8dc6..e0c7ae02 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -50,6 +50,19 @@ def _strip_diacritics(text: str) -> str: return "".join(c for c in nfkd if not unicodedata.combining(c)) +def _query_terms(question: str) -> list[str]: + """Split a query into searchable terms, filtering only short English terms.""" + terms: list[str] = [] + for raw in question.split(): + term = raw.lower().strip() + if not term: + continue + is_english_only = all("a" <= ch <= "z" for ch in term) + if not is_english_only or len(term) > 2: + terms.append(term) + return terms + + _EXACT_MATCH_BONUS = 1000.0 _PREFIX_MATCH_BONUS = 100.0 _SUBSTRING_MATCH_BONUS = 1.0 @@ -306,7 +319,7 @@ def _query_graph_text( token_budget: int = 2000, context_filters: list[str] | None = None, ) -> str: - terms = [t.lower() for t in question.split() if len(t) > 2] + terms = _query_terms(question) scored = _score_nodes(G, terms) start_nodes = _pick_seeds(scored) if not start_nodes: diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index a1e3a4c1..0c40fa66 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -47,6 +47,13 @@ def test_query_bfs_expands_neighbors(): assert tokens_deep >= tokens_shallow +def test_query_keeps_short_non_english_terms(): + G = nx.Graph() + G.add_node("frontend", label="前端", source_file="docs/前端.md", source_location="L1", community=0) + tokens = _query_subgraph_tokens(G, "前端", depth=1) + assert tokens > 0 + + # --- run_benchmark --- def test_run_benchmark_returns_reduction(tmp_path): diff --git a/tests/test_serve.py b/tests/test_serve.py index e0298a52..9dd1c7ff 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -13,6 +13,7 @@ from graphify.serve import ( _dfs, _filter_graph_by_context, _infer_context_filters, + _query_terms, _query_graph_text, _resolve_context_filters, _subgraph_to_text, @@ -79,6 +80,19 @@ def test_score_nodes_source_file_partial(): assert "n2" in nids +def test_query_terms_filters_only_short_english_terms(): + terms = _query_terms("前端 dependency 依赖 install 安装 to of 包管理器 项目约定 a前") + assert terms == ["前端", "dependency", "依赖", "install", "安装", "包管理器", "项目约定", "a前"] + + +def test_query_graph_text_keeps_short_non_english_terms(): + G = nx.Graph() + G.add_node("frontend", label="前端", source_file="docs/前端.md", source_location="L1", community=0) + text = _query_graph_text(G, "前端", mode="bfs", depth=1) + assert "No matching nodes found." not in text + assert "NODE 前端" in text + + def test_infer_context_filters_for_calls_question(): assert _infer_context_filters("who calls extract") == ["call"]