Keep non-English query terms searchable (#964)

Graph queries filtered every token with len > 2, which dropped common two-character Chinese search terms while trying to suppress short English noise. Centralize query token selection and apply the length gate only to pure-English tokens so mixed or non-English terms remain searchable.

Constraint: Issue #962 reports space-separated Chinese query terms such as 前端, 依赖, and 安装 are lost by graphify query.

Rejected: Add Chinese segmentation now | the reported failure is fixed by preserving existing space-separated non-English tokens without expanding query behavior.

Confidence: high

Scope-risk: narrow

Directive: Keep CLI, MCP query, and benchmark query tokenization on one helper when changing query-term rules.

Tested: uv run --with pytest pytest tests/test_serve.py tests/test_query_cli.py tests/test_benchmark.py

Tested: graphify update .

Not-tested: Full test suite.

Co-authored-by: OmX <omx@oh-my-codex.dev>

graphify/benchmark.py

@@ -7,6 +7,7 @@ import networkx as nx
 from networkx.readwrite import json_graph
 
 from graphify.build import edge_data
+from graphify.serve import _query_terms
 
 
 _CHARS_PER_TOKEN = 4  # standard approximation
@@ -37,7 +38,7 @@ def _estimate_tokens(text: str) -> int:
 
 def _query_subgraph_tokens(G: nx.Graph, question: str, depth: int = 3) -> int:
     """Run BFS from best-matching nodes and return estimated tokens in the subgraph context."""
-    terms = [t.lower() for t in question.split() if len(t) > 2]
+    terms = _query_terms(question)
     scored = []
     for nid, data in G.nodes(data=True):
         label = data.get("label", "").lower()

graphify/serve.py

@@ -50,6 +50,19 @@ def _strip_diacritics(text: str) -> str:
     return "".join(c for c in nfkd if not unicodedata.combining(c))
 
 
+def _query_terms(question: str) -> list[str]:
+    """Split a query into searchable terms, filtering only short English terms."""
+    terms: list[str] = []
+    for raw in question.split():
+        term = raw.lower().strip()
+        if not term:
+            continue
+        is_english_only = all("a" <= ch <= "z" for ch in term)
+        if not is_english_only or len(term) > 2:
+            terms.append(term)
+    return terms
+
+
 _EXACT_MATCH_BONUS = 1000.0
 _PREFIX_MATCH_BONUS = 100.0
 _SUBSTRING_MATCH_BONUS = 1.0
@@ -306,7 +319,7 @@ def _query_graph_text(
     token_budget: int = 2000,
     context_filters: list[str] | None = None,
 ) -> str:
-    terms = [t.lower() for t in question.split() if len(t) > 2]
+    terms = _query_terms(question)
     scored = _score_nodes(G, terms)
     start_nodes = _pick_seeds(scored)
     if not start_nodes: