Clarify Chinese query segmentation scope (#1026)

Keep Graphify query segmentation focused on Chinese terms: rename the CJK helpers and extra to Chinese scope, cache the optional jieba import at module load, and keep a bigram fallback when jieba is unavailable.

Constraint: Reviewer asked either to broaden Hiragana/Katakana/Hangul support or rename CJK helpers; user chose Chinese-only because Japanese segmentation accuracy is uncertain.

Rejected: Broaden to Japanese and Korean segmentation | jieba is Chinese-oriented and the user explicitly limited scope to Chinese.

Confidence: high

Scope-risk: narrow

Directive: Do not label this path as CJK unless Hiragana/Katakana/Hangul segmentation is intentionally supported and tested.

Tested: uv run --with pytest pytest tests/test_serve.py tests/test_query_cli.py tests/test_benchmark.py

Tested: uv run --with pytest --with jieba pytest tests/test_serve.py -k "chinese or non_chinese"

Tested: graphify update .

Not-tested: Full test suite.

Co-authored-by: OmX <omx@oh-my-codex.dev>

graphify/serve.py

@@ -9,6 +9,11 @@ from networkx.readwrite import json_graph
 from graphify.security import sanitize_label, check_graph_file_size_cap
 from graphify.build import edge_data
 
+try:
+    import jieba as _jieba  # type: ignore[import-untyped]
+except ImportError:
+    _jieba = None
+
 
 def _load_graph(graph_path: str) -> nx.Graph:
     try:
@@ -51,16 +56,40 @@ def _strip_diacritics(text: str) -> str:
     return "".join(c for c in nfkd if not unicodedata.combining(c))
 
 
+def _has_chinese(text: str) -> bool:
+    return any("一" <= ch <= "鿿" for ch in text)
+
+
+def _segment_chinese(text: str) -> list[str]:
+    """Segment Chinese text and keep the original term for exact matching."""
+    if _jieba is not None:
+        segments = [w for w in _jieba.cut(text) if len(w.strip()) > 0]
+    else:
+        segments = [text[i:i + 2] for i in range(len(text) - 1)] or [text]
+    if len(text) > 1 and text not in segments:
+        segments.append(text)
+    return segments
+
+
+def _is_searchable(term: str) -> bool:
+    """True if term is Chinese, non-English, or an English word longer than 2 chars."""
+    if all("a" <= ch <= "z" for ch in term):
+        return len(term) > 2
+    return True
+
+
 def _query_terms(question: str) -> list[str]:
-    """Split a query into searchable terms, filtering only short English terms."""
+    """Split a query into searchable terms, segmenting Chinese text."""
     terms: list[str] = []
     for raw in question.split():
         term = raw.lower().strip()
         if not term:
             continue
-        is_english_only = all("a" <= ch <= "z" for ch in term)
-        if not is_english_only or len(term) > 2:
-            terms.append(term)
+        candidates = _segment_chinese(term) if _has_chinese(term) else [term]
+        for seg in candidates:
+            seg = seg.strip()
+            if seg and _is_searchable(seg):
+                terms.append(seg)
     return terms
 
 

pyproject.toml

@@ -62,8 +62,9 @@ ollama = ["openai"]
 bedrock = ["boto3"]
 gemini = ["openai", "tiktoken"]
 openai = ["openai", "tiktoken"]
+chinese = ["jieba"]
 sql = ["tree-sitter-sql"]
-all = ["mcp", "neo4j", "pypdf", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper", "yt-dlp", "matplotlib", "openai", "tiktoken", "boto3", "tree-sitter-sql"]
+all = ["mcp", "neo4j", "pypdf", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper", "yt-dlp", "matplotlib", "openai", "tiktoken", "boto3", "tree-sitter-sql", "jieba"]
 
 [project.scripts]
 graphify = "graphify.__main__:main"

tests/test_serve.py

@@ -80,9 +80,23 @@ def test_score_nodes_source_file_partial():
     assert "n2" in nids
 
 
-def test_query_terms_filters_only_short_english_terms():
+def test_query_terms_filters_only_short_english_terms(monkeypatch):
+    import graphify.serve as serve_mod
+
+    class FakeJieba:
+        def cut(self, text):
+            return {
+                "前端": ["前端"],
+                "依赖": ["依赖"],
+                "安装": ["安装"],
+                "包管理器": ["包", "管理器"],
+                "项目约定": ["项目", "约定"],
+                "a前": ["a", "前"],
+            }[text]
+
+    monkeypatch.setattr(serve_mod, "_jieba", FakeJieba())
     terms = _query_terms("前端 dependency 依赖 install 安装 to of 包管理器 项目约定 a前")
-    assert terms == ["前端", "dependency", "依赖", "install", "安装", "包管理器", "项目约定", "a前"]
+    assert terms == ["前端", "dependency", "依赖", "install", "安装", "包", "管理器", "包管理器", "项目", "约定", "项目约定", "前", "a前"]
 
 
 def test_query_graph_text_keeps_short_non_english_terms():
@@ -441,3 +455,71 @@ def test_query_graph_text_context_filter_aliases_resolve():
     # Pass-through for already-canonical values
     assert _normalize_context_filters(["parameter_type"]) == ["parameter_type"]
     assert _normalize_context_filters(["field"]) == ["field"]
+
+
+# --- Chinese segmentation ---
+
+def test_query_terms_chinese_segments_with_cached_jieba(monkeypatch):
+    """Chinese text should use the cached jieba module and keep the original term."""
+    import graphify.serve as serve_mod
+
+    class FakeJieba:
+        def cut(self, text):
+            assert text == "页面路由"
+            return ["页面", "路由"]
+
+    monkeypatch.setattr(serve_mod, "_jieba", FakeJieba())
+    terms = _query_terms("页面路由")
+    assert terms == ["页面", "路由", "页面路由"]
+
+
+def test_query_terms_chinese_mixed():
+    """Mixed Chinese and English text should be handled correctly."""
+    terms = _query_terms("前端 router 路由配置")
+    assert "前端" in terms
+    assert "router" in terms
+    assert "路由" in terms
+    assert "配置" in terms
+
+
+def test_query_terms_non_chinese_scripts_are_not_segmented():
+    """Japanese kana and Hangul are kept as terms but not segmented as Chinese."""
+    import graphify.serve as serve_mod
+
+    assert not serve_mod._has_chinese("かなカナ한글")
+    assert serve_mod._query_terms("かなカナ한글") == ["かなカナ한글"]
+
+
+def test_query_terms_chinese_no_jieba_fallback(monkeypatch):
+    """When jieba is not installed, fallback to character bigrams."""
+    import graphify.serve as serve_mod
+
+    monkeypatch.setattr(serve_mod, "_jieba", None)
+    terms = serve_mod._query_terms("页面路由")
+    # bigram fallback: ["页面", "面路", "路由"] + original "页面路由"
+    assert "页面" in terms
+    assert "路由" in terms
+    assert "页面路由" in terms
+    assert len(terms) == 4
+
+
+def test_score_nodes_chinese_substring_match():
+    """Searching for '路由' should match a node with label containing '路由'."""
+    G = nx.Graph()
+    G.add_node("n1", label="路由桥接核对表", source_file="doc.md", community=0)
+    G.add_node("n2", label="其他内容", source_file="doc.md", community=0)
+    scored = _score_nodes(G, ["路由"])
+    nids = [nid for _, nid in scored]
+    assert "n1" in nids
+    assert "n2" not in nids
+
+
+def test_query_text_chinese_finds_routing_nodes():
+    """Full pipeline: '页面路由' should find nodes with '路由' in label."""
+    G = nx.Graph()
+    G.add_node("parent", label="页面路由规范", source_file="doc.md", source_location="L1", community=0)
+    G.add_node("child", label="路由桥接核对表", source_file="doc.md", source_location="L10", community=0)
+    G.add_edge("parent", "child", relation="contains", confidence="EXTRACTED")
+    text = _query_graph_text(G, "页面路由", mode="bfs", depth=2)
+    assert "No matching nodes found." not in text
+    assert "路由" in text