fix(serve): drop question/filler stopwords from query terms

`graphify query "<question>"` tokenised the whole question and seeded BFS on
every word, so natural-language scaffolding dominated retrieval. "how does the
frontier cache work" seeded on "how"/"the"/"work" — which prefix-match prose
labels like "Working Principles" at the 100x prefix tier — instead of on
"frontier"/"cache", landing in the wrong part of the graph.

Filter a set of English question/filler words from the query terms so content
words drive seeding, with a fallback to the unfiltered terms when a query is all
stopwords ("how does it work"). Applied to query terms only — node text is never
filtered, so a symbol literally named `work` stays findable via explain/path.

Updates the one test that pinned "what" as a kept term and adds coverage for the
new drop + all-stopword fallback. Full suite green (2745 passed, 28 skipped).
This commit is contained in:
Paul Young
2026-07-02 11:50:17 +01:00
committed by safishamsi
parent 1256d65214
commit 6e97088493
2 changed files with 38 additions and 3 deletions
+24 -2
View File
@@ -107,8 +107,29 @@ def _is_searchable(term: str) -> bool:
return True
# English question/filler words dropped from query terms so content words drive
# BFS seeding. Without this, "how does the frontier cache work" seeds on "how"/
# "the"/"work" (which prefix-match prose labels like "Working Principles" at 100x)
# instead of "frontier"/"cache", and lands in the wrong part of the graph. Applied
# to query terms only — node text is never filtered, so a symbol literally named
# `work` stays findable via explain/path. `work`/`works`/`working` are included
# because "how does X work" / "how X works" is the most common question phrasing.
_QUERY_STOPWORDS = frozenset({
"how", "what", "why", "when", "where", "which", "who", "whom", "whose",
"does", "did", "is", "are", "was", "were", "be", "been", "being",
"can", "could", "should", "would", "will", "shall", "may", "might", "must",
"has", "have", "had", "the", "and", "but", "not", "for", "from", "with",
"without", "into", "onto", "off", "that", "this", "these", "those", "there",
"here", "its", "their", "them", "they", "about", "any", "all", "some",
"work", "works", "working",
})
def _query_terms(question: str) -> list[str]:
"""Split a query into searchable terms, segmenting Chinese text."""
"""Split a query into searchable terms, segmenting Chinese text, then drop
English question/filler words (`_QUERY_STOPWORDS`) so content words drive
seeding. Falls back to the unfiltered terms if the query is all stopwords, so
a question like "how does it work" still seeds on something."""
terms: list[str] = []
for raw in question.split():
if _has_chinese(raw):
@@ -121,7 +142,8 @@ def _query_terms(question: str) -> list[str]:
for tok in re.findall(r"\w+", raw.lower()):
if _is_searchable(tok):
terms.append(tok)
return terms
content = [t for t in terms if t not in _QUERY_STOPWORDS]
return content or terms
_EXACT_MATCH_BONUS = 1000.0
+14 -1
View File
@@ -262,7 +262,20 @@ def test_trigram_index_cached_and_rebuilt_per_graph():
def test_query_terms_strips_search_punctuation():
assert _query_terms("what calls extract?") == ["what", "calls", "extract"]
# "what" is a question stopword (dropped); punctuation is still stripped from "extract?".
assert _query_terms("what calls extract?") == ["calls", "extract"]
def test_query_terms_drops_question_stopwords():
# Natural-language question words are dropped so content words drive seeding:
# "how does the frontier cache work" must reduce to the content terms, or it
# seeds on "how"/"the"/"work" (which prefix-match prose labels) instead.
assert _query_terms("how does the frontier cache work") == ["frontier", "cache"]
def test_query_terms_all_stopwords_falls_back_to_unfiltered():
# An all-stopword query keeps its terms rather than seeding on nothing.
assert _query_terms("how does it work") == ["how", "does", "work"]
def test_query_terms_filters_only_short_english_terms(monkeypatch):