mirror of
https://github.com/safishamsi/graphify.git
synced 2026-08-29 01:36:33 +00:00
fix(serve): drop question/filler stopwords from query terms
`graphify query "<question>"` tokenised the whole question and seeded BFS on
every word, so natural-language scaffolding dominated retrieval. "how does the
frontier cache work" seeded on "how"/"the"/"work" — which prefix-match prose
labels like "Working Principles" at the 100x prefix tier — instead of on
"frontier"/"cache", landing in the wrong part of the graph.
Filter a set of English question/filler words from the query terms so content
words drive seeding, with a fallback to the unfiltered terms when a query is all
stopwords ("how does it work"). Applied to query terms only — node text is never
filtered, so a symbol literally named `work` stays findable via explain/path.
Updates the one test that pinned "what" as a kept term and adds coverage for the
new drop + all-stopword fallback. Full suite green (2745 passed, 28 skipped).
This commit is contained in:
+24
-2
@@ -107,8 +107,29 @@ def _is_searchable(term: str) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
# English question/filler words dropped from query terms so content words drive
|
||||
# BFS seeding. Without this, "how does the frontier cache work" seeds on "how"/
|
||||
# "the"/"work" (which prefix-match prose labels like "Working Principles" at 100x)
|
||||
# instead of "frontier"/"cache", and lands in the wrong part of the graph. Applied
|
||||
# to query terms only — node text is never filtered, so a symbol literally named
|
||||
# `work` stays findable via explain/path. `work`/`works`/`working` are included
|
||||
# because "how does X work" / "how X works" is the most common question phrasing.
|
||||
_QUERY_STOPWORDS = frozenset({
|
||||
"how", "what", "why", "when", "where", "which", "who", "whom", "whose",
|
||||
"does", "did", "is", "are", "was", "were", "be", "been", "being",
|
||||
"can", "could", "should", "would", "will", "shall", "may", "might", "must",
|
||||
"has", "have", "had", "the", "and", "but", "not", "for", "from", "with",
|
||||
"without", "into", "onto", "off", "that", "this", "these", "those", "there",
|
||||
"here", "its", "their", "them", "they", "about", "any", "all", "some",
|
||||
"work", "works", "working",
|
||||
})
|
||||
|
||||
|
||||
def _query_terms(question: str) -> list[str]:
|
||||
"""Split a query into searchable terms, segmenting Chinese text."""
|
||||
"""Split a query into searchable terms, segmenting Chinese text, then drop
|
||||
English question/filler words (`_QUERY_STOPWORDS`) so content words drive
|
||||
seeding. Falls back to the unfiltered terms if the query is all stopwords, so
|
||||
a question like "how does it work" still seeds on something."""
|
||||
terms: list[str] = []
|
||||
for raw in question.split():
|
||||
if _has_chinese(raw):
|
||||
@@ -121,7 +142,8 @@ def _query_terms(question: str) -> list[str]:
|
||||
for tok in re.findall(r"\w+", raw.lower()):
|
||||
if _is_searchable(tok):
|
||||
terms.append(tok)
|
||||
return terms
|
||||
content = [t for t in terms if t not in _QUERY_STOPWORDS]
|
||||
return content or terms
|
||||
|
||||
|
||||
_EXACT_MATCH_BONUS = 1000.0
|
||||
|
||||
+14
-1
@@ -262,7 +262,20 @@ def test_trigram_index_cached_and_rebuilt_per_graph():
|
||||
|
||||
|
||||
def test_query_terms_strips_search_punctuation():
|
||||
assert _query_terms("what calls extract?") == ["what", "calls", "extract"]
|
||||
# "what" is a question stopword (dropped); punctuation is still stripped from "extract?".
|
||||
assert _query_terms("what calls extract?") == ["calls", "extract"]
|
||||
|
||||
|
||||
def test_query_terms_drops_question_stopwords():
|
||||
# Natural-language question words are dropped so content words drive seeding:
|
||||
# "how does the frontier cache work" must reduce to the content terms, or it
|
||||
# seeds on "how"/"the"/"work" (which prefix-match prose labels) instead.
|
||||
assert _query_terms("how does the frontier cache work") == ["frontier", "cache"]
|
||||
|
||||
|
||||
def test_query_terms_all_stopwords_falls_back_to_unfiltered():
|
||||
# An all-stopword query keeps its terms rather than seeding on nothing.
|
||||
assert _query_terms("how does it work") == ["how", "does", "work"]
|
||||
|
||||
|
||||
def test_query_terms_filters_only_short_english_terms(monkeypatch):
|
||||
|
||||
Reference in New Issue
Block a user