From 107cea044c6afe123592d4c14a398cf87068d4a6 Mon Sep 17 00:00:00 2001 From: safishamsi Date: Wed, 15 Jul 2026 00:35:59 +0100 Subject: [PATCH] fix(serve): drop German and Romance question stopwords from query terms (#1900) Full-sentence non-English queries picked wrong BFS seeds because _QUERY_STOPWORDS was English-only: in a mostly-English code corpus, fillers like wie/die/das/funktioniert are rare, get high IDF weight, and out-seed the real keyword (~175x score collapse). Extend the frozenset with a curated German set plus trimmed French/ Spanish/Portuguese/Italian question/filler words, diacritics intact. English-collision words (war, bald, comment, come, son, sin, con, pour, des) are deliberately omitted; die/hat are kept since the all-stopword fallback and unfiltered find_node protect English use. Filtering stays in _query_terms, so the per-term seed guarantee in _pick_seeds composes unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/serve.py | 47 ++++++++++++++++++++++++++++++++++++++++----- tests/test_serve.py | 33 +++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 5 deletions(-) diff --git a/graphify/serve.py b/graphify/serve.py index 33db0108f..c94b048fb 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -107,14 +107,30 @@ def _is_searchable(term: str) -> bool: return True -# English question/filler words dropped from query terms so content words drive -# BFS seeding. Without this, "how does the frontier cache work" seeds on "how"/ +# Question/filler words dropped from query terms so content words drive BFS +# seeding. Without this, "how does the frontier cache work" seeds on "how"/ # "the"/"work" (which prefix-match prose labels like "Working Principles" at 100x) # instead of "frontier"/"cache", and lands in the wrong part of the graph. Applied # to query terms only — node text is never filtered, so a symbol literally named # `work` stays findable via explain/path. `work`/`works`/`working` are included # because "how does X work" / "how X works" is the most common question phrasing. +# +# Non-English question words are just as damaging (#1900): in a mostly-English +# code corpus, German "wie"/"funktioniert" are rare, so they get HIGH IDF weight +# and out-seed the actual content noun by orders of magnitude. So this also +# carries a curated German set plus a trimmed French/Spanish/Portuguese/Italian +# set of question/filler words. Diacritics are kept intact (the query tokenizer +# does not NFKD-strip). +# +# Collision tradeoff: a few foreign stopwords are also English content words. +# We include high-German-value ones like "die"/"hat" (the all-stopword fallback +# in _query_terms and the unfiltered find_node path keep an English "die"/"hat" +# query workable), but deliberately OMIT "war"/"bald" (German was/soon) so +# English queries about "war" or "bald" are not clobbered. On the Romance side +# we likewise omit "comment" (FR how), "come" (IT how), "son"/"sin"/"con" (ES), +# and "pour"/"des" (FR) — all too common as English/code terms. _QUERY_STOPWORDS = frozenset({ + # English "how", "what", "why", "when", "where", "which", "who", "whom", "whose", "does", "did", "is", "are", "was", "were", "be", "been", "being", "can", "could", "should", "would", "will", "shall", "may", "might", "must", @@ -122,14 +138,35 @@ _QUERY_STOPWORDS = frozenset({ "without", "into", "onto", "off", "that", "this", "these", "those", "there", "here", "its", "their", "them", "they", "about", "any", "all", "some", "work", "works", "working", + # German (articles/conjunctions/question words/auxiliaries/prepositions) + "der", "die", "das", "den", "dem", "ein", "eine", "und", "oder", "nicht", + "wie", "wer", "wann", "wo", "warum", "wieso", + "welche", "welcher", "welches", + "ist", "sind", "wird", "wurde", "hat", "haben", + "kann", "koennen", "können", "soll", "muss", "sich", + "bei", "mit", "von", "fuer", "für", "ueber", "über", "nach", "aus", + "gibt", "es", + "funktioniert", "geaendert", "geändert", "aendert", "ändert", + # French + "pourquoi", "quand", "quel", "quelle", "quels", "quelles", "quoi", + "qui", "que", "est", "sont", "fonctionne", "cette", "dans", "avec", "où", + # Spanish + "cómo", "como", "qué", "cuál", "cuáles", "cuándo", "dónde", "donde", + "porque", "por", "para", "funciona", "está", "están", "hay", + # Portuguese + "qual", "quais", "quando", "onde", "são", "estão", "tem", "uma", "não", + # Italian + "perché", "cosa", "quale", "quali", "dove", "funziona", "sono", "che", + "della", }) def _query_terms(question: str) -> list[str]: """Split a query into searchable terms, segmenting Chinese text, then drop - English question/filler words (`_QUERY_STOPWORDS`) so content words drive - seeding. Falls back to the unfiltered terms if the query is all stopwords, so - a question like "how does it work" still seeds on something.""" + question/filler words (`_QUERY_STOPWORDS`, English plus common German/ + Romance-language fillers) so content words drive seeding. Falls back to the + unfiltered terms if the query is all stopwords, so a question like "how does + it work" or "wie funktioniert das" still seeds on something.""" terms: list[str] = [] for raw in question.split(): if _has_chinese(raw): diff --git a/tests/test_serve.py b/tests/test_serve.py index 859cde026..3a9656124 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -364,6 +364,39 @@ def test_query_terms_all_stopwords_falls_back_to_unfiltered(): assert _query_terms("how does it work") == ["how", "does", "work"] +def test_query_terms_drops_german_question_stopwords(): + # #1900: German full-sentence queries must reduce to the content noun. + # In a mostly-English corpus "wie"/"funktioniert" are rare, get high IDF + # weight, and out-seed the actual keyword unless dropped here. + assert _query_terms("Wie funktioniert die Authentifizierung?") == ["authentifizierung"] + + +def test_query_terms_all_german_stopwords_falls_back_to_unfiltered(): + # Existing all-stopword fallback applies to German fillers too: the query + # keeps its terms rather than seeding on nothing. + terms = _query_terms("wie funktioniert das") + assert terms == ["wie", "funktioniert", "das"] + + +def test_pick_seeds_german_query_seeds_content_node_not_heading_noise(): + """End-to-end for #1900: a German question over a graph with German + heading-noise nodes must seed on the content noun, not on nodes that + happen to contain 'die'/'wie'/'wird'.""" + G = nx.DiGraph() + G.add_node("cfg", label="Die Konfiguration", source_file="docs/konfiguration.md") + G.add_node("sec", label="Wie wird gesichert", source_file="docs/sicherheit.md") + G.add_node("auth", label="Authentifizierung", source_file="src/auth.py") + G.add_node("helper", label="login_helper", source_file="src/auth.py") + G.add_edge("helper", "auth") + + q = "Wie funktioniert die Authentifizierung?" + terms = _query_terms(q) + seeds = _pick_seeds(_score_nodes(G, terms), G=G, terms=terms) + assert "auth" in seeds + assert "cfg" not in seeds + assert "sec" not in seeds + + def test_query_terms_filters_only_short_english_terms(monkeypatch): import graphify.serve as serve_mod