From fa70449d80cb9ef7ca32f08e978651bbd121ea02 Mon Sep 17 00:00:00 2001 From: Safi Date: Sat, 16 May 2026 00:01:57 +0100 Subject: [PATCH] Suppress autogenerated module docstrings from rationale extraction (#882) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Alembic/Flask-Migrate revisions, Django migrations, and protobuf/OpenAPI generated files produce hundreds of degree-1 rationale nodes labeled as 'possible documentation gaps'. Their module docstrings are revision annotations or boilerplate, not architectural rationale. - Add _is_autogenerated_python() in extract.py detecting Alembic, Django migrations, and generic DO-NOT-EDIT markers; skip module docstring only - Function/class docstrings inside those files still extracted as normal - report.py: exclude file_type=rationale nodes from isolated-node gaps section — rationale nodes are degree-1 by construction; flagging them as missing edges was always wrong - 5 new tests covering Alembic, Django, protobuf, false-positive guard, and function-docstring passthrough --- graphify/extract.py | 32 ++++++++++++++-- graphify/report.py | 5 ++- tests/test_rationale.py | 85 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 117 insertions(+), 5 deletions(-) diff --git a/graphify/extract.py b/graphify/extract.py index bc7ea874..937d52db 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -1866,6 +1866,27 @@ def _extract_generic(path: Path, config: LanguageConfig) -> dict: _RATIONALE_PREFIXES = ("# NOTE:", "# IMPORTANT:", "# HACK:", "# WHY:", "# RATIONALE:", "# TODO:", "# FIXME:") +def _is_autogenerated_python(source: bytes) -> bool: + """Return True if this Python file is auto-generated and its module docstring is noise. + + Covers: Alembic/Flask-Migrate revisions, Django migrations, protobuf/gRPC/OpenAPI stubs. + Module docstrings in these files are change annotations or boilerplate, not rationale. + """ + head = source[:2048].decode("utf-8", errors="replace") + # Generic generated-file markers (protobuf, gRPC, OpenAPI codegen, etc.) + if any(m in head for m in ("DO NOT EDIT", "@generated", "Generated by the protocol buffer")): + return True + # Alembic / Flask-Migrate revision files + if (re.search(r"^revision\s*[:=]", head, re.MULTILINE) + and "def upgrade(" in head + and "down_revision" in head): + return True + # Django migrations + if "class Migration(migrations.Migration)" in head and "operations" in head: + return True + return False + + def _extract_python_rationale(path: Path, result: dict) -> None: """Post-pass: extract docstrings and rationale comments from Python source. Mutates result in-place by appending to result['nodes'] and result['edges']. @@ -1924,10 +1945,13 @@ def _extract_python_rationale(path: Path, result: dict) -> None: "weight": 1.0, }) - # Module-level docstring - ds = _get_docstring(root) - if ds: - _add_rationale(ds[0], ds[1], file_nid) + # Module-level docstring — skip for auto-generated files (Alembic, Django + # migrations, protobuf stubs, etc.) whose module docstrings are revision + # annotations, not architectural rationale. + if not _is_autogenerated_python(source): + ds = _get_docstring(root) + if ds: + _add_rationale(ds[0], ds[1], file_nid) # Class and function docstrings def walk_docstrings(node, parent_nid: str) -> None: diff --git a/graphify/report.py b/graphify/report.py index fed26fa5..8aa51e7c 100644 --- a/graphify/report.py +++ b/graphify/report.py @@ -164,7 +164,10 @@ def generate( isolated = [ n for n in G.nodes() - if G.degree(n) <= 1 and not _is_file_node(G, n) and not _is_concept_node(G, n) + if G.degree(n) <= 1 + and not _is_file_node(G, n) + and not _is_concept_node(G, n) + and G.nodes[n].get("file_type") != "rationale" ] thin_communities = { cid: nodes for cid, nodes in communities.items() diff --git a/tests/test_rationale.py b/tests/test_rationale.py index 010493e3..67bd3df9 100644 --- a/tests/test_rationale.py +++ b/tests/test_rationale.py @@ -87,3 +87,88 @@ def test_rationale_confidence_is_extracted(tmp_path): result = extract_python(path) rationale_edges = [e for e in result["edges"] if e.get("relation") == "rationale_for"] assert all(e.get("confidence") == "EXTRACTED" for e in rationale_edges) + + +def test_alembic_module_docstring_suppressed(tmp_path): + path = _write_py(tmp_path, ''' + """initial schema + + Revision ID: 0001abcd + Revises: + Create Date: 2023-01-01 00:00:00 + """ + revision = "0001abcd" + down_revision = None + branch_labels = None + + def upgrade(): + pass + + def downgrade(): + pass + ''') + result = extract_python(path) + rationale = [n for n in result["nodes"] if n.get("file_type") == "rationale"] + assert not any("Revision ID" in n["label"] for n in rationale) + + +def test_alembic_function_docstrings_still_extracted(tmp_path): + """Function docstrings inside upgrade/downgrade should still be captured.""" + path = _write_py(tmp_path, ''' + """Revision ID: 0002 Revises: 0001""" + revision = "0002" + down_revision = "0001" + + def upgrade(): + """Add users table because auth was added in this release.""" + pass + + def downgrade(): + pass + ''') + result = extract_python(path) + rationale = [n for n in result["nodes"] if n.get("file_type") == "rationale"] + # module docstring suppressed + assert not any("Revision ID" in n["label"] for n in rationale) + # function docstring still captured + assert any("auth" in n["label"] for n in rationale) + + +def test_non_migration_revision_var_not_suppressed(tmp_path): + """A file with a `revision` variable but no Alembic markers keeps its docstring.""" + path = _write_py(tmp_path, ''' + """This module tracks document revisions because we need audit history.""" + revision = 42 + + def get_revision(): pass + ''') + result = extract_python(path) + rationale = [n for n in result["nodes"] if n.get("file_type") == "rationale"] + assert any("audit history" in n["label"] for n in rationale) + + +def test_django_migration_module_docstring_suppressed(tmp_path): + path = _write_py(tmp_path, ''' + """Add post_priority_config table.""" + from django.db import migrations + + class Migration(migrations.Migration): + dependencies = [("myapp", "0001_initial")] + operations = [] + ''') + result = extract_python(path) + rationale = [n for n in result["nodes"] if n.get("file_type") == "rationale"] + assert not any("post_priority" in n["label"] for n in rationale) + + +def test_generated_file_module_docstring_suppressed(tmp_path): + path = _write_py(tmp_path, ''' + """Generated by the protocol buffer compiler. DO NOT EDIT!""" + from google.protobuf import descriptor as _descriptor + + class UserMessage: + pass + ''') + result = extract_python(path) + rationale = [n for n in result["nodes"] if n.get("file_type") == "rationale"] + assert not any("protocol buffer" in n["label"].lower() for n in rationale)