mirror of
https://github.com/safishamsi/graphify.git
synced 2026-07-13 02:47:00 +00:00
Suppress autogenerated module docstrings from rationale extraction (#882)
Alembic/Flask-Migrate revisions, Django migrations, and protobuf/OpenAPI generated files produce hundreds of degree-1 rationale nodes labeled as 'possible documentation gaps'. Their module docstrings are revision annotations or boilerplate, not architectural rationale. - Add _is_autogenerated_python() in extract.py detecting Alembic, Django migrations, and generic DO-NOT-EDIT markers; skip module docstring only - Function/class docstrings inside those files still extracted as normal - report.py: exclude file_type=rationale nodes from isolated-node gaps section — rationale nodes are degree-1 by construction; flagging them as missing edges was always wrong - 5 new tests covering Alembic, Django, protobuf, false-positive guard, and function-docstring passthrough
This commit is contained in:
+28
-4
@@ -1866,6 +1866,27 @@ def _extract_generic(path: Path, config: LanguageConfig) -> dict:
|
||||
_RATIONALE_PREFIXES = ("# NOTE:", "# IMPORTANT:", "# HACK:", "# WHY:", "# RATIONALE:", "# TODO:", "# FIXME:")
|
||||
|
||||
|
||||
def _is_autogenerated_python(source: bytes) -> bool:
|
||||
"""Return True if this Python file is auto-generated and its module docstring is noise.
|
||||
|
||||
Covers: Alembic/Flask-Migrate revisions, Django migrations, protobuf/gRPC/OpenAPI stubs.
|
||||
Module docstrings in these files are change annotations or boilerplate, not rationale.
|
||||
"""
|
||||
head = source[:2048].decode("utf-8", errors="replace")
|
||||
# Generic generated-file markers (protobuf, gRPC, OpenAPI codegen, etc.)
|
||||
if any(m in head for m in ("DO NOT EDIT", "@generated", "Generated by the protocol buffer")):
|
||||
return True
|
||||
# Alembic / Flask-Migrate revision files
|
||||
if (re.search(r"^revision\s*[:=]", head, re.MULTILINE)
|
||||
and "def upgrade(" in head
|
||||
and "down_revision" in head):
|
||||
return True
|
||||
# Django migrations
|
||||
if "class Migration(migrations.Migration)" in head and "operations" in head:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _extract_python_rationale(path: Path, result: dict) -> None:
|
||||
"""Post-pass: extract docstrings and rationale comments from Python source.
|
||||
Mutates result in-place by appending to result['nodes'] and result['edges'].
|
||||
@@ -1924,10 +1945,13 @@ def _extract_python_rationale(path: Path, result: dict) -> None:
|
||||
"weight": 1.0,
|
||||
})
|
||||
|
||||
# Module-level docstring
|
||||
ds = _get_docstring(root)
|
||||
if ds:
|
||||
_add_rationale(ds[0], ds[1], file_nid)
|
||||
# Module-level docstring — skip for auto-generated files (Alembic, Django
|
||||
# migrations, protobuf stubs, etc.) whose module docstrings are revision
|
||||
# annotations, not architectural rationale.
|
||||
if not _is_autogenerated_python(source):
|
||||
ds = _get_docstring(root)
|
||||
if ds:
|
||||
_add_rationale(ds[0], ds[1], file_nid)
|
||||
|
||||
# Class and function docstrings
|
||||
def walk_docstrings(node, parent_nid: str) -> None:
|
||||
|
||||
+4
-1
@@ -164,7 +164,10 @@ def generate(
|
||||
|
||||
isolated = [
|
||||
n for n in G.nodes()
|
||||
if G.degree(n) <= 1 and not _is_file_node(G, n) and not _is_concept_node(G, n)
|
||||
if G.degree(n) <= 1
|
||||
and not _is_file_node(G, n)
|
||||
and not _is_concept_node(G, n)
|
||||
and G.nodes[n].get("file_type") != "rationale"
|
||||
]
|
||||
thin_communities = {
|
||||
cid: nodes for cid, nodes in communities.items()
|
||||
|
||||
@@ -87,3 +87,88 @@ def test_rationale_confidence_is_extracted(tmp_path):
|
||||
result = extract_python(path)
|
||||
rationale_edges = [e for e in result["edges"] if e.get("relation") == "rationale_for"]
|
||||
assert all(e.get("confidence") == "EXTRACTED" for e in rationale_edges)
|
||||
|
||||
|
||||
def test_alembic_module_docstring_suppressed(tmp_path):
|
||||
path = _write_py(tmp_path, '''
|
||||
"""initial schema
|
||||
|
||||
Revision ID: 0001abcd
|
||||
Revises:
|
||||
Create Date: 2023-01-01 00:00:00
|
||||
"""
|
||||
revision = "0001abcd"
|
||||
down_revision = None
|
||||
branch_labels = None
|
||||
|
||||
def upgrade():
|
||||
pass
|
||||
|
||||
def downgrade():
|
||||
pass
|
||||
''')
|
||||
result = extract_python(path)
|
||||
rationale = [n for n in result["nodes"] if n.get("file_type") == "rationale"]
|
||||
assert not any("Revision ID" in n["label"] for n in rationale)
|
||||
|
||||
|
||||
def test_alembic_function_docstrings_still_extracted(tmp_path):
|
||||
"""Function docstrings inside upgrade/downgrade should still be captured."""
|
||||
path = _write_py(tmp_path, '''
|
||||
"""Revision ID: 0002 Revises: 0001"""
|
||||
revision = "0002"
|
||||
down_revision = "0001"
|
||||
|
||||
def upgrade():
|
||||
"""Add users table because auth was added in this release."""
|
||||
pass
|
||||
|
||||
def downgrade():
|
||||
pass
|
||||
''')
|
||||
result = extract_python(path)
|
||||
rationale = [n for n in result["nodes"] if n.get("file_type") == "rationale"]
|
||||
# module docstring suppressed
|
||||
assert not any("Revision ID" in n["label"] for n in rationale)
|
||||
# function docstring still captured
|
||||
assert any("auth" in n["label"] for n in rationale)
|
||||
|
||||
|
||||
def test_non_migration_revision_var_not_suppressed(tmp_path):
|
||||
"""A file with a `revision` variable but no Alembic markers keeps its docstring."""
|
||||
path = _write_py(tmp_path, '''
|
||||
"""This module tracks document revisions because we need audit history."""
|
||||
revision = 42
|
||||
|
||||
def get_revision(): pass
|
||||
''')
|
||||
result = extract_python(path)
|
||||
rationale = [n for n in result["nodes"] if n.get("file_type") == "rationale"]
|
||||
assert any("audit history" in n["label"] for n in rationale)
|
||||
|
||||
|
||||
def test_django_migration_module_docstring_suppressed(tmp_path):
|
||||
path = _write_py(tmp_path, '''
|
||||
"""Add post_priority_config table."""
|
||||
from django.db import migrations
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [("myapp", "0001_initial")]
|
||||
operations = []
|
||||
''')
|
||||
result = extract_python(path)
|
||||
rationale = [n for n in result["nodes"] if n.get("file_type") == "rationale"]
|
||||
assert not any("post_priority" in n["label"] for n in rationale)
|
||||
|
||||
|
||||
def test_generated_file_module_docstring_suppressed(tmp_path):
|
||||
path = _write_py(tmp_path, '''
|
||||
"""Generated by the protocol buffer compiler. DO NOT EDIT!"""
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
|
||||
class UserMessage:
|
||||
pass
|
||||
''')
|
||||
result = extract_python(path)
|
||||
rationale = [n for n in result["nodes"] if n.get("file_type") == "rationale"]
|
||||
assert not any("protocol buffer" in n["label"].lower() for n in rationale)
|
||||
|
||||
Reference in New Issue
Block a user