Preserve Terraform attributes during extraction

graphify/extractors/terraform.py

@@ -7,6 +7,13 @@ import json
 from collections.abc import Sequence
 from pathlib import Path
 from graphify.extractors.base import _make_id
+from graphify.security import (
+    _CONTROL_CHAR_RE,
+    _METADATA_MAX_ATTRIBUTES,
+    _METADATA_MAX_LIST_ITEMS,
+    _METADATA_MAX_VALUE_LEN,
+)
+
 
 
 _TF_META_HEADS = frozenset({"count", "each", "self", "path", "terraform"})
@@ -248,6 +255,85 @@ def extract_terraform(path: Path) -> dict:
             if c.is_named:
                 _collect_refs(c, owner_nid, rel)
 
+    def _clean_val_str(text: str) -> str:
+        text = _CONTROL_CHAR_RE.sub("", str(text))
+        if len(text) > _METADATA_MAX_VALUE_LEN:
+            text = text[:_METADATA_MAX_VALUE_LEN]
+        return text
+
+    def _parse_attr_value(node, depth: int = 0):
+        if depth > 5:
+            return _clean_val_str(_read(node).strip())
+        cur = node
+        while cur.type in ("expression", "literal_value", "collection_value") and len(cur.named_children) == 1:
+            cur = cur.named_children[0]
+
+        t = cur.type
+        raw = _read(cur).strip()
+
+        if t == "bool_lit":
+            return raw == "true"
+        elif t == "numeric_lit":
+            try:
+                return int(raw)
+            except ValueError:
+                try:
+                    return float(raw)
+                except ValueError:
+                    return _clean_val_str(raw)
+        elif t == "null_lit":
+            return None
+        elif t == "string_lit":
+            if raw.startswith('"') and raw.endswith('"'):
+                try:
+                    return _clean_val_str(json.loads(raw))
+                except Exception:
+                    return _clean_val_str(raw[1:-1])
+            return _clean_val_str(raw)
+        elif t == "tuple":
+            items = []
+            for c in cur.named_children:
+                if c.type not in ("tuple_start", "tuple_end"):
+                    items.append(_parse_attr_value(c, depth + 1))
+                    if len(items) >= _METADATA_MAX_LIST_ITEMS:
+                        break
+            return items
+        elif t == "object":
+            obj = {}
+            for c in cur.children:
+                if c.type == "object_elem":
+                    k_node = c.child_by_field_name("key") or (c.named_children[0] if c.named_children else None)
+                    v_node = c.child_by_field_name("val") or (c.named_children[-1] if len(c.named_children) > 1 else None)
+                    if k_node and v_node:
+                        k_val = _parse_attr_value(k_node, depth + 1)
+                        k_str = str(k_val) if k_val is not None else _read(k_node).strip().strip('"')
+                        k_str = _clean_val_str(k_str)
+                        obj[k_str] = _parse_attr_value(v_node, depth + 1)
+                        if len(obj) >= _METADATA_MAX_LIST_ITEMS:
+                            break
+            return obj
+        else:
+            return _clean_val_str(raw)
+
+    def _collect_direct_attributes(blk_body) -> dict:
+        attrs = {}
+        for child in blk_body.children:
+            if child.type != "attribute":
+                continue
+            k_node = child.child_by_field_name("key") or (child.children[0] if child.children else None)
+            if k_node is None:
+                continue
+            key = _clean_val_str(_read(k_node).strip())
+            if not key:
+                continue
+            val_node = child.named_children[-1] if child.named_children else None
+            if val_node is None:
+                continue
+            attrs[key] = _parse_attr_value(val_node)
+            if len(attrs) >= _METADATA_MAX_ATTRIBUTES:
+                break
+        return attrs
+
     def _body_of(block):
         for c in block.children:
             if c.type == "body":
@@ -306,6 +392,9 @@ def extract_terraform(path: Path) -> dict:
         else:
             continue
         if blk_body is not None:
+            attrs = _collect_direct_attributes(blk_body)
+            if attrs:
+                nodes_by_id[owner]["attributes"] = attrs
             _collect_refs(blk_body, owner, "references")
 
     return {"nodes": nodes, "edges": edges}

graphify/security.py

@@ -411,6 +411,7 @@ def sanitize_label(text: str | None) -> str:
 
 _METADATA_MAX_VALUE_LEN = 512
 _METADATA_MAX_LIST_ITEMS = 50
+_METADATA_MAX_ATTRIBUTES = 100
 
 
 def _sanitize_metadata_string(value: object) -> str:

graphify/serve.py

@@ -13,7 +13,7 @@ import threading
 from typing import NamedTuple
 import networkx as nx
 from networkx.readwrite import json_graph
-from graphify.security import sanitize_label, check_graph_file_size_cap
+from graphify.security import sanitize_label, check_graph_file_size_cap, _CONTROL_CHAR_RE
 from graphify.build import edge_data, edge_datas
 from graphify.paths import default_graph_json as _default_graph_json
 
@@ -312,6 +312,35 @@ _SOURCE_MATCH_BONUS = 0.5
 # toward term coverage, so a long rationale adds recall without winning back
 # an exact-label tier it did not earn.
 _RATIONALE_MATCH_BONUS = 0.75
+_ATTRIBUTE_MATCH_BONUS = 0.75
+
+
+def _flatten_attr_text(val, parts: list[str]) -> None:
+    if isinstance(val, dict):
+        for k, v in val.items():
+            parts.append(str(k))
+            _flatten_attr_text(v, parts)
+    elif isinstance(val, (list, tuple)):
+        for item in val:
+            _flatten_attr_text(item, parts)
+    elif val is not None:
+        parts.append(str(val))
+
+
+def _node_attributes_text(data: dict) -> tuple[str, str]:
+    """The node's `attributes` key-value pairs normalized and tokenized for search."""
+    attrs = data.get("attributes")
+    if not isinstance(attrs, dict) or not attrs:
+        return "", ""
+    parts: list[str] = []
+    _flatten_attr_text(attrs, parts)
+    if not parts:
+        return "", ""
+    raw_text = " ".join(parts)
+    norm = _strip_diacritics(raw_text).lower()
+    tokens = " ".join(_search_tokens(norm))
+    return norm, tokens
+
 
 
 def _compute_idf(G: nx.Graph, terms: list[str]) -> dict[str, float]:
@@ -401,6 +430,9 @@ def _node_search_text(data: dict, nid: str) -> str:
     rationale = _node_rationale_text(data)
     if rationale:
         fields += (rationale,)
+    attr_norm, attr_tokens = _node_attributes_text(data)
+    if attr_norm:
+        fields += (attr_norm, attr_tokens)
     return "\x00".join(fields)
 
 
@@ -578,6 +610,7 @@ def _score_query(
         label_tokens = " ".join(_search_tokens(data.get("label") or ""))
         source = (data.get("source_file") or "").lower()
         rationale = _node_rationale_text(data)
+        attr_norm, attr_tokens = _node_attributes_text(data)
         # `nid_lower` is needed both by the full-query tier (`if joined`) and by
         # the per-token singleton tier (joined-singlet exact-match check). When
         # neither runs (`joined` empty AND not collecting seeds) skip the call;
@@ -643,6 +676,11 @@ def _score_query(
             if rationale and t in rationale:
                 rationale_value = _RATIONALE_MATCH_BONUS * w
                 score += rationale_value
+            # Attribute tier (#3625): recall for questions naming attributes or literal values.
+            attr_value = 0.0
+            if attr_norm and (t in attr_norm or t in attr_tokens):
+                attr_value = _ATTRIBUTE_MATCH_BONUS * w
+                score += attr_value
             tiered += tier_value
             if collect_per_term_seeds and best_by_term is not None:
                 # Singleton score for [t] on this node, mirroring
@@ -661,7 +699,7 @@ def _score_query(
                     singleton = _PREFIX_MATCH_BONUS * 10 * w
                 else:
                     singleton = 0.0
-                singleton += tier_value + substr_value + source_value + rationale_value
+                singleton += tier_value + substr_value + source_value + rationale_value + attr_value
                 if singleton > 0:
                     # Tie-break key mirrors the legacy sort+max(degree):
                     # (-singleton, -degree, label_len, nid) — the minimum
@@ -1089,12 +1127,38 @@ def _subgraph_to_text(G: nx.Graph, nodes: set[str], edges: list[tuple], token_bu
             status = sanitize_label(str(entry.get("status", "")))
             if status:
                 learning_suffix = f" learning={status}{':stale' if entry.get('stale') else ''}"
+        attrs = d.get("attributes")
+        attrs_suffix = ""
+        if isinstance(attrs, dict) and attrs:
+            pairs = []
+            for k, v in sorted(attrs.items()):
+                if isinstance(v, (dict, list)):
+                    v_str = json.dumps(v, sort_keys=True)
+                elif isinstance(v, str):
+                    v_str = f'"{v}"'
+                elif isinstance(v, bool):
+                    v_str = "true" if v else "false"
+                elif v is None:
+                    v_str = "null"
+                else:
+                    v_str = str(v)
+                if len(v_str) > 60:
+                    v_str = v_str[:57] + "..."
+                pairs.append(f"{k}={v_str}")
+                if len(pairs) >= 10:
+                    pairs.append("...")
+                    break
+            attrs_summary = ", ".join(pairs)
+            if len(attrs_summary) > 200:
+                attrs_summary = attrs_summary[:197] + "..."
+            attrs_suffix = f" attrs={{{sanitize_label(attrs_summary)}}}"
         line = (
             f"NODE {sanitize_label(d.get('label', nid))} "
             f"[src={sanitize_label(str(d.get('source_file', '')))} "
             f"loc={sanitize_label(str(d.get('source_location', '')))} "
             f"community={sanitize_label(str(d.get('community_name') or d.get('community', '')))}"
-            f"{learning_suffix}]"
+            f"{learning_suffix}"
+            f"{attrs_suffix}]"
         )
         lines.append(line)
     for u, v in edges:
@@ -1990,6 +2054,13 @@ def _build_server(graph_path: str):
         if err:
             return err
         d = G.nodes[nid]
+        attrs = d.get("attributes")
+        attrs_line = []
+        if isinstance(attrs, dict) and attrs:
+            raw_attrs = json.dumps(attrs, sort_keys=True)
+            if len(raw_attrs) > 1000:
+                raw_attrs = raw_attrs[:997] + "..."
+            attrs_line = [f"  Attributes: {_CONTROL_CHAR_RE.sub('', raw_attrs)}"]
         # Sanitise every LLM-derived field before concatenation (F-010).
         return "\n".join([
             f"Node: {sanitize_label(d.get('label', nid))}",
@@ -2004,6 +2075,7 @@ def _build_server(graph_path: str):
             f"  Type: {sanitize_label(str(d.get('file_type', '')))}",
             f"  Community: {sanitize_label(str(d.get('community_name') or d.get('community', '')))}",
             f"  Degree: {G.degree(nid)}",
+            *attrs_line,
         ])
 
     def _tool_get_neighbors(arguments: dict) -> str:

tests/test_terraform.py

@@ -156,3 +156,168 @@ def test_tfvars_key_value_is_safe(tmp_path):
     r = extract_terraform(_write(tmp_path, "terraform.tfvars", 'region = "us-east-1"\nenv = "prod"\n'))
     assert r.get("error") is None
     assert len(r["nodes"]) == 1  # only the file node, no variable nodes
+
+
+def test_terraform_attributes_primitive_literals(tmp_path):
+    body = """\
+resource "aws_s3_bucket" "example" {
+  bucket        = "my-test-bucket"
+  force_destroy = true
+  versioning    = false
+  max_size      = 42
+  ratio         = 3.14
+  logging       = null
+}
+"""
+    r = extract_terraform(_write(tmp_path, "main.tf", body))
+    node = next(n for n in r["nodes"] if n["label"] == "aws_s3_bucket.example")
+    attrs = node.get("attributes")
+    assert attrs is not None
+    assert attrs["bucket"] == "my-test-bucket"
+    assert attrs["force_destroy"] is True
+    assert attrs["versioning"] is False
+    assert attrs["max_size"] == 42
+    assert attrs["ratio"] == 3.14
+    assert attrs["logging"] is None
+
+
+def test_terraform_attributes_collections(tmp_path):
+    body = """\
+resource "aws_security_group" "sg" {
+  name          = "sg"
+  tags          = { Environment = "production", ManagedBy = "terraform" }
+  ingress_cidrs = ["10.0.0.0/16", "192.168.1.0/24"]
+}
+"""
+    r = extract_terraform(_write(tmp_path, "main.tf", body))
+    node = next(n for n in r["nodes"] if n["label"] == "aws_security_group.sg")
+    attrs = node.get("attributes")
+    assert attrs is not None
+    assert attrs["name"] == "sg"
+    assert attrs["tags"] == {"Environment": "production", "ManagedBy": "terraform"}
+    assert attrs["ingress_cidrs"] == ["10.0.0.0/16", "192.168.1.0/24"]
+
+
+def test_terraform_attributes_expressions_and_references(tmp_path):
+    body = """\
+variable "bucket_name" { default = "my-bucket" }
+data "aws_ami" "ubuntu" { most_recent = true }
+
+resource "aws_s3_bucket" "example" {
+  bucket = var.bucket_name
+  ami    = data.aws_ami.ubuntu.id
+}
+"""
+    r = extract_terraform(_write(tmp_path, "main.tf", body))
+    bucket_node = next(n for n in r["nodes"] if n["label"] == "aws_s3_bucket.example")
+    attrs = bucket_node.get("attributes")
+    assert attrs is not None
+    assert attrs["bucket"] == "var.bucket_name"
+    assert attrs["ami"] == "data.aws_ami.ubuntu.id"
+
+    # Verify existing reference edges remain intact
+    refs = _rel_pairs(r, "references")
+    assert ("aws_s3_bucket.example", "var.bucket_name") in refs
+    assert ("aws_s3_bucket.example", "data.aws_ami.ubuntu") in refs
+
+
+def test_terraform_multiple_attributes_and_nested_block_isolation(tmp_path):
+    body = """\
+resource "aws_instance" "web" {
+  ami           = "ami-12345"
+  instance_type = "t3.micro"
+  lifecycle {
+    prevent_destroy = true
+  }
+}
+"""
+    r = extract_terraform(_write(tmp_path, "main.tf", body))
+    node = next(n for n in r["nodes"] if n["label"] == "aws_instance.web")
+    attrs = node.get("attributes")
+    assert attrs is not None
+    assert attrs["ami"] == "ami-12345"
+    assert attrs["instance_type"] == "t3.micro"
+    # Ownership rule: only direct attributes of the block are captured.
+    # Nested blocks (e.g. lifecycle) must NOT be flattened into parent attributes.
+    assert "prevent_destroy" not in attrs
+    assert "lifecycle" not in attrs
+
+
+def test_terraform_attributes_query_and_search_discovery(tmp_path):
+    from graphify.serve import _query_graph_text
+
+    body = """\
+resource "aws_s3_bucket" "prod" {
+  bucket         = "company-prod-assets"
+  force_destroy  = true
+  engine_version = "15.4"
+}
+
+resource "aws_s3_bucket" "backup" {
+  bucket         = "company-backup-assets"
+  force_destroy  = false
+  engine_version = "14.2"
+}
+"""
+    r = extract_terraform(_write(tmp_path, "main.tf", body))
+    G = build_from_json(r)
+
+    # 1. Attribute name contributes to searchability
+    res_key = _query_graph_text(G, "force_destroy", depth=1)
+    assert "aws_s3_bucket.prod" in res_key
+    assert "aws_s3_bucket.backup" in res_key
+
+    # 2. Attribute literal value contributes to retrieval
+    res_val = _query_graph_text(G, "company-prod-assets", depth=1)
+    assert "aws_s3_bucket.prod" in res_val
+
+    # 3. Attribute name contributes to searchability
+    res_engine = _query_graph_text(G, "engine_version", depth=1)
+    assert "aws_s3_bucket.prod" in res_engine
+
+    # 4. Verified node retrieval and serialized attribute map content
+    prod_node = next(n for n in r["nodes"] if n["label"] == "aws_s3_bucket.prod")
+    assert prod_node["attributes"]["force_destroy"] is True
+    assert prod_node["attributes"]["engine_version"] == "15.4"
+
+
+def test_terraform_attributes_context_rendering(tmp_path):
+    from graphify.serve import _subgraph_to_text
+
+    body = """\
+resource "aws_s3_bucket" "example" {
+  bucket        = "var.bucket_name"
+  force_destroy = true
+  tags = {
+    Environment = "production"
+  }
+}
+"""
+    r = extract_terraform(_write(tmp_path, "main.tf", body))
+    G = build_from_json(r)
+    node_id = next(n["id"] for n in r["nodes"] if n["label"] == "aws_s3_bucket.example")
+
+    rendered = _subgraph_to_text(G, {node_id}, set(), token_budget=2000)
+    assert "attrs={" in rendered
+    assert 'bucket="var.bucket_name"' in rendered
+    assert "force_destroy=true" in rendered
+    assert '"Environment": "production"' in rendered
+
+
+def test_terraform_attributes_round_trip_serialization(tmp_path):
+    from graphify.export import to_json
+    from graphify.serve import _load_graph
+
+    body = """\
+resource "aws_s3_bucket" "example" {
+  bucket        = "my-bucket"
+  force_destroy = true
+}
+"""
+    r = extract_terraform(_write(tmp_path, "main.tf", body))
+    G = build_from_json(r)
+    out_file = tmp_path / "graph.json"
+    to_json(G, {}, str(out_file), force=True)
+    loaded_G = _load_graph(str(out_file))
+    node_id = next(n["id"] for n in r["nodes"] if n["label"] == "aws_s3_bucket.example")
+    assert loaded_G.nodes[node_id].get("attributes") == {"bucket": "my-bucket", "force_destroy": True}