diff --git a/graphify/extractors/terraform.py b/graphify/extractors/terraform.py index 8b3fb4d8f..a224e9fb7 100644 --- a/graphify/extractors/terraform.py +++ b/graphify/extractors/terraform.py @@ -7,6 +7,13 @@ import json from collections.abc import Sequence from pathlib import Path from graphify.extractors.base import _make_id +from graphify.security import ( + _CONTROL_CHAR_RE, + _METADATA_MAX_ATTRIBUTES, + _METADATA_MAX_LIST_ITEMS, + _METADATA_MAX_VALUE_LEN, +) + _TF_META_HEADS = frozenset({"count", "each", "self", "path", "terraform"}) @@ -248,6 +255,85 @@ def extract_terraform(path: Path) -> dict: if c.is_named: _collect_refs(c, owner_nid, rel) + def _clean_val_str(text: str) -> str: + text = _CONTROL_CHAR_RE.sub("", str(text)) + if len(text) > _METADATA_MAX_VALUE_LEN: + text = text[:_METADATA_MAX_VALUE_LEN] + return text + + def _parse_attr_value(node, depth: int = 0): + if depth > 5: + return _clean_val_str(_read(node).strip()) + cur = node + while cur.type in ("expression", "literal_value", "collection_value") and len(cur.named_children) == 1: + cur = cur.named_children[0] + + t = cur.type + raw = _read(cur).strip() + + if t == "bool_lit": + return raw == "true" + elif t == "numeric_lit": + try: + return int(raw) + except ValueError: + try: + return float(raw) + except ValueError: + return _clean_val_str(raw) + elif t == "null_lit": + return None + elif t == "string_lit": + if raw.startswith('"') and raw.endswith('"'): + try: + return _clean_val_str(json.loads(raw)) + except Exception: + return _clean_val_str(raw[1:-1]) + return _clean_val_str(raw) + elif t == "tuple": + items = [] + for c in cur.named_children: + if c.type not in ("tuple_start", "tuple_end"): + items.append(_parse_attr_value(c, depth + 1)) + if len(items) >= _METADATA_MAX_LIST_ITEMS: + break + return items + elif t == "object": + obj = {} + for c in cur.children: + if c.type == "object_elem": + k_node = c.child_by_field_name("key") or (c.named_children[0] if c.named_children else None) + v_node = c.child_by_field_name("val") or (c.named_children[-1] if len(c.named_children) > 1 else None) + if k_node and v_node: + k_val = _parse_attr_value(k_node, depth + 1) + k_str = str(k_val) if k_val is not None else _read(k_node).strip().strip('"') + k_str = _clean_val_str(k_str) + obj[k_str] = _parse_attr_value(v_node, depth + 1) + if len(obj) >= _METADATA_MAX_LIST_ITEMS: + break + return obj + else: + return _clean_val_str(raw) + + def _collect_direct_attributes(blk_body) -> dict: + attrs = {} + for child in blk_body.children: + if child.type != "attribute": + continue + k_node = child.child_by_field_name("key") or (child.children[0] if child.children else None) + if k_node is None: + continue + key = _clean_val_str(_read(k_node).strip()) + if not key: + continue + val_node = child.named_children[-1] if child.named_children else None + if val_node is None: + continue + attrs[key] = _parse_attr_value(val_node) + if len(attrs) >= _METADATA_MAX_ATTRIBUTES: + break + return attrs + def _body_of(block): for c in block.children: if c.type == "body": @@ -306,6 +392,9 @@ def extract_terraform(path: Path) -> dict: else: continue if blk_body is not None: + attrs = _collect_direct_attributes(blk_body) + if attrs: + nodes_by_id[owner]["attributes"] = attrs _collect_refs(blk_body, owner, "references") return {"nodes": nodes, "edges": edges} diff --git a/graphify/security.py b/graphify/security.py index 2dbe5bd77..ab052e954 100644 --- a/graphify/security.py +++ b/graphify/security.py @@ -411,6 +411,7 @@ def sanitize_label(text: str | None) -> str: _METADATA_MAX_VALUE_LEN = 512 _METADATA_MAX_LIST_ITEMS = 50 +_METADATA_MAX_ATTRIBUTES = 100 def _sanitize_metadata_string(value: object) -> str: diff --git a/graphify/serve.py b/graphify/serve.py index 9301a4a32..95bb63b88 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -13,7 +13,7 @@ import threading from typing import NamedTuple import networkx as nx from networkx.readwrite import json_graph -from graphify.security import sanitize_label, check_graph_file_size_cap +from graphify.security import sanitize_label, check_graph_file_size_cap, _CONTROL_CHAR_RE from graphify.build import edge_data, edge_datas from graphify.paths import default_graph_json as _default_graph_json @@ -312,6 +312,35 @@ _SOURCE_MATCH_BONUS = 0.5 # toward term coverage, so a long rationale adds recall without winning back # an exact-label tier it did not earn. _RATIONALE_MATCH_BONUS = 0.75 +_ATTRIBUTE_MATCH_BONUS = 0.75 + + +def _flatten_attr_text(val, parts: list[str]) -> None: + if isinstance(val, dict): + for k, v in val.items(): + parts.append(str(k)) + _flatten_attr_text(v, parts) + elif isinstance(val, (list, tuple)): + for item in val: + _flatten_attr_text(item, parts) + elif val is not None: + parts.append(str(val)) + + +def _node_attributes_text(data: dict) -> tuple[str, str]: + """The node's `attributes` key-value pairs normalized and tokenized for search.""" + attrs = data.get("attributes") + if not isinstance(attrs, dict) or not attrs: + return "", "" + parts: list[str] = [] + _flatten_attr_text(attrs, parts) + if not parts: + return "", "" + raw_text = " ".join(parts) + norm = _strip_diacritics(raw_text).lower() + tokens = " ".join(_search_tokens(norm)) + return norm, tokens + def _compute_idf(G: nx.Graph, terms: list[str]) -> dict[str, float]: @@ -401,6 +430,9 @@ def _node_search_text(data: dict, nid: str) -> str: rationale = _node_rationale_text(data) if rationale: fields += (rationale,) + attr_norm, attr_tokens = _node_attributes_text(data) + if attr_norm: + fields += (attr_norm, attr_tokens) return "\x00".join(fields) @@ -578,6 +610,7 @@ def _score_query( label_tokens = " ".join(_search_tokens(data.get("label") or "")) source = (data.get("source_file") or "").lower() rationale = _node_rationale_text(data) + attr_norm, attr_tokens = _node_attributes_text(data) # `nid_lower` is needed both by the full-query tier (`if joined`) and by # the per-token singleton tier (joined-singlet exact-match check). When # neither runs (`joined` empty AND not collecting seeds) skip the call; @@ -643,6 +676,11 @@ def _score_query( if rationale and t in rationale: rationale_value = _RATIONALE_MATCH_BONUS * w score += rationale_value + # Attribute tier (#3625): recall for questions naming attributes or literal values. + attr_value = 0.0 + if attr_norm and (t in attr_norm or t in attr_tokens): + attr_value = _ATTRIBUTE_MATCH_BONUS * w + score += attr_value tiered += tier_value if collect_per_term_seeds and best_by_term is not None: # Singleton score for [t] on this node, mirroring @@ -661,7 +699,7 @@ def _score_query( singleton = _PREFIX_MATCH_BONUS * 10 * w else: singleton = 0.0 - singleton += tier_value + substr_value + source_value + rationale_value + singleton += tier_value + substr_value + source_value + rationale_value + attr_value if singleton > 0: # Tie-break key mirrors the legacy sort+max(degree): # (-singleton, -degree, label_len, nid) — the minimum @@ -1089,12 +1127,38 @@ def _subgraph_to_text(G: nx.Graph, nodes: set[str], edges: list[tuple], token_bu status = sanitize_label(str(entry.get("status", ""))) if status: learning_suffix = f" learning={status}{':stale' if entry.get('stale') else ''}" + attrs = d.get("attributes") + attrs_suffix = "" + if isinstance(attrs, dict) and attrs: + pairs = [] + for k, v in sorted(attrs.items()): + if isinstance(v, (dict, list)): + v_str = json.dumps(v, sort_keys=True) + elif isinstance(v, str): + v_str = f'"{v}"' + elif isinstance(v, bool): + v_str = "true" if v else "false" + elif v is None: + v_str = "null" + else: + v_str = str(v) + if len(v_str) > 60: + v_str = v_str[:57] + "..." + pairs.append(f"{k}={v_str}") + if len(pairs) >= 10: + pairs.append("...") + break + attrs_summary = ", ".join(pairs) + if len(attrs_summary) > 200: + attrs_summary = attrs_summary[:197] + "..." + attrs_suffix = f" attrs={{{sanitize_label(attrs_summary)}}}" line = ( f"NODE {sanitize_label(d.get('label', nid))} " f"[src={sanitize_label(str(d.get('source_file', '')))} " f"loc={sanitize_label(str(d.get('source_location', '')))} " f"community={sanitize_label(str(d.get('community_name') or d.get('community', '')))}" - f"{learning_suffix}]" + f"{learning_suffix}" + f"{attrs_suffix}]" ) lines.append(line) for u, v in edges: @@ -1990,6 +2054,13 @@ def _build_server(graph_path: str): if err: return err d = G.nodes[nid] + attrs = d.get("attributes") + attrs_line = [] + if isinstance(attrs, dict) and attrs: + raw_attrs = json.dumps(attrs, sort_keys=True) + if len(raw_attrs) > 1000: + raw_attrs = raw_attrs[:997] + "..." + attrs_line = [f" Attributes: {_CONTROL_CHAR_RE.sub('', raw_attrs)}"] # Sanitise every LLM-derived field before concatenation (F-010). return "\n".join([ f"Node: {sanitize_label(d.get('label', nid))}", @@ -2004,6 +2075,7 @@ def _build_server(graph_path: str): f" Type: {sanitize_label(str(d.get('file_type', '')))}", f" Community: {sanitize_label(str(d.get('community_name') or d.get('community', '')))}", f" Degree: {G.degree(nid)}", + *attrs_line, ]) def _tool_get_neighbors(arguments: dict) -> str: diff --git a/tests/test_terraform.py b/tests/test_terraform.py index 2049078b3..c30c0c379 100644 --- a/tests/test_terraform.py +++ b/tests/test_terraform.py @@ -156,3 +156,168 @@ def test_tfvars_key_value_is_safe(tmp_path): r = extract_terraform(_write(tmp_path, "terraform.tfvars", 'region = "us-east-1"\nenv = "prod"\n')) assert r.get("error") is None assert len(r["nodes"]) == 1 # only the file node, no variable nodes + + +def test_terraform_attributes_primitive_literals(tmp_path): + body = """\ +resource "aws_s3_bucket" "example" { + bucket = "my-test-bucket" + force_destroy = true + versioning = false + max_size = 42 + ratio = 3.14 + logging = null +} +""" + r = extract_terraform(_write(tmp_path, "main.tf", body)) + node = next(n for n in r["nodes"] if n["label"] == "aws_s3_bucket.example") + attrs = node.get("attributes") + assert attrs is not None + assert attrs["bucket"] == "my-test-bucket" + assert attrs["force_destroy"] is True + assert attrs["versioning"] is False + assert attrs["max_size"] == 42 + assert attrs["ratio"] == 3.14 + assert attrs["logging"] is None + + +def test_terraform_attributes_collections(tmp_path): + body = """\ +resource "aws_security_group" "sg" { + name = "sg" + tags = { Environment = "production", ManagedBy = "terraform" } + ingress_cidrs = ["10.0.0.0/16", "192.168.1.0/24"] +} +""" + r = extract_terraform(_write(tmp_path, "main.tf", body)) + node = next(n for n in r["nodes"] if n["label"] == "aws_security_group.sg") + attrs = node.get("attributes") + assert attrs is not None + assert attrs["name"] == "sg" + assert attrs["tags"] == {"Environment": "production", "ManagedBy": "terraform"} + assert attrs["ingress_cidrs"] == ["10.0.0.0/16", "192.168.1.0/24"] + + +def test_terraform_attributes_expressions_and_references(tmp_path): + body = """\ +variable "bucket_name" { default = "my-bucket" } +data "aws_ami" "ubuntu" { most_recent = true } + +resource "aws_s3_bucket" "example" { + bucket = var.bucket_name + ami = data.aws_ami.ubuntu.id +} +""" + r = extract_terraform(_write(tmp_path, "main.tf", body)) + bucket_node = next(n for n in r["nodes"] if n["label"] == "aws_s3_bucket.example") + attrs = bucket_node.get("attributes") + assert attrs is not None + assert attrs["bucket"] == "var.bucket_name" + assert attrs["ami"] == "data.aws_ami.ubuntu.id" + + # Verify existing reference edges remain intact + refs = _rel_pairs(r, "references") + assert ("aws_s3_bucket.example", "var.bucket_name") in refs + assert ("aws_s3_bucket.example", "data.aws_ami.ubuntu") in refs + + +def test_terraform_multiple_attributes_and_nested_block_isolation(tmp_path): + body = """\ +resource "aws_instance" "web" { + ami = "ami-12345" + instance_type = "t3.micro" + lifecycle { + prevent_destroy = true + } +} +""" + r = extract_terraform(_write(tmp_path, "main.tf", body)) + node = next(n for n in r["nodes"] if n["label"] == "aws_instance.web") + attrs = node.get("attributes") + assert attrs is not None + assert attrs["ami"] == "ami-12345" + assert attrs["instance_type"] == "t3.micro" + # Ownership rule: only direct attributes of the block are captured. + # Nested blocks (e.g. lifecycle) must NOT be flattened into parent attributes. + assert "prevent_destroy" not in attrs + assert "lifecycle" not in attrs + + +def test_terraform_attributes_query_and_search_discovery(tmp_path): + from graphify.serve import _query_graph_text + + body = """\ +resource "aws_s3_bucket" "prod" { + bucket = "company-prod-assets" + force_destroy = true + engine_version = "15.4" +} + +resource "aws_s3_bucket" "backup" { + bucket = "company-backup-assets" + force_destroy = false + engine_version = "14.2" +} +""" + r = extract_terraform(_write(tmp_path, "main.tf", body)) + G = build_from_json(r) + + # 1. Attribute name contributes to searchability + res_key = _query_graph_text(G, "force_destroy", depth=1) + assert "aws_s3_bucket.prod" in res_key + assert "aws_s3_bucket.backup" in res_key + + # 2. Attribute literal value contributes to retrieval + res_val = _query_graph_text(G, "company-prod-assets", depth=1) + assert "aws_s3_bucket.prod" in res_val + + # 3. Attribute name contributes to searchability + res_engine = _query_graph_text(G, "engine_version", depth=1) + assert "aws_s3_bucket.prod" in res_engine + + # 4. Verified node retrieval and serialized attribute map content + prod_node = next(n for n in r["nodes"] if n["label"] == "aws_s3_bucket.prod") + assert prod_node["attributes"]["force_destroy"] is True + assert prod_node["attributes"]["engine_version"] == "15.4" + + +def test_terraform_attributes_context_rendering(tmp_path): + from graphify.serve import _subgraph_to_text + + body = """\ +resource "aws_s3_bucket" "example" { + bucket = "var.bucket_name" + force_destroy = true + tags = { + Environment = "production" + } +} +""" + r = extract_terraform(_write(tmp_path, "main.tf", body)) + G = build_from_json(r) + node_id = next(n["id"] for n in r["nodes"] if n["label"] == "aws_s3_bucket.example") + + rendered = _subgraph_to_text(G, {node_id}, set(), token_budget=2000) + assert "attrs={" in rendered + assert 'bucket="var.bucket_name"' in rendered + assert "force_destroy=true" in rendered + assert '"Environment": "production"' in rendered + + +def test_terraform_attributes_round_trip_serialization(tmp_path): + from graphify.export import to_json + from graphify.serve import _load_graph + + body = """\ +resource "aws_s3_bucket" "example" { + bucket = "my-bucket" + force_destroy = true +} +""" + r = extract_terraform(_write(tmp_path, "main.tf", body)) + G = build_from_json(r) + out_file = tmp_path / "graph.json" + to_json(G, {}, str(out_file), force=True) + loaded_G = _load_graph(str(out_file)) + node_id = next(n["id"] for n in r["nodes"] if n["label"] == "aws_s3_bucket.example") + assert loaded_G.nodes[node_id].get("attributes") == {"bucket": "my-bucket", "force_destroy": True}