graphify/tests/test_llm_parser.py

"""Tests for `_parse_llm_json` robustness and the `_call_claude_cli`
subprocess argv shape introduced in the hollow-response fix.

These tests cover:
- The four parser failure modes described in PR #1062
- Extraction instructions delivered in the user turn (Claude Code >= 2.1)
- The GRAPHIFY_CLAUDE_CLI_MODEL env-var passthrough
"""
from __future__ import annotations

import json
from unittest.mock import patch

import pytest

from graphify import llm


# ---------- _parse_llm_json: the four canonical failure modes ----------


def test_preamble_then_fence_is_parsed():
    """Claude often prefixes the JSON with a short preamble before the
    ```json fence. The original parser only stripped fences at offset 0,
    so any preamble caused json.loads to fail and the chunk to be
    dropped as a hollow response. The robust parser handles fences
    anywhere in the text."""
    raw = (
        "Here are the extracted entities:\n\n"
        '```json\n{"nodes": [{"id": "a"}], "edges": []}\n```'
    )
    result = llm._parse_llm_json(raw)
    assert result["nodes"] == [{"id": "a"}]
    assert result["edges"] == []


def test_prose_wrapped_json_without_fence_is_parsed():
    """Some models return prose around bare JSON with no markdown fence.
    The balanced-brace fallback extracts the first complete object."""
    raw = (
        'The extracted graph is {"nodes": [{"id": "b"}], "edges": []}. '
        "Hope this helps!"
    )
    result = llm._parse_llm_json(raw)
    assert result["nodes"] == [{"id": "b"}]


def test_raw_json_still_works():
    """Regression: clean JSON input (the original happy path) must keep
    parsing exactly as before."""
    raw = '{"nodes": [], "edges": [], "hyperedges": []}'
    result = llm._parse_llm_json(raw)
    assert result == {"nodes": [], "edges": [], "hyperedges": []}


def test_total_refusal_returns_empty_fragment():
    """When the model refuses or returns unrelated prose, the parser
    must degrade gracefully — return the empty fragment so the hollow
    detector takes over, never raise."""
    raw = "I cannot extract structured data from this content."
    result = llm._parse_llm_json(raw)
    assert result == {"nodes": [], "edges": [], "hyperedges": []}


# ---------- _parse_llm_json: secondary cases worth pinning ----------


def test_fence_with_uppercase_language_tag():
    raw = '```JSON\n{"nodes": [{"id": "x"}], "edges": []}\n```'
    result = llm._parse_llm_json(raw)
    assert result["nodes"] == [{"id": "x"}]


def test_fence_without_closing_backticks():
    """Truncated response: the model started the fence but ran out of
    tokens before closing it. We should still recover the JSON body."""
    raw = '```json\n{"nodes": [{"id": "y"}], "edges": []}'
    result = llm._parse_llm_json(raw)
    assert result["nodes"] == [{"id": "y"}]


def test_empty_response_returns_empty_fragment():
    assert llm._parse_llm_json("") == {"nodes": [], "edges": [], "hyperedges": []}


# ---------- _call_claude_cli: argv shape ----------


def _make_envelope(result_obj: dict) -> str:
    return json.dumps({
        "type": "result",
        "subtype": "success",
        "is_error": False,
        "result": json.dumps(result_obj),
        "usage": {"input_tokens": 1, "output_tokens": 1,
                  "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0},
        "modelUsage": {"claude-opus-4-7": {}},
        "stop_reason": "end_turn",
    })


@patch("shutil.which", return_value="/usr/local/bin/claude")
@patch("subprocess.run")
def test_instructions_ride_in_user_turn_not_system_prompt(mock_run, _which):
    """Extraction instructions must be delivered in the user turn, not via
    --system-prompt.

    History: the original hollow-response cause was --append-system-prompt
    layering graphify's prompt on top of Claude Code's default agent prompt;
    the first fix switched to --system-prompt (replace). But newer Claude Code
    CLIs (>= ~2.1) don't treat --system-prompt as the sole authority — they
    keep the coding-agent context and reply conversationally to a bare file
    dump, which parses to zero nodes and gets bisected forever. The instructions
    now ride in the user turn (stdin) and neither system-prompt flag is used."""
    mock_run.return_value.returncode = 0
    mock_run.return_value.stdout = _make_envelope({"nodes": [], "edges": [], "hyperedges": []})
    mock_run.return_value.stderr = ""
    llm._call_claude_cli("payload")
    argv = mock_run.call_args.args[0]
    assert "--system-prompt" not in argv, (
        f"--system-prompt is ignored by Claude Code >= 2.1; argv: {argv}"
    )
    assert "--append-system-prompt" not in argv
    sent = mock_run.call_args.kwargs["input"]
    assert "graphify semantic extraction agent" in sent
    assert "output ONLY the JSON object" in sent
    assert "payload" in sent


@patch("shutil.which", return_value="/usr/local/bin/claude")
@patch("subprocess.run")
def test_model_env_var_adds_model_flag(mock_run, _which, monkeypatch):
    """GRAPHIFY_CLAUDE_CLI_MODEL must be forwarded to claude -p --model."""
    monkeypatch.setenv("GRAPHIFY_CLAUDE_CLI_MODEL", "haiku")
    mock_run.return_value.returncode = 0
    mock_run.return_value.stdout = _make_envelope({"nodes": [], "edges": [], "hyperedges": []})
    mock_run.return_value.stderr = ""
    llm._call_claude_cli("payload")
    argv = mock_run.call_args.args[0]
    assert "--model" in argv
    assert argv[argv.index("--model") + 1] == "haiku"


@patch("shutil.which", return_value="/usr/local/bin/claude")
@patch("subprocess.run")
def test_no_model_flag_when_env_var_unset(mock_run, _which, monkeypatch):
    """Default behaviour: when the env var is not set, --model is not
    added so claude-cli's own default kicks in."""
    monkeypatch.delenv("GRAPHIFY_CLAUDE_CLI_MODEL", raising=False)
    mock_run.return_value.returncode = 0
    mock_run.return_value.stdout = _make_envelope({"nodes": [], "edges": [], "hyperedges": []})
    mock_run.return_value.stderr = ""
    llm._call_claude_cli("payload")
    argv = mock_run.call_args.args[0]
    assert "--model" not in argv