From 0e8d92cf5fc8b87199d7df362590e53f2de0b151 Mon Sep 17 00:00:00 2001
From: nuthalapativarun <nuthalapativarun@users.noreply.github.com>
Date: Sun, 28 Jun 2026 10:32:57 +0100
Subject: [PATCH] fix(llm): tolerate non-UTF8 claude-cli output on Windows GBK
 systems (#1505)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

On Windows where claude.cmd emits GBK/cp936 bytes, the claude-cli subprocess
decoding raised UnicodeDecodeError and crashed extraction. Both claude-cli
subprocess.run sites (_call_claude_cli and the claude-cli branch of _call_llm) now
pass errors="replace", so incidental non-UTF8 console chatter decodes to
replacement chars instead of crashing — the structured JSON payload (ASCII/UTF-8
on stdout) is unaffected. capture_output=True means the single errors= covers both
stdout and stderr.

Ported from PR #1507 by @nuthalapativarun (dropped an unrelated .gitignore change).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 graphify/llm.py            |  2 ++
 tests/test_llm_backends.py | 52 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+)

diff --git a/graphify/llm.py b/graphify/llm.py
index bc1964e6..2b08532d 100644
--- a/graphify/llm.py
+++ b/graphify/llm.py
@@ -1160,6 +1160,7 @@ def _call_claude_cli(user_message: str, max_tokens: int = 8192, *, deep_mode: bo
         capture_output=True,
         text=True,
         encoding="utf-8",  # Force UTF-8 — prevents UnicodeEncodeError on Windows cp1252
+        errors="replace",  # Tolerate non-UTF-8 bytes (e.g. GBK/cp936 from claude.cmd on Chinese Windows)
         timeout=_resolve_api_timeout(),
         check=False,
         **_no_window_kwargs(),
@@ -1913,6 +1914,7 @@ def _call_llm(
             capture_output=True,
             text=True,
             encoding="utf-8",  # Force UTF-8 — prevents UnicodeEncodeError on Windows cp1252
+            errors="replace",  # Tolerate non-UTF-8 bytes (e.g. GBK/cp936 from claude.cmd on Chinese Windows)
             timeout=_resolve_api_timeout(),
             check=False,
             **_no_window_kwargs(),
diff --git a/tests/test_llm_backends.py b/tests/test_llm_backends.py
index c9638443..6ce5c7f3 100644
--- a/tests/test_llm_backends.py
+++ b/tests/test_llm_backends.py
@@ -931,3 +931,55 @@ def test_base_url_defaults_without_env(backend, default):
         env=env, capture_output=True, text=True, check=True,
     )
     assert out.stdout.strip() == default
+
+
+# ---------------------------------------------------------------------------
+# #1505: claude-cli subprocess.run must use errors="replace" so non-UTF-8
+# bytes from claude.cmd on Chinese Windows (GBK/cp936) don't crash the reader
+# thread.
+# ---------------------------------------------------------------------------
+
+import json as _json
+
+
+def _make_cli_envelope(result_text: str) -> str:
+    """Return a minimal claude -p --output-format json envelope."""
+    return _json.dumps({"type": "result", "result": result_text, "usage": {}, "modelUsage": {}})
+
+
+def test_call_claude_cli_passes_errors_replace_to_subprocess():
+    """subprocess.run must be called with errors='replace' so non-UTF-8 output
+    bytes (e.g. GBK from claude.cmd on Chinese Windows) are tolerated instead
+    of crashing the reader thread with UnicodeDecodeError (#1505)."""
+    from unittest.mock import patch, MagicMock
+
+    valid_envelope = _make_cli_envelope('{"nodes":[],"edges":[],"hyperedges":[]}')
+    mock_proc = MagicMock()
+    mock_proc.returncode = 0
+    mock_proc.stdout = valid_envelope
+    mock_proc.stderr = ""
+
+    with patch("platform.system", return_value="Linux"), \
+         patch("shutil.which", return_value="/usr/bin/claude"), \
+         patch("subprocess.run", return_value=mock_proc) as mock_run:
+        llm._call_claude_cli("test prompt")
+
+    assert mock_run.call_args.kwargs.get("errors") == "replace", \
+        "subprocess.run missing errors='replace' — non-UTF-8 bytes will crash the reader thread"
+
+
+def test_call_claude_cli_tolerates_non_utf8_in_stderr():
+    """When errors='replace' is set, non-UTF-8 bytes in stderr produce replacement
+    chars instead of UnicodeDecodeError, allowing the error path to report cleanly."""
+    from unittest.mock import patch, MagicMock
+
+    mock_proc = MagicMock()
+    mock_proc.returncode = 1
+    mock_proc.stdout = ""
+    mock_proc.stderr = "GBK error: ��"  # replacement chars after decode
+
+    with patch("platform.system", return_value="Linux"), \
+         patch("shutil.which", return_value="/usr/bin/claude"), \
+         patch("subprocess.run", return_value=mock_proc):
+        with pytest.raises(RuntimeError, match="claude -p exited 1"):
+            llm._call_claude_cli("test prompt")