From 3d49402ad2ad4bd88ec238b6a0f804038f875a78 Mon Sep 17 00:00:00 2001 From: mohamed zwanski Date: Thu, 22 Oct 2026 22:21:06 +0100 Subject: [PATCH] Fix #138: Force UTF-8 on agent subprocess pipes on Windows The orchestrator already decoded child output as UTF-8 with errors=replace, but Python children still encoded their piped stdout with the Windows ANSI code page (CP1252). Set PYTHONIOENCODING=utf-8 for every agent subprocess (parallel orchestrator and the server's process manager) so both ends agree, and decode claude CLI stderr in start.py as UTF-8 instead of the locale codec. Adds test_subprocess_encoding.py covering the spawn kwargs, a non-ASCII round trip, and invalid bytes such as the 0x8f from the report. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 1 + parallel_orchestrator.py | 11 +-- server/services/process_manager.py | 4 ++ start.py | 8 ++- test_subprocess_encoding.py | 104 +++++++++++++++++++++++++++++ 5 files changed, 122 insertions(+), 6 deletions(-) create mode 100644 test_subprocess_encoding.py diff --git a/CLAUDE.md b/CLAUDE.md index 6f59910be..d2d59659c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -120,6 +120,7 @@ python test_security_integration.py # Integration tests (9 tests) python -m pytest test_client.py # Client tests (20 tests) python -m pytest test_dependency_resolver.py # Dependency resolver tests (12 tests) python -m pytest test_rate_limit_utils.py # Rate limit tests (22 tests) +python -m pytest test_subprocess_encoding.py # Subprocess UTF-8 encoding tests (6 tests) ``` ### React UI diff --git a/parallel_orchestrator.py b/parallel_orchestrator.py index e39ef7995..7c0fd9430 100644 --- a/parallel_orchestrator.py +++ b/parallel_orchestrator.py @@ -848,6 +848,7 @@ def _spawn_coding_agent(self, feature_id: int) -> tuple[bool, str]: # CREATE_NO_WINDOW on Windows prevents console window pop-ups # stdin=DEVNULL prevents blocking on stdin reads # encoding="utf-8" and errors="replace" fix Windows CP1252 issues + # PYTHONIOENCODING=utf-8 makes the child write UTF-8 to the pipe (Windows defaults to CP1252) popen_kwargs: dict[str, Any] = { "stdin": subprocess.DEVNULL, "stdout": subprocess.PIPE, @@ -856,7 +857,7 @@ def _spawn_coding_agent(self, feature_id: int) -> tuple[bool, str]: "encoding": "utf-8", "errors": "replace", "cwd": str(self.project_dir), # Run from project dir so CLI creates .claude/ in project - "env": {**os.environ, "PYTHONUNBUFFERED": "1", "NODE_COMPILE_CACHE": "", "PLAYWRIGHT_CLI_SESSION": f"coding-{feature_id}"}, + "env": {**os.environ, "PYTHONUNBUFFERED": "1", "PYTHONIOENCODING": "utf-8", "NODE_COMPILE_CACHE": "", "PLAYWRIGHT_CLI_SESSION": f"coding-{feature_id}"}, } if sys.platform == "win32": popen_kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW @@ -919,7 +920,7 @@ def _spawn_coding_agent_batch(self, feature_ids: list[int]) -> tuple[bool, str]: "encoding": "utf-8", "errors": "replace", "cwd": str(self.project_dir), # Run from project dir so CLI creates .claude/ in project - "env": {**os.environ, "PYTHONUNBUFFERED": "1", "NODE_COMPILE_CACHE": "", "PLAYWRIGHT_CLI_SESSION": f"coding-{primary_id}"}, + "env": {**os.environ, "PYTHONUNBUFFERED": "1", "PYTHONIOENCODING": "utf-8", "NODE_COMPILE_CACHE": "", "PLAYWRIGHT_CLI_SESSION": f"coding-{primary_id}"}, } if sys.platform == "win32": popen_kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW @@ -1015,6 +1016,7 @@ def _spawn_testing_agent(self) -> tuple[bool, str]: # CREATE_NO_WINDOW on Windows prevents console window pop-ups # stdin=DEVNULL prevents blocking on stdin reads # encoding="utf-8" and errors="replace" fix Windows CP1252 issues + # PYTHONIOENCODING=utf-8 makes the child write UTF-8 to the pipe (Windows defaults to CP1252) popen_kwargs: dict[str, Any] = { "stdin": subprocess.DEVNULL, "stdout": subprocess.PIPE, @@ -1023,7 +1025,7 @@ def _spawn_testing_agent(self) -> tuple[bool, str]: "encoding": "utf-8", "errors": "replace", "cwd": str(self.project_dir), # Run from project dir so CLI creates .claude/ in project - "env": {**os.environ, "PYTHONUNBUFFERED": "1", "NODE_COMPILE_CACHE": "", "PLAYWRIGHT_CLI_SESSION": f"testing-{self._testing_session_counter}"}, + "env": {**os.environ, "PYTHONUNBUFFERED": "1", "PYTHONIOENCODING": "utf-8", "NODE_COMPILE_CACHE": "", "PLAYWRIGHT_CLI_SESSION": f"testing-{self._testing_session_counter}"}, } self._testing_session_counter += 1 if sys.platform == "win32": @@ -1077,6 +1079,7 @@ async def _run_initializer(self) -> bool: # CREATE_NO_WINDOW on Windows prevents console window pop-ups # stdin=DEVNULL prevents blocking on stdin reads # encoding="utf-8" and errors="replace" fix Windows CP1252 issues + # PYTHONIOENCODING=utf-8 makes the child write UTF-8 to the pipe (Windows defaults to CP1252) popen_kwargs: dict[str, Any] = { "stdin": subprocess.DEVNULL, "stdout": subprocess.PIPE, @@ -1085,7 +1088,7 @@ async def _run_initializer(self) -> bool: "encoding": "utf-8", "errors": "replace", "cwd": str(AUTOFORGE_ROOT), - "env": {**os.environ, "PYTHONUNBUFFERED": "1", "NODE_COMPILE_CACHE": ""}, + "env": {**os.environ, "PYTHONUNBUFFERED": "1", "PYTHONIOENCODING": "utf-8", "NODE_COMPILE_CACHE": ""}, } if sys.platform == "win32": popen_kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW diff --git a/server/services/process_manager.py b/server/services/process_manager.py index 0a7b8feea..8ea6e19bb 100644 --- a/server/services/process_manager.py +++ b/server/services/process_manager.py @@ -477,6 +477,10 @@ async def start( subprocess_env = { **os.environ, "PYTHONUNBUFFERED": "1", + # Force UTF-8 stdout in the child; on Windows a piped stdout otherwise + # defaults to the ANSI code page (e.g. CP1252) and mangles/crashes on + # non-ASCII agent output. The reader below decodes as UTF-8. + "PYTHONIOENCODING": "utf-8", "PLAYWRIGHT_CLI_SESSION": f"agent-{self.project_name}-{os.getpid()}", "NODE_COMPILE_CACHE": "", # Disable V8 compile caching to prevent .node file accumulation in %TEMP% **api_env, diff --git a/start.py b/start.py index 9a5007b09..fdde2e482 100644 --- a/start.py +++ b/start.py @@ -243,7 +243,9 @@ def run_spec_creation(project_dir: Path) -> bool: check=False, # Don't raise on non-zero exit cwd=str(Path(__file__).parent), # Run from project root stderr=subprocess.PIPE, - text=True + text=True, + encoding="utf-8", + errors="replace", ) # Check for authentication errors in stderr @@ -415,7 +417,9 @@ def run_agent(project_name: str, project_dir: Path) -> None: cmd, check=False, stderr=subprocess.PIPE, - text=True + text=True, + encoding="utf-8", + errors="replace", ) # Check for authentication errors diff --git a/test_subprocess_encoding.py b/test_subprocess_encoding.py new file mode 100644 index 000000000..f1aedc6fa --- /dev/null +++ b/test_subprocess_encoding.py @@ -0,0 +1,104 @@ +""" +Subprocess Encoding Tests +========================= + +Regression tests for issue #138: on Windows, agent subprocess output was +decoded with the ANSI code page (CP1252), crashing the orchestrator's output +reader thread with UnicodeDecodeError on non-ASCII output. + +Two halves must agree on UTF-8: +- the child must *write* UTF-8 to the pipe (PYTHONIOENCODING=utf-8) +- the parent must *read* UTF-8 from the pipe (encoding="utf-8", errors="replace") + +Run with: python -m pytest test_subprocess_encoding.py -v +""" + +import os +import subprocess +import sys +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +import parallel_orchestrator +from parallel_orchestrator import ParallelOrchestrator + +NON_ASCII = "done ✓ — café 🚀 日本語" + + +@pytest.fixture +def orchestrator(tmp_path: Path) -> ParallelOrchestrator: + return ParallelOrchestrator(project_dir=tmp_path, max_concurrency=1) + + +def _capture_popen_kwargs(spawn) -> dict: + """Run a spawn function with Popen/Thread mocked and return Popen's kwargs.""" + with patch.object(parallel_orchestrator.subprocess, "Popen", return_value=MagicMock()) as popen, \ + patch.object(parallel_orchestrator.threading, "Thread"): + ok, _ = spawn() + assert ok + popen.assert_called_once() + return dict(popen.call_args.kwargs) + + +def _assert_utf8_pipe(kwargs: dict) -> None: + assert kwargs["encoding"] == "utf-8" + assert kwargs["errors"] == "replace" + assert kwargs["env"]["PYTHONIOENCODING"] == "utf-8" + + +def test_coding_agent_uses_utf8_pipe(orchestrator: ParallelOrchestrator) -> None: + _assert_utf8_pipe(_capture_popen_kwargs(lambda: orchestrator._spawn_coding_agent(1))) + + +def test_coding_agent_batch_uses_utf8_pipe(orchestrator: ParallelOrchestrator) -> None: + _assert_utf8_pipe(_capture_popen_kwargs(lambda: orchestrator._spawn_coding_agent_batch([1, 2]))) + + +def test_testing_agent_uses_utf8_pipe(orchestrator: ParallelOrchestrator) -> None: + with patch.object(orchestrator, "_get_test_batch", return_value=[1]): + _assert_utf8_pipe(_capture_popen_kwargs(orchestrator._spawn_testing_agent)) + + +def test_child_env_overrides_legacy_parent_encoding(orchestrator: ParallelOrchestrator) -> None: + """Even if the parent environment forces CP1252, children get UTF-8.""" + with patch.dict(os.environ, {"PYTHONIOENCODING": "cp1252"}): + kwargs = _capture_popen_kwargs(lambda: orchestrator._spawn_coding_agent(1)) + assert kwargs["env"]["PYTHONIOENCODING"] == "utf-8" + + +def test_non_ascii_child_output_round_trips() -> None: + """End to end: a Python child on a CP1252-like setup prints non-ASCII text + that the parent reads back intact, as the orchestrator does.""" + env = {**os.environ, "PYTHONIOENCODING": "cp1252"} # simulate the Windows default + env.update({"PYTHONUNBUFFERED": "1", "PYTHONIOENCODING": "utf-8"}) # orchestrator overrides + proc = subprocess.Popen( + [sys.executable, "-c", f"print({NON_ASCII!r})"], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding="utf-8", + errors="replace", + env=env, + ) + assert proc.stdout is not None + lines = [line.rstrip("\n") for line in proc.stdout] + assert proc.wait() == 0 + assert lines == [NON_ASCII] + + +def test_reader_survives_invalid_utf8_bytes() -> None: + """Bytes that are invalid UTF-8 (like 0x8f from the issue) must not raise.""" + proc = subprocess.Popen( + [sys.executable, "-c", "import sys; sys.stdout.buffer.write(b'ok \\x8f\\xff end\\n')"], + stdout=subprocess.PIPE, + text=True, + encoding="utf-8", + errors="replace", + ) + assert proc.stdout is not None + lines = list(proc.stdout) + assert proc.wait() == 0 + assert lines == ["ok �� end\n"]