Fix #138: Force UTF-8 on agent subprocess pipes on Windows

CLAUDE.md

@@ -120,6 +120,7 @@ python test_security_integration.py   # Integration tests (9 tests)
 python -m pytest test_client.py       # Client tests (20 tests)
 python -m pytest test_dependency_resolver.py  # Dependency resolver tests (12 tests)
 python -m pytest test_rate_limit_utils.py     # Rate limit tests (22 tests)
+python -m pytest test_subprocess_encoding.py  # Subprocess UTF-8 encoding tests (6 tests)
 ```
 
 ### React UI

parallel_orchestrator.py

@@ -848,6 +848,7 @@ def _spawn_coding_agent(self, feature_id: int) -> tuple[bool, str]:
             # CREATE_NO_WINDOW on Windows prevents console window pop-ups
             # stdin=DEVNULL prevents blocking on stdin reads
             # encoding="utf-8" and errors="replace" fix Windows CP1252 issues
+            # PYTHONIOENCODING=utf-8 makes the child write UTF-8 to the pipe (Windows defaults to CP1252)
             popen_kwargs: dict[str, Any] = {
                 "stdin": subprocess.DEVNULL,
                 "stdout": subprocess.PIPE,
@@ -856,7 +857,7 @@ def _spawn_coding_agent(self, feature_id: int) -> tuple[bool, str]:
                 "encoding": "utf-8",
                 "errors": "replace",
                 "cwd": str(self.project_dir),  # Run from project dir so CLI creates .claude/ in project
-                "env": {**os.environ, "PYTHONUNBUFFERED": "1", "NODE_COMPILE_CACHE": "", "PLAYWRIGHT_CLI_SESSION": f"coding-{feature_id}"},
+                "env": {**os.environ, "PYTHONUNBUFFERED": "1", "PYTHONIOENCODING": "utf-8", "NODE_COMPILE_CACHE": "", "PLAYWRIGHT_CLI_SESSION": f"coding-{feature_id}"},
             }
             if sys.platform == "win32":
                 popen_kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW
@@ -919,7 +920,7 @@ def _spawn_coding_agent_batch(self, feature_ids: list[int]) -> tuple[bool, str]:
                 "encoding": "utf-8",
                 "errors": "replace",
                 "cwd": str(self.project_dir),  # Run from project dir so CLI creates .claude/ in project
-                "env": {**os.environ, "PYTHONUNBUFFERED": "1", "NODE_COMPILE_CACHE": "", "PLAYWRIGHT_CLI_SESSION": f"coding-{primary_id}"},
+                "env": {**os.environ, "PYTHONUNBUFFERED": "1", "PYTHONIOENCODING": "utf-8", "NODE_COMPILE_CACHE": "", "PLAYWRIGHT_CLI_SESSION": f"coding-{primary_id}"},
             }
             if sys.platform == "win32":
                 popen_kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW
@@ -1015,6 +1016,7 @@ def _spawn_testing_agent(self) -> tuple[bool, str]:
                 # CREATE_NO_WINDOW on Windows prevents console window pop-ups
                 # stdin=DEVNULL prevents blocking on stdin reads
                 # encoding="utf-8" and errors="replace" fix Windows CP1252 issues
+                # PYTHONIOENCODING=utf-8 makes the child write UTF-8 to the pipe (Windows defaults to CP1252)
                 popen_kwargs: dict[str, Any] = {
                     "stdin": subprocess.DEVNULL,
                     "stdout": subprocess.PIPE,
@@ -1023,7 +1025,7 @@ def _spawn_testing_agent(self) -> tuple[bool, str]:
                     "encoding": "utf-8",
                     "errors": "replace",
                     "cwd": str(self.project_dir),  # Run from project dir so CLI creates .claude/ in project
-                    "env": {**os.environ, "PYTHONUNBUFFERED": "1", "NODE_COMPILE_CACHE": "", "PLAYWRIGHT_CLI_SESSION": f"testing-{self._testing_session_counter}"},
+                    "env": {**os.environ, "PYTHONUNBUFFERED": "1", "PYTHONIOENCODING": "utf-8", "NODE_COMPILE_CACHE": "", "PLAYWRIGHT_CLI_SESSION": f"testing-{self._testing_session_counter}"},
                 }
                 self._testing_session_counter += 1
                 if sys.platform == "win32":
@@ -1077,6 +1079,7 @@ async def _run_initializer(self) -> bool:
         # CREATE_NO_WINDOW on Windows prevents console window pop-ups
         # stdin=DEVNULL prevents blocking on stdin reads
         # encoding="utf-8" and errors="replace" fix Windows CP1252 issues
+        # PYTHONIOENCODING=utf-8 makes the child write UTF-8 to the pipe (Windows defaults to CP1252)
         popen_kwargs: dict[str, Any] = {
             "stdin": subprocess.DEVNULL,
             "stdout": subprocess.PIPE,
@@ -1085,7 +1088,7 @@ async def _run_initializer(self) -> bool:
             "encoding": "utf-8",
             "errors": "replace",
             "cwd": str(AUTOFORGE_ROOT),
-            "env": {**os.environ, "PYTHONUNBUFFERED": "1", "NODE_COMPILE_CACHE": ""},
+            "env": {**os.environ, "PYTHONUNBUFFERED": "1", "PYTHONIOENCODING": "utf-8", "NODE_COMPILE_CACHE": ""},
         }
         if sys.platform == "win32":
             popen_kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW

server/services/process_manager.py

@@ -477,6 +477,10 @@ async def start(
             subprocess_env = {
                 **os.environ,
                 "PYTHONUNBUFFERED": "1",
+                # Force UTF-8 stdout in the child; on Windows a piped stdout otherwise
+                # defaults to the ANSI code page (e.g. CP1252) and mangles/crashes on
+                # non-ASCII agent output. The reader below decodes as UTF-8.
+                "PYTHONIOENCODING": "utf-8",
                 "PLAYWRIGHT_CLI_SESSION": f"agent-{self.project_name}-{os.getpid()}",
                 "NODE_COMPILE_CACHE": "",  # Disable V8 compile caching to prevent .node file accumulation in %TEMP%
                 **api_env,

start.py

@@ -243,7 +243,9 @@ def run_spec_creation(project_dir: Path) -> bool:
             check=False,  # Don't raise on non-zero exit
             cwd=str(Path(__file__).parent),  # Run from project root
             stderr=subprocess.PIPE,
-            text=True
+            text=True,
+            encoding="utf-8",
+            errors="replace",
         )
 
         # Check for authentication errors in stderr
@@ -415,7 +417,9 @@ def run_agent(project_name: str, project_dir: Path) -> None:
             cmd,
             check=False,
             stderr=subprocess.PIPE,
-            text=True
+            text=True,
+            encoding="utf-8",
+            errors="replace",
         )
 
         # Check for authentication errors

test_subprocess_encoding.py

@@ -0,0 +1,104 @@
+"""
+Subprocess Encoding Tests
+=========================
+
+Regression tests for issue #138: on Windows, agent subprocess output was
+decoded with the ANSI code page (CP1252), crashing the orchestrator's output
+reader thread with UnicodeDecodeError on non-ASCII output.
+
+Two halves must agree on UTF-8:
+- the child must *write* UTF-8 to the pipe (PYTHONIOENCODING=utf-8)
+- the parent must *read* UTF-8 from the pipe (encoding="utf-8", errors="replace")
+
+Run with: python -m pytest test_subprocess_encoding.py -v
+"""
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+import parallel_orchestrator
+from parallel_orchestrator import ParallelOrchestrator
+
+NON_ASCII = "done ✓ — café 🚀 日本語"
+
+
+@pytest.fixture
+def orchestrator(tmp_path: Path) -> ParallelOrchestrator:
+    return ParallelOrchestrator(project_dir=tmp_path, max_concurrency=1)
+
+
+def _capture_popen_kwargs(spawn) -> dict:
+    """Run a spawn function with Popen/Thread mocked and return Popen's kwargs."""
+    with patch.object(parallel_orchestrator.subprocess, "Popen", return_value=MagicMock()) as popen, \
+            patch.object(parallel_orchestrator.threading, "Thread"):
+        ok, _ = spawn()
+    assert ok
+    popen.assert_called_once()
+    return dict(popen.call_args.kwargs)
+
+
+def _assert_utf8_pipe(kwargs: dict) -> None:
+    assert kwargs["encoding"] == "utf-8"
+    assert kwargs["errors"] == "replace"
+    assert kwargs["env"]["PYTHONIOENCODING"] == "utf-8"
+
+
+def test_coding_agent_uses_utf8_pipe(orchestrator: ParallelOrchestrator) -> None:
+    _assert_utf8_pipe(_capture_popen_kwargs(lambda: orchestrator._spawn_coding_agent(1)))
+
+
+def test_coding_agent_batch_uses_utf8_pipe(orchestrator: ParallelOrchestrator) -> None:
+    _assert_utf8_pipe(_capture_popen_kwargs(lambda: orchestrator._spawn_coding_agent_batch([1, 2])))
+
+
+def test_testing_agent_uses_utf8_pipe(orchestrator: ParallelOrchestrator) -> None:
+    with patch.object(orchestrator, "_get_test_batch", return_value=[1]):
+        _assert_utf8_pipe(_capture_popen_kwargs(orchestrator._spawn_testing_agent))
+
+
+def test_child_env_overrides_legacy_parent_encoding(orchestrator: ParallelOrchestrator) -> None:
+    """Even if the parent environment forces CP1252, children get UTF-8."""
+    with patch.dict(os.environ, {"PYTHONIOENCODING": "cp1252"}):
+        kwargs = _capture_popen_kwargs(lambda: orchestrator._spawn_coding_agent(1))
+    assert kwargs["env"]["PYTHONIOENCODING"] == "utf-8"
+
+
+def test_non_ascii_child_output_round_trips() -> None:
+    """End to end: a Python child on a CP1252-like setup prints non-ASCII text
+    that the parent reads back intact, as the orchestrator does."""
+    env = {**os.environ, "PYTHONIOENCODING": "cp1252"}  # simulate the Windows default
+    env.update({"PYTHONUNBUFFERED": "1", "PYTHONIOENCODING": "utf-8"})  # orchestrator overrides
+    proc = subprocess.Popen(
+        [sys.executable, "-c", f"print({NON_ASCII!r})"],
+        stdin=subprocess.DEVNULL,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
+        env=env,
+    )
+    assert proc.stdout is not None
+    lines = [line.rstrip("\n") for line in proc.stdout]
+    assert proc.wait() == 0
+    assert lines == [NON_ASCII]
+
+
+def test_reader_survives_invalid_utf8_bytes() -> None:
+    """Bytes that are invalid UTF-8 (like 0x8f from the issue) must not raise."""
+    proc = subprocess.Popen(
+        [sys.executable, "-c", "import sys; sys.stdout.buffer.write(b'ok \\x8f\\xff end\\n')"],
+        stdout=subprocess.PIPE,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
+    )
+    assert proc.stdout is not None
+    lines = list(proc.stdout)
+    assert proc.wait() == 0
+    assert lines == ["ok �� end\n"]