[object Object]

← back to Cli Printing Press

fix(cli): force UTF-8 stdio in verify-skill Python subprocess (#985)

cb1697ed2fcc914b9f53592ba2b2f65ff7de6488 · 2026-05-10 16:47:55 -0700 · Trevin Chow

Windows consoles default to cp1252, which cannot encode the ✓/✘ glyphs
the verify-skill Python script prints. The subprocess crashed with
UnicodeEncodeError mid-print and the shipcheck umbrella reported FAIL
even when the underlying checks all passed.

Belt-and-suspenders fix:
- Python side: sys.stdout/stderr.reconfigure(encoding="utf-8") at the
  top of main() in scripts/verify-skill/verify_skill.py (and the bundled
  copy at internal/cli/verify_skill_bundled.py). Self-contained, no env
  dependency, covers direct python3 invocations.
- Go side: pythonUTF8Env() helper forces PYTHONIOENCODING=utf-8 and
  PYTHONUTF8=1 on the subprocess env, overriding any conflicting parent
  values. Robust even if a future edit removes the Python-side guard.

Either change alone closes the bug; both together is defense in depth.

Closes #976
Closes #819

Files touched

Diff

commit cb1697ed2fcc914b9f53592ba2b2f65ff7de6488
Author: Trevin Chow <trevin@trevinchow.com>
Date:   Sun May 10 16:47:55 2026 -0700

    fix(cli): force UTF-8 stdio in verify-skill Python subprocess (#985)
    
    Windows consoles default to cp1252, which cannot encode the ✓/✘ glyphs
    the verify-skill Python script prints. The subprocess crashed with
    UnicodeEncodeError mid-print and the shipcheck umbrella reported FAIL
    even when the underlying checks all passed.
    
    Belt-and-suspenders fix:
    - Python side: sys.stdout/stderr.reconfigure(encoding="utf-8") at the
      top of main() in scripts/verify-skill/verify_skill.py (and the bundled
      copy at internal/cli/verify_skill_bundled.py). Self-contained, no env
      dependency, covers direct python3 invocations.
    - Go side: pythonUTF8Env() helper forces PYTHONIOENCODING=utf-8 and
      PYTHONUTF8=1 on the subprocess env, overriding any conflicting parent
      values. Robust even if a future edit removes the Python-side guard.
    
    Either change alone closes the bug; both together is defense in depth.
    
    Closes #976
    Closes #819
---
 internal/cli/verify_skill.go               | 19 +++++++++
 internal/cli/verify_skill_bundled.py       | 14 +++++++
 internal/cli/verify_skill_internal_test.go | 66 ++++++++++++++++++++++++++++++
 scripts/verify-skill/verify_skill.py       | 14 +++++++
 4 files changed, 113 insertions(+)

diff --git a/internal/cli/verify_skill.go b/internal/cli/verify_skill.go
index 1f26e771..2f4e05d4 100644
--- a/internal/cli/verify_skill.go
+++ b/internal/cli/verify_skill.go
@@ -73,6 +73,7 @@ func runVerifySkillScript(dir string, only []string, asJSON bool, strict bool) (
 
 	py := exec.Command("python3", pyArgs...)
 	py.Stdin = os.Stdin
+	py.Env = pythonUTF8Env(os.Environ())
 	var stdout, stderr bytes.Buffer
 	py.Stdout = &stdout
 	py.Stderr = &stderr
@@ -334,6 +335,24 @@ func emitMergedJSON(pyResult verifySkillRunResult, runCanonical, hasCanonicalFin
 	return nil
 }
 
+// pythonUTF8Env returns base with PYTHONIOENCODING and PYTHONUTF8 forced to
+// UTF-8. Windows consoles default to cp1252, which cannot encode the ✓/✘
+// glyphs the Python script prints; without these env vars the subprocess
+// crashes with UnicodeEncodeError even though the underlying checks passed.
+// The Python script also calls sys.stdout.reconfigure() as a self-contained
+// defense; this env propagation is the belt-and-suspenders half.
+func pythonUTF8Env(base []string) []string {
+	env := make([]string, 0, len(base)+2)
+	for _, kv := range base {
+		if strings.HasPrefix(kv, "PYTHONIOENCODING=") || strings.HasPrefix(kv, "PYTHONUTF8=") {
+			continue
+		}
+		env = append(env, kv)
+	}
+	env = append(env, "PYTHONIOENCODING=utf-8", "PYTHONUTF8=1")
+	return env
+}
+
 // indentLines prefixes every line of s with prefix. Used for human-readable
 // evidence blocks that need indentation matching the rest of the output.
 func indentLines(s, prefix string) string {
diff --git a/internal/cli/verify_skill_bundled.py b/internal/cli/verify_skill_bundled.py
index 60a9bca6..9f9d28d6 100755
--- a/internal/cli/verify_skill_bundled.py
+++ b/internal/cli/verify_skill_bundled.py
@@ -1019,7 +1019,21 @@ def format_json(report: Report) -> str:
     return json.dumps(out, indent=2)
 
 
+def _force_utf8_stdio() -> None:
+    # Windows consoles default to cp1252, which cannot encode the ✓/✘ glyphs
+    # this script prints. Reconfigure stdout/stderr to UTF-8 so the human
+    # output renders cleanly instead of raising UnicodeEncodeError mid-print.
+    # The Go wrapper also sets PYTHONIOENCODING/PYTHONUTF8 as belt-and-suspenders;
+    # this call covers direct `python3 verify_skill.py ...` invocations.
+    for stream in (sys.stdout, sys.stderr):
+        try:
+            stream.reconfigure(encoding="utf-8")  # type: ignore[attr-defined]
+        except (AttributeError, OSError):
+            pass
+
+
 def main():
+    _force_utf8_stdio()
     p = argparse.ArgumentParser(
         description="Verify SKILL.md matches shipped CLI source."
     )
diff --git a/internal/cli/verify_skill_internal_test.go b/internal/cli/verify_skill_internal_test.go
new file mode 100644
index 00000000..7302cb86
--- /dev/null
+++ b/internal/cli/verify_skill_internal_test.go
@@ -0,0 +1,66 @@
+// Copyright 2026 trevin-chow. Licensed under Apache-2.0. See LICENSE.
+
+package cli
+
+import (
+	"slices"
+	"testing"
+)
+
+// TestPythonUTF8Env_AppendsEncodingVarsWhenAbsent guards the Windows cp1252
+// crash fix: the verify-skill Python subprocess must receive UTF-8 stdio
+// env vars so its ✓/✘ glyph prints don't raise UnicodeEncodeError on Windows.
+func TestPythonUTF8Env_AppendsEncodingVarsWhenAbsent(t *testing.T) {
+	t.Parallel()
+
+	got := pythonUTF8Env([]string{"PATH=/usr/bin", "HOME=/tmp"})
+
+	if !slices.Contains(got, "PYTHONIOENCODING=utf-8") {
+		t.Errorf("expected PYTHONIOENCODING=utf-8 in env, got %v", got)
+	}
+	if !slices.Contains(got, "PYTHONUTF8=1") {
+		t.Errorf("expected PYTHONUTF8=1 in env, got %v", got)
+	}
+	if !slices.Contains(got, "PATH=/usr/bin") {
+		t.Errorf("expected PATH=/usr/bin to be preserved, got %v", got)
+	}
+}
+
+// TestPythonUTF8Env_OverridesUserSetting ensures a user's existing
+// PYTHONIOENCODING (e.g. inherited cp1252 on Windows) cannot defeat the fix:
+// the helper drops conflicting entries and re-appends the UTF-8 values.
+func TestPythonUTF8Env_OverridesUserSetting(t *testing.T) {
+	t.Parallel()
+
+	got := pythonUTF8Env([]string{
+		"PYTHONIOENCODING=cp1252",
+		"PYTHONUTF8=0",
+		"PATH=/usr/bin",
+	})
+
+	for _, kv := range got {
+		if kv == "PYTHONIOENCODING=cp1252" || kv == "PYTHONUTF8=0" {
+			t.Errorf("conflicting env entry %q should have been removed, got env %v", kv, got)
+		}
+	}
+	if !slices.Contains(got, "PYTHONIOENCODING=utf-8") {
+		t.Errorf("expected PYTHONIOENCODING=utf-8 in env, got %v", got)
+	}
+	if !slices.Contains(got, "PYTHONUTF8=1") {
+		t.Errorf("expected PYTHONUTF8=1 in env, got %v", got)
+	}
+}
+
+// TestPythonUTF8Env_EmptyBase covers the edge case where os.Environ()
+// returns nothing — the helper should still emit both UTF-8 vars.
+func TestPythonUTF8Env_EmptyBase(t *testing.T) {
+	t.Parallel()
+
+	got := pythonUTF8Env(nil)
+	if len(got) != 2 {
+		t.Fatalf("expected exactly the two UTF-8 entries, got %v", got)
+	}
+	if !slices.Contains(got, "PYTHONIOENCODING=utf-8") || !slices.Contains(got, "PYTHONUTF8=1") {
+		t.Errorf("expected both UTF-8 entries, got %v", got)
+	}
+}
diff --git a/scripts/verify-skill/verify_skill.py b/scripts/verify-skill/verify_skill.py
index 60a9bca6..9f9d28d6 100755
--- a/scripts/verify-skill/verify_skill.py
+++ b/scripts/verify-skill/verify_skill.py
@@ -1019,7 +1019,21 @@ def format_json(report: Report) -> str:
     return json.dumps(out, indent=2)
 
 
+def _force_utf8_stdio() -> None:
+    # Windows consoles default to cp1252, which cannot encode the ✓/✘ glyphs
+    # this script prints. Reconfigure stdout/stderr to UTF-8 so the human
+    # output renders cleanly instead of raising UnicodeEncodeError mid-print.
+    # The Go wrapper also sets PYTHONIOENCODING/PYTHONUTF8 as belt-and-suspenders;
+    # this call covers direct `python3 verify_skill.py ...` invocations.
+    for stream in (sys.stdout, sys.stderr):
+        try:
+            stream.reconfigure(encoding="utf-8")  # type: ignore[attr-defined]
+        except (AttributeError, OSError):
+            pass
+
+
 def main():
+    _force_utf8_stdio()
     p = argparse.ArgumentParser(
         description="Verify SKILL.md matches shipped CLI source."
     )

← e0240cea fix(skills): forward --research-dir to scorecard --live-chec  ·  back to Cli Printing Press  ·  fix(cli): reject reserved placeholder hosts in spec validati 5f8dae13 →