From fdb3184c71f72514a7b22964d25a082dc60d062a Mon Sep 17 00:00:00 2001 From: thibautbus <310327033+thibautbus@users.noreply.github.com> Date: Sat, 8 Aug 2026 13:13:10 +0200 Subject: [PATCH] Fix Windows crash: decode LuaJIT dump output as UTF-8, not the locale codepage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit subprocess.run(..., capture_output=True, text=True) with no explicit encoding falls back to locale.getpreferredencoding(False) -- the OS default codepage. On Windows that's a legacy single-byte codepage (e.g. cp1252), never UTF-8. When the LuaJIT dump contains a byte with no mapping in that codepage, subprocess's internal _readerthread crashes with an uncaught UnicodeDecodeError in a background thread; the thread dies silently and the caller gets back stdout=None instead of a string, crashing one line later with AttributeError: 'NoneType' object has no attribute 'splitlines'. Concretely, the Yellow-side imported dataset contains: "_ColosseumHeightText" -> "...6’8” tall!" The right double quotation mark (U+201D) encodes in UTF-8 as E2 80 9D; 0x9D has no defined character in cp1252, so decoding as cp1252 fails outright. Verified against the real imported dataset: the Red/Blue-only dump has zero bytes outside cp1252's defined range; the Yellow dump has exactly one, at this row. UTF-8 is the actual encoding these dumps are produced in -- the driver Lua sources are read/written as UTF-8 throughout this file, and LuaJIT writes those source strings' bytes back out verbatim -- so passing encoding="utf-8" explicitly at the three affected call sites (run_loader, check_data_dump, dump_dataset) is a no-op on platforms whose default codepage is already UTF-8 (Linux/macOS) and a correctness fix on Windows. Co-Authored-By: Claude Sonnet 5 --- tools/modkit.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/modkit.py b/tools/modkit.py index 4eaf7f8b..09f868a5 100644 --- a/tools/modkit.py +++ b/tools/modkit.py @@ -716,7 +716,7 @@ def run_loader(repo, mod_dir, findings, base="fixture", notes=None): driver_path = handle.name try: proc = subprocess.run([LUAJIT, driver_path], cwd=repo, - capture_output=True, text=True, timeout=120) + capture_output=True, text=True, encoding="utf-8", timeout=120) except FileNotFoundError: findings.append(Finding("MK100", "error", f"cannot run {LUAJIT} (install luajit or " @@ -1056,7 +1056,7 @@ def check_data_dump(repo, path, base, rel): driver = DUMP_DRIVER % (lua_quote(path), lua_quote(vanilla)) try: proc = subprocess.run([LUAJIT, "-e", driver], cwd=repo, - capture_output=True, text=True, timeout=60) + capture_output=True, text=True, encoding="utf-8", timeout=60) except FileNotFoundError: # the gate must fail closed: a missing interpreter is a broken # environment, not a clean mod @@ -1416,7 +1416,7 @@ def dump_dataset(repo, base): handle.close() try: proc = subprocess.run([os.environ.get("LUA", "luajit"), handle.name], - cwd=repo, capture_output=True, text=True) + cwd=repo, capture_output=True, text=True, encoding="utf-8") finally: os.unlink(handle.name) if proc.returncode != 0: