read filing text as UTF-8 with replace fallback to avoid decode crashes

b05a674a60b69610c68b9090d832e844e05b1894 · 2026-05-18 20:07:40 -0700 · SteveStudio2

Files touched

M src/classify.py
M src/extract.py

Diff

commit b05a674a60b69610c68b9090d832e844e05b1894
Author: SteveStudio2 <steve@designerwallcoverings.com>
Date:   Mon May 18 20:07:40 2026 -0700

    read filing text as UTF-8 with replace fallback to avoid decode crashes
---
 src/classify.py | 2 +-
 src/extract.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/classify.py b/src/classify.py
index 84bc2e1..a84b2b5 100644
--- a/src/classify.py
+++ b/src/classify.py
@@ -68,7 +68,7 @@ def main() -> None:
             rows = cur.fetchall()
 
         for filing_id, text_path in rows:
-            text = Path(text_path).read_text()
+            text = Path(text_path).read_text(encoding="utf-8", errors="replace")
             doc_type = classify(text)
             with conn.cursor() as cur:
                 cur.execute(
diff --git a/src/extract.py b/src/extract.py
index 6e0cbe8..bc9a07c 100644
--- a/src/extract.py
+++ b/src/extract.py
@@ -61,7 +61,7 @@ def load_filing_text(filing_id: str) -> tuple[str, str]:
         if not row:
             raise SystemExit(f"filing {filing_id} not found")
         case_id, text_path = row
-    return case_id, Path(text_path).read_text()
+    return case_id, Path(text_path).read_text(encoding="utf-8", errors="replace")
 
 
 def extract(filing_text: str) -> list[dict]:

← 924a861 add canonical .gitignore per standing rule · back to Bankrupt Leads · (newest)