chore: lint + light refactor, v1.0.0 (session close)

7ed04ec8a194a611ed567e68a74668f9866a7c00 · 2026-07-22 09:23:50 -0700 · Steve Abrams
Files touched

A VERSION
A __pycache__/load_staging.cpython-314.pyc
A __pycache__/scrape_details.cpython-314.pyc
A __pycache__/scrape_listing.cpython-314.pyc
M load_staging.py
M scrape_details.py
M scrape_listing.py
Diff

commit 7ed04ec8a194a611ed567e68a74668f9866a7c00
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Wed Jul 22 09:23:50 2026 -0700

    chore: lint + light refactor, v1.0.0 (session close)
---
 VERSION                                    |   1 +
 __pycache__/load_staging.cpython-314.pyc   | Bin 0 -> 7233 bytes
 __pycache__/scrape_details.cpython-314.pyc | Bin 0 -> 6018 bytes
 __pycache__/scrape_listing.cpython-314.pyc | Bin 0 -> 7825 bytes
 load_staging.py                            |   4 +++-
 scrape_details.py                          |  34 +++++++++++++++--------------
 scrape_listing.py                          |   2 +-
 7 files changed, 23 insertions(+), 18 deletions(-)

diff --git a/VERSION b/VERSION
new file mode 100644
index 0000000..3eefcb9
--- /dev/null
+++ b/VERSION
@@ -0,0 +1 @@
+1.0.0
diff --git a/__pycache__/load_staging.cpython-314.pyc b/__pycache__/load_staging.cpython-314.pyc
new file mode 100644
index 0000000..71950d2
Binary files /dev/null and b/__pycache__/load_staging.cpython-314.pyc differ
diff --git a/__pycache__/scrape_details.cpython-314.pyc b/__pycache__/scrape_details.cpython-314.pyc
new file mode 100644
index 0000000..ed5ecec
Binary files /dev/null and b/__pycache__/scrape_details.cpython-314.pyc differ
diff --git a/__pycache__/scrape_listing.cpython-314.pyc b/__pycache__/scrape_listing.cpython-314.pyc
new file mode 100644
index 0000000..d2bffb9
Binary files /dev/null and b/__pycache__/scrape_listing.cpython-314.pyc differ
diff --git a/load_staging.py b/load_staging.py
index 2566fe3..def9a46 100644
--- a/load_staging.py
+++ b/load_staging.py
@@ -52,10 +52,12 @@ def main():
     stro = [r for r in rows if r.get("brand") == "stroheim"]
     print(f"true Stroheim rows: {len(stro)}")
 
+    def c(x):
+        return (str(x) if x else "").replace("\t", " ").replace("\n", " ").replace("\r", " ")
+
     with tempfile.NamedTemporaryFile("w", suffix=".tsv", delete=False) as tf:
         for r_ in stro:
             d = details.get(r_["mfr_sku"], {})
-            def c(x): return (str(x) if x else "").replace("\t"," ").replace("\n"," ").replace("\r"," ")
             enriched = "t" if d.get("width") else "f"
             tf.write("\t".join([c(r_["mfr_sku"]), c(r_["pattern_name"]), c(r_["color_name"]),
                                 c(r_["product_type"] or "Wallcovering"),
diff --git a/scrape_details.py b/scrape_details.py
index 92cde3f..d20f5fb 100644
--- a/scrape_details.py
+++ b/scrape_details.py
@@ -58,25 +58,27 @@ def main():
     done = set()
     if os.path.exists('data/details.jsonl'):
         for l in open('data/details.jsonl'):
-            try: done.add(json.loads(l)['mfr_sku'])
-            except: pass
+            try:
+                done.add(json.loads(l)['mfr_sku'])
+            except (json.JSONDecodeError, KeyError):
+                pass
     todo = [r for r in listing if r['mfr_sku'] not in done]
     sys.stderr.write(f"details: {len(todo)} to fetch ({len(done)} already done)\n")
-    out = open('data/details.jsonl', 'a')
     n = 0
-    with ThreadPoolExecutor(max_workers=10) as ex:
-        futs = {ex.submit(fetch, r['product_url']): r for r in todo}
-        for f in as_completed(futs):
-            r = futs[f]
-            html = f.result()
-            rec = {"mfr_sku": r['mfr_sku']}
-            if html:
-                rec.update(parse_detail(html))
-            out.write(json.dumps(rec) + "\n"); out.flush()
-            n += 1
-            if n % 200 == 0:
-                sys.stderr.write(f"  {n}/{len(todo)}\n")
-    out.close()
+    with open('data/details.jsonl', 'a') as out:
+        with ThreadPoolExecutor(max_workers=10) as ex:
+            futs = {ex.submit(fetch, r['product_url']): r for r in todo}
+            for f in as_completed(futs):
+                r = futs[f]
+                html = f.result()
+                rec = {"mfr_sku": r['mfr_sku']}
+                if html:
+                    rec.update(parse_detail(html))
+                out.write(json.dumps(rec) + "\n")
+                out.flush()
+                n += 1
+                if n % 200 == 0:
+                    sys.stderr.write(f"  {n}/{len(todo)}\n")
     print(f"DONE details: {n} fetched")
 
 if __name__ == "__main__":
diff --git a/scrape_listing.py b/scrape_listing.py
index 675e79e..c7a8add 100644
--- a/scrape_listing.py
+++ b/scrape_listing.py
@@ -10,6 +10,7 @@ card and let the loader filter to Stroheim-only.
 Writes data/listing.jsonl (all brands, brand-tagged).
 """
 import re, json, sys, time
+from collections import Counter
 from concurrent.futures import ThreadPoolExecutor, as_completed
 import urllib.request
 
@@ -85,7 +86,6 @@ def main():
     with open("data/listing.jsonl", "w") as fh:
         for r in results.values():
             fh.write(json.dumps(r) + "\n")
-    from collections import Counter
     bc = Counter(r["brand"] or "(no-href)" for r in results.values())
     print(f"DONE: {len(results)} unique SKUs -> data/listing.jsonl")
     print("BY BRAND:", dict(bc))
← 06dec99 Stroheim onboarding: brand-scoped to 782 true Stroheim SKUs, · back to Stroheim Onboard · (newest)