chore: lint, refactor (file-handle hygiene), v1.0.1 (session close)

846688e43509ceaee5b2b313d09c4a5911404791 · 2026-07-22 12:38:56 -0700 · Steve Abrams
Files touched

A VERSION
A data/NEEDS_REAUTH
A scripts/__pycache__/extract.cpython-314.pyc
A scripts/__pycache__/palette.cpython-314.pyc
A scripts/__pycache__/parse_trade.cpython-314.pyc
M scripts/extract.py
M scripts/harvest_retail.sh
M scripts/palette.py
M scripts/parse_trade.py
Diff

commit 846688e43509ceaee5b2b313d09c4a5911404791
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Wed Jul 22 12:38:56 2026 -0700

    chore: lint, refactor (file-handle hygiene), v1.0.1 (session close)
---
 VERSION                                         |   1 +
 data/NEEDS_REAUTH                               |   1 +
 scripts/__pycache__/extract.cpython-314.pyc     | Bin 0 -> 9319 bytes
 scripts/__pycache__/palette.cpython-314.pyc     | Bin 0 -> 7193 bytes
 scripts/__pycache__/parse_trade.cpython-314.pyc | Bin 0 -> 5328 bytes
 scripts/extract.py                              |  15 ++++++++++-----
 scripts/harvest_retail.sh                       |   2 +-
 scripts/palette.py                              |   2 +-
 scripts/parse_trade.py                          |   8 +++++---
 9 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/VERSION b/VERSION
new file mode 100644
index 0000000..7dea76e
--- /dev/null
+++ b/VERSION
@@ -0,0 +1 @@
+1.0.1
diff --git a/data/NEEDS_REAUTH b/data/NEEDS_REAUTH
new file mode 100644
index 0000000..64c25bf
--- /dev/null
+++ b/data/NEEDS_REAUTH
@@ -0,0 +1 @@
+2026-07-22T19:15:21.739Z trade portal session expired -> run scripts/reauth.sh (headed login)
diff --git a/scripts/__pycache__/extract.cpython-314.pyc b/scripts/__pycache__/extract.cpython-314.pyc
new file mode 100644
index 0000000..f044f77
Binary files /dev/null and b/scripts/__pycache__/extract.cpython-314.pyc differ
diff --git a/scripts/__pycache__/palette.cpython-314.pyc b/scripts/__pycache__/palette.cpython-314.pyc
new file mode 100644
index 0000000..80e2889
Binary files /dev/null and b/scripts/__pycache__/palette.cpython-314.pyc differ
diff --git a/scripts/__pycache__/parse_trade.cpython-314.pyc b/scripts/__pycache__/parse_trade.cpython-314.pyc
new file mode 100644
index 0000000..09ca889
Binary files /dev/null and b/scripts/__pycache__/parse_trade.cpython-314.pyc differ
diff --git a/scripts/extract.py b/scripts/extract.py
index f842ef0..685f4d0 100644
--- a/scripts/extract.py
+++ b/scripts/extract.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """Feed-first Sanderson extractor: plain-fetch product pages, parse JSON-LD, emit CSV for PG staging. $0 (no proxy/API)."""
-import json, re, sys, time, urllib.request, csv, html
+import json, os, re, sys, time, urllib.request, csv, html
 
 BASE = "https://www.sanderson.design"
 UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36"
@@ -18,9 +18,14 @@ def load_map(path):
                 m[parts[0].upper()] = parts[1]
     return m
 
-pattern_map = load_map("pattern_map.tsv")
-settlement = set(l.strip() for l in open("settlement_urls.txt") if l.strip())
-urls = [l.strip() for l in open(URLFILE) if l.strip()]
+pattern_map = load_map("pattern_map.tsv") if os.path.exists("pattern_map.tsv") else {}
+if os.path.exists("settlement_urls.txt"):
+    with open("settlement_urls.txt") as _sf: settlement = set(l.strip() for l in _sf if l.strip())
+else:
+    settlement = set()
+if not os.path.exists(URLFILE):
+    sys.exit(f"ERROR: url file not found: {URLFILE}")
+with open(URLFILE) as _uf: urls = [l.strip() for l in _uf if l.strip()]
 urls = [u if u.startswith("/") else "/wallpaper/"+u for u in urls]
 if LIMIT:
     urls = urls[:LIMIT]
@@ -102,4 +107,4 @@ with open(OUTFILE, "w", newline="") as f:
     w = csv.DictWriter(f, fieldnames=cols)
     w.writeheader()
     for r in rows: w.writerow(r)
-sys.stderr.write(f"DONE: {len(rows)} rows -> rows.csv\n")
+sys.stderr.write(f"DONE: {len(rows)} rows -> {OUTFILE}\n")
diff --git a/scripts/harvest_retail.sh b/scripts/harvest_retail.sh
index 2960b68..b05bb2a 100644
--- a/scripts/harvest_retail.sh
+++ b/scripts/harvest_retail.sh
@@ -20,7 +20,7 @@ echo "[retail] $(wc -l < all_prod_urls.txt) unique colorway URLs"
 echo "[retail] pattern map (cn_ page h1)..."
 : > pattern_map.tsv
 while read -r u; do
-  pref=$(echo "$u" | grep -oiE 'saw[0-9]{4}' | tr 'a-z' 'A-Z' || true)
+  pref=$(echo "$u" | grep -oiE 'saw[0-9]{4}' | tr '[:lower:]' '[:upper:]' || true)
   h1=$(curl -s -m 20 -A "$UA" "$u" | grep -oiE '<h1[^>]*>[^<]+</h1>' | head -1 | sed -E 's/<[^>]+>//g' | sed 's/^ *//;s/ *$//' || true)
   [ -n "${pref:-}" ] && [ -n "${h1:-}" ] && printf '%s\t%s\n' "$pref" "$h1" >> pattern_map.tsv || true
 done < pattern_pages.txt
diff --git a/scripts/palette.py b/scripts/palette.py
index 438d469..44e61f5 100644
--- a/scripts/palette.py
+++ b/scripts/palette.py
@@ -27,7 +27,7 @@ def palette(bs,k=6):
 def esc(s): return s.replace("'","''")
 inp=os.path.join(DATA,'palette_input.csv')
 if not os.path.exists(inp): print("no palette_input.csv (nothing new)"); raise SystemExit
-rows=list(csv.DictReader(open(inp))); out=open(os.path.join(DATA,'palette_updates.sql'),'w'); ok=fail=0
+_inp_fh=open(inp); rows=list(csv.DictReader(_inp_fh)); _inp_fh.close(); out=open(os.path.join(DATA,'palette_updates.sql'),'w'); ok=fail=0
 for r in rows:
     sku,url=r['mfr_sku'],r['image_url']
     if not url: fail+=1; continue
diff --git a/scripts/parse_trade.py b/scripts/parse_trade.py
index 72eb9f9..0e13f3f 100644
--- a/scripts/parse_trade.py
+++ b/scripts/parse_trade.py
@@ -7,11 +7,12 @@ CODE = re.compile(r'^([A-Z0-9]{5,12})\s-\s(Wallpaper|Fabric)\s-\s(Live|Discontin
 PRICE = re.compile(r'^\$([\d,]+(?:\.\d+)?)$')
 def num(s): return float(s.replace(',',''))
 rows=[]; seen=set()
-open(os.path.join(DATA,'parse_dropped.log'),'w').close()
+DROPLOG=os.path.join(DATA,'parse_dropped.log')
+with open(DROPLOG,'w'): pass
 for cat in ['wallpaper','fabric']:
     fp=os.path.join(DATA,f'sanderson_{cat}_raw.txt')
     if not os.path.exists(fp): continue
-    L=[x.strip() for x in open(fp).read().split('\n')]
+    with open(fp) as fh: L=[x.strip() for x in fh.read().split('\n')]
     for i,line in enumerate(L):
         m=CODE.match(line)
         if not m: continue
@@ -31,7 +32,8 @@ for cat in ['wallpaper','fabric']:
                     if pm: ssp=num(pm.group(1)); break
             if x.startswith('Per '): unit=x
         if trade is None or ssp is None:
-            open(os.path.join(DATA,'parse_dropped.log'),'a').write(f"{cat} {code} {name} - {status} (no price)\n"); continue
+            with open(DROPLOG,'a') as dl: dl.write(f"{cat} {code} {name} - {status} (no price)\n")
+            continue
         seen.add(code)
         rows.append({'mfr_code':code,'pattern_name':name,'product_type':ptype,'status':status,
                      'trade_usd':trade,'ssp_usd':ssp,'unit':unit})
← 1c623a5 Sanderson daily-cadence staging pipeline (retail feed-first · back to Sanderson Onboard · auto-save: 2026-07-23T04:48:39 (1 files) — data/pattern_map. 1ee3998 →