← back to Japan Enrich
auto-save: 2026-07-01T10:11:20 (1 files) — scrape_sangetsu_backfill.py
501439c600ac59bf2dec907a44ba14bbec66499b · 2026-07-01 10:11:26 -0700 · Steve Abrams
Files touched
A scrape_sangetsu_backfill.py
Diff
commit 501439c600ac59bf2dec907a44ba14bbec66499b
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Wed Jul 1 10:11:26 2026 -0700
auto-save: 2026-07-01T10:11:20 (1 files) — scrape_sangetsu_backfill.py
---
scrape_sangetsu_backfill.py | 127 ++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 127 insertions(+)
diff --git a/scrape_sangetsu_backfill.py b/scrape_sangetsu_backfill.py
new file mode 100644
index 0000000..89076ec
--- /dev/null
+++ b/scrape_sangetsu_backfill.py
@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+"""
+Sangetsu colorway/image BACKFILL — fills the 257 patterns whose original scrape
+left colorway_skus:[] (they render as blank cards on japan.).
+
+Source: sangetsu-goodrich.co.th is WooCommerce; its Store API is public
+(the WP REST proper is locked by Kadence security, but /wc/store/v1 is open):
+ GET /wc/store/v1/products?slug=<slug> → parent (attributes = SKU list)
+ GET /wc/store/v1/products?type=variation&parent=ID → variation images (filename = SKU)
+
+A variable product's "Full Collection" attribute holds the clean colorway SKUs
+(A222-10, A222-35, …); each variation's image filename embeds its SKU. We map
+SKU→image by "basename starts with SKU" so the SKU's own trailing -NN isn't
+confused with WordPress's -1 dedup suffix.
+
+SAFE: only fills rows currently missing colorways; never touches populated rows.
+Writes a SEPARATE merged file by default (staging/sangetsu-staging.backfilled.jsonl);
+pass --apply to atomically replace staging/sangetsu-staging.jsonl (temp+rename).
+
+Usage: scrape_sangetsu_backfill.py [--limit N] [--apply]
+"""
+import json, os, re, sys, time, urllib.request, urllib.error
+
+ROOT = os.path.dirname(__file__)
+STAGING = os.path.join(ROOT, "staging", "sangetsu-staging.jsonl")
+OUT = os.path.join(ROOT, "staging", "sangetsu-staging.backfilled.jsonl")
+API = "https://www.sangetsu-goodrich.co.th/wp-json/wc/store/v1"
+UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) japan-enrich/sangetsu-backfill"
+
+LIMIT = int(sys.argv[sys.argv.index("--limit") + 1]) if "--limit" in sys.argv else None
+APPLY = "--apply" in sys.argv
+
+DIM = re.compile(r"-\d+x\d+(?=\.\w+$)") # WP thumbnail dimension suffix
+def full_src(src): # strip -150x150 → full-size upload
+ return DIM.sub("", src or "")
+def stem(src):
+ return os.path.basename(full_src(src)).rsplit(".", 1)[0]
+def slug_of(url):
+ return (url or "").rstrip("/").rsplit("/", 1)[-1]
+
+
+def get(path, tries=4):
+ for i in range(tries):
+ try:
+ req = urllib.request.Request(API + path, headers={"User-Agent": UA, "Accept": "application/json"})
+ with urllib.request.urlopen(req, timeout=45) as r:
+ return json.loads(r.read().decode("utf-8"))
+ except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as e:
+ if i == tries - 1:
+ return None
+ time.sleep(1.2 * (i + 1))
+ return None
+
+
+def backfill_one(url):
+ """Return (colorway_skus, sku_images, mfr_prefix) or (None,None,None)."""
+ slug = slug_of(url)
+ plist = get(f"/products?slug={slug}")
+ if not (isinstance(plist, list) and plist):
+ return None, None, None
+ p = plist[0]
+ # colorway SKUs: the attribute with the most term entries (the "Full Collection")
+ skus = []
+ for a in p.get("attributes") or []:
+ terms = [t.get("name", "").strip() for t in (a.get("terms") or []) if t.get("name")]
+ if len(terms) > len(skus):
+ skus = terms
+ # variation images: filename stem embeds the SKU
+ var = get(f"/products?type=variation&parent={p['id']}&per_page=100") or []
+ srcs = []
+ for v in (var if isinstance(var, list) else []):
+ for im in (v.get("images") or []):
+ if im.get("src"):
+ srcs.append(full_src(im["src"]))
+ # parent gallery images too (some variations reuse the parent image)
+ for im in (p.get("images") or []):
+ if im.get("src"):
+ srcs.append(full_src(im["src"]))
+ srcs = list(dict.fromkeys(srcs)) # dedupe, keep order
+ # fallback: if no attribute SKUs, derive from image stems that look like SKUs
+ if not skus:
+ cand = [stem(s) for s in srcs]
+ skus = [c for c in dict.fromkeys(cand) if re.match(r"^[A-Z0-9]+(-[A-Z0-9]+)*$", c) and any(ch.isdigit() for ch in c)]
+ imgs = {}
+ for sku in skus:
+ # prefer exact "<sku>.jpg", else a stem that starts with the sku (WP -1 dedup)
+ exact = next((s for s in srcs if stem(s) == sku), None)
+ pref = exact or next((s for s in srcs if stem(s).startswith(sku)), None)
+ if pref:
+ imgs[sku] = pref
+ pm = re.match(r"^[A-Za-z]+", skus[0]) if skus else None
+ prefix = pm.group(0) if pm else None
+ return skus, imgs, prefix
+
+
+def main():
+ rows = [json.loads(l) for l in open(STAGING) if l.strip()]
+ empties = [r for r in rows if not (r.get("colorway_skus"))]
+ todo = empties[:LIMIT] if LIMIT else empties
+ print(f"sangetsu backfill: {len(rows)} rows, {len(empties)} imageless, doing {len(todo)}", flush=True)
+ filled = still = 0
+ for i, r in enumerate(todo, 1):
+ skus, imgs, prefix = backfill_one(r.get("source_url"))
+ if skus and imgs:
+ r["colorway_skus"] = skus
+ r["colorway_count"] = len(skus)
+ r["sku_images"] = imgs
+ if prefix:
+ r["mfr_prefix"] = prefix
+ r["backfilled"] = True
+ filled += 1
+ print(f" [{i}/{len(todo)}] {(r.get('pattern') or '?'):28} +{len(skus)} colorways / {len(imgs)} imgs", flush=True)
+ else:
+ still += 1
+ print(f" [{i}/{len(todo)}] {(r.get('pattern') or '?'):28} — no data ({slug_of(r.get('source_url'))})", flush=True)
+ time.sleep(0.25) # polite
+ dest = STAGING if APPLY else OUT
+ tmp = dest + ".tmp"
+ with open(tmp, "w") as f:
+ for r in rows:
+ f.write(json.dumps(r, ensure_ascii=False) + "\n")
+ os.replace(tmp, dest)
+ print(f"\nbackfilled {filled}, still-empty {still} → {dest}{' (APPLIED to staging)' if APPLY else ' (dry-run; --apply to merge)'}", flush=True)
+
+
+if __name__ == "__main__":
+ main()
← b644473 greenland: per-colorway color enrichment (Pillow, $0 local,
·
back to Japan Enrich
·
sangetsu: backfill colorways/images for 254 imageless patter 1f25bc6 →