japan: Store-API crawler finds full Sangetsu-Goodrich catalog (878 products vs 623 in sitemap); feed-first, $0

f05a2b565c9a036d6091d818d01c5f11ad35c396 · 2026-07-06 18:04:14 -0700 · Steve

Files touched

A scrape_sangetsu_storeapi.py

Diff

commit f05a2b565c9a036d6091d818d01c5f11ad35c396
Author: Steve <steve@designerwallcoverings.com>
Date:   Mon Jul 6 18:04:14 2026 -0700

    japan: Store-API crawler finds full Sangetsu-Goodrich catalog (878 products vs 623 in sitemap); feed-first, $0
---
 scrape_sangetsu_storeapi.py | 52 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/scrape_sangetsu_storeapi.py b/scrape_sangetsu_storeapi.py
new file mode 100644
index 0000000..7648b1d
--- /dev/null
+++ b/scrape_sangetsu_storeapi.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+"""Find MORE Sangetsu products via the WooCommerce Store API (feed-first, $0, no anti-bot).
+The WP product sitemap listed 623; the Store API reports ~879 — so ~256 more patterns.
+Paginates /wp-json/wc/store/v1/products, writes full product rows to
+staging/sangetsu-storeapi.jsonl, and reports how many are NEW vs our current 623."""
+import json, os, re, time, urllib.request
+
+HERE = os.path.dirname(__file__)
+BASE = "https://www.sangetsu-goodrich.co.th/wp-json/wc/store/v1/products"
+OUT = os.path.join(HERE, 'staging', 'sangetsu-storeapi.jsonl')
+STAGING = os.path.join(HERE, 'staging', 'sangetsu-staging.jsonl')
+UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)'
+
+def fetch(page):
+    req = urllib.request.Request(f"{BASE}?per_page=100&page={page}", headers={'User-Agent': UA})
+    return json.loads(urllib.request.urlopen(req, timeout=30).read().decode('utf-8', 'ignore'))
+
+def slug_of(u): return (u or '').rstrip('/').split('/')[-1].lower()
+
+def main():
+    ours = set()
+    for l in open(STAGING):
+        l = l.strip()
+        if l: ours.add(slug_of(json.loads(l).get('source_url', '')))
+    all_prod, page = [], 1
+    while True:
+        try: batch = fetch(page)
+        except Exception as e: print(f"page {page} stop: {e}"); break
+        if not batch: break
+        all_prod.extend(batch)
+        print(f"  page {page}: +{len(batch)} (total {len(all_prod)})")
+        if len(batch) < 100: break
+        page += 1; time.sleep(0.4)
+    with open(OUT, 'w') as f:
+        for p in all_prod:
+            f.write(json.dumps({
+                'id': p.get('id'), 'name': p.get('name'), 'slug': p.get('slug'),
+                'sku': p.get('sku'), 'type': p.get('type'), 'permalink': p.get('permalink'),
+                'price': (p.get('prices') or {}).get('price'),
+                'images': [im.get('src') for im in (p.get('images') or [])],
+                'variations': p.get('variations') or [],
+                'attributes': p.get('attributes') or [],
+            }, ensure_ascii=False) + '\n')
+    slugs = {slug_of(p.get('permalink')) or (p.get('slug') or '').lower() for p in all_prod}
+    new = slugs - ours
+    print(f"\nStore API products: {len(all_prod)} | distinct slugs: {len(slugs)}")
+    print(f"already have (of 623): {len(slugs & ours)} | GENUINELY NEW: {len(new)}")
+    for s in sorted(new)[:25]: print('  +', s)
+    print(f"\n→ {OUT}")
+
+if __name__ == '__main__':
+    main()

← a3d28d9 japan: loader creates vendor_never_on_shopify if missing (ca · back to Japan Enrich · japan: +9 new Sangetsu wallcoverings (198 colorways) the sit c0773d3 →