japan: scrape_gg_sg_brands.py — resolve the 242 Singapore distributors (correct selector: psg-single-brands + /product_brand/)

d0a51a931cf8cb15a0824e9f12844464fabe3c2f · 2026-07-06 18:46:01 -0700 · Steve

Files touched

A scrape_gg_sg_brands.py

Diff

commit d0a51a931cf8cb15a0824e9f12844464fabe3c2f
Author: Steve <steve@designerwallcoverings.com>
Date:   Mon Jul 6 18:46:01 2026 -0700

    japan: scrape_gg_sg_brands.py — resolve the 242 Singapore distributors (correct selector: psg-single-brands + /product_brand/)
---
 scrape_gg_sg_brands.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)

diff --git a/scrape_gg_sg_brands.py b/scrape_gg_sg_brands.py
new file mode 100644
index 0000000..ceffa7b
--- /dev/null
+++ b/scrape_gg_sg_brands.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+"""Scrape the distributor brand for the 242 Singapore (goodrichglobal) wallcovering patterns
+that came in blank. Singapore markup differs from Thailand: class="psg-single-brands" +
+/product_brand/<slug>/ (Thailand was ...-main + /brand/). Updates the distributor field on
+those staging rows + the brand map. Parallel headless, $0 local, resumable-ish (idempotent)."""
+import json, os, re, subprocess, html as H
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+HERE = os.path.dirname(__file__)
+STAGING = os.path.join(HERE, 'staging', 'sangetsu-staging.jsonl')
+BRANDS = os.path.join(HERE, 'staging', 'sangetsu-brands.json')
+CHROME = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
+# brand link: /singapore/product_brand/<slug>/ ... >Name</a>  (class psg-single-brands)
+LINK = re.compile(r'/product_brand/([a-z0-9-]+)/"[^>]*class="psg-single-brands"[^>]*>([^<]+)<', re.I)
+ALT = re.compile(r'class="psg-single-brands"[^>]*>([^<]+)</a>', re.I)
+
+def brand(url):
+    try:
+        h = subprocess.run([CHROME,"--headless=new","--disable-gpu","--no-sandbox","--virtual-time-budget=12000","--dump-dom",url],
+                           capture_output=True, timeout=45, text=True, errors='ignore').stdout
+        m = LINK.search(h or '')
+        if m: return url, H.unescape(m.group(2).strip()), m.group(1)
+        a = ALT.search(h or '')
+        if a: return url, H.unescape(a.group(1).strip()), None
+        return url, None, None
+    except Exception:
+        return url, None, None
+
+def main():
+    rows = [json.loads(l) for l in open(STAGING) if l.strip()]
+    targets = [r['source_url'] for r in rows if r.get('status') == 'gg-sg-new' and not r.get('distributor')]
+    print(f"scraping brand for {len(targets)} Singapore patterns")
+    got = {}
+    with ThreadPoolExecutor(max_workers=6) as ex:
+        done = 0
+        for f in as_completed({ex.submit(brand, u): u for u in targets}):
+            u, dist, slug = f.result()
+            if dist: got[u] = {"distributor": dist, "slug": slug}
+            done += 1
+            if done % 25 == 0: print(f"  {done}/{len(targets)} … last={dist}")
+    # rewrite staging distributor for those rows
+    for r in rows:
+        if r['source_url'] in got:
+            r['distributor'] = got[r['source_url']]['distributor']
+    with open(STAGING, 'w') as out:
+        for r in rows: out.write(json.dumps(r, ensure_ascii=False) + '\n')
+    # merge brand map
+    bm = json.load(open(BRANDS)) if os.path.exists(BRANDS) else {}
+    bm.update(got)
+    json.dump(bm, open(BRANDS, 'w'), ensure_ascii=False, indent=0)
+    from collections import Counter
+    c = Counter(v['distributor'] for v in got.values())
+    print(f"\nresolved {len(got)}/{len(targets)} distributors:")
+    for k, n in c.most_common(): print(f"  {n:4d}  {k}")
+
+if __name__ == '__main__':
+    main()

← d2fb29e japan: nginx-fix-conflict.sh — strip the 19 distributor host · back to Japan Enrich · japan: get-all-live.sh — push all new patterns (Thailand-unc 52c7000 →