← back to Japan Enrich
japan: scrape REAL per-pattern Sangetsu distributor (.psg-single-brands-main → Koroseal/Eijffinger/J.Josephson/Omexco/…) into sangetsu-brands.json; viewer reads the map (blank until scraped), replacing wrong host-derived Goodrich
dcf30727feba4b03474566919a61e27c919eca2a · 2026-07-06 15:11:48 -0700 · Steve
Files touched
A scrape_sangetsu_brands.pyM viewer-local/server.js
Diff
commit dcf30727feba4b03474566919a61e27c919eca2a
Author: Steve <steve@designerwallcoverings.com>
Date: Mon Jul 6 15:11:48 2026 -0700
japan: scrape REAL per-pattern Sangetsu distributor (.psg-single-brands-main → Koroseal/Eijffinger/J.Josephson/Omexco/…) into sangetsu-brands.json; viewer reads the map (blank until scraped), replacing wrong host-derived Goodrich
---
scrape_sangetsu_brands.py | 69 +++++++++++++++++++++++++++++++++++++++++++++++
viewer-local/server.js | 15 +++++++++--
2 files changed, 82 insertions(+), 2 deletions(-)
diff --git a/scrape_sangetsu_brands.py b/scrape_sangetsu_brands.py
new file mode 100644
index 0000000..24362a3
--- /dev/null
+++ b/scrape_sangetsu_brands.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""Scrape the REAL per-pattern distributor (the .psg-single-brands-main brand link:
+Koroseal / Eijffinger / J.Josephson / Omexco / Versa / Cole & Son / ...) from each
+Sangetsu pattern page on sangetsu-goodrich.co.th, into staging/sangetsu-brands.json
+as {source_url: {distributor, slug}}. Headless Chrome (JS-rendered site), parallel,
+RESUMABLE (skips URLs already in the output). $0 local."""
+import json, os, re, subprocess, sys
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+HERE = os.path.dirname(__file__)
+STAGING = os.path.join(HERE, 'staging', 'sangetsu-staging.jsonl')
+OUT = os.path.join(HERE, 'staging', 'sangetsu-brands.json')
+CHROME = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
+WORKERS = int(os.environ.get('WORKERS', '6'))
+
+def urls():
+ s = set()
+ for line in open(STAGING):
+ line = line.strip()
+ if line:
+ u = json.loads(line).get('source_url')
+ if u: s.add(u)
+ return sorted(s)
+
+BRAND_RE = re.compile(r'psg-single-brands-main"?>(.*?)</div>', re.S | re.I)
+LINK_RE = re.compile(r'/brand/([a-z0-9-]+)/"[^>]*>([^<]+)<', re.I)
+
+def scrape(u):
+ try:
+ html = subprocess.run(
+ [CHROME, "--headless=new", "--disable-gpu", "--no-sandbox",
+ "--virtual-time-budget=11000", "--dump-dom", u],
+ capture_output=True, timeout=45, text=True, errors='ignore').stdout
+ m = BRAND_RE.search(html or '')
+ if not m: return u, None, None
+ a = LINK_RE.search(m.group(1))
+ if a: return u, a.group(2).strip(), a.group(1).strip()
+ txt = re.sub(r'\s+', ' ', re.sub(r'<[^>]+>', ' ', m.group(1))).strip()
+ return u, (txt.split(' ')[0] or None), None
+ except Exception:
+ return u, None, None
+
+def main():
+ out = {}
+ if os.path.exists(OUT):
+ try: out = json.load(open(OUT))
+ except Exception: out = {}
+ todo = [u for u in urls() if u not in out]
+ print(f"total {len(out)+len(todo)} patterns | already {len(out)} | scraping {len(todo)} | {WORKERS} workers")
+ done = 0
+ with ThreadPoolExecutor(max_workers=WORKERS) as ex:
+ futs = {ex.submit(scrape, u): u for u in todo}
+ for f in as_completed(futs):
+ u, dist, slug = f.result()
+ out[u] = {"distributor": dist, "slug": slug}
+ done += 1
+ if done % 25 == 0 or done == len(todo):
+ json.dump(out, open(OUT, 'w'), ensure_ascii=False, indent=0)
+ print(f" {done}/{len(todo)} … last: {dist!r}")
+ json.dump(out, open(OUT, 'w'), ensure_ascii=False, indent=0)
+ from collections import Counter
+ c = Counter(v.get('distributor') for v in out.values())
+ print("DONE. distributor tally:")
+ for k, n in c.most_common(): print(f" {n:4d} {k!r}")
+ miss = sum(1 for v in out.values() if not v.get('distributor'))
+ print(f"missing/none: {miss}")
+
+if __name__ == '__main__':
+ main()
diff --git a/viewer-local/server.js b/viewer-local/server.js
index 80af25c..627b8f1 100644
--- a/viewer-local/server.js
+++ b/viewer-local/server.js
@@ -75,6 +75,13 @@ const loadSangSwatches = () => {
sangSwatches = s;
};
loadSangSwatches();
+// REAL per-pattern Sangetsu distributor scraped from the .psg-single-brands-main brand
+// link (Koroseal / Eijffinger / J.Josephson / Omexco / …), keyed by source_url. Replaces
+// the old host-derived 'Goodrich (Thailand)'. Blank when a pattern isn't scraped yet.
+const SANG_BRANDS_FILE = path.join(DIR, 'sangetsu-brands.json');
+let SANG_BRANDS = {};
+const loadSangBrands = () => { try { SANG_BRANDS = JSON.parse(fs.readFileSync(SANG_BRANDS_FILE, 'utf8')); } catch { SANG_BRANDS = {}; } };
+loadSangBrands();
const ENRICH_FILES = [
process.env.ENRICH_FILE, // explicit override
path.join(DIR, 'lilycolor-enriched.jsonl'), // post-deploy: beside staging
@@ -226,7 +233,10 @@ function buildRows() {
image: sImg,
image_state: sImg ? 'Has swatch' : 'No image', // facet: swatch availability
url: r.source_url || null,
- ...distInfo('sangetsu', r.source_url), // facet/chip: buy-from channel (Sangetsu → Goodrich distributor)
+ // real distributor from the scraped brand map (blank until scraped) — NOT host-derived
+ distributor: (SANG_BRANDS[r.source_url] && SANG_BRANDS[r.source_url].distributor) || null,
+ dist_direct: false,
+ distributor_slug: (SANG_BRANDS[r.source_url] && SANG_BRANDS[r.source_url].slug) || null,
is_sample: false,
colorway_count: cw.length,
siblings,
@@ -290,7 +300,8 @@ function buildRows() {
}
let ROWS = buildRows();
-const reload = () => { loadGreenSwatches(); loadSangSwatches(); ENRICH = loadEnrich(); ROWS = buildRows(); };
+const reload = () => { loadGreenSwatches(); loadSangSwatches(); loadSangBrands(); ENRICH = loadEnrich(); ROWS = buildRows(); };
+try { if (fs.existsSync(SANG_BRANDS_FILE)) fs.watchFile(SANG_BRANDS_FILE, { interval: 5000 }, reload); } catch {}
fs.watchFile(SANG, { interval: 4000 }, reload); // pick up Sangetsu staging as it grows
fs.watchFile(LILY, { interval: 8000 }, reload);
try { if (fs.existsSync(GREEN)) fs.watchFile(GREEN, { interval: 8000 }, reload); } catch {}
← 5b7654c japan: Tier-1 canonical dw_unified stage runbook (approved;
·
back to Japan Enrich
·
japan: real Sangetsu distributor brand map (619/623 patterns b29fdd5 →