← back to Japan Enrich
japan: overnight vendor-corporate crawl — feed-first harvest of 25 vendors' full catalogs (Shopify products.json / sitemap) + local Pillow color-ID, $0 no-tokens, resumable loop (overnight_yolo.sh)
2eb6f859123a1156aa34e6d598299def9fdf5852 · 2026-07-06 19:44:05 -0700 · Steve
Files touched
A config/vendor-corporate.jsonA crawl_vendor_corporate.pyA enrich_corporate_colors.pyA overnight_yolo.sh
Diff
commit 2eb6f859123a1156aa34e6d598299def9fdf5852
Author: Steve <steve@designerwallcoverings.com>
Date: Mon Jul 6 19:44:05 2026 -0700
japan: overnight vendor-corporate crawl — feed-first harvest of 25 vendors' full catalogs (Shopify products.json / sitemap) + local Pillow color-ID, $0 no-tokens, resumable loop (overnight_yolo.sh)
---
config/vendor-corporate.json | 27 ++++++++++++++
crawl_vendor_corporate.py | 88 ++++++++++++++++++++++++++++++++++++++++++++
enrich_corporate_colors.py | 60 ++++++++++++++++++++++++++++++
overnight_yolo.sh | 26 +++++++++++++
4 files changed, 201 insertions(+)
diff --git a/config/vendor-corporate.json b/config/vendor-corporate.json
new file mode 100644
index 0000000..3ecc70a
--- /dev/null
+++ b/config/vendor-corporate.json
@@ -0,0 +1,27 @@
+{
+ "Koroseal": "https://www.koroseal.com",
+ "Eijffinger": "https://www.eijffinger.com",
+ "J.Josephson": "https://www.jjinc.com",
+ "Omexco": "https://www.omexco.com",
+ "Cole & Son": "https://www.cole-and-son.com",
+ "Harlequin": "https://www.sandersondesigngroup.com",
+ "Versa Design Surfaces": "https://www.versa1.com",
+ "Carlisle & Co": "https://www.carlisleandco.com",
+ "Zambaiti Parati": "https://www.zambaitiparati.com",
+ "La Scala Milano": "https://www.lascalamilano.com",
+ "P3TEC": "https://www.p3tec.com",
+ "Vahallan": "https://vahallan.com",
+ "Zintra": "https://www.acoufelt.com",
+ "Texam": "https://www.texamhome.com",
+ "Texdecor": "https://www.texdecor.com",
+ "Greenland": "https://www.greenlandwallcoverings.com",
+ "Casadeco": "https://www.casadeco.fr",
+ "Zoffany": "https://www.sandersondesigngroup.com",
+ "Sanderson": "https://www.sandersondesigngroup.com",
+ "Morris & Co": "https://www.sandersondesigngroup.com",
+ "Scion": "https://www.sandersondesigngroup.com",
+ "Missoni": "https://www.missonihome.com",
+ "Jannelli & Volpi": "https://www.jannellievolpi.it",
+ "Jwall": "https://www.jwall.it",
+ "Armani Casa": "https://www.armani.com"
+}
diff --git a/crawl_vendor_corporate.py b/crawl_vendor_corporate.py
new file mode 100644
index 0000000..e55e949
--- /dev/null
+++ b/crawl_vendor_corporate.py
@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+"""Overnight feed-first crawl of the 25 vendor CORPORATE sites for their FULL catalogs
+(products NOT on Sangetsu/Goodrich). $0 — pure HTTP (Shopify products.json OR sitemap product
+URLs + page meta). NO paid API, NO tokens. Per-vendor resumable. Enrichment (color via Pillow,
+style via local Ollama qwen2.5vl) is a SEPARATE local pass — this file just harvests products."""
+import json, os, re, time, gzip, urllib.request
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+HERE = os.path.dirname(__file__)
+SITES = json.load(open(os.path.join(HERE, 'config', 'vendor-corporate.json')))
+OUTDIR = os.path.join(HERE, 'staging', 'corporate')
+UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)'
+
+def get(u, timeout=25):
+ r = urllib.request.urlopen(urllib.request.Request(u, headers={'User-Agent': UA}), timeout=timeout)
+ data = r.read()
+ if r.headers.get('Content-Encoding') == 'gzip': data = gzip.decompress(data)
+ return data
+def getjson(u): return json.loads(get(u).decode('utf-8', 'ignore'))
+def gettext(u): return get(u).decode('utf-8', 'ignore')
+
+def shopify(base):
+ out, page = [], 1
+ while page <= 60:
+ try: d = getjson(f"{base}/products.json?limit=250&page={page}")
+ except Exception: break
+ ps = d.get('products', [])
+ if not ps: break
+ for p in ps:
+ out.append({'title': p.get('title'), 'handle': p.get('handle'), 'type': p.get('product_type'),
+ 'url': f"{base}/products/{p.get('handle')}", 'image': (p.get('images') or [{}])[0].get('src'),
+ 'variants': len(p.get('variants') or []), 'src': 'shopify'})
+ if len(ps) < 250: break
+ page += 1; time.sleep(0.3)
+ return out
+
+def sitemap_urls(base):
+ urls = set()
+ for sm in ['/product-sitemap.xml', '/wp-sitemap-posts-product-1.xml', '/sitemap_index.xml', '/sitemap.xml']:
+ try: xml = gettext(base + sm)
+ except Exception: continue
+ for s in re.findall(r'<loc>([^<]+sitemap[^<]*\.xml)</loc>', xml)[:15]:
+ if any(k in s.lower() for k in ('product', 'wallcover', 'wallpaper', 'collection')):
+ try: xml += gettext(s)
+ except Exception: pass
+ for loc in re.findall(r'<loc>([^<]+)</loc>', xml):
+ if re.search(r'/(product|products|wallcovering|wallpaper|collection)s?/[^/?<]+/?$', loc, re.I):
+ urls.add(loc.strip())
+ if len(urls) > 5: break
+ return sorted(urls)
+
+def page_meta(u):
+ try:
+ h = gettext(u)
+ t = re.search(r'<title>([^<]+)', h); og = re.search(r'og:image"[^>]*content="([^"]+)"', h)
+ return {'url': u, 'title': (t.group(1).strip() if t else None), 'image': (og.group(1) if og else None), 'src': 'sitemap'}
+ except Exception:
+ return {'url': u, 'title': None, 'image': None, 'src': 'sitemap'}
+
+def crawl(vendor, base):
+ out_f = os.path.join(OUTDIR, re.sub(r'[^a-z0-9]+', '-', vendor.lower()).strip('-') + '.jsonl')
+ if os.path.exists(out_f) and os.path.getsize(out_f) > 0: return 'skip(done)', 0
+ prods = []
+ try:
+ if getjson(f"{base}/products.json?limit=1").get('products') is not None:
+ prods = shopify(base)
+ except Exception: pass
+ if not prods:
+ urls = sitemap_urls(base)
+ with ThreadPoolExecutor(max_workers=6) as ex:
+ prods = [f.result() for f in as_completed({ex.submit(page_meta, u): u for u in urls[:2500]})]
+ with open(out_f, 'w') as o:
+ for p in prods: o.write(json.dumps(p, ensure_ascii=False) + '\n')
+ return 'ok', len(prods)
+
+def main():
+ os.makedirs(OUTDIR, exist_ok=True)
+ total = 0
+ for vendor, base in SITES.items():
+ try:
+ status, n = crawl(vendor, base); total += n
+ print(f"[{time.strftime('%H:%M:%S')}] {vendor:22s} {status:12s} {n:5d} products", flush=True)
+ except Exception as e:
+ print(f"[{time.strftime('%H:%M:%S')}] {vendor:22s} ERROR {str(e)[:60]}", flush=True)
+ print(f"ALL VENDORS DONE — {total} products harvested (offline, $0)", flush=True)
+
+if __name__ == '__main__':
+ main()
diff --git a/enrich_corporate_colors.py b/enrich_corporate_colors.py
new file mode 100644
index 0000000..3313c81
--- /dev/null
+++ b/enrich_corporate_colors.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+"""Local color-ID enrichment for the harvested vendor-corporate products ($0, Pillow — same
+logic as the distributor-color pipeline). Reads staging/corporate/*.jsonl, computes dominant
+hex + palette + family + hue for each product image, writes staging/corporate-colors.jsonl.
+Resumable. NO paid API, NO tokens."""
+import json, os, sys, glob, tempfile, colorsys, urllib.request
+from concurrent.futures import ThreadPoolExecutor, as_completed
+sys.path.insert(0, os.path.dirname(__file__))
+from enrich import hex_palette, dominant_family
+
+HERE = os.path.dirname(__file__)
+CORP = os.path.join(HERE, 'staging', 'corporate')
+OUT = os.path.join(HERE, 'staging', 'corporate-colors.jsonl')
+UA = 'Mozilla/5.0 (Macintosh)'
+
+def hue_of(hx):
+ h = hx.lstrip('#'); r, g, b = [int(h[i:i+2], 16)/255 for i in (0, 2, 4)]
+ return round(colorsys.rgb_to_hsv(r, g, b)[0]*360)
+
+def process(vendor, p):
+ img = p.get('image')
+ if not img or not img.startswith('http'): return None
+ try:
+ data = urllib.request.urlopen(urllib.request.Request(img, headers={'User-Agent': UA}), timeout=20).read()
+ if len(data) < 200: return None
+ f = tempfile.NamedTemporaryFile(suffix='.jpg', delete=False); f.write(data); f.close()
+ pal = hex_palette(f.name); os.unlink(f.name)
+ if not pal: return None
+ return {'vendor': vendor, 'title': p.get('title'), 'url': p.get('url'), 'image': img,
+ 'hex': pal[0]['hex'], 'palette': [x['hex'] for x in pal[:4]],
+ 'family': dominant_family(pal), 'hue': hue_of(pal[0]['hex'])}
+ except Exception:
+ return None
+
+def main():
+ done = set()
+ if os.path.exists(OUT):
+ for l in open(OUT):
+ try: done.add(json.loads(l)['url'])
+ except Exception: pass
+ jobs = []
+ for f in glob.glob(os.path.join(CORP, '*.jsonl')):
+ vendor = os.path.basename(f)[:-6]
+ for l in open(f):
+ l = l.strip()
+ if not l: continue
+ p = json.loads(l)
+ if p.get('url') not in done and p.get('image'): jobs.append((vendor, p))
+ print(f"color-enriching {len(jobs)} corporate products ({len(done)} done)", flush=True)
+ n = 0
+ with open(OUT, 'a') as out, ThreadPoolExecutor(max_workers=8) as ex:
+ for fut in as_completed({ex.submit(process, v, p): 1 for v, p in jobs}):
+ r = fut.result()
+ if r: out.write(json.dumps(r, ensure_ascii=False) + '\n'); out.flush()
+ n += 1
+ if n % 300 == 0: print(f" {n}/{len(jobs)}", flush=True)
+ print("corporate color enrichment pass done", flush=True)
+
+if __name__ == '__main__':
+ main()
diff --git a/overnight_yolo.sh b/overnight_yolo.sh
new file mode 100755
index 0000000..128edff
--- /dev/null
+++ b/overnight_yolo.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Overnight autonomous vendor-corporate harvest + local color enrichment loop.
+# $0 — pure HTTP crawl + local Pillow color (NO paid API, NO tokens). Resumable each cycle.
+# Runs unattended (keep-awake pinned). Launch: nohup bash overnight_yolo.sh &
+cd "$(dirname "$0")"
+LOG="staging/overnight-yolo.log"
+LOCK="staging/.overnight.lock"
+exec >> "$LOG" 2>&1
+if [ -f "$LOCK" ] && kill -0 "$(cat "$LOCK" 2>/dev/null)" 2>/dev/null; then echo "already running"; exit 0; fi
+echo $$ > "$LOCK"; trap 'rm -f "$LOCK"' EXIT
+echo "=== [$(date)] overnight_yolo START (pid $$) ==="
+for cycle in $(seq 1 24); do
+ echo "--- cycle $cycle [$(date '+%F %H:%M')] ---"
+ # 1) harvest full corporate catalogs (resumable — completes remaining vendors)
+ python3 crawl_vendor_corporate.py
+ # 2) local color-ID enrichment on everything harvested ($0 Pillow)
+ python3 enrich_corporate_colors.py
+ # 3) summary
+ V=$(ls staging/corporate/*.jsonl 2>/dev/null | wc -l | tr -d ' ')
+ P=$(cat staging/corporate/*.jsonl 2>/dev/null | wc -l | tr -d ' ')
+ C=$(wc -l < staging/corporate-colors.jsonl 2>/dev/null | tr -d ' ')
+ echo "cycle $cycle done: $V vendors / $P products harvested / $C color-enriched"
+ # if a full harvest + enrich is complete and stable, longer sleep; else keep cycling
+ sleep 2400 # 40 min between cycles (retry failed vendors, catch site changes)
+done
+echo "=== [$(date)] overnight_yolo END ==="
← b6d69f0 japan: expand visual-match to ALL sub-vendors — +Morris&Co/M
·
back to Japan Enrich
·
chore: v1.1.0 (session close) — distributor microsites + cat d1e46a8 →