← back to Japan Enrich
auto-save: 2026-07-01T11:42:20 (3 files) — __pycache__/scrape_sangetsu_specs.cpython-314.pyc out/officer-yolo-ledger.md scrape_sangetsu_specs.py
1027fc6781c37988dc7c76012130c03c751a2a28 · 2026-07-01 11:42:27 -0700 · Steve Abrams
Files touched
A __pycache__/scrape_sangetsu_specs.cpython-314.pycA out/officer-yolo-ledger.mdA scrape_sangetsu_specs.py
Diff
commit 1027fc6781c37988dc7c76012130c03c751a2a28
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Wed Jul 1 11:42:27 2026 -0700
auto-save: 2026-07-01T11:42:20 (3 files) — __pycache__/scrape_sangetsu_specs.cpython-314.pyc out/officer-yolo-ledger.md scrape_sangetsu_specs.py
---
__pycache__/scrape_sangetsu_specs.cpython-314.pyc | Bin 0 -> 16546 bytes
out/officer-yolo-ledger.md | 7 +
scrape_sangetsu_specs.py | 290 ++++++++++++++++++++++
3 files changed, 297 insertions(+)
diff --git a/__pycache__/scrape_sangetsu_specs.cpython-314.pyc b/__pycache__/scrape_sangetsu_specs.cpython-314.pyc
new file mode 100644
index 0000000..fbfe970
Binary files /dev/null and b/__pycache__/scrape_sangetsu_specs.cpython-314.pyc differ
diff --git a/out/officer-yolo-ledger.md b/out/officer-yolo-ledger.md
new file mode 100644
index 0000000..9c8032f
--- /dev/null
+++ b/out/officer-yolo-ledger.md
@@ -0,0 +1,7 @@
+# officer-yolo loop ledger — japan-enrich (japan. side only)
+
+## Cycle 1 — 2026-07-01
+- **DTD verdict: A** (Sangetsu spec backfill, 557 patterns). Tally: Codex=A, Qwen=TIMEOUT(abstain), Claude=slow/abstain. No dissent against A. Data-confirmed: WC Store API `description` carries Material/Width/Type-Fire/Repeat/Backing.
+- **Officer: vp-dw-commerce** (DW catalog/vendor data owner).
+- **Gate check: NONE gated** — read-only WC API fetch + local staging write (reversible via git). No prod/dw_unified/Shopify/Greenland.
+- **Action: EXECUTE spec backfill.**
diff --git a/scrape_sangetsu_specs.py b/scrape_sangetsu_specs.py
new file mode 100644
index 0000000..c14c155
--- /dev/null
+++ b/scrape_sangetsu_specs.py
@@ -0,0 +1,290 @@
+#!/usr/bin/env python3
+"""
+Sangetsu product-SPEC backfill — fills spec.width / spec.type (and material,
+weight, backing, repeat, match) for the ~557 patterns whose original scrape
+left spec.width AND spec.type both null. Those empty specs leave the japan.
+viewer's Width facet and Fire-class facet blank.
+
+Source: sangetsu-goodrich.co.th WooCommerce Store API (public; WP REST proper
+is Kadence-locked, only /wc/store/v1 is open):
+ GET /wc/store/v1/products?slug=<slug> → parent; its "description" HTML holds
+ the spec block.
+
+The description HTML contains a spec block after either a "Specifications" or a
+"Technical Specifications" header, as label:value pairs, e.g.:
+ Material: Vinyl Width1: 54 IN (1.37 M) Type / Fire Rating: Type II / Class A
+ Vertical Repeat: 3.0 in Horizontal Repeat: 3.375 in
+ Hang: Straight Match, Non-Reverse Backing: Non-woven
+A few residential rows use a freeform metric form ("... width 0.91 m") with no
+label:value block — we recover width from that as a fallback.
+
+SAFE / reversible: read-only public fetch + local atomic rewrite (temp+rename)
+of the git-tracked staging/sangetsu-staging.jsonl. Only rows currently missing
+BOTH spec.width and spec.type are touched; every other row is written back
+byte-identical.
+
+Usage: scrape_sangetsu_specs.py [--limit N]
+"""
+import gzip
+import html
+import json
+import os
+import re
+import sys
+import time
+import urllib.error
+import urllib.request
+
+ROOT = os.path.dirname(os.path.abspath(__file__))
+STAGING = os.path.join(ROOT, "staging", "sangetsu-staging.jsonl")
+API = "https://www.sangetsu-goodrich.co.th/wp-json/wc/store/v1"
+UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) japan-enrich/sangetsu-specs"
+
+LIMIT = int(sys.argv[sys.argv.index("--limit") + 1]) if "--limit" in sys.argv else None
+
+
+def slug_of(url):
+ return (url or "").rstrip("/").rsplit("/", 1)[-1]
+
+
+def get(path, tries=4):
+ for i in range(tries):
+ try:
+ req = urllib.request.Request(
+ API + path,
+ headers={"User-Agent": UA, "Accept": "application/json",
+ "Accept-Encoding": "gzip"},
+ )
+ with urllib.request.urlopen(req, timeout=45) as r:
+ data = r.read()
+ if r.headers.get("Content-Encoding") == "gzip":
+ data = gzip.decompress(data)
+ return json.loads(data.decode("utf-8"))
+ except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, OSError):
+ if i == tries - 1:
+ return None
+ time.sleep(1.2 * (i + 1))
+ return None
+
+
+def clean_text(desc_html):
+ """Strip tags + decode entities + collapse whitespace from description HTML.
+
+ Entity decode is critical: widths are published with ″ (double-prime
+ ″) and ” etc., which the width regex must see decoded, not raw.
+ """
+ t = re.sub(r"<[^>]+>", " ", desc_html or "")
+ t = html.unescape(t) # ″->″ ->\xa0 ®->® &->&
+ t = t.replace("\xa0", " ").replace("®", "")
+ t = re.sub(r"\s+", " ", t).strip()
+ return t
+
+
+# Labels we harvest from the spec block, in the order they appear.
+# Each entry: (regex label alternation) -> canonical key. Multiple layouts exist
+# on the vendor site, so this list is deliberately broad; width is resolved by a
+# dedicated priority ladder below rather than a single label.
+_LABELS = [
+ (r"Material", "material"),
+ (r"Width1|Width", "width_lbl"),
+ (r"Type\s*/\s*Fire\s*Rating|Type\s*/\s*Fire|Type\s*&\s*Fire\s*Rating|Fire\s*Rating|Type", "type"),
+ (r"Vertical\s*Repeat", "vrepeat"),
+ (r"Horizontal\s*Repeat", "hrepeat"),
+ (r"Hang(?:\s*/\s*Match)?|Match", "match"),
+ (r"Backing", "backing"),
+ (r"Weight", "weight"),
+ (r"Brand", "brand"),
+ (r"Origin", "origin"),
+ (r"Bolt\s*Size", "bolt"),
+ (r"Dimension\s*Per\s*Unit|Dimensions|Dimension", "dimension"),
+ (r"Description", "descr"),
+]
+
+# Union of every label token, used to find where the NEXT field starts so a
+# value doesn't greedily swallow the following label.
+_ALL_LABEL_ALT = "|".join(alt for alt, _ in _LABELS)
+_NEXT = re.compile(r"\s*(?:%s)\s*:" % _ALL_LABEL_ALT, re.I)
+
+
+def _value_after(text, label_alt):
+ """Return the value string following 'label:' up to the next known label."""
+ m = re.search(r"(?:%s)\s*:\s*" % label_alt, text, re.I)
+ if not m:
+ return None
+ rest = text[m.end():]
+ nm = _NEXT.search(rest)
+ val = rest[: nm.start()] if nm else rest
+ val = val.strip(" .,–-")
+ return val or None
+
+
+# A width token: a number + a length unit (in / inches / IN / cm / m / mm / ″ / ")
+# — deliberately EXCLUDES "yd/yards" so "54″ x 30 yards" yields the 54″ width,
+# not the 30-yard bolt length. An optional bracketed metric equivalent
+# ("(1.37 M)") is captured whole.
+_WIDTH_TOK = re.compile(
+ r"(\d+(?:\.\d+)?\s*(?:inch(?:es)?|in\.?|IN|cm|mm|m|″|\")(?![a-z])"
+ r"(?:\s*\([^)]{0,24}\))?)",
+ re.I,
+)
+
+
+def _first_width(s):
+ """Extract the first plausible width token from a string, or None."""
+ if not s:
+ return None
+ m = _WIDTH_TOK.search(s)
+ if not m:
+ return None
+ return m.group(1).strip(" .,")
+
+
+def parse_spec(desc_html):
+ """Parse a spec dict from a product description. Returns {} if no block."""
+ text = clean_text(desc_html)
+ # Isolate the region after a Specifications / Technical Specifications header
+ hm = re.search(r"(Technical\s+Specifications|Specifications)", text, re.I)
+ region = text[hm.end():] if hm else text
+
+ raw = {}
+ for alt, key in _LABELS:
+ v = _value_after(region, alt)
+ if v:
+ raw[key] = v
+
+ out = {}
+ for k in ("material", "type", "vrepeat", "hrepeat", "match", "weight"):
+ if raw.get(k):
+ out[k] = raw[k]
+
+ # Backing: reject when the field is actually a full material sentence
+ # ("Vinyl Fabric Back", "non-woven backing") that duplicates Material.
+ b = raw.get("backing")
+ if b and len(b) <= 40 and not re.search(r"\bfabric back\b", b, re.I):
+ out["backing"] = b
+
+ # ---- width priority ladder --------------------------------------------
+ # 1) explicit Width/Width1 label
+ # 2) "... width 0.52 m" freeform (Length L x width W)
+ # 3) Dimension / Dimension Per Unit / Dimensions field
+ # 4) Description field ("... on non-woven backing Dimensions 91cm")
+ width = _first_width(raw.get("width_lbl"))
+ if not width:
+ fm = re.search(r"\bwidth\s+([\d.]+\s*(?:m|cm|mm|in\.?|inch(?:es)?|IN)\b)", region, re.I)
+ if fm:
+ width = fm.group(1).strip()
+ if not width:
+ width = _first_width(raw.get("dimension"))
+ if not width:
+ # standalone "WIDTH: 54 INCHES" all-caps variant already covered by
+ # width_lbl, but Description-embedded dimensions are not:
+ width = _first_width(raw.get("descr"))
+ if width:
+ out["width"] = width
+
+ return out
+
+
+GRASS_KW = re.compile(r"grasscloth|grass\s*cloth|sisal|jute|hemp|paperweave|natural\s*fiber", re.I)
+METAL_KW = re.compile(r"metallic|foil|mylar|gilt|gold\s*leaf|silver\s*leaf", re.I)
+
+
+def apply_spec(row, parsed):
+ """Merge parsed values into row['spec']; return True if width or type set."""
+ spec = row.get("spec") or {}
+ changed = False
+
+ if parsed.get("width") and not spec.get("width"):
+ spec["width"] = parsed["width"]
+ changed = True
+ if parsed.get("type") and not spec.get("type"):
+ spec["type"] = parsed["type"]
+ changed = True
+ # NA repeats -> skip; only meaningful values
+ if parsed.get("weight") and not spec.get("weight"):
+ spec["weight"] = parsed["weight"]
+ if parsed.get("backing") and not spec.get("backing"):
+ spec["backing"] = parsed["backing"]
+ if parsed.get("material"):
+ spec["material"] = parsed["material"]
+ if parsed.get("match"):
+ spec["match"] = parsed["match"]
+
+ vr = parsed.get("vrepeat")
+ hr = parsed.get("hrepeat")
+ def _real(x):
+ return x and not re.fullmatch(r"n/?a", x.strip(), re.I)
+ if _real(vr) or _real(hr):
+ parts = [p for p in (vr, hr) if _real(p)]
+ spec["repeat"] = " x ".join(parts)
+
+ # material-keyword flags (only flip to True when obvious; never clear)
+ mat = (parsed.get("material") or "") + " " + (parsed.get("backing") or "")
+ if GRASS_KW.search(mat):
+ spec["is_grasscloth"] = True
+ if METAL_KW.search(mat):
+ spec["is_metallic"] = True
+
+ row["spec"] = spec
+ return changed
+
+
+def main():
+ rows = [json.loads(l) for l in open(STAGING) if l.strip()]
+
+ def sp(r):
+ return r.get("spec") or {}
+
+ targets = [r for r in rows if not sp(r).get("width") and not sp(r).get("type")]
+ todo = targets[:LIMIT] if LIMIT else targets
+ print(f"sangetsu specs: {len(rows)} rows, {len(targets)} missing width+type, doing {len(todo)}",
+ flush=True)
+
+ got_width = got_type = parsed_any = no_block = fetch_fail = 0
+ for i, r in enumerate(todo, 1):
+ slug = slug_of(r.get("source_url"))
+ plist = get(f"/products?slug={slug}")
+ if not (isinstance(plist, list) and plist):
+ fetch_fail += 1
+ print(f" [{i}/{len(todo)}] {(r.get('pattern') or '?'):26} — FETCH FAIL ({slug})", flush=True)
+ time.sleep(0.25)
+ continue
+ parsed = parse_spec(plist[0].get("description") or "")
+ if parsed:
+ changed = apply_spec(r, parsed)
+ r["spec_backfilled"] = True
+ if r["spec"].get("width"):
+ got_width += 1
+ if r["spec"].get("type"):
+ got_type += 1
+ if changed:
+ parsed_any += 1
+ bits = []
+ if r["spec"].get("width"):
+ bits.append("w=" + r["spec"]["width"])
+ if r["spec"].get("type"):
+ bits.append("type=" + r["spec"]["type"])
+ print(f" [{i}/{len(todo)}] {(r.get('pattern') or '?'):26} {' '.join(bits) or '(spec fields, no w/type)'}",
+ flush=True)
+ else:
+ no_block += 1
+ print(f" [{i}/{len(todo)}] {(r.get('pattern') or '?'):26} — no spec block", flush=True)
+ time.sleep(0.25) # polite
+
+ tmp = STAGING + ".tmp"
+ with open(tmp, "w") as f:
+ for r in rows:
+ f.write(json.dumps(r, ensure_ascii=False) + "\n")
+ os.replace(tmp, STAGING)
+
+ total_width = sum(1 for r in rows if sp(r).get("width"))
+ total_type = sum(1 for r in rows if sp(r).get("type"))
+ still = sum(1 for r in rows if not sp(r).get("width") and not sp(r).get("type"))
+ print(f"\ndone. this pass: +{got_width} width, +{got_type} type "
+ f"({parsed_any} rows changed), {no_block} no-block, {fetch_fail} fetch-fail")
+ print(f"catalog now: {total_width} patterns with width, {total_type} with type, "
+ f"{still} still missing both → {STAGING}", flush=True)
+
+
+if __name__ == "__main__":
+ main()
← 0f42607 chore: lint (node -c + pyflakes clean), v0.3.0 (Greenland ch
·
back to Japan Enrich
·
sangetsu: backfill product spec (width+type) for 557 missing beb1d0c →