sangetsu: store-api detail enrich — descriptions 4->614 patterns + specs backfill (623/623)

1f7a06432a0d76c6373743f3d76ebdda23337fe1 · 2026-07-02 07:42:42 -0700 · Steve
Files touched

A onboarding/sangetsu-lilycolor/scripts/sangetsu-storeapi-enrich.py
Diff

commit 1f7a06432a0d76c6373743f3d76ebdda23337fe1
Author: Steve <steve@designerwallcoverings.com>
Date:   Thu Jul 2 07:42:42 2026 -0700

    sangetsu: store-api detail enrich — descriptions 4->614 patterns + specs backfill (623/623)
---
 .../scripts/sangetsu-storeapi-enrich.py            | 87 ++++++++++++++++++++++
 1 file changed, 87 insertions(+)

diff --git a/onboarding/sangetsu-lilycolor/scripts/sangetsu-storeapi-enrich.py b/onboarding/sangetsu-lilycolor/scripts/sangetsu-storeapi-enrich.py
new file mode 100644
index 00000000..a3b913fa
--- /dev/null
+++ b/onboarding/sangetsu-lilycolor/scripts/sangetsu-storeapi-enrich.py
@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""Sangetsu detail backfill from the WooCommerce Store API.
+
+The sitemap scraper's page-regex leaves specs sparse (source pages are thin).
+The Store API's product `description` carries a richer prose+spec block, so we
+reuse japan-enrich's proven parse_spec()/apply_spec() to fill spec fields AND
+capture the marketing Overview text (which the page scrape misses — only 4/623).
+
+Reversible: read-only public fetch + atomic rewrite of staging/sangetsu-staging.jsonl.
+Runs over ALL patterns (not just missing-both). $0 (no paid API). No ollama.
+Banned-word guard: never emit "Wallpaper" -> "Wallcovering".
+"""
+import json, os, re, sys, time, html
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+PROJ = os.path.dirname(HERE)
+STAGING = os.path.join(PROJ, "staging", "sangetsu-staging.jsonl")
+
+# reuse the proven japan-enrich parser
+sys.path.insert(0, os.path.expanduser("~/Projects/japan-enrich"))
+from scrape_sangetsu_specs import get, slug_of, clean_text, parse_spec, apply_spec  # noqa: E402
+
+
+def extract_overview(desc_html):
+    t = clean_text(desc_html)
+    if not t:
+        return None
+    # take text after an "Overview" header up to the spec block
+    m = re.search(r"\bOverview\b", t, re.I)
+    body = t[m.end():] if m else t
+    body = re.split(r"(?:Technical\s+Specifications|Specifications)\b", body, 1, re.I)[0].strip(" .:-")
+    if not body:
+        return None
+    body = body[:600].strip()
+    # banned-word guard
+    body = re.sub(r"\bWallpapers?\b", "Wallcovering", body)
+    return body or None
+
+
+def main():
+    rows = [json.loads(l) for l in open(STAGING) if l.strip()]
+    limit = int(sys.argv[sys.argv.index("--limit") + 1]) if "--limit" in sys.argv else len(rows)
+    before_spec = sum(1 for r in rows if (r.get("spec") or {}).get("width") or (r.get("spec") or {}).get("type"))
+    before_ov = sum(1 for r in rows if str(r.get("overview") or "").strip())
+
+    hit = miss = 0
+    for i, r in enumerate(rows[:limit]):
+        slug = slug_of(r.get("source_url"))
+        d = get(f"/products?slug={slug}")
+        if not d:
+            miss += 1
+            if i % 40 == 0:
+                print(f"[{i+1}/{limit}] {r.get('pattern')}: no store-api", flush=True)
+            continue
+        desc = d[0].get("description", "")
+        parsed = parse_spec(desc)
+        apply_spec(r, parsed)
+        ov = extract_overview(desc)
+        if ov and not str(r.get("overview") or "").strip():
+            r["overview"] = ov
+        hit += 1
+        if i % 40 == 0:
+            sp = r.get("spec") or {}
+            print(f"[{i+1}/{limit}] {r.get('pattern')}: w={sp.get('width')} t={sp.get('type')} mat={sp.get('material')} ov={'Y' if ov else '-'}", flush=True)
+        time.sleep(0.15)
+
+    # atomic write-back
+    tmp = STAGING + ".tmp"
+    with open(tmp, "w") as f:
+        for r in rows:
+            f.write(json.dumps(r) + "\n")
+    os.replace(tmp, STAGING)
+
+    after_spec = sum(1 for r in rows if (r.get("spec") or {}).get("width") or (r.get("spec") or {}).get("type"))
+    after_ov = sum(1 for r in rows if str(r.get("overview") or "").strip())
+    def cov(k):
+        return sum(1 for r in rows if str((r.get("spec") or {}).get(k) or "").strip())
+    print(json.dumps({
+        "patterns": len(rows), "store_api_hit": hit, "miss": miss,
+        "spec_width_or_type": f"{before_spec} -> {after_spec}",
+        "overview": f"{before_ov} -> {after_ov}",
+        "coverage": {k: cov(k) for k in ("width", "type", "material", "weight", "backing", "match", "repeat")},
+    }, indent=2))
+
+
+if __name__ == "__main__":
+    main()
← d613c488 sangetsu: parse 5 pricebook PDFs -> 1,007 domestic-SKU specs · back to Designer Wallcoverings · auto-save: 2026-07-02T07:46:12 (8 files) — DW-MCP/daily-sale ee401ffe →