`onboarding/sangetsu-lilycolor/scripts/sangetsu-storeapi-enrich.py`

93 lines
#!/usr/bin/env python3
"""Sangetsu detail backfill from the WooCommerce Store API.

The sitemap scraper's page-regex leaves specs sparse (source pages are thin).
The Store API's product `description` carries a richer prose+spec block, so we
reuse japan-enrich's proven parse_spec()/apply_spec() to fill spec fields AND
capture the marketing Overview text (which the page scrape misses — only 4/623).

Reversible: read-only public fetch + atomic rewrite of staging/sangetsu-staging.jsonl.
Runs over ALL patterns (not just missing-both). $0 (no paid API). No ollama.
Banned-word guard: never emit "Wallpaper" -> "Wallcovering".
"""
import json, os, re, sys, time, html

HERE = os.path.dirname(os.path.abspath(__file__))
PROJ = os.path.dirname(HERE)
STAGING = os.path.join(PROJ, "staging", "sangetsu-staging.jsonl")

# reuse the proven japan-enrich parser
sys.path.insert(0, os.path.expanduser("~/Projects/japan-enrich"))
from scrape_sangetsu_specs import get, slug_of, clean_text, parse_spec, apply_spec  # noqa: E402


def extract_overview(desc_html):
    t = clean_text(desc_html)
    if not t:
        return None
    # take text after an "Overview" header up to the spec block
    m = re.search(r"\bOverview\b", t, re.I)
    body = t[m.end():] if m else t
    body = re.split(r"(?:Technical\s+Specifications|Specifications)\b", body, 1, re.I)[0].strip(" .:-")
    if not body:
        return None
    body = body[:600].strip()
    # banned-word guard (case-preserving, all cases — never emit "wallpaper")
    body = re.sub(r"WALLPAPERS", "WALLCOVERINGS", body)
    body = re.sub(r"WALLPAPER", "WALLCOVERING", body)
    body = re.sub(r"Wallpapers", "Wallcoverings", body)
    body = re.sub(r"Wallpaper", "Wallcovering", body)
    body = re.sub(r"wallpapers", "wallcoverings", body)
    body = re.sub(r"wallpaper", "wallcovering", body)
    return body or None


def main():
    rows = [json.loads(l) for l in open(STAGING) if l.strip()]
    limit = int(sys.argv[sys.argv.index("--limit") + 1]) if "--limit" in sys.argv else len(rows)
    before_spec = sum(1 for r in rows if (r.get("spec") or {}).get("width") or (r.get("spec") or {}).get("type"))
    before_ov = sum(1 for r in rows if str(r.get("overview") or "").strip())

    hit = miss = 0
    for i, r in enumerate(rows[:limit]):
        slug = slug_of(r.get("source_url"))
        d = get(f"/products?slug={slug}")
        if not d:
            miss += 1
            if i % 40 == 0:
                print(f"[{i+1}/{limit}] {r.get('pattern')}: no store-api", flush=True)
            continue
        desc = d[0].get("description", "")
        parsed = parse_spec(desc)
        apply_spec(r, parsed)
        ov = extract_overview(desc)
        if ov and not str(r.get("overview") or "").strip():
            r["overview"] = ov
        hit += 1
        if i % 40 == 0:
            sp = r.get("spec") or {}
            print(f"[{i+1}/{limit}] {r.get('pattern')}: w={sp.get('width')} t={sp.get('type')} mat={sp.get('material')} ov={'Y' if ov else '-'}", flush=True)
        time.sleep(0.15)

    # atomic write-back
    tmp = STAGING + ".tmp"
    with open(tmp, "w") as f:
        for r in rows:
            f.write(json.dumps(r) + "\n")
    os.replace(tmp, STAGING)

    after_spec = sum(1 for r in rows if (r.get("spec") or {}).get("width") or (r.get("spec") or {}).get("type"))
    after_ov = sum(1 for r in rows if str(r.get("overview") or "").strip())
    def cov(k):
        return sum(1 for r in rows if str((r.get("spec") or {}).get(k) or "").strip())
    print(json.dumps({
        "patterns": len(rows), "store_api_hit": hit, "miss": miss,
        "spec_width_or_type": f"{before_spec} -> {after_spec}",
        "overview": f"{before_ov} -> {after_ov}",
        "coverage": {k: cov(k) for k in ("width", "type", "material", "weight", "backing", "match", "repeat")},
    }, indent=2))


if __name__ == "__main__":
    main()