`build_vendor_profiles.py`

149 lines
#!/usr/bin/env python3
"""Build data/vendor_profiles.json — learn HOW EACH VENDOR LABELS THEIR SAMPLES from the
live catalog, so the SKU scanner can detect the brand and lock onto the right code fast.

For every Shopify `vendor` we group all of that vendor's manufacturer codes (+ house SKUs)
and induce a profile:
  • aliases  — brand-name strings to look for in the OCR text (regex-ready)
  • prefixes — histogram of the leading alpha run of their codes (WD, WDW, WHF, …)
  • numeric  — share of codes that are bare numbers (Schumacher 5255-16 style)
  • shape    — typical length of a code
  • examples — a few real codes
This is the deterministic counterpart to live scan-learning (/api/learn): the catalog
teaches the broad strokes instantly; live scans refine the on-swatch reality over time.

Uses one Bulk Operation (server-side full export). Read-only. $0 (local + Shopify read)."""
import json, os, re, sys, time, urllib.request
from collections import defaultdict, Counter
from datetime import datetime, timezone

SHOP = "designer-laboratory-sandbox.myshopify.com"; API = "2024-10"
ROOT = os.path.dirname(os.path.abspath(__file__))
OUT = os.path.join(ROOT, "data", "vendor_profiles.json")
TOKEN = os.environ.get("SHOPIFY_ADMIN_TOKEN", "")
if not TOKEN:
    for line in open(os.path.expanduser("~/Projects/secrets-manager/.env")):
        if line.startswith("SHOPIFY_ADMIN_TOKEN="):
            TOKEN = line.strip().split("=", 1)[1]; break
assert TOKEN, "no SHOPIFY_ADMIN_TOKEN"

# Front-facing private-label rule: these upstream names must NEVER be surfaced to a
# customer. The scanner is an internal admin tool so it MAY detect them, but we tag the
# profile so any public consumer can skip it. (See MEMORY: Brewster/York/WallQuest…)
PRIVATE_LABEL = re.compile(r"BREWSTER|YORK|WALLQUEST|CHESAPEAKE|NEXTWALL|SEABROOK|COMMAND ?54|DESIMA|CARLSTEN|NICOLETTE ?MAYER", re.I)

def gql(query):
    req = urllib.request.Request(f"https://{SHOP}/admin/api/{API}/graphql.json",
        data=json.dumps({"query": query}).encode(),
        headers={"X-Shopify-Access-Token": TOKEN, "Content-Type": "application/json"})
    return json.load(urllib.request.urlopen(req, timeout=90))

def parts(val):                       # "WDW-2290; Diamond WMT5001" -> ["WDW-2290","Diamond WMT5001"]
    return [p for p in re.split(r"[;,/|]+", val or "") if p.strip()]

def norm(s):                          # display code, trimmed (keep dashes for shape)
    return re.sub(r"\s+", "", (s or "").upper())

def alpha_prefix(code):               # leading letters of a code: WDW2310 -> WDW ; 5255-16 -> ""
    m = re.match(r"[A-Za-z]+", code.strip())
    return m.group(0).upper() if m else ""

def brand_regex(name):                # vendor name -> flexible OCR detection pattern
    words = [w.upper() for w in re.findall(r"[A-Za-z]+", name) if len(w) >= 3]
    if not words: return None
    # single short word (ARTE, ROMO) is noisy — require ≥4 chars; multi-word is safe at ≥3
    if len(words) == 1 and len(words[0]) < 4: return None
    return r"\s*".join(re.escape(w) for w in words)

# ── 1) bulk export: vendor + mfr metafields + variant skus ────────────────────
BULK_Q = '''
{
  products {
    edges { node {
      id status vendor
      mf: metafield(namespace:"custom", key:"manufacturer_sku"){ value }
      mf2: metafield(namespace:"dwc", key:"manufacturer_sku"){ value }
      variants { edges { node { sku } } }
    }}
  }
}'''
start = gql('mutation { bulkOperationRunQuery(query: """%s""") { bulkOperation{ id status } userErrors{ field message } } }' % BULK_Q)
ue = start.get("data", {}).get("bulkOperationRunQuery", {}).get("userErrors", [])
if ue:
    print("bulk start errors:", ue)
    print("current:", gql('{ currentBulkOperation { id status } }')["data"]["currentBulkOperation"])
    sys.exit(1)
print("bulk op started, polling…")

url = None
for _ in range(180):
    time.sleep(5)
    cur = gql('{ currentBulkOperation { id status errorCode objectCount url } }')["data"]["currentBulkOperation"]
    print(f"  status={cur['status']} objects={cur.get('objectCount')}")
    if cur["status"] == "COMPLETED": url = cur["url"]; break
    if cur["status"] in ("FAILED", "CANCELED"): print("bulk failed:", cur); sys.exit(1)
assert url, "bulk op did not complete in time"

# ── 2) parse JSONL: collect codes per vendor ──────────────────────────────────
# Keep the two code namespaces SEPARATE: the manufacturer code (printed on the physical
# swatch — what the scanner reads) vs the DW house SKU (what we resolve to). The profile
# describes the SAMPLE, so we learn from mfr codes first and only fall back to house codes.
data = urllib.request.urlopen(url, timeout=300).read().decode()
prod_vendor = {}                       # gid -> vendor
mfr_by_vendor = defaultdict(list)      # vendor -> [manufacturer code, …]  (on the swatch)
house_by_vendor = defaultdict(list)    # vendor -> [DW house sku, …]       (fallback)
for line in data.splitlines():
    if not line.strip(): continue
    o = json.loads(line)
    if "__parentId" in o and "sku" in o:                 # variant child = house sku
        ven = prod_vendor.get(o["__parentId"])
        sku = (o.get("sku") or "")
        if ven and sku and not sku.lower().endswith("-sample"):
            house_by_vendor[ven].append(sku)
    elif "id" in o and ("vendor" in o or "status" in o): # product line = mfr metafields
        ven = (o.get("vendor") or "").strip()
        prod_vendor[o["id"]] = ven
        if not ven: continue
        for k in ("mf", "mf2"):
            v = (o.get(k) or {}).get("value")
            if v:
                for c in parts(v): mfr_by_vendor[ven].append(c)

# ── 3) induce a profile per vendor ────────────────────────────────────────────
profiles = {}
for vendor in set(mfr_by_vendor) | set(house_by_vendor):
    mfr = [norm(c) for c in mfr_by_vendor.get(vendor, []) if len(norm(c)) >= 4]
    house = [norm(c) for c in house_by_vendor.get(vendor, []) if len(norm(c)) >= 4]
    # learn from the manufacturer codes (the swatch); only fall back to house if too thin
    codes = mfr if len(set(mfr)) >= 3 else house
    code_source = "mfr" if len(set(mfr)) >= 3 else "house"
    if len(codes) < 3: continue                          # too thin to learn anything reliable
    uniq = sorted(set(codes))
    pre = Counter(alpha_prefix(c) for c in codes if alpha_prefix(c))
    numeric = sum(1 for c in codes if not alpha_prefix(c))
    # keep prefixes that are common enough to be a real signal (≥3 hits AND ≥4% of codes)
    keep = [(p, n) for p, n in pre.most_common(8) if n >= 3 and n >= 0.04 * len(codes)]
    lens = Counter(len(c) for c in codes)
    profiles[vendor] = {
        "n": len(uniq),
        "aliases": [vendor.upper()],
        "alias_re": brand_regex(vendor),
        "prefixes": {p: n for p, n in keep},
        "numeric_share": round(numeric / max(1, len(codes)), 2),
        "common_len": lens.most_common(1)[0][0] if lens else 0,
        "examples": uniq[:6],
        "private_label": bool(PRIVATE_LABEL.search(vendor)),
        "source": "catalog",
        "code_source": code_source,        # "mfr" = learned from real swatch codes; "house" = fallback
        "learned_scans": 0,
        "updated_at": datetime.now(timezone.utc).isoformat(),
    }

meta = {"_meta": {"vendors": len(profiles), "products": len(prod_vendor),
                  "built_at": datetime.now(timezone.utc).isoformat(), "source": "catalog-bulk"}}
json.dump({**meta, "profiles": profiles}, open(OUT, "w"), indent=0)
print(f"\nlearned {len(profiles)} vendor profiles from {len(prod_vendor)} products -> {OUT}")
for probe in ("Winfield Thybony", "Schumacher", "Phillip Jeffries", "Thibaut", "Kravet"):
    p = profiles.get(probe)
    if p: print(f"  {probe:22} n={p['n']:5}  prefixes={list(p['prefixes'])}  ex={p['examples'][:3]}")