`scripts/scrape_momentum_feed.py`

83 lines
#!/usr/bin/env python3
"""
TK-11331 D3b — feed-first FULL Momentum scrape ($0, Meilisearch, no browser).
Pages the `redesign-colors` index sharded by category_id (each shard < 20k, so the
maxTotalHits=20000 offset cap never truncates), capturing every field the D3b recovery
join + manifest need. READ-ONLY against the feed; writes ONE local TSV artifact.

list_price = pre_discount_price if >0 else price   (verified: on-sale rows carry the
             discounted price in `price` and the true wholesale list in `pre_discount_price`;
             non-sale rows carry list in `price` with pre_discount_price=0).

OUTPUT: data/momentum_feed_full.tsv  (tab-separated, one row per colorway)
  cols: alt_product_description, number, preferred_color_number, list_price,
        is_roll_price, uom, base_width, category_name, pattern_name,
        preferred_color_name, product_line_code
"""
import json, os, time, urllib.request

HOST = "https://ms-e886719d86e7-4256.sfo.meilisearch.io"
# NOT A SECRET — vendor-owned PUBLIC search key. Momentum ships this exact literal in
# their own ANONYMOUS browser bundle (momentumco.com/build/assets/app-*.js) and it is
# search-scoped only (GET /keys -> 403 invalid_api_key). We do NOT own the Meilisearch
# account, so it is not ours to revoke or rotate. Do not route it through the secrets
# manager and do not re-open it as a leak. Verified read-only 2026-09-10 — TK-11133/TK-11415.
KEY  = os.environ.get("MOMENTUM_MS_KEY",
        "95fe8376edc78d49e787db49edda68426b21649f6271f49e2f1412118612fbd6")
INDEX = "redesign-colors"
HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
OUT  = os.path.join(HERE, "data", "momentum_feed_full.tsv")
CAT_IDS = [1, 2, 3, 4, 5, 6, 7, 8]
FIELDS = ["number", "preferred_color_number", "alt_product_description", "price",
          "pre_discount_price", "is_roll_price", "UOM", "base_width",
          "category_name", "pattern_name", "preferred_color_name", "product_line"]

def page(cid, offset, limit=200):
    body = json.dumps({"q": "", "filter": f"category_id = {cid}", "limit": limit,
                       "offset": offset, "attributesToRetrieve": FIELDS}).encode()
    req = urllib.request.Request(f"{HOST}/indexes/{INDEX}/search", data=body,
            headers={"Authorization": f"Bearer {KEY}", "Content-Type": "application/json"},
            method="POST")
    with urllib.request.urlopen(req, timeout=60) as r:
        return json.loads(r.read()).get("hits", [])

def clean(s):
    return ("" if s is None else str(s)).replace("\t", " ").replace("\n", " ").strip()

rows, seen_ids = [], set()
for cid in CAT_IDS:
    off = 0
    while off < 20000:
        hits = page(cid, off)
        if not hits:
            break
        for h in hits:
            hid = h.get("number")  # de-dupe key (a record can theoretically appear once)
            price = h.get("price")
            pdp = h.get("pre_discount_price")
            lp = pdp if (pdp is not None and float(pdp) > 0) else price
            rows.append([
                clean(h.get("alt_product_description")),
                clean(h.get("number")),
                clean(h.get("preferred_color_number")),
                clean(lp),
                clean(h.get("is_roll_price")),
                clean(h.get("UOM")),
                clean(h.get("base_width")),
                clean(h.get("category_name")),
                clean(h.get("pattern_name")),
                clean(h.get("preferred_color_name")),
                clean((h.get("product_line") or {}).get("code") if isinstance(h.get("product_line"), dict) else h.get("product_line")),
            ])
        off += 200
        print(f"\r  cat {cid} offset {off} total-rows {len(rows)}   ", end="", flush=True)
    print(f"  [cat {cid} done]")

with open(OUT, "w") as f:
    f.write("\t".join(["alt_product_description","number","preferred_color_number","list_price",
                       "is_roll_price","uom","base_width","category_name","pattern_name",
                       "preferred_color_name","product_line_code"]) + "\n")
    for r in rows:
        f.write("\t".join(r) + "\n")
print(f"\nWROTE {len(rows)} feed rows -> {OUT}")