← back to Dw Kravet Hires
scripts/build-batchD-measure.py
212 lines
#!/usr/bin/env python3
"""TK-12097 Batch D (Gaston Y Daniela) — measure current featured images for ALL active
products, and measure the local-staging (gaston_daniela_catalog) Brandfolder hi-res
candidates. Mirrors build-batchA-map.py (READ pattern), adapted for Gaston:
* low-res class = live max-dim <= MAX_LOWRES (500) per TK-12097 Batch D.
* staging hi-res candidate = the staging Brandfolder URL with its ?width=650&height=650&pad=true
query REWRITTEN to ?width=2048&height=2048&fit=bound (bigger original), then MEASURED.
(batchA-map.py rejected any querystring; Batch D's staging candidate intentionally
carries the resize param, so that guard is removed here.)
Inputs (data/tk12097/):
batchD-all-active.tsv shopify_id \t vendor \t mfr_sku \t dw_sku \t current_url (2148 active)
batchD-candidates.tsv shopify_id \t vendor \t mfr_sku \t dw_sku \t cat_mfr \t current_url \t hires_url(width=2048)
Outputs (data/tk12097/):
batchD-lowres-all.json authoritative low-res set (<=500px) [{shopify_id,vendor,mfr_sku,dw_sku,rollback_url,cur_le}]
batchD-phase2-targets.json phase-2 targets (low-res only) for phase2-recover.py
batchD-swap-map.json staging Brandfolder swappable (>cur AND >=800), apply-hires row schema
batchD-measured.json full per-row current-featured measurement evidence
All $0 (network reads only; MEASURED, never claimed).
"""
import json, io, sys, re, os, datetime, concurrent.futures as cf
from urllib.request import Request, urlopen
from urllib.error import HTTPError, URLError
from PIL import Image
HERE = os.path.dirname(os.path.abspath(__file__))
PROJ = os.path.dirname(HERE)
D = os.path.join(PROJ, "data/tk12097")
ACTIVE = os.path.join(D, "batchD-all-active.tsv")
CANDS = os.path.join(D, "batchD-candidates.tsv")
OUT_LOWRES = os.path.join(D, "batchD-lowres-all.json")
OUT_P2TARG = os.path.join(D, "batchD-phase2-targets.json")
OUT_SWAP = os.path.join(D, "batchD-swap-map.json")
OUT_MEAS = os.path.join(D, "batchD-measured.json")
MAX_LOWRES = int(os.environ.get("MAX_LOWRES", "500")) # Batch D low-res class
HIRES_FLOOR = int(os.environ.get("HIRES_FLOOR", "800")) # genuine-upgrade floor
SHOP_PX = 5000
SHOP_BYTES = 20 * 1024 * 1024
UA = "Mozilla/5.0 (dw-preflight TK-12097 batchD)"
def fetch_head_bytes(url, nbytes):
req = Request(url, headers={"User-Agent": UA, "Range": f"bytes=0-{nbytes-1}"})
with urlopen(req, timeout=45) as r:
code = r.status
cr = r.headers.get("Content-Range", "")
total = None
if "/" in cr:
tail = cr.rsplit("/", 1)[1].strip()
if tail.isdigit():
total = int(tail)
if total is None:
gcl = r.headers.get("x-goog-stored-content-length")
if gcl and gcl.isdigit():
total = int(gcl)
if total is None:
cl = r.headers.get("Content-Length")
if code == 200 and cl and cl.isdigit():
total = int(cl)
data = r.read()
return code, total, data
def measure_url(url):
res = {"url": url}
try:
code, total, data = fetch_head_bytes(url, 65536)
dims = None
for rng in (65536, 262144, 1048576):
try:
if rng > len(data):
code2, total2, data = fetch_head_bytes(url, rng)
if total2:
total = total2
im = Image.open(io.BytesIO(data)); im.draft(None, None)
dims = im.size
break
except Exception:
continue
res.update(http=code, meas_bytes=total,
meas_w=(dims[0] if dims else None),
meas_h=(dims[1] if dims else None))
if code not in (200, 206):
res.update(status=f"HTTP_{code}", ok=False); return res
if dims is None:
res.update(status="NO_DIMS", ok=False); return res
res["meas_le"] = max(dims)
res.update(status="OK", ok=True)
except HTTPError as e:
res.update(http=e.code, status=f"HTTP_{e.code}", ok=False)
except (URLError, Exception) as e:
res.update(status="ERR", ok=False, err=str(e)[:140])
return res
def load_tsv(path, ncols):
rows = []
with open(path) as f:
for line in f:
line = line.rstrip("\n")
if not line:
continue
p = line.split("\t")
if len(p) < ncols:
continue
rows.append(p)
return rows
def measure_current(r):
m = measure_url(r["current_url"])
r["cur_measure"] = m
r["cur_le"] = m.get("meas_le")
return r
def measure_hires(r):
raw = r["hires_url"] or ""
if not raw or "placeholder" in raw.lower():
r["hires_status"] = "BAD_SOURCE_PLACEHOLDER"; r["ok"] = False; return r
m = measure_url(raw)
r["hires_measure"] = m
if not m.get("ok"):
r["hires_status"] = m["status"]; r["ok"] = False; return r
le = m["meas_le"]
if (le > SHOP_PX) or (m.get("meas_bytes") and m["meas_bytes"] > SHOP_BYTES):
# request was width=2048 bound so this should not happen; if it does, exclude (never edit engine)
r["hires_status"] = f"OVER_CEILING(le={le})"; r["ok"] = False; return r
r["final_url"] = raw; r["final_le"] = le; r["capped"] = False
if r["final_le"] <= (r.get("cur_le") or 0) or r["final_le"] < HIRES_FLOOR:
r["hires_status"] = f"NO_UPGRADE(cur={r.get('cur_le')},hires={r['final_le']})"; r["ok"] = False; return r
r["hires_status"] = "PASS"; r["ok"] = True
return r
def main():
# ---- Phase 1: measure current featured for ALL active ----
active = [{"shopify_id": p[0], "vendor": p[1], "mfr_sku": p[2],
"dw_sku": p[3] or None, "current_url": p[4]} for p in load_tsv(ACTIVE, 5)]
N = len(active)
print(f"phase1: measuring current featured for {N} active products", file=sys.stderr)
with cf.ThreadPoolExecutor(max_workers=24) as ex:
for i, _ in enumerate(ex.map(measure_current, active), 1):
if i % 200 == 0:
print(f" phase1 {i}/{N}", file=sys.stderr)
cur_unreachable = [r for r in active if r["cur_le"] is None]
lowres = [r for r in active if r["cur_le"] is not None and r["cur_le"] <= MAX_LOWRES]
already_hi = [r for r in active if r["cur_le"] is not None and r["cur_le"] > MAX_LOWRES]
print(f" lowres(<= {MAX_LOWRES}px)={len(lowres)} already-hi={len(already_hi)} cur-unreachable={len(cur_unreachable)}", file=sys.stderr)
lowres_out = [{"shopify_id": r["shopify_id"], "vendor": r["vendor"], "mfr_sku": r["mfr_sku"],
"dw_sku": r["dw_sku"], "rollback_url": r["current_url"], "cur_le": r["cur_le"]}
for r in lowres]
json.dump(lowres_out, open(OUT_LOWRES, "w"), indent=1)
p2targets = [{"job": "TK-12097-batchD", "vendor": r["vendor"], "dw_sku": r["dw_sku"],
"mfr_sku": r["mfr_sku"], "shopify_id": r["shopify_id"],
"rollback_url": r["current_url"], "cur_le": r["cur_le"]} for r in lowres]
json.dump(p2targets, open(OUT_P2TARG, "w"), indent=1)
# ---- staging candidates: measure hires (only for low-res survivors) ----
lowres_ids = {r["shopify_id"] for r in lowres}
cand_rows = [{"shopify_id": p[0], "vendor": p[1], "mfr_sku": p[2], "dw_sku": p[3] or None,
"cat_mfr": p[4], "current_url": p[5], "hires_url": p[6]} for p in load_tsv(CANDS, 7)]
# attach cur_le measured above
cur_le_by = {r["shopify_id"]: r["cur_le"] for r in active}
for c in cand_rows:
c["cur_le"] = cur_le_by.get(c["shopify_id"])
cand_lowres = [c for c in cand_rows if c["shopify_id"] in lowres_ids]
print(f"staging candidates: {len(cand_rows)} total, {len(cand_lowres)} are low-res", file=sys.stderr)
with cf.ThreadPoolExecutor(max_workers=12) as ex:
list(ex.map(measure_hires, cand_lowres))
swappable = [c for c in cand_lowres if c.get("ok")]
map_rows = []
for r in swappable:
map_rows.append({
"shopify_id": r["shopify_id"], "vendor": r["vendor"], "dw_sku": r["dw_sku"],
"mfr_sku": r["mfr_sku"], "cat_mfr": r["cat_mfr"], "cur_width": r["cur_le"],
"current_400px_url": r["current_url"], "rollback_url": r["current_url"],
"proposed_hires_url": r["final_url"], "hires_le": r["final_le"],
"hires_source": "gaston_daniela_catalog_brandfolder",
"preflight_capped": False, "swappable_from_local_staging": True,
})
json.dump({"ticket": "TK-12097", "batch": "D",
"generated_at": datetime.datetime.utcnow().isoformat() + "Z",
"rows": map_rows}, open(OUT_SWAP, "w"), indent=1)
# evidence
json.dump({"ticket": "TK-12097", "batch": "D",
"measured_at_utc": datetime.datetime.utcnow().isoformat() + "Z",
"max_lowres": MAX_LOWRES, "hires_floor": HIRES_FLOOR,
"counts": {"active_total": N, "cur_unreachable": len(cur_unreachable),
"lowres": len(lowres), "already_hi": len(already_hi),
"staging_candidates": len(cand_rows),
"staging_lowres": len(cand_lowres),
"staging_swappable": len(swappable)},
"active": active, "staging_candidates": cand_rows},
open(OUT_MEAS, "w"), indent=1)
print(json.dumps({"active_total": N, "cur_unreachable": len(cur_unreachable),
"lowres_total": len(lowres), "already_hi": len(already_hi),
"staging_lowres": len(cand_lowres), "staging_swappable": len(swappable)}, indent=2))
if __name__ == "__main__":
main()