← back to Dw Kravet Hires
scripts/preflight-phase1.py
248 lines
#!/usr/bin/env python3
"""TK-11658 Cycle 3 pre-flight + NORMALIZE the ORIGINAL Phase-1 621 swappable-now rows.
Problem (seeded by cycle 2): the 621 swappable_from_local_staging:true rows in
data/kravet-hires-swap-map.json carry RAW un-normalized source URLs (0/621 have any
resize cap). Some Brandfolder originals EXCEED Shopify's ceiling (long-edge>5000px /
>20MP or >20MB), so apply-hires.mjs (MAX_HIRES=5000 + oversized self-heal) SILENTLY
no-ops them exactly like the Phase-1 applied=0 run. This MEASURES all 621 (never trust),
caps the oversized ones with the cycle-2/TK-11740-verified Brandfolder resize param,
RE-MEASURES the capped URL to confirm it now passes, and writes a NORMALIZED map that
apply-hires.mjs can drive one-paste-from-live.
MEASURE method: ranged GET (first N KB) -> real HTTP status, true total bytes
(content-range), real pixel dims (PIL). ~64KB-1MB/row. $0 (network only).
CAP params (TK-11740 verified, per source format):
JPEG: ?width=4472&height=4472&fit=bound&quality=90 (~6MB @ 20MP; lossy compresses)
PNG : ?width=3000&height=3000&fit=bound (Brandfolder ignores &quality on PNG,
4472px PNG ~20-24MB fails 20MB -> 3000px)
"""
import json, io, sys, re, datetime, concurrent.futures as cf
from urllib.request import Request, urlopen
from urllib.error import HTTPError, URLError
from PIL import Image
SRC = "data/kravet-hires-swap-map.json"
OUT_MEAS = "data/phase1-preflight-measured.json"
OUT_MAP = "data/kravet-hires-phase1-normalized-map.json"
OUT_SPLIT = "data/phase1-clean-flagged-split.json"
SHOP_PX = 5000
SHOP_BYTES = 20 * 1024 * 1024
RESIZE_JPEG = "?width=4472&height=4472&fit=bound&quality=90"
RESIZE_PNG = "?width=3000&height=3000&fit=bound"
UA = "Mozilla/5.0 (dw-preflight TK-11658 cycle3)"
def resize_for(src):
return RESIZE_PNG if re.search(r"\.png($|\?)", src, re.I) else RESIZE_JPEG
def fetch_head_bytes(url, nbytes):
req = Request(url, headers={"User-Agent": UA, "Range": f"bytes=0-{nbytes-1}"})
with urlopen(req, timeout=45) as r:
code = r.status
cr = r.headers.get("Content-Range", "")
total = None
if "/" in cr:
tail = cr.rsplit("/", 1)[1].strip()
if tail.isdigit():
total = int(tail)
if total is None:
gcl = r.headers.get("x-goog-stored-content-length")
if gcl and gcl.isdigit():
total = int(gcl)
if total is None:
cl = r.headers.get("Content-Length")
# only trust Content-Length as total when server ignored Range (200, full body)
if code == 200 and cl and cl.isdigit():
total = int(cl)
data = r.read()
return code, total, data
def measure_url(url):
"""Return dict: http, meas_bytes, meas_w, meas_h, meas_le, status, ok."""
res = {"url": url}
try:
code, total, data = fetch_head_bytes(url, 65536)
dims = None
for rng in (65536, 262144, 1048576):
try:
if rng > len(data):
code2, total2, data = fetch_head_bytes(url, rng)
if total2:
total = total2
im = Image.open(io.BytesIO(data)); im.draft(None, None)
dims = im.size
break
except Exception:
continue
res.update(http=code, meas_bytes=total,
meas_w=(dims[0] if dims else None),
meas_h=(dims[1] if dims else None))
if code not in (200, 206):
res.update(status=f"HTTP_{code}", ok=False); return res
if dims is None:
res.update(status="NO_DIMS", ok=False); return res
le = max(dims)
res["meas_le"] = le
over_px = le > SHOP_PX
over_bytes = (total is not None and total > SHOP_BYTES)
if over_px or over_bytes:
res.update(status="OVER_LIMIT", ok=False, over_px=over_px, over_bytes=over_bytes)
elif total is None:
res.update(status="BYTES_UNKNOWN", ok=False) # not-measured != pass
else:
res.update(status="PASS", ok=True)
except HTTPError as e:
# An HTTP error IS a measurement result (MEASURED-BAD, not un-measured): record http.
res.update(http=e.code, status=f"HTTP_{e.code}", ok=False)
except (URLError, Exception) as e:
res.update(status="ERR", ok=False, err=str(e)[:140])
return res
def process(row):
raw = row.get("proposed_hires_url") or ""
out = {"mfr_sku": row.get("mfr_sku"), "vendor": row.get("vendor"),
"shopify_id": row.get("shopify_id"), "hires_source": row.get("hires_source"),
"raw_url": raw}
if not raw:
out.update(status="NO_URL", ok=False, disposition="flagged"); return out
# Guard: placeholder sources must NEVER swap a real image for a placeholder.
if "placeholder" in raw.lower():
out.update(status="BAD_SOURCE_PLACEHOLDER", ok=False, disposition="flagged"); return out
# Guard: cannot append a second query string safely -> flag for manual review.
if "?" in raw:
out.update(status="SOURCE_HAS_QUERYSTRING", ok=False, disposition="flagged"); return out
m = measure_url(raw)
out.update(raw_measure=m)
if m["ok"]:
out.update(status="PASS", ok=True, disposition="clean", normalized_url=raw, capped=False)
return out
# Only OVER_LIMIT / BYTES_UNKNOWN are cap-fixable; HTTP/ERR are dead sources.
if m["status"] not in ("OVER_LIMIT", "BYTES_UNKNOWN"):
out.update(status=m["status"], ok=False, disposition="flagged"); return out
# Apply the per-format cap and RE-MEASURE.
rp = resize_for(raw)
capped_url = raw + rp
cm = measure_url(capped_url)
out.update(cap_param=rp, capped_url=capped_url, capped_measure=cm)
if cm["ok"]:
out.update(status="PASS_AFTER_CAP", ok=True, disposition="clean",
normalized_url=capped_url, capped=True)
else:
out.update(status="STILL_OVER_AFTER_CAP:" + cm["status"], ok=False, disposition="flagged")
return out
def main():
doc = json.load(open(SRC))
rows = [r for r in doc["rows"] if r.get("swappable_from_local_staging") is True]
N = len(rows)
results = [None] * N
done = 0
with cf.ThreadPoolExecutor(max_workers=16) as ex:
futs = {ex.submit(process, r): i for i, r in enumerate(rows)}
for f in cf.as_completed(futs):
i = futs[f]; results[i] = f.result(); done += 1
if done % 100 == 0:
print(f" measured {done}/{N}", file=sys.stderr)
clean = [r for r in results if r["ok"]]
flagged = [r for r in results if not r["ok"]]
# every row that produced ANY http response counts as measured
measured = sum(1 for r in results
if (r.get("raw_measure", {}).get("http") or r.get("capped_measure", {}).get("http")))
# placeholder/querystring/no_url rows never hit the network; count them measured=deterministic
static_flag = sum(1 for r in results if r["status"] in
("BAD_SOURCE_PLACEHOLDER", "SOURCE_HAS_QUERYSTRING", "NO_URL"))
from collections import Counter
breakdown = dict(Counter(r["status"] for r in results))
pass_asis = sum(1 for r in results if r["status"] == "PASS")
pass_capped = sum(1 for r in results if r["status"] == "PASS_AFTER_CAP")
meas_out = {
"ticket": "TK-11658", "cycle": 3,
"measured_at_utc": datetime.datetime.utcnow().isoformat() + "Z",
"method": "ranged GET (64KB-1MB) content-range total + PIL dims; MEASURED not claimed; oversized capped+re-measured",
"ceilings": {"long_edge_px": SHOP_PX, "bytes": SHOP_BYTES},
"cap_params": {"jpeg": RESIZE_JPEG, "png": RESIZE_PNG},
"total": N,
"measured": measured + static_flag,
"network_measured": measured,
"deterministic_flagged_no_network": static_flag,
"clean": len(clean), "flagged": len(flagged),
"pass_as_is": pass_asis, "pass_after_cap": pass_capped,
"status_breakdown": breakdown,
"rows": results,
}
json.dump(meas_out, open(OUT_MEAS, "w"), indent=2)
# Build the NORMALIZED apply-hires map from clean rows only, keyed to the source doc rows.
src_by_id = {r["shopify_id"]: r for r in doc["rows"]}
norm_rows = []
for c in clean:
base = dict(src_by_id[c["shopify_id"]])
base["proposed_hires_url"] = c["normalized_url"]
base["preflight_capped"] = c["capped"]
if c["capped"]:
base["original_oversized_url"] = c["raw_url"]
base["resize_param"] = c["cap_param"]
base["swappable_from_local_staging"] = True
norm_rows.append(base)
norm_map = {
"ticket": "TK-11658", "cycle": 3, "parent_map": SRC,
"generated_at": datetime.datetime.utcnow().isoformat() + "Z",
"generated_by": "vp-dw-commerce cycle3",
"store": doc.get("store"),
"scope": ("Phase-1 swappable_from_local_staging rows, PRE-FLIGHTED: every proposed_hires_url "
"MEASURED (HTTP 200 + long-edge<=5000px + <=20MB); oversized sources capped with the "
"TK-11740 Brandfolder resize param and RE-MEASURED to confirm pass. Flagged rows "
"(placeholder sources, dead sources, still-over-after-cap) EXCLUDED. This map is "
"one-paste-from-live: apply-hires.mjs --only-swappable will apply every row without "
"a silent oversized no-op."),
"cap_params": {"jpeg": RESIZE_JPEG, "png": RESIZE_PNG},
"rollback": doc.get("rollback"),
"counts": {"phase1_swappable_total": N, "clean_applyable": len(clean),
"flagged_excluded": len(flagged),
"pass_as_is": pass_asis, "pass_after_cap": pass_capped},
"rows": norm_rows,
}
json.dump(norm_map, open(OUT_MAP, "w"), indent=1)
split = {"ticket": "TK-11658", "cycle": 3,
"clean": [{"mfr_sku": r["mfr_sku"], "vendor": r["vendor"], "shopify_id": r["shopify_id"],
"status": r["status"], "capped": r.get("capped", False),
"normalized_url": r["normalized_url"],
"meas_le": (r.get("capped_measure") or r.get("raw_measure", {})).get("meas_le"),
"meas_bytes": (r.get("capped_measure") or r.get("raw_measure", {})).get("meas_bytes")}
for r in clean],
"flagged": [{"mfr_sku": r["mfr_sku"], "vendor": r["vendor"], "shopify_id": r["shopify_id"],
"status": r["status"], "raw_url": r["raw_url"]} for r in flagged]}
json.dump(split, open(OUT_SPLIT, "w"), indent=2)
print(json.dumps({
"phase1_swappable_total": N,
"measured": measured + static_flag,
"network_measured": measured,
"clean_applyable": len(clean),
"flagged_excluded": len(flagged),
"pass_as_is": pass_asis,
"pass_after_cap": pass_capped,
"status_breakdown": breakdown,
}, indent=2))
les = [r.get("raw_measure", {}).get("meas_le") for r in results if r.get("raw_measure", {}).get("meas_le")]
if les:
print("MAX raw long-edge:", max(les), "px")
cles = [r.get("capped_measure", {}).get("meas_le") for r in results if r.get("capped_measure", {}).get("meas_le")]
if cles:
print("MAX capped long-edge:", max(cles), "px (<=5000 required)")
if __name__ == "__main__":
main()