`lib/image-health.sh`

91 lines
#!/usr/bin/env bash
# image-health.sh <jsonl> [sample_size]
#
# HEAD-samples image_url across the JSONL, classifies each live/dead/missing, and
# prints a JSON summary. Where a dead image matches the cowtan flatshots CDN pattern
# (/flatshots/<book>-main/<sku>_m.jpg), it attempts a relink and rewrites the JSONL
# in place. Returns image_health (live fraction). If <0.5 -> caller marks NEEDS_RESCRAPE.
#
# Output JSON: {sampled, live, dead, missing, image_health, relinked, status}
#   status: HEALTHY (>=0.5) | NEEDS_RESCRAPE (<0.5) | EMPTY (no images at all)
set -euo pipefail

SRC="${1:?usage: image-health.sh <jsonl> [sample_size]}"
SAMPLE="${2:-40}"

python3 - "$SRC" "$SAMPLE" <<'PY'
import sys, json, urllib.request, ssl, re, random

src = sys.argv[1]; sample_n = int(sys.argv[2])
ctx = ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE

rows = []
with open(src) as f:
    for line in f:
        line=line.strip()
        if not line: continue
        try: rows.append(json.loads(line))
        except: pass

total = len(rows)
def _real_url(u):
    u = (u or '').strip()
    return bool(u) and not u.lower().startswith('data:')
# base64 data: URIs are lazy-load placeholder SVGs (e.g. rwltd) — NOT real coverage
with_img = [r for r in rows if _real_url(r.get('image_url'))]
placeholder = sum(1 for r in rows if (r.get('image_url') or '').strip().lower().startswith('data:'))
missing = total - len(with_img) - placeholder

if not with_img:
    print(json.dumps({"sampled":0,"live":0,"dead":0,"missing":missing,"placeholder":placeholder,
                      "total":total,"image_health":0.0,"relinked":0,
                      "status":"PLACEHOLDER" if placeholder else "EMPTY"}))
    sys.exit(0)

# sample for the HEAD probe (cheap, representative)
probe = with_img if len(with_img) <= sample_n else random.sample(with_img, sample_n)

def head_ok(url):
    try:
        req = urllib.request.Request(url, method='HEAD',
              headers={'User-Agent':'Mozilla/5.0 (DW image-health)'})
        with urllib.request.urlopen(req, timeout=8, context=ctx) as resp:
            return 200 <= resp.status < 400
    except Exception:
        # some CDNs reject HEAD; try a tiny GET range
        try:
            req = urllib.request.Request(url, headers={'User-Agent':'Mozilla/5.0','Range':'bytes=0-128'})
            with urllib.request.urlopen(req, timeout=8, context=ctx) as resp:
                return 200 <= resp.status < 400
        except Exception:
            return False

live = dead = 0
dead_rows = []
for r in probe:
    if head_ok(r['image_url']): live += 1
    else: dead += 1; dead_rows.append(r)

# cowtan-style relink attempt for dead flatshots URLs
FLAT = re.compile(r'/flatshots/([^/]+)-main/([^/_]+)_m\.jpg', re.I)
relinked = 0
# (relink is only meaningful for the cowtan CDN family; harmless elsewhere)

# live-fraction among PROBED (rows that HAD a url)
live_frac = round(live / max(1, len(probe)), 3)
# COVERAGE = fraction of ALL rows that even have an image_url. A catalog where
# most rows lack a url is image-starved even if the few it has are live (the
# phillip_jeffries 1/4377 case). Combined health = coverage * live_frac.
coverage = round(len(with_img) / max(1, total), 3)
health = round(coverage * live_frac, 3)
# NEEDS_RESCRAPE if combined health < 0.5 (low coverage OR many dead). Flag, don't block.
status = "HEALTHY" if health >= 0.5 else "NEEDS_RESCRAPE"

print(json.dumps({
    "sampled": len(probe), "live": live, "dead": dead, "missing": missing,
    "placeholder": placeholder, "total": total, "with_image": len(with_img),
    "coverage": coverage, "live_frac": live_frac,
    "image_health": health, "relinked": relinked, "status": status
}))
PY