`scrape_captions.py`

78 lines
#!/usr/bin/env python3
"""Fetch every Quadrille detail page (Editorials-Pages + Product-Shot-Pages),
extract each Editorial-Image URL -> caption (designer-name span + alt text).
Free HTTP only. Builds data/image_captions.json."""
import re, json, html as H, os, sys, time
from concurrent.futures import ThreadPoolExecutor
import urllib.request

BASE = "https://www.quadrillefabrics.com"
UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
OUT = "/Users/macstudio3/Projects/quadrille-room-credit-enrich/data"
urls = [u.strip() for u in open(f"{OUT}/detail_urls.txt") if u.strip()]

S3 = "quadrillefabrics.s3.us-east-2.amazonaws.com/Editorial-Images/"

def fetch(u):
    try:
        req = urllib.request.Request(u, headers={"User-Agent": UA})
        with urllib.request.urlopen(req, timeout=25) as r:
            return u, r.read().decode("utf-8", "replace")
    except Exception as e:
        return u, None

def parse(u, html):
    """Return list of (image_url, caption_text, alt_text, source_page)."""
    out = []
    if not html:
        return out
    # remove scripts/styles for clean text
    body = re.sub(r"<script.*?</script>", "", html, flags=re.S | re.I)
    body = re.sub(r"<style.*?</style>", "", body, flags=re.S | re.I)
    # Find <img ... alt="..." src="...Editorial-Images/...">  capturing alt + src in any order
    for m in re.finditer(r"<img\b[^>]*?>", body, re.S | re.I):
        tag = m.group(0)
        src = re.search(r"src=[\"']([^\"']*Editorial-Images/[^\"']+)[\"']", tag, re.I)
        if not src:
            continue
        img = src.group(1)
        alt = re.search(r"alt=[\"']([^\"']*)[\"']", tag, re.I)
        alt = H.unescape(alt.group(1)).strip() if alt else ""
        # caption: nearest designer-name span AFTER the img (within 1200 chars)
        seg = body[m.end():m.end() + 1500]
        cap = ""
        cm = re.search(r"<span[^>]*class=[\"'][^\"']*designer-name[^\"']*[\"'][^>]*>(.*?)</span>", seg, re.S | re.I)
        if cm:
            cap = H.unescape(re.sub(r"<[^>]+>", " ", cm.group(1)))
            cap = re.sub(r"\s+", " ", cap).strip()
        out.append((img, cap, alt, u))
    return out

def main():
    results = {}
    done = 0
    with ThreadPoolExecutor(max_workers=12) as ex:
        for u, html in ex.map(fetch, urls):
            done += 1
            if done % 200 == 0:
                print(f"  {done}/{len(urls)}", file=sys.stderr)
            for img, cap, alt, page in parse(u, html):
                # keep the richest caption seen for each image
                prev = results.get(img)
                score = len(cap) + len(alt)
                if prev is None or score > prev["_score"]:
                    results[img] = {"caption": cap, "alt": alt, "page": page, "_score": score}
    # strip score
    for v in results.values():
        v.pop("_score", None)
    json.dump(results, open(f"{OUT}/image_captions.json", "w"), indent=0)
    print(f"images captured: {len(results)}")
    rich = sum(1 for v in results.values() if "Photography by" in v["caption"] or "As Seen in" in v["caption"]
               or "Photography by" in v["alt"] or "As Seen in" in v["alt"])
    print(f"images with photographer/publication text: {rich}")

if __name__ == "__main__":
    t = time.time()
    main()
    print(f"elapsed {time.time()-t:.0f}s")