`scripts/harvest_retail.sh`

37 lines
#!/bin/bash
# Feed-first ($0) Sanderson retail harvest -> data/*.txt for extract.py. No auth, no proxy, no API.
set -euo pipefail
UA='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36'
D="$(cd "$(dirname "$0")/.." && pwd)/data"; cd "$D"
BASE=https://www.sanderson.design

echo "[retail] sitemap categories..."
curl -s -m 30 -A "$UA" "$BASE/sitemap.xml" | grep -oE "<loc>$BASE/wallpaper[^<]*</loc>" | sed -E 's#</?loc>##g' | sort -u > cats.txt
grep -E '/wallpaper/(colour|style)/[a-z-]+$' cats.txt > filters.txt
grep -E '/wallpaper/cn_saw[0-9]+' cats.txt > pattern_pages.txt || true

echo "[retail] harvesting colorway URLs from filter pages..."
: > prod_urls.txt
while read -r u; do curl -s -m 30 -A "$UA" "$u" | grep -oE '/wallpaper/[a-z0-9-]+-saw[0-9]{4}-[0-9]{2}' >> prod_urls.txt || true; done < filters.txt
while read -r u; do curl -s -m 30 -A "$UA" "$u" | grep -oE '/wallpaper/[a-z0-9-]+-saw[0-9]{4}-[0-9]{2}' >> prod_urls.txt || true; done < pattern_pages.txt
sort -u prod_urls.txt -o all_prod_urls.txt
echo "[retail] $(wc -l < all_prod_urls.txt) unique colorway URLs"

echo "[retail] pattern map (cn_ page h1)..."
: > pattern_map.tsv
while read -r u; do
  pref=$(echo "$u" | grep -oiE 'saw[0-9]{4}' | tr '[:lower:]' '[:upper:]' || true)
  h1=$(curl -s -m 20 -A "$UA" "$u" | grep -oiE '<h1[^>]*>[^<]+</h1>' | head -1 | sed -E 's/<[^>]+>//g' | sed 's/^ *//;s/ *$//' || true)
  [ -n "${pref:-}" ] && [ -n "${h1:-}" ] && printf '%s\t%s\n' "$pref" "$h1" >> pattern_map.tsv || true
done < pattern_pages.txt
sort -u pattern_map.tsv -o pattern_map.tsv
echo "[retail] $(wc -l < pattern_map.tsv) patterns mapped"

echo "[retail] settlement style membership..."
: > settlement_urls.txt
for s in botanical florals birds animals leaves-and-trees animal-print landscapes; do
  curl -s -m 30 -A "$UA" "$BASE/wallpaper/style/$s" | grep -oE '/wallpaper/[a-z0-9-]+-saw[0-9]{4}-[0-9]{2}' >> settlement_urls.txt || true
done
sort -u settlement_urls.txt -o settlement_urls.txt
echo "[retail] $(wc -l < settlement_urls.txt) settlement-review colorways"