`tk11041-innovations-reconcile/rescrape/sweep.sh`

21 lines
#!/bin/bash
UA="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36"
mkdir -p pages
: > codes.tsv
n=0
while IFS= read -r slug; do
  [ -z "$slug" ] && continue
  n=$((n+1))
  enc=$(printf '%s' "$slug" | sed 's/ /%20/g; s/&#039;/%27/g')
  f="pages/$(printf '%s' "$slug" | tr ' /' '__').html"
  if [ ! -s "$f" ]; then
    curl -s -A "$UA" -m 30 "https://www.innovationsusa.com/item/$enc" -o "$f"
    sleep 0.35
  fi
  grep -oE '\b[A-Z]{2,4}-[0-9]{2,3}\b' "$f" 2>/dev/null | sort -u | while read -r c; do
    printf '%s\t%s\n' "$c" "$slug" >> codes.tsv
  done
  [ $((n % 50)) -eq 0 ] && echo "...$n" >&2
done < inn_slugs.txt
echo "DONE $n pages, $(wc -l < codes.tsv) code rows" >&2