`scripts/download.sh`

32 lines
#!/bin/bash
set -euo pipefail
cd "$(dirname "$0")/.."
UA="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36"
URL="https://www.cslb.ca.gov/onlineservices/dataportal/ContractorList"
vs(){ grep -oE "id=\"$1\" value=\"[^\"]*\"" "$2" | sed 's/.*value="//;s/"$//'; }
# 1) session + viewstate
curl -s --max-time 60 -c data/cj.txt -A "$UA" "$URL" -o data/s1.html
# 2) select License Master (autopostback)
curl -s --max-time 60 -b data/cj.txt -c data/cj.txt -A "$UA" \
  --data-urlencode "__EVENTTARGET=ctl00\$MainContent\$ddlStatus" --data-urlencode "__EVENTARGUMENT=" \
  --data-urlencode "__VIEWSTATE=$(vs __VIEWSTATE data/s1.html)" \
  --data-urlencode "__VIEWSTATEGENERATOR=$(vs __VIEWSTATEGENERATOR data/s1.html)" \
  --data-urlencode "__EVENTVALIDATION=$(vs __EVENTVALIDATION data/s1.html)" \
  --data-urlencode 'ctl00$MainContent$ddlStatus=M' "$URL" -o data/s2.html
# 3) fire CSV link -> 302 to DownLoadFile.ashx
LOC=$(curl -s --max-time 60 -b data/cj.txt -c data/cj.txt -A "$UA" -D - -o /dev/null \
  --data-urlencode "__EVENTTARGET=ctl00\$MainContent\$lbMasterCSV" --data-urlencode "__EVENTARGUMENT=" \
  --data-urlencode "__VIEWSTATE=$(vs __VIEWSTATE data/s2.html)" \
  --data-urlencode "__VIEWSTATEGENERATOR=$(vs __VIEWSTATEGENERATOR data/s2.html)" \
  --data-urlencode "__EVENTVALIDATION=$(vs __EVENTVALIDATION data/s2.html)" \
  --data-urlencode 'ctl00$MainContent$ddlStatus=M' "$URL" | grep -i '^location:' | tr -d '\r' | awk '{print $2}')
[ -z "$LOC" ] && { echo "ERROR: no redirect location"; exit 1; }
# 4) download the file
curl -s --max-time 590 -b data/cj.txt -A "$UA" -e "$URL" \
  "https://www.cslb.ca.gov${LOC}" -o data/MasterLicenseData.new
sz=$(wc -c < data/MasterLicenseData.new)
[ "$sz" -lt 20000000 ] && { echo "ERROR: file too small ($sz bytes) - refusing to overwrite"; exit 1; }
head -1 data/MasterLicenseData.new | grep -q "LicenseNo,LastUpdate" || { echo "ERROR: bad header"; exit 1; }
mv data/MasterLicenseData.new data/MasterLicenseData.csv
echo "OK: $(wc -l < data/MasterLicenseData.csv) rows, $sz bytes"