`src/cslb/download.js`

83 lines
// CSLB License Master downloader — the free public portal is an ASP.NET
// session/postback flow (no static URL). Flow:
//   1. GET the page -> session cookie + viewstate tokens
//   2. POST ddlStatus=M (dropdown-change postback) -> page with the CSV button + new tokens
//   3. POST the CSV linkbutton -> streams the License Master CSV
// Writes the CSV to tmp/cslb-master.csv.
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';

const URL = 'https://www.cslb.ca.gov/onlineservices/dataportal/ContractorList';
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const OUT = path.resolve(__dirname, '../../tmp/cslb-master.csv');

const pick = (html, name) => {
  const m = html.match(new RegExp(`id="${name}"[^>]*value="([^"]*)"`)) ||
            html.match(new RegExp(`name="${name}"[^>]*value="([^"]*)"`));
  return m ? m[1] : '';
};
const tokens = (html) => ({
  __VIEWSTATE: pick(html, '__VIEWSTATE'),
  __VIEWSTATEGENERATOR: pick(html, '__VIEWSTATEGENERATOR'),
  __EVENTVALIDATION: pick(html, '__EVENTVALIDATION'),
});
// find the CSV download linkbutton's postback target from the page
function findCsvTarget(html) {
  const targets = [...html.matchAll(/__doPostBack\(&?#?39?;?'?([^'&]*(?:CSV|Csv)[^'&]*)/g)].map((m) => m[1]);
  // prefer a Master CSV button
  return targets.find((t) => /master/i.test(t)) || targets.find((t) => /csv/i.test(t)) || 'ctl00$MainContent$lbMasterCSV';
}

async function post(cookie, fields) {
  const body = new URLSearchParams(fields).toString();
  return fetch(URL, {
    method: 'POST',
    headers: { 'User-Agent': UA, 'Content-Type': 'application/x-www-form-urlencoded', Cookie: cookie, Referer: URL },
    body,
  });
}

async function main() {
  // 1. GET
  const r1 = await fetch(URL, { headers: { 'User-Agent': UA } });
  const cookie = (r1.headers.get('set-cookie') || '').split(',').map((c) => c.split(';')[0].trim()).filter((c) => /=/.test(c)).join('; ');
  const html1 = await r1.text();
  const t1 = tokens(html1);
  console.log('step1: session', cookie ? 'ok' : 'MISSING', '| viewstate', t1.__VIEWSTATE ? t1.__VIEWSTATE.length + 'b' : 'MISSING');

  // 2. select License Master (dropdown-change postback)
  const r2 = await post(cookie, {
    __EVENTTARGET: 'ctl00$MainContent$ddlStatus', __EVENTARGUMENT: '', __LASTFOCUS: '',
    ...t1, 'ctl00$MainContent$ddlStatus': 'M',
  });
  const html2 = await r2.text();
  const t2 = tokens(html2);
  const csvTarget = findCsvTarget(html2);
  console.log('step2: master selected | csv target =', csvTarget, '| viewstate', t2.__VIEWSTATE ? t2.__VIEWSTATE.length + 'b' : 'MISSING');

  // 3. click the CSV download button -> stream to file
  const r3 = await post(cookie, {
    __EVENTTARGET: csvTarget, __EVENTARGUMENT: '', __LASTFOCUS: '',
    ...t2, 'ctl00$MainContent$ddlStatus': 'M',
  });
  const ct = r3.headers.get('content-type') || '';
  const cd = r3.headers.get('content-disposition') || '';
  console.log('step3: status', r3.status, '| content-type', ct, '| disposition', cd.slice(0, 60));

  if (!/csv|octet-stream|text\/plain|application\/vnd/i.test(ct) && !/attachment/i.test(cd)) {
    const txt = await r3.text();
    fs.writeFileSync(OUT.replace('.csv', '-debug.html'), txt);
    throw new Error(`step3 did not return a file (got ${ct}); wrote debug HTML. First 200: ${txt.slice(0, 200)}`);
  }
  const buf = Buffer.from(await r3.arrayBuffer());
  fs.mkdirSync(path.dirname(OUT), { recursive: true });
  fs.writeFileSync(OUT, buf);
  const lines = buf.toString('utf8', 0, Math.min(buf.length, 400)).split('\n')[0];
  console.log(`✔ wrote ${OUT} — ${(buf.length / 1e6).toFixed(1)} MB`);
  console.log('  header:', lines.slice(0, 180));
}

main().catch((e) => { console.error('✖', e.message); process.exit(1); });