`scripts/dw-stale-urls-head-check.js`

58 lines
// HEAD-verify all swap candidates in parallel (concurrency=20)
// Persists head_check_status + head_checked_at to staging table.
const { Client } = require('pg');

const CONCURRENCY = 20;
const TIMEOUT_MS = 8000;

async function headCheck(url) {
  const ctrl = new AbortController();
  const t = setTimeout(() => ctrl.abort(), TIMEOUT_MS);
  try {
    let res = await fetch(url, { method: 'HEAD', signal: ctrl.signal, redirect: 'follow' });
    // Some CDNs reject HEAD with 405 — fall back to GET range
    if (res.status === 405 || res.status === 403) {
      const ctrl2 = new AbortController();
      const t2 = setTimeout(() => ctrl2.abort(), TIMEOUT_MS);
      try {
        res = await fetch(url, { method: 'GET', signal: ctrl2.signal, redirect: 'follow', headers: { Range: 'bytes=0-0' } });
      } finally { clearTimeout(t2); }
    }
    return res.status;
  } catch (e) {
    return 0; // network/timeout failure
  } finally {
    clearTimeout(t);
  }
}

async function worker(client, queue, progress) {
  while (queue.length > 0) {
    const row = queue.shift();
    if (!row) break;
    const status = await headCheck(row.new_url);
    await client.query(
      'UPDATE image_url_swap_proposals_2026_05_19 SET head_check_status=$1, head_checked_at=NOW() WHERE id=$2',
      [status, row.id]
    );
    progress.done++;
    if (status === 200) progress.ok++;
    if (progress.done % 25 === 0) console.log(`[${progress.done}/${progress.total}] ok=${progress.ok}`);
  }
}

(async () => {
  const client = new Client();
  await client.connect();
  const { rows } = await client.query(
    "SELECT id, new_url FROM image_url_swap_proposals_2026_05_19 WHERE new_url IS NOT NULL AND head_check_status IS NULL"
  );
  console.log(`To check: ${rows.length}`);
  const queue = [...rows];
  const progress = { done: 0, ok: 0, total: rows.length };
  const workers = Array.from({ length: CONCURRENCY }, () => worker(client, queue, progress));
  await Promise.all(workers);
  console.log(`DONE: ${progress.done} checked, ${progress.ok} returned 200`);
  await client.end();
})().catch(e => { console.error(e); process.exit(1); });