`scripts/download-images.js`

104 lines
#!/usr/bin/env node
/**
 * Download phase: pull image bytes for any pattern_images row missing local_path.
 * Stores under data/images/<pattern_id>/<position>.<ext>
 */
require('dotenv').config({ path: require('path').join(__dirname, '..', '.env') });
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const { withPage, sleep, UA } = require('../src/browser');
const db = require('../src/db');

const RATE = Number(process.env.RATE_LIMIT_MS || 2500);
const ROOT = path.join(__dirname, '..', 'data', 'images');
fs.mkdirSync(ROOT, { recursive: true });

function args() {
  const a = {};
  for (const arg of process.argv.slice(2)) {
    const m = arg.match(/^--([^=]+)=(.+)$/);
    if (m) a[m[1]] = m[2];
  }
  return a;
}

function extFromCT(ct, url) {
  if (ct?.includes('jpeg') || /\.jpe?g(\?|$)/i.test(url)) return 'jpg';
  if (ct?.includes('png')  || /\.png(\?|$)/i.test(url))   return 'png';
  if (ct?.includes('webp') || /\.webp(\?|$)/i.test(url))  return 'webp';
  if (ct?.includes('gif')  || /\.gif(\?|$)/i.test(url))   return 'gif';
  return 'bin';
}

async function fetchImage(url) {
  // Use browserbase page.context().request to inherit any CF cookies / UA
  return await require('../src/browser').withPage(async (page, session) => {
    // Warm the origin so CF challenge is solved if needed
    try { await page.goto('https://patternbank.com/', { waitUntil: 'domcontentloaded', timeout: 30000 }); } catch {}
    const resp = await page.context().request.get(url, {
      headers: { 'User-Agent': UA, 'Accept': 'image/avif,image/webp,image/png,image/jpeg,*/*;q=0.8' },
      timeout: 60000,
    });
    const status = resp.status();
    const ct = resp.headers()['content-type'] || '';
    const buf = Buffer.from(await resp.body());
    return { status, ct, buf };
  });
}

(async () => {
  const a = args();
  const limit = Number(a.limit || 25);
  const runId = await db.startRun('images', `limit=${limit}`);
  console.log(`[img] run=${runId} limit=${limit}`);
  const r = await db.q(
    `SELECT image_id, pattern_id, position, remote_url
       FROM pattern_images
      WHERE local_path IS NULL
      ORDER BY image_id ASC
      LIMIT $1`, [limit]);
  if (!r.rows.length) {
    console.log(`[img] nothing pending`);
    await db.finishRun(runId, {});
    return db.pool.end();
  }
  let dl = 0, bytes = 0, err = 0;
  for (const row of r.rows) {
    try {
      const { status, ct, buf } = await fetchImage(row.remote_url);
      if (status >= 400 || !buf || buf.length < 64) {
        err++;
        console.warn(`[img] HTTP ${status} ${row.remote_url}`);
        await sleep(RATE);
        continue;
      }
      const ext = extFromCT(ct, row.remote_url);
      const dir = path.join(ROOT, String(row.pattern_id));
      fs.mkdirSync(dir, { recursive: true });
      const out = path.join(dir, `${row.position}.${ext}`);
      fs.writeFileSync(out, buf);
      const sha = crypto.createHash('sha256').update(buf).digest('hex');
      await db.q(
        `UPDATE pattern_images
            SET local_path=$1, bytes=$2, sha256=$3, content_type=$4
          WHERE image_id=$5`,
        [path.relative(path.join(__dirname,'..'), out), buf.length, sha, ct, row.image_id]
      );
      dl++; bytes += buf.length;
      console.log(`[img] ✓ ${row.pattern_id}/${row.position}.${ext} ${(buf.length/1024).toFixed(1)}KB sha=${sha.slice(0,8)}`);
    } catch (e) {
      err++;
      console.warn(`[img] ${row.remote_url}: ${e.message}`);
    }
    await sleep(RATE);
  }
  await db.finishRun(runId, {
    images_downloaded: dl,
    bytes_downloaded: bytes,
    errors: err,
  });
  console.log(`[img] done dl=${dl} bytes=${bytes} err=${err}`);
  await db.pool.end();
})().catch(e => { console.error(e); process.exit(1); });