`scripts/download-real-images.js`

102 lines
#!/usr/bin/env node
// download-real-images.js — pull each resolved candidate image down to public/assets/real/.
// Reads the fan-out agents' outputs (data/real-images/candidates/*.json), folds them into the
// ledger, then downloads every entry still at status candidate. $0 (plain HTTPS fetches).
// Idempotent: qc-pass/downloaded entries are skipped; re-run freely.
//
// Guards: browser UA (marketplace CDNs 403 default agents), ≥400px min dimension (sips),
// sha256 dedupe — the same byte-identical file arriving for >3 different items is a CDN
// placeholder, not 4 different products, and gets rejected.
const fs = require('fs');
const path = require('path');
const { execFileSync } = require('child_process');
const { CAND_DIR, ASSETS, loadLedger, saveLedger, sha256 } = require('./real-lib');

const DRY = process.argv.includes('--dry-run');
const LIMIT = (() => { const i = process.argv.indexOf('--limit'); return i > -1 ? parseInt(process.argv[i + 1], 10) : Infinity; })();
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36';
const EXT = { 'image/png': 'png', 'image/jpeg': 'jpg', 'image/webp': 'webp', 'image/gif': 'gif', 'image/avif': 'avif' };
const MIN_PX = 400;
const MAX_SAME_SHA = 3;

fs.mkdirSync(ASSETS, { recursive: true });

// ---- 1. fold candidate files into the ledger (agents never touch the ledger themselves) ----
const ledger = loadLedger();
let folded = 0;
if (fs.existsSync(CAND_DIR)){
  for (const f of fs.readdirSync(CAND_DIR).filter(f => f.endsWith('.json'))){
    let c; try { c = JSON.parse(fs.readFileSync(path.join(CAND_DIR, f), 'utf8')); } catch(e){ console.log(`  BAD candidate file ${f}: ${e.message}`); continue; }
    if (!c.fingerprint) continue;
    const l = ledger[c.fingerprint] || {};
    if (l.status === 'qc-pass') continue;                       // already settled — don't churn
    ledger[c.fingerprint] = { ...l, fingerprint: c.fingerprint, id: c.id, title: c.title || l.title,
      status: c.candidateImageUrl ? 'candidate' : 'unresolved',
      candidateUrl: c.candidateImageUrl || null, sourceUrl: c.sourcePageUrl || null,
      altCandidates: c.altCandidates || [], confidence: c.confidence ?? null,
      method: c.method || null, exaCalls: c.exaCalls ?? null, estCostUsd: c.estCostUsd ?? null,
      notes: c.notes || null, updatedAt: new Date().toISOString() };
    folded++;
  }
}

async function fetchImage(url){
  const res = await fetch(url, { headers: { 'User-Agent': UA, 'Accept': 'image/avif,image/webp,image/png,image/jpeg,*/*' }, redirect: 'follow', signal: AbortSignal.timeout(30000) });
  if (!res.ok) throw new Error(`HTTP ${res.status}`);
  const type = (res.headers.get('content-type') || '').split(';')[0].trim();
  const buf = Buffer.from(await res.arrayBuffer());
  if (!buf.length) throw new Error('empty body');
  if (!EXT[type] && !/^image\//.test(type)) throw new Error(`not an image (${type || 'no content-type'})`);
  return { buf, ext: EXT[type] || 'jpg' };
}

function dimensions(fp){
  try {
    const out = execFileSync('sips', ['-g', 'pixelWidth', '-g', 'pixelHeight', fp], { encoding: 'utf8' });
    const w = +(out.match(/pixelWidth: (\d+)/) || [])[1], h = +(out.match(/pixelHeight: (\d+)/) || [])[1];
    return { w, h };
  } catch(e){ return { w: 0, h: 0 }; }
}

(async () => {
  const todo = Object.values(ledger).filter(l => l.status === 'candidate' && l.candidateUrl).slice(0, LIMIT);
  console.log(`candidates folded: ${folded} · to download: ${todo.length}${DRY ? ' (dry-run)' : ''}`);
  if (DRY){ todo.forEach(l => console.log(`  ${l.id}  ${l.candidateUrl}`)); return; }

  const shaCount = {};                                          // sha256 -> #items this run+prior
  for (const l of Object.values(ledger)) if (l.sha256) shaCount[l.sha256] = (shaCount[l.sha256] || 0) + 1;

  let ok = 0, fail = 0;
  for (const l of todo){
    try {
      const { buf, ext } = await fetchImage(l.candidateUrl);
      const sha = sha256(buf);
      if ((shaCount[sha] || 0) >= MAX_SAME_SHA) throw new Error('placeholder: same bytes already used by ' + shaCount[sha] + ' items');
      let fname = `${l.fingerprint}.${ext}`;                    // fingerprint stem — survives TR-id churn
      let fp = path.join(ASSETS, fname);
      fs.writeFileSync(fp, buf);
      // ollama vision models 400 on webp/avif — normalize to png so the QC gate can see it
      if (ext === 'webp' || ext === 'avif' || ext === 'gif'){
        const pngName = `${l.fingerprint}.png`, pngPath = path.join(ASSETS, pngName);
        execFileSync('sips', ['-s', 'format', 'png', fp, '--out', pngPath], { stdio: 'ignore' });
        fs.unlinkSync(fp); fname = pngName; fp = pngPath;
      }
      const { w, h } = dimensions(fp);
      if (w < MIN_PX && h < MIN_PX){ fs.unlinkSync(fp); throw new Error(`too small (${w}×${h})`); }
      shaCount[sha] = (shaCount[sha] || 0) + 1;
      Object.assign(l, { status: 'downloaded', file: fname, sha256: sha, width: w, height: h, downloadedAt: new Date().toISOString() });
      ok++; console.log(`  ✓ ${l.id} ← ${l.candidateUrl.slice(0, 90)} (${w}×${h})`);
    } catch(e){
      fail++;
      // burn this candidate; promote the next alternate if the agent supplied one
      const alts = l.altCandidates || [];
      if (alts.length){ l.candidateUrl = alts.shift(); l.altCandidates = alts; l.status = 'candidate'; console.log(`  ↻ ${l.id} — ${e.message}; trying alt (${alts.length} left)`); }
      else { l.status = 'unresolved'; l.failReason = e.message; console.log(`  ✗ ${l.id} — ${e.message}`); }
      l.updatedAt = new Date().toISOString();
    }
  }
  saveLedger(ledger);
  const retriable = Object.values(ledger).filter(l => l.status === 'candidate').length;
  console.log(`downloaded ${ok}, failed ${fail}${retriable ? `, ${retriable} candidates remain (re-run to retry alts)` : ''}. $0 (plain fetches).`);
})();