`scripts/harvest_morris_images.mjs`

88 lines
#!/usr/bin/env node
// TK-10881 — Morris & Co image RE-HARVEST via openclaw real-Chrome (Cloudflare-cleared).
// wmorrisandco.com is a Magento 2 store (no Nucleus feed, no GraphQL). Image = PDP og:image
// on the un-challenged /media/catalog/product CDN. We already resolved SKU->product-URL from
// the sitemap (morris_sku_url_map.json). Here we in-page fetch each PDP HTML and regex og:image
// + <title>, running INSIDE the CF-cleared tab (fetch carries the clearance cookie). Serial by
// design (1 browser). Checkpoints to morris_image_harvest.jsonl (resumes, loses nothing).
//
// USAGE: node harvest_morris_images.mjs <target-id> [chunk_size]
import { execFileSync } from 'child_process';
import fs from 'fs';

const TID = process.argv[2];
const CHUNK = Number(process.argv[3] || 40);
if (!TID) { console.error('need <target-id>'); process.exit(1); }
const DIR = new URL('..', import.meta.url).pathname;
const MAP = JSON.parse(fs.readFileSync(`${DIR}pilot/morris_sku_url_map.json`, 'utf8'));
const CKPT = `${DIR}pilot/morris_image_harvest.jsonl`;

const done = new Set();
if (fs.existsSync(CKPT)) for (const l of fs.readFileSync(CKPT, 'utf8').split('\n').filter(Boolean)) {
  try { done.add(JSON.parse(l).base_code); } catch {}
}
const todo = Object.entries(MAP).filter(([sku]) => !done.has(sku));
console.log(`[morris-img] ${Object.keys(MAP).length} mapped, ${done.size} checkpointed, ${todo.length} to harvest`);

// NOTE: use appendFileSync, NOT createWriteStream — the loop blocks the event loop with
// synchronous execFileSync (openclaw) calls, so an async write stream never flushes until exit.
// appendFileSync flushes each row to disk immediately => true resumable checkpoint.
const out = { write: (s) => fs.appendFileSync(CKPT, s), end: () => {} };

function evalInPage(fnSrc) {
  // openclaw browser evaluate --target-id TID --fn '<src>' ; returns stdout (JSON-string)
  const res = execFileSync('openclaw', ['browser', 'evaluate', '--target-id', TID, '--timeout', '120000', '--fn', fnSrc],
    { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
  let s = res.trim();
  // openclaw prints the return value; if wrapped as a JSON string, decode once
  if (s.startsWith('"') && s.endsWith('"')) { try { s = JSON.parse(s); } catch {} }
  return s;
}

const chunks = [];
for (let i = 0; i < todo.length; i += CHUNK) chunks.push(todo.slice(i, i + CHUNK));

let ci = 0, ok = 0, noimg = 0, err = 0;
for (const ch of chunks) {
  ci++;
  const pairs = ch.map(([sku, url]) => [sku, url]);
  const fnSrc = `async () => {
    const pairs = ${JSON.stringify(pairs)};
    const results = [];
    // small concurrency to stay polite + avoid tab overload
    const CONC = 5;
    let idx = 0;
    async function worker() {
      while (idx < pairs.length) {
        const my = idx++; const [sku, url] = pairs[my];
        try {
          const r = await fetch(url, { headers: { 'Accept': 'text/html' } });
          const t = await r.text();
          const og = (t.match(/<meta[^>]+property=["']og:image["'][^>]+content=["']([^"']+)["']/i) || [])[1] || null;
          const title = (t.match(/<title>([^<]*)<\\/title>/i) || [])[1] || null;
          const sku_page = (t.match(/"sku"\\s*:\\s*"([^"]+)"/i) || [])[1] || null;
          results.push({ base_code: sku, url, image: og, title, sku_page, http: r.status });
        } catch (e) { results.push({ base_code: sku, url, image: null, error: String(e).slice(0, 100) }); }
      }
    }
    await Promise.all(Array.from({ length: CONC }, worker));
    return JSON.stringify(results);
  }`;
  let arr;
  try {
    const raw = evalInPage(fnSrc);
    arr = typeof raw === 'string' ? JSON.parse(raw) : raw;
  } catch (e) {
    console.error(`[morris-img] chunk ${ci}/${chunks.length} EVAL FAIL: ${String(e).slice(0,120)} — retrying once`);
    try { const raw = evalInPage(fnSrc); arr = JSON.parse(typeof raw==='string'?raw:JSON.stringify(raw)); }
    catch (e2) { console.error(`  retry failed, skipping chunk`); continue; }
  }
  for (const r of arr) {
    if (r.image) ok++; else if (r.error) err++; else noimg++;
    out.write(JSON.stringify(r) + '\n');
  }
  console.log(`[morris-img] chunk ${ci}/${chunks.length}  ok=${ok} noimg=${noimg} err=${err}`);
}
out.end();
console.log(`[morris-img] DONE ok=${ok} noimg=${noimg} err=${err}. Checkpoint: ${CKPT}`);