`scripts/pool_morris_search.mjs`

30 lines
#!/usr/bin/env node
// TK-10881 — pool live US product URLs via per-pattern search (server-rendered DOM), for the
// remaining SKUs whose sitemap URL 404s (stale "morris-" prefix) or was unresolvable. Appends
// each term's URLs to the pool file immediately (resumable / timeout-safe).
import { execFileSync } from 'child_process';
import fs from 'fs';
const TID = process.argv[2];
const plan = JSON.parse(fs.readFileSync('/tmp/morris_search_plan.json', 'utf8'));
const POOL = 'pilot/morris_search_pool.txt';
const DONE = 'pilot/morris_search_terms_done.txt';
const seen = new Set(fs.existsSync(POOL) ? fs.readFileSync(POOL, 'utf8').split('\n').filter(Boolean) : []);
const done = new Set(fs.existsSync(DONE) ? fs.readFileSync(DONE, 'utf8').split('\n').filter(Boolean) : []);
const oc = (a) => execFileSync('openclaw', a, { encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 });
for (const term of Object.keys(plan.terms)) {
  if (done.has(term)) continue;
  try {
    oc(['browser', 'navigate', '--target-id', TID, 'https://www.wmorrisandco.com/catalogsearch/result/?q=' + encodeURIComponent(term)]);
    execFileSync('sleep', ['6']);
    let raw = oc(['browser', 'evaluate', '--target-id', TID, '--timeout', '40000', '--fn',
      "() => { const a=[...document.querySelectorAll('a[href]')].map(x=>x.href); return JSON.stringify([...new Set(a)].filter(h=>/wmorrisandco\\.com\\/[a-z0-9-]+-(wallpaper|fabric)-[a-z0-9-]+-[a-z0-9]+\\/$/i.test(h))); }"]).trim();
    if (raw.startsWith('"') && raw.endsWith('"')) { try { raw = JSON.parse(raw); } catch {} }
    const urls = JSON.parse(raw);
    let add = 0;
    for (const u of urls) { if (!seen.has(u)) { seen.add(u); fs.appendFileSync(POOL, u + '\n'); add++; } }
    fs.appendFileSync(DONE, term + '\n');
    console.log(`  ${term}: +${add} (pool ${seen.size})`);
  } catch (e) { console.log(`  ${term}: ERR ${String(e).slice(0, 60)}`); }
}
console.log('POOL total', seen.size);