`build-manifest.js`

59 lines
#!/usr/bin/env node
// Generate data/manifest.json from vendor_registry — the single source of truth
// for the staggered monthly scrape. TIER RULES (conservative — unsure => excluded):
//   TIER_A  = feed-first: no creds, not skip_catalog_scrape, reachable platform we
//             have an adapter shape for (Shopify/WooCommerce/Magento).
//   headless = reachable-but-needs-render (Custom HTML/ASP.NET/TYPO3/Drupal/Next/Squarespace) => Tier-B (parked)
//   gated   = has_credentials OR login_url OR paid (Browserbase/2captcha) => NEVER auto-scheduled
//   skip    = Unreachable/Error/skip_catalog_scrape/archived
// Assigns a stable day-of-month: honor existing crawl_cron day; else hash vendor_code -> 1..28.
const fs = require('fs');
const path = require('path');
const { pool } = require('./lib/db');

const FEED_PLATFORMS = { shopify: 'shopify', 'shopify 2.0': 'shopify', woocommerce: 'woo', wordpress: 'woo', magento: 'magento' };
const HEADLESS = new Set(['custom html', 'asp.net', 'typo3', 'drupal', 'next.js', 'squarespace']);
const DEAD = new Set(['unreachable', 'error', '?', '']);
// Lines that must never be re-activated / are private-label-sensitive stay out of the auto scrape.
const HARD_EXCLUDE = new Set(['nicolette_mayer']); // hard-archived (Steve rule)

function dayFromCron(cron) {
  if (!cron) return null;
  const dom = String(cron).trim().split(/\s+/)[2];
  const n = Number(dom);
  return Number.isInteger(n) && n >= 1 && n <= 28 ? n : null;
}
function hashDay(code) { let h = 0; for (const c of code) h = (h * 31 + c.charCodeAt(0)) >>> 0; return (h % 28) + 1; }

(async () => {
  const { rows } = await pool.query(`
    select vendor_code, vendor_name, lower(coalesce(website_tech_stack,'')) stack,
           website_url, crawl_cron, coalesce(has_credentials,false) creds,
           login_url, coalesce(skip_catalog_scrape,false) skipc
    from vendor_registry
    where website_url is not null and website_url <> ''
    order by vendor_name`);

  const tier_a = [], parked_headless = [], gated = [], skipped = [];
  for (const r of rows) {
    const entry = { vendor_code: r.vendor_code, vendor: r.vendor_name, platform: r.stack,
                    base_url: r.website_url, day: dayFromCron(r.crawl_cron) || hashDay(r.vendor_code) };
    if (HARD_EXCLUDE.has(r.vendor_code)) { skipped.push({ ...entry, why: 'hard-archived' }); continue; }
    if (r.creds || r.login_url) { gated.push({ ...entry, why: 'portal-login/creds' }); continue; }
    if (r.skipc) { skipped.push({ ...entry, why: 'skip_catalog_scrape' }); continue; }
    if (DEAD.has(r.stack)) { skipped.push({ ...entry, why: 'dead/unknown platform' }); continue; }
    const adapter = FEED_PLATFORMS[r.stack];
    if (adapter) { tier_a.push({ ...entry, adapter, enabled: true }); continue; }
    if (HEADLESS.has(r.stack)) { parked_headless.push({ ...entry, adapter: 'headless', enabled: false }); continue; }
    skipped.push({ ...entry, why: `no adapter for '${r.stack}'` });
  }

  const manifest = { generated_note: 'built from vendor_registry by build-manifest.js',
    counts: { tier_a: tier_a.length, parked_headless: parked_headless.length, gated: gated.length, skipped: skipped.length },
    tier_a, parked_headless, gated, skipped };
  fs.writeFileSync(path.join(__dirname, 'data', 'manifest.json'), JSON.stringify(manifest, null, 2));
  console.log('manifest.json written:', JSON.stringify(manifest.counts));
  console.log('tier_a adapters:', tier_a.reduce((a, e) => { a[e.adapter] = (a[e.adapter] || 0) + 1; return a; }, {}));
  await pool.end();
})().catch(e => { console.error('FATAL', e); process.exit(1); });