`onboarding/sangetsu-lilycolor/lilycolor-feed-scraper.cjs`

89 lines
#!/usr/bin/env node
/**
 * Lilycolor feed-first scraper (STAGING ONLY — OFFLINE).
 * Pulls the full shop.lilycolor.co.jp Shopify feed, normalizes each product to a DW
 * staging record, and writes JSONL to staging/lilycolor-staging.jsonl.
 *
 * HARD: writes ONLY to a local staging file. No Shopify, no dw_unified, no publish.
 * JA->EN translation + final pricing/dedup happen at the gated enrichment/activation stage.
 *
 * Usage: node lilycolor-feed-scraper.cjs [maxPages]   (default 30 pages = full catalog)
 */
const https = require('https');
const fs = require('fs');
const path = require('path');

const BASE = 'https://shop.lilycolor.co.jp/products.json';
const MAX_PAGES = parseInt(process.argv[2] || '30', 10);
const OUT_DIR = path.join(__dirname, 'staging');
const OUT = path.join(OUT_DIR, 'lilycolor-staging.jsonl');

function get(url) {
  return new Promise((resolve, reject) => {
    https.get(url, { headers: { 'User-Agent': 'Mozilla/5.0 (DW-onboarding-recon)' } }, (res) => {
      let body = '';
      res.on('data', (c) => (body += c));
      res.on('end', () => { try { resolve(JSON.parse(body)); } catch (e) { reject(e); } });
    }).on('error', reject);
  });
}

// Pull a labelled spec out of the body_html spec table (有効巾 / 品番 / etc.)
function specFrom(html, label) {
  if (!html) return null;
  const re = new RegExp(`${label}[\\s\\S]{0,60}?<span>([^<]+)</span>`);
  const m = html.match(re);
  return m ? m[1].trim() : null;
}

function normalize(p) {
  const variants = p.variants || [];
  const sampleVar = variants.find((v) => /sample|サンプル/i.test(v.title || '') || /_{3}sample/.test(v.handle || ''));
  const productVar = variants.find((v) => v !== sampleVar) || variants[0] || {};
  const mfrSku = String((productVar.sku || '')).replace(/-nori$/i, '');
  const prefix = (mfrSku.match(/^([A-Za-z]+)/) || [])[1] || null;
  return {
    source: 'lilycolor',
    source_feed: 'shop.lilycolor.co.jp/products.json',
    shopify_source_id: p.id,
    handle: p.handle,
    title_ja: p.title,
    title_en: null,                 // TODO: JA->EN at enrichment stage (gated)
    mfr_sku: mfrSku || null,
    mfr_prefix: prefix,             // LW / LL / LV / LB / EC ... -> dedup key vs dw_sku_registry
    width: specFrom(p.body_html, '有効巾'),      // e.g. 92.5cm
    composition: /塩化ビニル/.test(p.body_html || '') ? 'Vinyl (塩化ビニル)' : null,
    fire_grade: (p.body_html || '').match(/F☆+|準不燃|不燃/)?.[0] || null,
    body_html_ja: p.body_html,
    images: (p.images || []).map((i) => i.src),
    has_sample_variant: !!sampleVar,
    sample_handle: sampleVar ? sampleVar.handle : null,
    vendor_raw: p.vendor || 'Lilycolor',
    product_type: p.product_type || null,
    // gates (all false until the gated stage runs):
    deduped: false, settlement_checked: false, cost_confirmed: false,
    activation_ready: false, status: 'staged-for-new',
  };
}

(async () => {
  if (!fs.existsSync(OUT_DIR)) fs.mkdirSync(OUT_DIR, { recursive: true });
  const out = fs.createWriteStream(OUT);
  let total = 0;
  const prefixCounts = {};
  for (let page = 1; page <= MAX_PAGES; page++) {
    const d = await get(`${BASE}?limit=250&page=${page}`);
    const ps = (d && d.products) || [];
    if (!ps.length) break;
    for (const p of ps) {
      const rec = normalize(p);
      if (rec.mfr_prefix) prefixCounts[rec.mfr_prefix] = (prefixCounts[rec.mfr_prefix] || 0) + 1;
      out.write(JSON.stringify(rec) + '\n');
      total++;
    }
    process.stderr.write(`page ${page}: +${ps.length} (staged ${total})\n`);
  }
  out.end();
  console.log(JSON.stringify({ staged: total, out: OUT, mfrPrefixCounts: prefixCounts }, null, 2));
})().catch((e) => { console.error('FAILED:', e.message); process.exit(1); });