[object Object]

← back to Designer Wallcoverings

Lilycolor: feed-first scraper staged 3,107 products offline (no Shopify); Andrew reply drafted via George

110f255db495e19ad635557db0af8c931f2cfb4e · 2026-06-29 15:37:54 -0700 · steve@designerwallcoverings.com

Files touched

Diff

commit 110f255db495e19ad635557db0af8c931f2cfb4e
Author: steve@designerwallcoverings.com <steve@designerwallcoverings.com>
Date:   Mon Jun 29 15:37:54 2026 -0700

    Lilycolor: feed-first scraper staged 3,107 products offline (no Shopify); Andrew reply drafted via George
---
 onboarding/sangetsu-lilycolor/.gitignore           |  1 +
 .../sangetsu-lilycolor/lilycolor-feed-scraper.cjs  | 88 ++++++++++++++++++++++
 onboarding/sangetsu-lilycolor/staging-summary.json | 30 ++++++++
 3 files changed, 119 insertions(+)

diff --git a/onboarding/sangetsu-lilycolor/.gitignore b/onboarding/sangetsu-lilycolor/.gitignore
new file mode 100644
index 00000000..6065974d
--- /dev/null
+++ b/onboarding/sangetsu-lilycolor/.gitignore
@@ -0,0 +1 @@
+staging/*.jsonl
diff --git a/onboarding/sangetsu-lilycolor/lilycolor-feed-scraper.cjs b/onboarding/sangetsu-lilycolor/lilycolor-feed-scraper.cjs
new file mode 100644
index 00000000..bc0377d5
--- /dev/null
+++ b/onboarding/sangetsu-lilycolor/lilycolor-feed-scraper.cjs
@@ -0,0 +1,88 @@
+#!/usr/bin/env node
+/**
+ * Lilycolor feed-first scraper (STAGING ONLY — OFFLINE).
+ * Pulls the full shop.lilycolor.co.jp Shopify feed, normalizes each product to a DW
+ * staging record, and writes JSONL to staging/lilycolor-staging.jsonl.
+ *
+ * HARD: writes ONLY to a local staging file. No Shopify, no dw_unified, no publish.
+ * JA->EN translation + final pricing/dedup happen at the gated enrichment/activation stage.
+ *
+ * Usage: node lilycolor-feed-scraper.cjs [maxPages]   (default 30 pages = full catalog)
+ */
+const https = require('https');
+const fs = require('fs');
+const path = require('path');
+
+const BASE = 'https://shop.lilycolor.co.jp/products.json';
+const MAX_PAGES = parseInt(process.argv[2] || '30', 10);
+const OUT_DIR = path.join(__dirname, 'staging');
+const OUT = path.join(OUT_DIR, 'lilycolor-staging.jsonl');
+
+function get(url) {
+  return new Promise((resolve, reject) => {
+    https.get(url, { headers: { 'User-Agent': 'Mozilla/5.0 (DW-onboarding-recon)' } }, (res) => {
+      let body = '';
+      res.on('data', (c) => (body += c));
+      res.on('end', () => { try { resolve(JSON.parse(body)); } catch (e) { reject(e); } });
+    }).on('error', reject);
+  });
+}
+
+// Pull a labelled spec out of the body_html spec table (有効巾 / 品番 / etc.)
+function specFrom(html, label) {
+  if (!html) return null;
+  const re = new RegExp(`${label}[\\s\\S]{0,60}?<span>([^<]+)</span>`);
+  const m = html.match(re);
+  return m ? m[1].trim() : null;
+}
+
+function normalize(p) {
+  const variants = p.variants || [];
+  const sampleVar = variants.find((v) => /sample|サンプル/i.test(v.title || '') || /_{3}sample/.test(v.handle || ''));
+  const productVar = variants.find((v) => v !== sampleVar) || variants[0] || {};
+  const mfrSku = String((productVar.sku || '')).replace(/-nori$/i, '');
+  const prefix = (mfrSku.match(/^([A-Za-z]+)/) || [])[1] || null;
+  return {
+    source: 'lilycolor',
+    source_feed: 'shop.lilycolor.co.jp/products.json',
+    shopify_source_id: p.id,
+    handle: p.handle,
+    title_ja: p.title,
+    title_en: null,                 // TODO: JA->EN at enrichment stage (gated)
+    mfr_sku: mfrSku || null,
+    mfr_prefix: prefix,             // LW / LL / LV / LB / EC ... -> dedup key vs dw_sku_registry
+    width: specFrom(p.body_html, '有効巾'),      // e.g. 92.5cm
+    composition: /塩化ビニル/.test(p.body_html || '') ? 'Vinyl (塩化ビニル)' : null,
+    fire_grade: (p.body_html || '').match(/F☆+|準不燃|不燃/)?.[0] || null,
+    body_html_ja: p.body_html,
+    images: (p.images || []).map((i) => i.src),
+    has_sample_variant: !!sampleVar,
+    sample_handle: sampleVar ? sampleVar.handle : null,
+    vendor_raw: p.vendor || 'Lilycolor',
+    product_type: p.product_type || null,
+    // gates (all false until the gated stage runs):
+    deduped: false, settlement_checked: false, cost_confirmed: false,
+    activation_ready: false, status: 'staged-for-new',
+  };
+}
+
+(async () => {
+  if (!fs.existsSync(OUT_DIR)) fs.mkdirSync(OUT_DIR, { recursive: true });
+  const out = fs.createWriteStream(OUT);
+  let total = 0;
+  const prefixCounts = {};
+  for (let page = 1; page <= MAX_PAGES; page++) {
+    const d = await get(`${BASE}?limit=250&page=${page}`);
+    const ps = (d && d.products) || [];
+    if (!ps.length) break;
+    for (const p of ps) {
+      const rec = normalize(p);
+      if (rec.mfr_prefix) prefixCounts[rec.mfr_prefix] = (prefixCounts[rec.mfr_prefix] || 0) + 1;
+      out.write(JSON.stringify(rec) + '\n');
+      total++;
+    }
+    process.stderr.write(`page ${page}: +${ps.length} (staged ${total})\n`);
+  }
+  out.end();
+  console.log(JSON.stringify({ staged: total, out: OUT, mfrPrefixCounts: prefixCounts }, null, 2));
+})().catch((e) => { console.error('FAILED:', e.message); process.exit(1); });
diff --git a/onboarding/sangetsu-lilycolor/staging-summary.json b/onboarding/sangetsu-lilycolor/staging-summary.json
new file mode 100644
index 00000000..efef692a
--- /dev/null
+++ b/onboarding/sangetsu-lilycolor/staging-summary.json
@@ -0,0 +1,30 @@
+{
+  "staged": 3107,
+  "out": "/Users/stevestudio2/Projects/Designer-Wallcoverings/onboarding/sangetsu-lilycolor/staging/lilycolor-staging.jsonl",
+  "mfrPrefixCounts": {
+    "LV": 1003,
+    "LB": 69,
+    "LW": 628,
+    "LWA": 12,
+    "LYT": 167,
+    "T": 1,
+    "EC": 36,
+    "LLA": 4,
+    "toilet": 2,
+    "washroom": 1,
+    "living": 4,
+    "LHP": 12,
+    "LHG": 2,
+    "midium": 1,
+    "light": 1,
+    "wood": 1,
+    "mortar": 1,
+    "XB": 32,
+    "XR": 90,
+    "LL": 882,
+    "souryou": 1,
+    "LH": 114,
+    "LAY": 24,
+    "LYH": 10
+  }
+}

← 0db1c131 appointments: export freeBusy alias for calendar busy() (fre  ·  back to Designer Wallcoverings  ·  Sangetsu: sitemap discovery (879 wallpaper pattern pages, re 5ac4e209 →