← back to Designer Wallcoverings
onboarding/sangetsu-lilycolor/sangetsu-sitemap-scraper.cjs
162 lines
#!/usr/bin/env node
/**
* Sangetsu (Goodrich) scraper — DISCOVERY + per-page PARSE (OFFLINE/STAGING).
*
* Discovery: US site blocks wp-json/sitemap, but the regional .co.th site exposes
* wp-sitemap-posts-product-1.xml (~879 pattern pages, 623 wallpaper). The Sangetsu/Goodrich
* catalog is shared across regions, so this is a valid seed for the global product set.
*
* Per-page parse: each PATTERN page is server-rendered WooCommerce (NO JS render needed —
* just send Accept-Encoding and decompress). Colorway SKUs live in <option> values, full-res
* images at /wp-content/uploads/YYYY/MM/<SKU>.jpg, specs + overview in static HTML, and the
* full WooCommerce variation set in the data-product_variations JSON blob.
*
* HARD: writes ONLY to local staging. No Shopify, no dw_unified, no publish.
* Usage:
* node sangetsu-sitemap-scraper.cjs discover # enumerate + filter wallpaper URLs
* node sangetsu-sitemap-scraper.cjs parse <url> # parse one pattern page -> JSON (stdout)
* node sangetsu-sitemap-scraper.cjs stage [limit] # parse N pattern pages -> staging JSONL
*/
const https = require('https');
const zlib = require('zlib');
const fs = require('fs');
const path = require('path');
const SITEMAP = 'https://www.sangetsu-goodrich.co.th/wp-sitemap-posts-product-1.xml';
const OUT_DIR = path.join(__dirname, 'staging');
const URLS_OUT = path.join(OUT_DIR, 'sangetsu-wallpaper-urls.txt');
const STAGE_OUT = path.join(OUT_DIR, 'sangetsu-staging.jsonl');
// fetch with gzip/br/deflate decompression (Node https does NOT auto-decode)
function get(url) {
return new Promise((resolve, reject) => {
https.get(url, {
headers: {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) AppleWebKit/537.36 Chrome/120 Safari/537.36',
'Accept-Encoding': 'gzip, deflate, br',
},
}, (res) => {
const chunks = [];
res.on('data', (c) => chunks.push(c));
res.on('end', () => {
let buf = Buffer.concat(chunks);
const enc = (res.headers['content-encoding'] || '').toLowerCase();
try {
if (enc === 'gzip') buf = zlib.gunzipSync(buf);
else if (enc === 'br') buf = zlib.brotliDecompressSync(buf);
else if (enc === 'deflate') buf = zlib.inflateSync(buf);
} catch (e) { /* fall through with raw */ }
resolve(buf.toString('utf8'));
});
}).on('error', reject);
});
}
const decodeEntities = (s) => (s || '')
.replace(/"/g, '"').replace(/–/g, '–').replace(/’/g, "'")
.replace(/&/g, '&').replace(/’/g, "'").replace(/</g, '<').replace(/>/g, '>');
const stripTags = (s) => (s || '').replace(/<[^>]+>/g, '').replace(/\s+/g, ' ').trim();
function parsePage(html, url) {
const name = stripTags((html.match(/<h1[^>]*class="product_title[^"]*"[^>]*>([\s\S]*?)<\/h1>/i) || [])[1])
.replace(/\s+Wallpapers?$/i, '').trim() || null;
// SKU -> full-res image (strip -WxH thumbnails). Works for AZ#####, Y#####, numeric 384500, G0167.
const imgMap = {};
for (const m of html.matchAll(/(https:\/\/[^"' ]*\/uploads\/[0-9]{4}\/[0-9]{2}\/([A-Za-z]{0,4}[0-9]{3,6}))(?:-\d+x\d+)?\.jpe?g/gi)) {
imgMap[m[2].toUpperCase()] = m[1] + '.jpg';
}
// colorway SKUs = union of <option> values + image-filename SKUs (covers dropdown AND swatch pages)
const fromOptions = [...html.matchAll(/<option[^>]*>\s*([A-Za-z]{0,4}[0-9]{4,6})\s*</gi)].map((m) => m[1].toUpperCase());
const skus = [...new Set([...fromOptions, ...Object.keys(imgMap)])].filter((s) => /[0-9]{4,}/.test(s)).sort();
// overview: short-description (percent- or entity-encoded in places) -> best-effort plain text
let overview = stripTags((html.match(/woocommerce-product-details__short-description[^>]*>([\s\S]*?)<\/div>/i) || [])[1]);
if (!overview) {
const enc = (html.match(/Mimicking[\s\S]{0,400}?(?:%2E|\.)/) || [])[0];
if (enc) { try { overview = decodeURIComponent(enc.replace(/\+/g, ' ')); } catch (_) {} }
}
overview = stripTags(decodeEntities(overview || '')) || null;
// specs: pull common wallcovering spec tokens from static text
const text = stripTags(html);
const spec = {
width: (() => {
const m = text.match(/Width[:\s]*(\d{2}(?:\.\d)?)\s*"/i) || text.match(/\b(\d{2}(?:\.\d)?)"\s*(?:untrimmed|Type\s+I|wide)/i);
return m ? `${m[1]}"` : null;
})(),
type: (text.match(/Type\s+I{1,3}/i) || [])[0] || null,
weight: (text.match(/\b\d{1,2}\s*oz\b/i) || [])[0] || null,
// stop the backing capture at the next Capitalized label (avoids "OsnaburgWidth")
backing: (text.match(/Backing[:\s]*([A-Z][a-z]+(?:[-\s][a-z]+)?)/) || [])[1] || null,
is_grasscloth: /grasscloth/i.test(text),
is_metallic: /metallic|shimmer/i.test(text),
};
return {
source: 'sangetsu',
source_url: url,
pattern: name,
overview,
spec,
colorway_skus: skus,
colorway_count: skus.length,
sku_images: imgMap,
mfr_prefix: (skus[0] || '').match(/^([A-Z]+)/)?.[1] || null, // dedup key
settlement_checked: false, deduped: false, cost_confirmed: false,
activation_ready: false, status: 'staged-for-new',
};
}
async function discover() {
const xml = await get(SITEMAP);
const all = [...xml.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1]);
const wp = all.filter((u) => /\/products\/wallpaper-wallcovering\//.test(u));
if (!fs.existsSync(OUT_DIR)) fs.mkdirSync(OUT_DIR, { recursive: true });
fs.writeFileSync(URLS_OUT, wp.join('\n') + '\n');
console.log(JSON.stringify({ totalProductPages: all.length, wallpaperPages: wp.length, out: URLS_OUT }, null, 2));
return wp;
}
(async () => {
const cmd = process.argv[2] || 'discover';
if (cmd === 'discover') return discover();
if (cmd === 'parse') {
const url = process.argv[3];
if (!url) { console.error('parse needs <url>'); process.exit(1); }
const html = await get(url);
console.log(JSON.stringify(parsePage(html, url), null, 2));
return;
}
if (cmd === 'stage') {
const limit = parseInt(process.argv[3] || '50', 10);
const wp = fs.existsSync(URLS_OUT) ? fs.readFileSync(URLS_OUT, 'utf8').trim().split('\n') : await discover();
if (!fs.existsSync(OUT_DIR)) fs.mkdirSync(OUT_DIR, { recursive: true });
const out = fs.createWriteStream(STAGE_OUT);
let n = 0, skuTotal = 0;
for (const url of wp.slice(0, limit)) {
try {
const rec = parsePage(await get(url), url);
out.write(JSON.stringify(rec) + '\n'); n++; skuTotal += rec.colorway_count;
process.stderr.write(`[${n}] ${rec.pattern} (${rec.colorway_count} skus)\n`);
} catch (e) { process.stderr.write(`SKIP ${url}: ${e.message}\n`); }
}
out.end();
console.log(JSON.stringify({ patternsStaged: n, colorwaySkus: skuTotal, out: STAGE_OUT }, null, 2));
return;
}
if (cmd === 'hunt') {
// hunt <urls-file> : parse each URL, print ONE compact JSON line (pattern/width/grasscloth/skus)
const file = process.argv[3];
const urls = fs.readFileSync(file, 'utf8').trim().split('\n').filter(Boolean);
for (const url of urls) {
try {
const r = parsePage(await get(url), url);
console.log(JSON.stringify({ pattern: r.pattern, width: r.spec.width, grasscloth: r.spec.is_grasscloth, metallic: r.spec.is_metallic, skus: r.colorway_count, url }));
} catch (e) { console.log(JSON.stringify({ pattern: null, url, err: e.message })); }
}
return;
}
console.error('unknown command:', cmd); process.exit(1);
})().catch((e) => { console.error('FAILED:', e.message); process.exit(1); });