← back to Atomic50 Onboard
atomic50: feed-first scraper CLI + wire cadence re-scrape
138c9031e71d7ed6d4dae6594589391e60597119 · 2026-07-14 09:48:20 -0700 · Steve Abrams
- scraper/scrape-atomic50.mjs: $0 plain-fetch (sitemap + ?format=json, NO headless
despite skill note). URL-based upsert (slug->mfr_sku unreliable: at50g34 vs AT50-G34).
Non-destructive: preserves shopify_product_id/on_shopify/product_type/dw_sku/ai_*.
Disco is 404-CONFIRMED + scoped to individual pages (accessory /molding-accessories
line-items excluded) + 70% floor guard. Idempotent (0 inserts on unchanged site).
- cadence: RESCRAPE_CMD wired -> re-scrape now runs (was skipped).
- First live run flagged AT50-32 discontinued (404 on vendor; draft on Shopify so
cadence stays silent, correctly).
Files touched
M cadence/atomic50-cadence.mjsA scraper/scrape-atomic50.mjs
Diff
commit 138c9031e71d7ed6d4dae6594589391e60597119
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Tue Jul 14 09:48:20 2026 -0700
atomic50: feed-first scraper CLI + wire cadence re-scrape
- scraper/scrape-atomic50.mjs: $0 plain-fetch (sitemap + ?format=json, NO headless
despite skill note). URL-based upsert (slug->mfr_sku unreliable: at50g34 vs AT50-G34).
Non-destructive: preserves shopify_product_id/on_shopify/product_type/dw_sku/ai_*.
Disco is 404-CONFIRMED + scoped to individual pages (accessory /molding-accessories
line-items excluded) + 70% floor guard. Idempotent (0 inserts on unchanged site).
- cadence: RESCRAPE_CMD wired -> re-scrape now runs (was skipped).
- First live run flagged AT50-32 discontinued (404 on vendor; draft on Shopify so
cadence stays silent, correctly).
---
cadence/atomic50-cadence.mjs | 6 +-
scraper/scrape-atomic50.mjs | 159 +++++++++++++++++++++++++++++++++++++++++++
2 files changed, 162 insertions(+), 3 deletions(-)
diff --git a/cadence/atomic50-cadence.mjs b/cadence/atomic50-cadence.mjs
index fcd778e..703d714 100644
--- a/cadence/atomic50-cadence.mjs
+++ b/cadence/atomic50-cadence.mjs
@@ -32,9 +32,9 @@ const DATA = ROOT + '/cadence/data';
const SHOP = 'designer-laboratory-sandbox.myshopify.com', VER = '2024-10';
const URL = `https://${SHOP}/admin/api/${VER}/graphql.json`;
const VENDOR = 'Atomic 50 Ceilings';
-// Set this to a runnable scraper entrypoint (e.g. `${process.env.HOME}/.../scrape.mjs`)
-// to enable best-effort re-scrape. Leave null to skip re-scrape (current state).
-const RESCRAPE_CMD = null;
+// Runnable feed-first scraper entrypoint (sitemap + ?format=json, $0 plain-fetch).
+// Wired 2026-07-14. Best-effort: a failure here does not crash the cadence.
+const RESCRAPE_CMD = `/opt/homebrew/bin/node ${process.env.HOME}/Projects/atomic50-onboard/scraper/scrape-atomic50.mjs`;
// ---- token (parse .env directly; do NOT shell-source — unquoted-value bug) ----
function readToken() {
diff --git a/scraper/scrape-atomic50.mjs b/scraper/scrape-atomic50.mjs
new file mode 100644
index 0000000..870441c
--- /dev/null
+++ b/scraper/scrape-atomic50.mjs
@@ -0,0 +1,159 @@
+#!/usr/bin/env node
+/**
+ * scrape-atomic50.mjs — feed-first Atomic 50 Ceilings refresh.
+ *
+ * Atomic 50 is a Squarespace site. Despite the skill note ("requires headless
+ * browser"), it exposes the feed-first crack: sitemap.xml lists every INDIVIDUAL
+ * product URL, and each returns full structured JSON at `<url>?format=json`.
+ * $0 plain-fetch — NO Puppeteer, NO captcha, NO login.
+ *
+ * CATALOG SHAPE (important): atomic50_catalog AT50-* rows are TWO kinds:
+ * 1. individual product pages (/at50-24, /at50-c1, /at50-m11, /at50g34) — crawlable.
+ * 2. accessory/molding line-items (SP1..SP9, NAIL, SNIP, CLIP, TCRO, APAD, ...) that
+ * ALL share ONE page (/molding-accessories) — NOT individually crawlable.
+ * So matching + disco are scoped to INDIVIDUAL pages, matched BY product_url (slug->mfr_sku
+ * is unreliable: /at50g34 vs catalog AT50-G34). Accessory-page rows are never disco-touched.
+ *
+ * Behaviour (idempotent, NON-DESTRUCTIVE):
+ * - UPSERT by product_url: existing individual page -> refresh all_images/image_url/
+ * last_scraped, discontinued=false; PRESERVE shopify_product_id, on_shopify,
+ * product_type, dw_sku, ai_*. New individual page -> INSERT with next DWJT-6000xx.
+ * - DISCO: an individual-page catalog row NOT seen in the sitemap is re-checked with a
+ * live GET; only a real 404/410 sets discontinued=true. Never deletes. Guarded by a
+ * 70%-of-individual-count floor (transient-hiccup guard). Set env DISCO=0 to skip.
+ *
+ * Cost: $0 (local PG socket + plain HTTPS). Never writes Shopify.
+ */
+import { createRequire } from 'node:module';
+const require = createRequire(import.meta.url);
+const { Client } = require('pg');
+
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36';
+const SITEMAP = 'https://www.atomic50ceilings.com/sitemap.xml';
+const IMG_CAP = 40;
+const DISCO_ON = process.env.DISCO !== '0';
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+const normUrl = (u) => String(u || '').trim().replace(/\/+$/, '').replace(/\?.*$/, '').toLowerCase();
+
+async function fetchText(url, tries = 4) {
+ for (let a = 0; a < tries; a++) {
+ try { const r = await fetch(url, { headers: { 'User-Agent': UA } }); if (r.ok) return await r.text(); }
+ catch (_) { /* retry */ }
+ await sleep(800 * (a + 1));
+ }
+ throw new Error('fetch failed: ' + url);
+}
+async function statusOf(url) {
+ try { const r = await fetch(url, { headers: { 'User-Agent': UA } }); return r.status; }
+ catch (_) { return 0; }
+}
+
+function extractImages(rawJson) {
+ const out = [], seen = new Set();
+ const re = /https:(?:\\\/|\/){2}images\.squarespace-cdn\.com(?:\\\/|\/)[^"'\\)\s]+/g;
+ let m;
+ while ((m = re.exec(rawJson))) {
+ let u = m[0].replace(/\\\//g, '/').replace(/\?.*$/, '') + '?format=2500w';
+ const key = u.replace(/\?format=2500w$/, '');
+ if (u.includes('/content/v1/') && !seen.has(key)) { seen.add(key); out.push(u); }
+ if (out.length >= IMG_CAP) break;
+ }
+ return out;
+}
+function extractTitle(rawJson, slug) {
+ const m = rawJson.match(/"(?:title|websiteTitle|seoTitle)"\s*:\s*"([^"]{2,120})"/);
+ if (m && m[1] && !/atomic\s*50/i.test(m[1])) return m[1];
+ return slug.toUpperCase();
+}
+
+async function main() {
+ const t0 = Date.now();
+ const xml = await fetchText(SITEMAP);
+ const locs = [...xml.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1]);
+ const products = [];
+ const seenUrl = new Set();
+ for (const u of locs) {
+ if (!/\/at50[-a-z0-9]+$/i.test(u)) continue; // individual product pages (incl. at50g34)
+ const nu = normUrl(u);
+ if (seenUrl.has(nu)) continue; seenUrl.add(nu);
+ products.push({ url: u, nurl: nu, slug: u.split('/').pop() });
+ }
+ console.log(`sitemap: ${locs.length} locs -> ${products.length} individual at50 product pages`);
+
+ const scraped = [];
+ for (const p of products) {
+ try {
+ const raw = await fetchText(p.url + '?format=json');
+ scraped.push({ ...p, images: extractImages(raw), title: extractTitle(raw, p.slug) });
+ } catch (e) { console.log(` ! skip ${p.slug}: ${e.message}`); }
+ await sleep(150);
+ }
+ const scrapedByUrl = new Map(scraped.map((s) => [s.nurl, s]));
+
+ const db = new Client({ host: '/tmp', database: 'dw_unified' });
+ await db.connect();
+
+ const existing = (await db.query(
+ `SELECT dw_sku, mfr_sku, product_url FROM atomic50_catalog WHERE mfr_sku IS NOT NULL`
+ )).rows;
+ // individual-page catalog rows = those whose product_url is an /at50* page (not /molding-accessories)
+ const indivRows = existing.filter((r) => /\/at50[-a-z0-9]+$/i.test(r.product_url || ''));
+ const haveByUrl = new Map(indivRows.map((r) => [normUrl(r.product_url), r]));
+
+ const maxRow = await db.query(
+ `SELECT MAX((regexp_replace(dw_sku,'\\D','','g'))::bigint) AS n FROM atomic50_catalog WHERE dw_sku ~ '^DWJT-'`
+ );
+ let nextNum = (Number(maxRow.rows[0].n) || 600059) + 1;
+
+ let inserted = 0, updated = 0;
+ for (const s of scraped) {
+ const imgsJson = JSON.stringify(s.images);
+ const firstImg = s.images[0] || null;
+ const row = haveByUrl.get(s.nurl);
+ if (row) {
+ await db.query(
+ `UPDATE atomic50_catalog SET all_images=$2::jsonb, image_url=COALESCE($3,image_url),
+ discontinued=false, last_scraped=now(), updated_at=now() WHERE dw_sku=$1`,
+ [row.dw_sku, imgsJson, firstImg]
+ );
+ updated++;
+ } else {
+ const dw = `DWJT-${nextNum++}`;
+ const mfr = s.slug.toUpperCase();
+ await db.query(
+ `INSERT INTO atomic50_catalog
+ (mfr_sku,dw_sku,pattern_name,product_type,all_images,image_url,product_url,
+ discontinued,on_shopify,last_scraped,created_at,updated_at)
+ VALUES ($1,$2,$3,'Tin Ceiling Tile',$4::jsonb,$5,$6,false,false,now(),now(),now())`,
+ [mfr, dw, s.title, imgsJson, firstImg, s.url]
+ );
+ inserted++; console.log(` + NEW ${mfr} -> ${dw} (${s.url})`);
+ }
+ }
+
+ // DISCO: individual-page rows absent from sitemap -> confirm with live 404 before flagging
+ let discoFlagged = 0, missing = indivRows.filter((r) => !scrapedByUrl.has(normUrl(r.product_url)));
+ const floorOk = scraped.length >= Math.floor(indivRows.length * 0.7);
+ if (!DISCO_ON) {
+ console.log(` · disco DISABLED (DISCO=0); ${missing.length} individual rows not in sitemap this pass`);
+ } else if (!floorOk) {
+ console.log(` ! disco SKIPPED — scraped ${scraped.length} < 70% of ${indivRows.length} individual rows (hiccup guard)`);
+ } else {
+ for (const r of missing) {
+ const st = await statusOf(r.product_url);
+ if (st === 404 || st === 410) {
+ await db.query(`UPDATE atomic50_catalog SET discontinued=true, updated_at=now()
+ WHERE dw_sku=$1 AND discontinued IS DISTINCT FROM true`, [r.dw_sku]);
+ discoFlagged++; console.log(` - DISCO ${r.mfr_sku} (${r.product_url} -> ${st})`);
+ } else {
+ console.log(` · ${r.mfr_sku} not in sitemap but url still ${st} — left as-is`);
+ }
+ await sleep(120);
+ }
+ }
+
+ await db.end();
+ const secs = ((Date.now() - t0) / 1000).toFixed(1);
+ console.log(`\nDONE (${secs}s, $0 local): scraped=${scraped.length} inserted=${inserted} updated=${updated} disco_flagged=${discoFlagged}`);
+}
+main().catch((e) => { console.error('FATAL', e); process.exit(1); });
← a321e2f Add monthly propose-only Atomic 50 cadence (15th @ 08:00 PT,
·
back to Atomic50 Onboard
·
auto-save: 2026-07-14T10:00:24 (1 files) — cadence/data/late 193c5c9 →