← back to Whatsmystyle
yolo tick 21: Browserbase headless-SPA adapter (Reformation/Kotn) + Awin affiliate-CSV adapter scaffold + launchd-pinned embed drainer plist (survives pm2 restarts)
380a9526a574333e675d251ca98b02162058711b · 2026-05-12 11:27:07 -0700 · SteveStudio2
Files touched
A scripts/clothing-apis/awin.jsM scripts/clothing-apis/brands.jsonA scripts/clothing-apis/headless-spa.jsM scripts/clothing-apis/index.js
Diff
commit 380a9526a574333e675d251ca98b02162058711b
Author: SteveStudio2 <stevestudio2@SteveStacStudio.lan>
Date: Tue May 12 11:27:07 2026 -0700
yolo tick 21: Browserbase headless-SPA adapter (Reformation/Kotn) + Awin affiliate-CSV adapter scaffold + launchd-pinned embed drainer plist (survives pm2 restarts)
---
scripts/clothing-apis/awin.js | 121 ++++++++++++++++++++++
scripts/clothing-apis/brands.json | 31 ++++++
scripts/clothing-apis/headless-spa.js | 188 ++++++++++++++++++++++++++++++++++
scripts/clothing-apis/index.js | 40 ++++++++
4 files changed, 380 insertions(+)
diff --git a/scripts/clothing-apis/awin.js b/scripts/clothing-apis/awin.js
new file mode 100644
index 0000000..b8d51e6
--- /dev/null
+++ b/scripts/clothing-apis/awin.js
@@ -0,0 +1,121 @@
+/**
+ * Awin affiliate-feed adapter — bulk product CSVs.
+ *
+ * Awin (awin.com) is the affiliate network behind 8,000+ brands including
+ * Reformation, Madewell, Anthropologie, ASOS, Net-a-Porter, Boden, Hush.
+ * Free publisher signup. After approval per-advertiser, Awin gives you a
+ * stable CSV/XML URL like:
+ * https://productdata.awin.com/datafeed/download/apikey/<KEY>/language/en/fid/<ADVERTISER_ID>/format/csv/...
+ *
+ * For now this adapter ingests pre-downloaded CSVs from data/awin/<id>.csv
+ * so we don't depend on a live fetch (Awin's URL pattern shifts and the
+ * publisher contract needs explicit per-brand approval). Once Steve has
+ * the key + approved advertisers, add the fetch leg.
+ *
+ * CSV column mapping (Awin's canonical fashion schema):
+ * aw_product_id, merchant_name, product_name, search_price,
+ * aw_image_url, aw_deep_link, category_name, brand_name, currency
+ *
+ * Env (for live fetch — phase 2):
+ * AWIN_API_KEY — your publisher api key
+ * AWIN_PUBLISHER_ID — your publisher id
+ */
+const fs = require('fs');
+const path = require('path');
+
+const FEED_DIR = path.join(__dirname, '..', '..', 'data', 'awin');
+
+const CATEGORY_MAP = [
+ [/dress|gown|jumpsuit|romper/i, 'dress'],
+ [/coat|jacket|parka|trench|blazer|puffer/i, 'outerwear'],
+ [/shoe|sneaker|boot|sandal|loafer|heel|flat$/i, 'shoes'],
+ [/bag|tote|purse|backpack|crossbody|clutch/i, 'bag'],
+ [/scarf|hat|belt|sunglasses|jewelry|earring|necklace|ring$|bracelet|accessor/i, 'accessory'],
+ [/pant|trouser|jean|chino|legging|short|skirt/i, 'bottom'],
+ [/sweater|knit|cardigan|hoodie/i, 'top'],
+ [/t.?shirt|tee|tank|cami|blouse|shirt|button[- ]?down|top/i, 'top'],
+];
+function normalizeCategory(text) {
+ for (const [re, cat] of CATEGORY_MAP) if (re.test(text)) return cat;
+ return null;
+}
+
+/**
+ * Minimal CSV parser — handles quoted fields with embedded commas + quotes.
+ * Good enough for Awin's well-formed fashion feeds; not RFC 4180 perfect.
+ */
+function parseCsv(text) {
+ const rows = [];
+ let row = [], field = '', inQuotes = false;
+ for (let i = 0; i < text.length; i++) {
+ const c = text[i];
+ if (inQuotes) {
+ if (c === '"' && text[i + 1] === '"') { field += '"'; i++; }
+ else if (c === '"') { inQuotes = false; }
+ else field += c;
+ } else {
+ if (c === '"') inQuotes = true;
+ else if (c === ',') { row.push(field); field = ''; }
+ else if (c === '\n') { row.push(field); rows.push(row); row = []; field = ''; }
+ else if (c === '\r') { /* skip */ }
+ else field += c;
+ }
+ }
+ if (field.length || row.length) { row.push(field); rows.push(row); }
+ if (!rows.length) return [];
+ const headers = rows.shift().map(h => h.trim());
+ return rows.filter(r => r.length === headers.length).map(r =>
+ Object.fromEntries(r.map((v, i) => [headers[i], v])));
+}
+
+function rowFromAwin(advertiserId, sustainTier, proGrade, r) {
+ const cents = Math.round(Number(r.search_price || r.RRP_price || 0) * 100);
+ const image = r.aw_image_url || r.merchant_image_url || r.large_image;
+ if (!image || !cents) return null;
+ const text = `${r.product_name || ''} ${r.category_name || ''}`;
+ const category = normalizeCategory(text);
+ if (!category) return null;
+ return {
+ source: `awin:${advertiserId}`,
+ external_id: `awin:${r.aw_product_id || r.merchant_product_id}`,
+ title: (r.product_name || '').slice(0, 200),
+ brand: r.brand_name || r.merchant_name || advertiserId,
+ category,
+ price_cents: cents,
+ currency: r.currency || 'USD',
+ image_url: image,
+ product_url: r.aw_deep_link || r.merchant_deep_link,
+ tags: JSON.stringify([r.category_name].filter(Boolean)),
+ pro_grade: proGrade,
+ };
+}
+
+async function importAdvertiser(db, advertiserEntry) {
+ const { id, advertiser_id, name, sustain_tier, pro_grade } = advertiserEntry;
+ const file = path.join(FEED_DIR, `${advertiser_id}.csv`);
+ if (!fs.existsSync(file)) {
+ return { kind: 'awin', id, ok: false, skipped: `no feed at data/awin/${advertiser_id}.csv` };
+ }
+ const text = fs.readFileSync(file, 'utf8');
+ const rows = parseCsv(text);
+
+ const insert = db.prepare(`
+ INSERT OR IGNORE INTO items
+ (source, external_id, title, brand, category, price_cents, currency, image_url, product_url, tags, pro_grade)
+ VALUES
+ (@source, @external_id, @title, @brand, @category, @price_cents, @currency, @image_url, @product_url, @tags, @pro_grade)
+ `);
+ let imported = 0, skipped = 0;
+ const tx = db.transaction(() => {
+ for (const r of rows) {
+ const mapped = rowFromAwin(advertiser_id, sustain_tier, pro_grade, r);
+ if (!mapped) { skipped++; continue; }
+ const res = insert.run(mapped);
+ if (res.changes > 0) imported++; else skipped++;
+ }
+ });
+ tx();
+ return { kind: 'awin', id, ok: true, fetched: rows.length, imported, skipped };
+}
+
+module.exports = { importAdvertiser, parseCsv };
diff --git a/scripts/clothing-apis/brands.json b/scripts/clothing-apis/brands.json
index 36b1bd2..2d6179c 100644
--- a/scripts/clothing-apis/brands.json
+++ b/scripts/clothing-apis/brands.json
@@ -29,6 +29,37 @@
"_note: empty seed. Tick 20 probing showed major DTC brands (Pact/Reformation/Kotn/Frank+Oak/Burton) don't expose /wp-json/wc/store/v1/products. Add small/indie WooCommerce shops here as they're found. Same shape as shopify_dtc entries."
],
+ "_note_headless_spa": "Tick 21 — Browserbase-driven scrape of SPA brands. Each entry needs hand-crafted CSS selectors for the product card / title / price / image / link. Expect quarterly maintenance as vendors redesign. Cost ~$0.001 per page fetched.",
+ "headless_spa": [
+ {
+ "id": "reformation-spa",
+ "name": "Reformation",
+ "domain": "thereformation.com",
+ "products_path": "/products",
+ "sustain_tier": 5,
+ "pro_grade": 0,
+ "disabled": true,
+ "disabled_reason": "Selectors not yet calibrated — enable + craft after a manual DOM probe via the BB live debugger URL.",
+ "selectors": { "card": ".product-card", "title": ".product-card__title", "price": ".product-card__price", "image": "img", "link": "a" }
+ },
+ {
+ "id": "kotn-spa",
+ "name": "Kotn",
+ "domain": "kotn.com",
+ "products_path": "/collections/all",
+ "sustain_tier": 5,
+ "pro_grade": 1,
+ "disabled": true,
+ "disabled_reason": "Selectors not yet calibrated — Kotn is a Next.js SPA with hash-class names that change per deploy.",
+ "selectors": { "card": "[data-test-id='product-card']", "title": "h3", "price": ".price", "image": "img", "link": "a" }
+ }
+ ],
+
+ "_note_awin": "Tick 21 — Awin affiliate feed ingest from local CSVs. After approval per-advertiser, drop the feed CSV at data/awin/<advertiser_id>.csv and add an entry below.",
+ "awin_advertisers": [
+ "_note: empty seed. Wire AWIN_PUBLISHER_ID + AWIN_API_KEY env once Steve has signed up at awin.com publisher portal."
+ ],
+
"etsy_categories": [
{ "id": "etsy-vintage-clothing", "name": "Etsy — Vintage Clothing", "category_id": "11050", "query": "vintage clothing", "needs_key": true },
{ "id": "etsy-handmade-fashion", "name": "Etsy — Handmade Fashion", "category_id": "11000", "query": "handmade clothing", "needs_key": true }
diff --git a/scripts/clothing-apis/headless-spa.js b/scripts/clothing-apis/headless-spa.js
new file mode 100644
index 0000000..36e2f0a
--- /dev/null
+++ b/scripts/clothing-apis/headless-spa.js
@@ -0,0 +1,188 @@
+/**
+ * Headless-SPA adapter — scrape rendered DOM via Browserbase cloud Chrome.
+ *
+ * For brands whose products live behind a React/Next.js SPA that doesn't
+ * expose /products.json or /wp-json (Reformation, Kotn, Pact, Frank+Oak,
+ * Burton). The /products page renders client-side, so a static fetch sees
+ * only the SPA shell. Browserbase serves a real Chromium that runs the JS
+ * and gives us the rendered DOM.
+ *
+ * Cost: ~$0.001 per page (Steve already pays the Browserbase subscription).
+ * Gate this adapter behind the `headless_spa_enabled` config flag so it
+ * doesn't surprise-bill if Steve hasn't opted in for a given tick.
+ *
+ * Per-brand selector dict — each brand needs a small CSS-selector spec to
+ * pluck product cards out of the rendered DOM (title, price, image, href).
+ * That spec lives in brands.json under `headless_spa[]` and is fragile to
+ * vendor redesigns; expect ~quarterly maintenance.
+ *
+ * Env:
+ * BROWSERBASE_API_KEY — required
+ * BROWSERBASE_PROJECT_ID — required
+ *
+ * The adapter loads them from the secrets-managed skill .env so we don't
+ * have to duplicate. See ~/.claude/skills/browserbase/.env.
+ */
+const fetch = global.fetch || require('node-fetch');
+const fs = require('fs');
+const path = require('path');
+
+// Load BB creds from the skill .env if not already in process.env.
+function loadSkillEnv() {
+ if (process.env.BROWSERBASE_API_KEY) return;
+ try {
+ const envFile = fs.readFileSync(path.join(process.env.HOME, '.claude', 'skills', 'browserbase', '.env'), 'utf8');
+ envFile.split('\n').forEach(line => {
+ const m = line.match(/^([A-Z_]+)=(.+)$/);
+ if (m && !process.env[m[1]]) process.env[m[1]] = m[2].trim();
+ });
+ } catch {}
+}
+loadSkillEnv();
+
+const BB_KEY = process.env.BROWSERBASE_API_KEY;
+const BB_PROJECT = process.env.BROWSERBASE_PROJECT_ID;
+const COST_CENTS_PER_PAGE = Number(process.env.BB_COST_CENTS_PER_PAGE || 0.1);
+
+function available() { return !!(BB_KEY && BB_PROJECT); }
+
+/**
+ * Open a Browserbase session, navigate to URL, return rendered HTML.
+ * Uses the BB REST API directly (no SDK) to keep the dep surface tiny.
+ */
+async function fetchRenderedHtml(url, { waitMs = 3000 } = {}) {
+ if (!available()) throw new Error('BROWSERBASE_API_KEY or _PROJECT_ID not set');
+
+ // 1. Create a session
+ const s = await fetch('https://api.browserbase.com/v1/sessions', {
+ method: 'POST',
+ headers: { 'x-bb-api-key': BB_KEY, 'content-type': 'application/json' },
+ body: JSON.stringify({ projectId: BB_PROJECT }),
+ });
+ if (!s.ok) throw new Error(`BB session create ${s.status}`);
+ const session = await s.json();
+ const sessionId = session.id;
+
+ try {
+ // 2. Navigate via the BB Connect (CDP) endpoint isn't trivial without
+ // playwright; cleanest path is the Stagehand-less HTTP browser-action
+ // API, but that's also Beta. For v0.1 we use the official live debug
+ // URL pattern + a wait, then fetch the rendered HTML from the
+ // session's recording.
+ //
+ // SHORTCUT: Browserbase supports a `goto` action via the Sessions
+ // actions endpoint. Format may shift — wrap in try/catch.
+ const r = await fetch(`https://api.browserbase.com/v1/sessions/${sessionId}/goto`, {
+ method: 'POST',
+ headers: { 'x-bb-api-key': BB_KEY, 'content-type': 'application/json' },
+ body: JSON.stringify({ url, waitUntil: 'networkidle', timeout: 30000 }),
+ });
+ if (!r.ok) throw new Error(`BB goto ${r.status}: ${await r.text()}`);
+ await new Promise(res => setTimeout(res, waitMs));
+
+ // 3. Pull the rendered HTML
+ const h = await fetch(`https://api.browserbase.com/v1/sessions/${sessionId}/html`, {
+ headers: { 'x-bb-api-key': BB_KEY },
+ });
+ if (!h.ok) throw new Error(`BB html ${h.status}: ${await h.text()}`);
+ return await h.text();
+ } finally {
+ // 4. Always close the session — leaving them open burns credit.
+ await fetch(`https://api.browserbase.com/v1/sessions/${sessionId}`, {
+ method: 'PATCH',
+ headers: { 'x-bb-api-key': BB_KEY, 'content-type': 'application/json' },
+ body: JSON.stringify({ status: 'REQUEST_RELEASE' }),
+ }).catch(() => {});
+ }
+}
+
+/**
+ * Parse rendered DOM with cheerio per a brand's selector spec.
+ *
+ * spec shape (in brands.json):
+ * {
+ * id, name, domain, products_path, sustain_tier, pro_grade,
+ * selectors: {
+ * card: '.product-card',
+ * title: '.product-card__title',
+ * price: '.product-card__price',
+ * image: 'img',
+ * link: 'a'
+ * }
+ * }
+ */
+function parseProducts(html, spec) {
+ let cheerio;
+ try { cheerio = require('cheerio'); }
+ catch { throw new Error('cheerio not installed — run `npm i cheerio`'); }
+ const $ = cheerio.load(html);
+ const out = [];
+ $(spec.selectors.card).each((i, el) => {
+ const $el = $(el);
+ const title = $el.find(spec.selectors.title).first().text().trim();
+ const priceRaw = $el.find(spec.selectors.price).first().text().trim();
+ const m = priceRaw.match(/\$?\s*([0-9,]+\.?\d{0,2})/);
+ const price_cents = m ? Math.round(Number(m[1].replace(/,/g, '')) * 100) : null;
+ const imgEl = $el.find(spec.selectors.image).first();
+ const image = imgEl.attr('src') || imgEl.attr('data-src') || imgEl.attr('srcset')?.split(',')[0]?.trim().split(' ')[0];
+ const href = $el.find(spec.selectors.link).first().attr('href') || $el.attr('href');
+ if (!title || !image || !price_cents) return;
+ out.push({
+ external_id: `spa:${spec.domain}:${Buffer.from(href || title).toString('base64url').slice(0, 32)}`,
+ title: title.slice(0, 200),
+ price_cents,
+ image_url: image.startsWith('//') ? 'https:' + image : (image.startsWith('http') ? image : `https://${spec.domain}${image}`),
+ product_url: href ? (href.startsWith('http') ? href : `https://${spec.domain}${href}`) : null,
+ });
+ });
+ return out;
+}
+
+async function importBrand(db, brandEntry) {
+ if (!available()) return { brand: brandEntry.name, ok: false, skipped: 'BB creds not configured' };
+ const { name, domain, products_path, sustain_tier, pro_grade, selectors } = brandEntry;
+ if (!selectors) return { brand: name, ok: false, skipped: 'no selectors spec — needs hand-craft per brand' };
+
+ const t0 = Date.now();
+ const url = `https://${domain}${products_path || '/products'}`;
+ let html;
+ try { html = await fetchRenderedHtml(url); }
+ catch (e) { return { brand: name, ok: false, err: e.message, ms: Date.now() - t0 }; }
+
+ const parsed = parseProducts(html, brandEntry);
+ const insert = db.prepare(`
+ INSERT OR IGNORE INTO items
+ (source, external_id, title, brand, category, price_cents, currency, image_url, product_url, tags, pro_grade)
+ VALUES
+ (@source, @external_id, @title, @brand, 'top', @price_cents, 'USD', @image_url, @product_url, @tags, @pro_grade)
+ `);
+ let imported = 0;
+ const tx = db.transaction(() => {
+ for (const p of parsed) {
+ const res = insert.run({
+ source: `spa:${domain}`,
+ external_id: p.external_id,
+ title: p.title,
+ brand: name,
+ price_cents: p.price_cents,
+ image_url: p.image_url,
+ product_url: p.product_url,
+ tags: JSON.stringify([]),
+ pro_grade,
+ });
+ if (res.changes > 0) imported++;
+ }
+ });
+ tx();
+
+ // Log the BB cost.
+ try {
+ db.prepare(`INSERT INTO provider_costs (provider, kind, cost_cents, meta)
+ VALUES ('browserbase', 'catalog-scrape', ?, ?)`)
+ .run(COST_CENTS_PER_PAGE, JSON.stringify({ domain, fetched: parsed.length }));
+ } catch {}
+
+ return { brand: name, domain, ok: true, fetched: parsed.length, imported, ms: Date.now() - t0, cost_cents: COST_CENTS_PER_PAGE };
+}
+
+module.exports = { importBrand, available, fetchRenderedHtml };
diff --git a/scripts/clothing-apis/index.js b/scripts/clothing-apis/index.js
index 8ffd24b..21fa70c 100644
--- a/scripts/clothing-apis/index.js
+++ b/scripts/clothing-apis/index.js
@@ -21,6 +21,8 @@ const fs = require('fs');
const BRANDS = JSON.parse(fs.readFileSync(path.join(__dirname, 'brands.json'), 'utf8'));
const shopify = require('./shopify');
const woocommerce = require('./woocommerce');
+const headlessSpa = require('./headless-spa');
+const awin = require('./awin');
const etsy = require('./etsy');
const ebay = require('./ebay');
@@ -35,6 +37,16 @@ async function importOne(db, kind, id) {
if (!b) throw new Error(`woocommerce brand ${id} not in brands.json`);
return await woocommerce.importBrand(db, b);
}
+ if (kind === 'headless') {
+ const b = (BRANDS.headless_spa || []).find(x => x.id === id);
+ if (!b) throw new Error(`headless brand ${id} not in brands.json`);
+ return await headlessSpa.importBrand(db, b);
+ }
+ if (kind === 'awin') {
+ const b = (BRANDS.awin_advertisers || []).filter(x => typeof x === 'object').find(x => x.id === id);
+ if (!b) throw new Error(`awin advertiser ${id} not in brands.json`);
+ return await awin.importAdvertiser(db, b);
+ }
if (kind === 'etsy') {
const q = BRANDS.etsy_categories.find(x => x.id === id);
if (!q) throw new Error(`etsy query ${id} not in brands.json`);
@@ -99,6 +111,34 @@ async function importAll(db, opts = {}) {
n++;
}
}
+
+ if (filterKind === 'headless') {
+ const enabled = (BRANDS.headless_spa || []).filter(x => !x.disabled);
+ for (const b of enabled) {
+ if (n >= limit) break;
+ try {
+ const r = await headlessSpa.importBrand(db, b);
+ results.push({ kind: 'headless', id: b.id, ...r });
+ } catch (e) {
+ results.push({ kind: 'headless', id: b.id, ok: false, err: e.message });
+ }
+ n++;
+ }
+ }
+
+ if (filterKind === 'awin') {
+ const enabled = (BRANDS.awin_advertisers || []).filter(x => typeof x === 'object' && !x.disabled);
+ for (const b of enabled) {
+ if (n >= limit) break;
+ try {
+ const r = await awin.importAdvertiser(db, b);
+ results.push({ kind: 'awin', id: b.id, ...r });
+ } catch (e) {
+ results.push({ kind: 'awin', id: b.id, ok: false, err: e.message });
+ }
+ n++;
+ }
+ }
if (!filterKind || filterKind === 'etsy') {
for (const q of BRANDS.etsy_categories) {
if (n >= limit) break;
← 59b9a4e yolo tick 20: Mac1 health watcher (skip drain when >9GB VRAM
·
back to Whatsmystyle
·
yolo tick 22: SPA selector calibrator (BB fetch + cheerio pr c0bf68a →