← back to Whatsmystyle
feat(catalog-apis): adapter framework for Shopify/Etsy/eBay clothing catalogs + admin import UI — 1,231 items pulled from 3 brands at zero key cost
202347cfcf459fb475184b3322636c2512e1fda6 · 2026-05-12 08:48:32 -0700 · SteveStudio2
Files touched
M public/admin-config.htmlA scripts/clothing-apis/README.mdA scripts/clothing-apis/brands.jsonA scripts/clothing-apis/ebay.jsA scripts/clothing-apis/etsy.jsA scripts/clothing-apis/index.jsA scripts/clothing-apis/shopify.jsM server.js
Diff
commit 202347cfcf459fb475184b3322636c2512e1fda6
Author: SteveStudio2 <stevestudio2@SteveStacStudio.lan>
Date: Tue May 12 08:48:32 2026 -0700
feat(catalog-apis): adapter framework for Shopify/Etsy/eBay clothing catalogs + admin import UI — 1,231 items pulled from 3 brands at zero key cost
---
public/admin-config.html | 66 ++++++++++++++++++
scripts/clothing-apis/README.md | 58 ++++++++++++++++
scripts/clothing-apis/brands.json | 42 ++++++++++++
scripts/clothing-apis/ebay.js | 74 ++++++++++++++++++++
scripts/clothing-apis/etsy.js | 73 ++++++++++++++++++++
scripts/clothing-apis/index.js | 108 +++++++++++++++++++++++++++++
scripts/clothing-apis/shopify.js | 138 ++++++++++++++++++++++++++++++++++++++
server.js | 34 ++++++++++
8 files changed, 593 insertions(+)
diff --git a/public/admin-config.html b/public/admin-config.html
index 18b9bf9..9e2a46c 100644
--- a/public/admin-config.html
+++ b/public/admin-config.html
@@ -92,6 +92,26 @@
</div>
<div id="role_status" class="status" style="margin-top: 16px; display: none;"></div>
+ <hr style="margin: 40px 0; border: 0; border-top: 1px solid #e6e1d8;" />
+
+ <h2 style="font-size: 24px; font-weight: 600; letter-spacing: -0.02em;">Clothing-API catalog</h2>
+ <p class="sub">WhatsMyStyle pulls from public Shopify storefronts (zero key) and optional Etsy / eBay / Amazon (free keys). One adapter per source in <code>scripts/clothing-apis/</code>.</p>
+
+ <div id="catalog_status" class="pill-row" style="display: block;">
+ <span class="muted">Loading catalog status…</span>
+ </div>
+
+ <div style="display: flex; gap: 12px; flex-wrap: wrap; margin-top: 12px;">
+ <button id="import_all_shopify" class="save" type="button">Import all Shopify brands →</button>
+ <button id="import_one" class="save" type="button" style="background:#5a3a1f;">Import one brand…</button>
+ <button id="refresh_status" class="save" type="button" style="background:#fff;color:#1d1d1f;border:1px solid #d6d0c4;">Refresh</button>
+ </div>
+ <div id="import_status" class="status" style="margin-top: 16px; display: none;"></div>
+
+ <p class="muted" style="margin-top: 14px; font-size: 13px;">
+ Note: newly imported items don't carry embeddings yet — they won't appear in duels or /For You until <code>node scripts/embed-items.js</code> runs (llava → 32-d projection). Drift sweep + nightly re-embed schedule planned.
+ </p>
+
<script>
const $ = (id) => document.getElementById(id);
const isAdminQ = location.search.includes('admin=1') ? '?admin=1' : '';
@@ -151,6 +171,52 @@
$('exit_preview').addEventListener('click', () => setViewAs(''));
loadRole();
+
+ // ---- Catalog status + import ------------------------------------------
+ async function loadCatalogStatus() {
+ const r = await fetch('/api/admin/catalog-status' + isAdminQ);
+ const j = await r.json();
+ if (!r.ok) { $('catalog_status').innerHTML = '<span class="muted">' + (j.error || 'failed') + '</span>'; return; }
+ const bySource = (j.by_source || []).map(s => `<span style="display:inline-block;background:#faf7f2;border:1px solid #e6e1d8;padding:3px 10px;border-radius:999px;font-size:12px;margin:2px;">${s.source}: <strong>${s.c}</strong></span>`).join('');
+ $('catalog_status').innerHTML = `
+ <div><strong>${j.total}</strong> items total · <strong>${j.embedded}</strong> embedded · <strong>${j.unembedded}</strong> pending re-embed</div>
+ <div style="margin-top:8px;">${bySource}</div>
+ `;
+ }
+
+ async function runImport(body) {
+ const s = $('import_status');
+ s.style.display = 'block';
+ s.className = 'status';
+ s.textContent = 'Importing… this can take 1-3 min for all brands.';
+ try {
+ const r = await fetch('/api/admin/import-catalog' + isAdminQ, {
+ method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(body || {}),
+ });
+ const j = await r.json();
+ if (r.ok) {
+ s.className = 'status ok';
+ const t = j.totals;
+ s.innerHTML = `Imported <strong>${t.imported}</strong> new items (fetched ${t.fetched}, errors ${t.errors}, skipped ${t.skipped}).`;
+ await loadCatalogStatus();
+ } else {
+ s.className = 'status err';
+ s.textContent = j.error || 'import failed';
+ }
+ } catch (e) {
+ s.className = 'status err';
+ s.textContent = e.message;
+ }
+ }
+
+ $('import_all_shopify').addEventListener('click', () => runImport({ kind: 'shopify' }));
+ $('import_one').addEventListener('click', () => {
+ const id = prompt('Shopify brand id (allbirds, reformation, pact, tradlands, cuyana, kotn, etc.):');
+ if (id) runImport({ kind: 'shopify', id: id.trim() });
+ });
+ $('refresh_status').addEventListener('click', () => loadCatalogStatus());
+
+ loadCatalogStatus();
</script>
</body>
</html>
diff --git a/scripts/clothing-apis/README.md b/scripts/clothing-apis/README.md
new file mode 100644
index 0000000..5408373
--- /dev/null
+++ b/scripts/clothing-apis/README.md
@@ -0,0 +1,58 @@
+# Clothing-API catalog ingestion
+
+Pulls real product catalogs into `items` from public + free clothing APIs.
+
+## Sources, ranked by friction
+
+| Tier | Source | Key needed? | Yield per brand |
+|---|---|---|---|
+| 🟢 None | Shopify `/products.json` | No | 250 / page × pages |
+| 🟡 Free | Etsy Open API v3 | `ETSY_API_KEY` | 25 / page × pages |
+| 🟡 Free | eBay Browse API | `EBAY_OAUTH_TOKEN` | 50 / page × pages |
+| 🟡 Free | Amazon PAAPI 5 | via `amazon-paapi-integration` skill | 10 / page (rate-limited) |
+| 🟡 Affiliate | Awin / Rakuten / CJ / Impact | network-specific | bulk feeds |
+
+## Run
+
+```bash
+# Import every Shopify brand in brands.json (zero key)
+node scripts/clothing-apis/index.js shopify
+
+# Import one brand
+node scripts/clothing-apis/index.js shopify allbirds
+
+# Programmatic — from server admin endpoint
+POST /api/admin/import-catalog { "kind": "shopify" }
+POST /api/admin/import-catalog { "kind": "shopify", "id": "cuyana" }
+```
+
+## Adapter rules
+
+1. **Identify the bot** in `User-Agent: WhatsMyStyleBot/0.1 (steve@designerwallcoverings.com)` so brands can rate-limit us if they need to.
+2. **Be nice** — 600-1000ms between pages, cap pages per brand.
+3. **Skip broken rows** — no image OR no price OR no recognized category → don't insert. Better a smaller, clean catalog.
+4. **Idempotent** — `INSERT OR IGNORE` on `UNIQUE(source, external_id)` so re-runs add only new SKUs.
+5. **Map to category enum** — `top` | `bottom` | `dress` | `outerwear` | `shoes` | `bag` | `accessory`. Anything that doesn't map is dropped.
+
+## After import
+
+Newly imported items have `embedding IS NULL`. They won't appear in duels or recs until embeddings are computed. Run:
+
+```bash
+node scripts/embed-items.js <limit> # llava → 32-d projection
+```
+
+## Adding a new brand
+
+1. Append to `scripts/clothing-apis/brands.json` `shopify_dtc` array:
+ ```json
+ { "id": "yourbrand", "name": "Your Brand", "domain": "yourbrand.com", "sustain_tier": 4, "pro_grade": 1 }
+ ```
+2. `node scripts/clothing-apis/index.js shopify yourbrand`
+3. That's it. The adapter is generic — every Shopify-powered DTC site Just Works.
+
+## Notes
+
+- The `pro_grade` flag is what set-decorator filter uses (`?pro_only=1`). Multi-stock brands get `pro_grade: 1`; resale-only (Etsy / eBay) stays at `0`.
+- The `sustain_tier` is informational — for now `scripts/sustainability.js` is the canonical lookup; this seed is a backstop.
+- We never persist email bodies, partner cookies, or anything besides the public product fields shown above. See `/privacy-policy`.
diff --git a/scripts/clothing-apis/brands.json b/scripts/clothing-apis/brands.json
new file mode 100644
index 0000000..164e12e
--- /dev/null
+++ b/scripts/clothing-apis/brands.json
@@ -0,0 +1,42 @@
+{
+ "_note": "Brand registry for catalog ingestion. Each entry names the source adapter + endpoint. 'shopify' means /products.json paginated, no key. Add new brands by appending — the importer reads this file at runtime.",
+ "_updated": "2026-05-12",
+
+ "shopify_dtc": [
+ { "id": "allbirds", "name": "Allbirds", "domain": "allbirds.com", "sustain_tier": 5, "pro_grade": 1 },
+ { "id": "reformation", "name": "Reformation", "domain": "thereformation.com", "sustain_tier": 5, "pro_grade": 0 },
+ { "id": "pact", "name": "Pact", "domain": "wearpact.com", "sustain_tier": 5, "pro_grade": 1 },
+ { "id": "outerknown", "name": "Outerknown", "domain": "outerknown.com", "sustain_tier": 5, "pro_grade": 1 },
+ { "id": "tradlands", "name": "Tradlands", "domain": "tradlands.com", "sustain_tier": 4, "pro_grade": 1 },
+ { "id": "whimsyandrow", "name": "Whimsy + Row", "domain": "whimsyandrow.com", "sustain_tier": 4, "pro_grade": 1 },
+ { "id": "cuyana", "name": "Cuyana", "domain": "cuyana.com", "sustain_tier": 4, "pro_grade": 1 },
+ { "id": "kotn", "name": "Kotn", "domain": "kotn.com", "sustain_tier": 5, "pro_grade": 1 },
+ { "id": "matethelabel", "name": "Mate the Label", "domain": "matethelabel.com", "sustain_tier": 5, "pro_grade": 1 },
+ { "id": "boody", "name": "Boody", "domain": "boody.com", "sustain_tier": 4, "pro_grade": 1 },
+ { "id": "naadam", "name": "Naadam", "domain": "naadam.co", "sustain_tier": 4, "pro_grade": 1 },
+ { "id": "christydawn", "name": "Christy Dawn", "domain": "christydawn.com", "sustain_tier": 5, "pro_grade": 0 },
+ { "id": "mottandbow", "name": "Mott & Bow", "domain": "mottandbow.com", "sustain_tier": 4, "pro_grade": 1 },
+ { "id": "saintjames", "name": "Saint James", "domain": "saintjames.us", "sustain_tier": 3, "pro_grade": 1 },
+ { "id": "tentree", "name": "Tentree", "domain": "tentree.com", "sustain_tier": 4, "pro_grade": 1 },
+ { "id": "mara-hoffman", "name": "Mara Hoffman", "domain": "marahoffman.com", "sustain_tier": 4, "pro_grade": 0 },
+ { "id": "knickey", "name": "Knickey", "domain": "knickey.com", "sustain_tier": 4, "pro_grade": 1 },
+ { "id": "boyish", "name": "Boyish", "domain": "boyish.com", "sustain_tier": 5, "pro_grade": 1 },
+ { "id": "fordays", "name": "For Days", "domain": "fordays.com", "sustain_tier": 5, "pro_grade": 1 },
+ { "id": "everybody-world","name": "Everybody.World", "domain": "everybody.world", "sustain_tier": 5, "pro_grade": 1 }
+ ],
+
+ "etsy_categories": [
+ { "id": "etsy-vintage-clothing", "name": "Etsy — Vintage Clothing", "category_id": "11050", "query": "vintage clothing", "needs_key": true },
+ { "id": "etsy-handmade-fashion", "name": "Etsy — Handmade Fashion", "category_id": "11000", "query": "handmade clothing", "needs_key": true }
+ ],
+
+ "ebay_categories": [
+ { "id": "ebay-vintage-fashion-women", "name": "eBay — Vintage Women's", "category_id": "175759", "needs_key": true },
+ { "id": "ebay-vintage-fashion-men", "name": "eBay — Vintage Men's", "category_id": "175760", "needs_key": true }
+ ],
+
+ "amazon_searches": [
+ { "id": "amazon-womens-tops", "name": "Amazon — Women's Tops", "node_id": "1045024", "needs_key": true },
+ { "id": "amazon-mens-shirts", "name": "Amazon — Men's Shirts", "node_id": "1045788", "needs_key": true }
+ ]
+}
diff --git a/scripts/clothing-apis/ebay.js b/scripts/clothing-apis/ebay.js
new file mode 100644
index 0000000..1ac4de6
--- /dev/null
+++ b/scripts/clothing-apis/ebay.js
@@ -0,0 +1,74 @@
+/**
+ * eBay Browse API adapter — secondhand / vintage.
+ *
+ * Endpoint: GET https://api.ebay.com/buy/browse/v1/item_summary/search
+ * Key: Bearer <OAuth app token>. Free dev account at developer.ebay.com.
+ *
+ * Env:
+ * EBAY_OAUTH_TOKEN — required (production application token, refreshed via
+ * client-credentials flow; out of scope for this file)
+ *
+ * One call yields up to 200 items per page. Free tier = 5000 calls/day, plenty.
+ */
+const fetch = global.fetch || require('node-fetch');
+
+const TOKEN = process.env.EBAY_OAUTH_TOKEN;
+const UA = 'WhatsMyStyleBot/0.1';
+const PAGE_SIZE = 50;
+const MAX_PAGES = 4;
+const PAGE_DELAY_MS = 800;
+
+async function importCategory(db, q) {
+ if (!TOKEN) return { skipped: 'EBAY_OAUTH_TOKEN not set', source: `ebay:${q.id}` };
+
+ const insert = db.prepare(`
+ INSERT OR IGNORE INTO items
+ (source, external_id, title, brand, category, price_cents, currency, image_url, product_url, tags, pro_grade)
+ VALUES
+ (@source, @external_id, @title, @brand, @category, @price_cents, @currency, @image_url, @product_url, @tags, @pro_grade)
+ `);
+
+ let imported = 0, fetched = 0;
+ for (let page = 1; page <= MAX_PAGES; page++) {
+ const offset = (page - 1) * PAGE_SIZE;
+ const url = `https://api.ebay.com/buy/browse/v1/item_summary/search?category_ids=${q.category_id}&limit=${PAGE_SIZE}&offset=${offset}`;
+ const r = await fetch(url, { headers: { Authorization: `Bearer ${TOKEN}`, 'User-Agent': UA, Accept: 'application/json' } });
+ if (!r.ok) return { source: `ebay:${q.id}`, ok: false, status: r.status, imported };
+ const j = await r.json();
+ const items = j.itemSummaries || [];
+ fetched += items.length;
+ const tx = db.transaction(() => {
+ for (const it of items) {
+ const img = it.image?.imageUrl;
+ const cents = it.price?.value ? Math.round(Number(it.price.value) * 100) : null;
+ if (!img || !cents) continue;
+ const category = /shoe|boot|sneaker/i.test(it.title) ? 'shoes'
+ : /dress|gown/i.test(it.title) ? 'dress'
+ : /coat|jacket|blazer/i.test(it.title) ? 'outerwear'
+ : /pant|trouser|jean|skirt/i.test(it.title) ? 'bottom'
+ : /bag|tote|purse/i.test(it.title) ? 'bag'
+ : 'top';
+ const res = insert.run({
+ source: `ebay:${q.id}`,
+ external_id: `ebay:${it.itemId}`,
+ title: (it.title || '').slice(0, 200),
+ brand: it.brand || (it.seller?.username ? `eBay: ${it.seller.username}` : 'eBay seller'),
+ category,
+ price_cents: cents,
+ currency: it.price?.currency || 'USD',
+ image_url: img,
+ product_url: it.itemWebUrl,
+ tags: JSON.stringify(it.conditionId ? [`condition:${it.condition}`] : []),
+ pro_grade: 0, // one-off auction item — not restockable
+ });
+ if (res.changes > 0) imported++;
+ }
+ });
+ tx();
+ if (items.length < PAGE_SIZE) break;
+ await new Promise(r => setTimeout(r, PAGE_DELAY_MS));
+ }
+ return { source: `ebay:${q.id}`, ok: true, fetched, imported };
+}
+
+module.exports = { importCategory };
diff --git a/scripts/clothing-apis/etsy.js b/scripts/clothing-apis/etsy.js
new file mode 100644
index 0000000..1dd8592
--- /dev/null
+++ b/scripts/clothing-apis/etsy.js
@@ -0,0 +1,73 @@
+/**
+ * Etsy Open API v3 adapter — vintage / handmade.
+ *
+ * Endpoint: GET https://openapi.etsy.com/v3/application/listings/active
+ * Key: x-api-key header. Free key — apply at https://www.etsy.com/developers/register
+ *
+ * Env:
+ * ETSY_API_KEY — required; without it this adapter returns { skipped: 'no key' }
+ *
+ * One-call yield is ~25 listings; we paginate up to MAX_PAGES.
+ */
+const fetch = global.fetch || require('node-fetch');
+
+const KEY = process.env.ETSY_API_KEY;
+const UA = 'WhatsMyStyleBot/0.1';
+const PAGE_SIZE = 25;
+const MAX_PAGES = 4;
+const PAGE_DELAY_MS = 1000;
+
+async function importQuery(db, q) {
+ if (!KEY) return { skipped: 'ETSY_API_KEY not set', source: `etsy:${q.id}` };
+
+ const insert = db.prepare(`
+ INSERT OR IGNORE INTO items
+ (source, external_id, title, brand, category, price_cents, currency, image_url, product_url, tags, pro_grade)
+ VALUES
+ (@source, @external_id, @title, @brand, @category, @price_cents, @currency, @image_url, @product_url, @tags, @pro_grade)
+ `);
+
+ let imported = 0, fetched = 0;
+ for (let page = 1; page <= MAX_PAGES; page++) {
+ const offset = (page - 1) * PAGE_SIZE;
+ const url = `https://openapi.etsy.com/v3/application/listings/active?keywords=${encodeURIComponent(q.query)}&limit=${PAGE_SIZE}&offset=${offset}&includes=Images`;
+ const r = await fetch(url, { headers: { 'x-api-key': KEY, 'User-Agent': UA } });
+ if (!r.ok) return { source: `etsy:${q.id}`, ok: false, status: r.status, imported };
+ const j = await r.json();
+ const items = j.results || [];
+ fetched += items.length;
+ const tx = db.transaction(() => {
+ for (const it of items) {
+ const img = it.images?.[0]?.url_fullxfull || it.images?.[0]?.url_570xN;
+ const cents = Math.round(Number(it.price?.amount || 0) / Number(it.price?.divisor || 1) * 100);
+ if (!img || !cents) continue;
+ // Etsy doesn't ship a tight category enum; bucket by query intent.
+ const category = /shoe|boot|sneaker/i.test(it.title) ? 'shoes'
+ : /dress|gown/i.test(it.title) ? 'dress'
+ : /coat|jacket|blazer/i.test(it.title) ? 'outerwear'
+ : /pant|trouser|jean|skirt/i.test(it.title) ? 'bottom'
+ : 'top';
+ const res = insert.run({
+ source: `etsy:${q.id}`,
+ external_id: `etsy:${it.listing_id}`,
+ title: (it.title || '').slice(0, 200),
+ brand: it.shop_id ? `Etsy shop #${it.shop_id}` : 'Etsy seller',
+ category,
+ price_cents: cents,
+ currency: it.price?.currency_code || 'USD',
+ image_url: img,
+ product_url: `https://www.etsy.com/listing/${it.listing_id}`,
+ tags: JSON.stringify(it.tags || []),
+ pro_grade: 0, // vintage / one-off — set decorators won't restock
+ });
+ if (res.changes > 0) imported++;
+ }
+ });
+ tx();
+ if (items.length < PAGE_SIZE) break;
+ await new Promise(r => setTimeout(r, PAGE_DELAY_MS));
+ }
+ return { source: `etsy:${q.id}`, ok: true, fetched, imported };
+}
+
+module.exports = { importQuery };
diff --git a/scripts/clothing-apis/index.js b/scripts/clothing-apis/index.js
new file mode 100644
index 0000000..27ce442
--- /dev/null
+++ b/scripts/clothing-apis/index.js
@@ -0,0 +1,108 @@
+/**
+ * Catalog import orchestrator.
+ *
+ * Reads scripts/clothing-apis/brands.json and dispatches each entry to the
+ * right adapter (shopify, etsy, ebay, amazon). Idempotent — adapters use
+ * INSERT OR IGNORE on items(source, external_id), so re-runs add only new
+ * SKUs.
+ *
+ * CLI:
+ * node scripts/clothing-apis/index.js # import everything
+ * node scripts/clothing-apis/index.js shopify # only Shopify brands
+ * node scripts/clothing-apis/index.js shopify allbirds # one brand
+ *
+ * Programmatic (called from server admin endpoint):
+ * const { importAll, importOne } = require('./scripts/clothing-apis');
+ */
+const Database = require('better-sqlite3');
+const path = require('path');
+const fs = require('fs');
+
+const BRANDS = JSON.parse(fs.readFileSync(path.join(__dirname, 'brands.json'), 'utf8'));
+const shopify = require('./shopify');
+const etsy = require('./etsy');
+const ebay = require('./ebay');
+
+async function importOne(db, kind, id) {
+ if (kind === 'shopify') {
+ const b = BRANDS.shopify_dtc.find(x => x.id === id);
+ if (!b) throw new Error(`shopify brand ${id} not in brands.json`);
+ return await shopify.importBrand(db, b);
+ }
+ if (kind === 'etsy') {
+ const q = BRANDS.etsy_categories.find(x => x.id === id);
+ if (!q) throw new Error(`etsy query ${id} not in brands.json`);
+ return await etsy.importQuery(db, q);
+ }
+ if (kind === 'ebay') {
+ const q = BRANDS.ebay_categories.find(x => x.id === id);
+ if (!q) throw new Error(`ebay category ${id} not in brands.json`);
+ return await ebay.importCategory(db, q);
+ }
+ throw new Error(`unknown kind: ${kind}`);
+}
+
+async function importAll(db, opts = {}) {
+ const results = [];
+ const filterKind = opts.kind || null; // 'shopify' | 'etsy' | 'ebay' | null
+ const limit = Number.isFinite(opts.limit) ? opts.limit : Infinity;
+ let n = 0;
+
+ if (!filterKind || filterKind === 'shopify') {
+ for (const b of BRANDS.shopify_dtc) {
+ if (n >= limit) break;
+ try {
+ const r = await shopify.importBrand(db, b);
+ results.push({ kind: 'shopify', id: b.id, ...r });
+ } catch (e) {
+ results.push({ kind: 'shopify', id: b.id, ok: false, err: e.message });
+ }
+ n++;
+ }
+ }
+ if (!filterKind || filterKind === 'etsy') {
+ for (const q of BRANDS.etsy_categories) {
+ if (n >= limit) break;
+ const r = await etsy.importQuery(db, q);
+ results.push({ kind: 'etsy', id: q.id, ...r });
+ if (r.skipped) continue;
+ n++;
+ }
+ }
+ if (!filterKind || filterKind === 'ebay') {
+ for (const q of BRANDS.ebay_categories) {
+ if (n >= limit) break;
+ const r = await ebay.importCategory(db, q);
+ results.push({ kind: 'ebay', id: q.id, ...r });
+ if (r.skipped) continue;
+ n++;
+ }
+ }
+ return results;
+}
+
+async function main() {
+ const dbPath = path.join(__dirname, '..', '..', 'data', 'whatsmystyle.db');
+ const db = new Database(dbPath);
+ const [, , kind, id] = process.argv;
+ let results;
+ if (kind && id) {
+ results = [await importOne(db, kind, id)];
+ } else if (kind) {
+ results = await importAll(db, { kind });
+ } else {
+ results = await importAll(db, { kind: 'shopify' }); // default to free tier
+ }
+ const totals = results.reduce((acc, r) => {
+ acc.imported += r.imported || 0;
+ acc.fetched += r.fetched || 0;
+ if (r.ok === false || r.err) acc.errors++;
+ if (r.skipped) acc.skipped++;
+ return acc;
+ }, { imported: 0, fetched: 0, errors: 0, skipped: 0 });
+ console.log(JSON.stringify({ totals, results }, null, 2));
+ db.close();
+}
+
+if (require.main === module) main().catch(e => { console.error(e); process.exit(1); });
+module.exports = { importAll, importOne, BRANDS };
diff --git a/scripts/clothing-apis/shopify.js b/scripts/clothing-apis/shopify.js
new file mode 100644
index 0000000..68ea948
--- /dev/null
+++ b/scripts/clothing-apis/shopify.js
@@ -0,0 +1,138 @@
+/**
+ * Shopify /products.json adapter — zero key, paginated, public.
+ *
+ * Every Shopify-powered DTC storefront exposes /products.json by default.
+ * The endpoint is documented (Shopify's intentional API for the embedded
+ * Buy Button etc.), returns JSON, and accepts ?limit + ?page query args.
+ *
+ * Pull pattern: 250 per page (Shopify's cap), back off if the brand returns
+ * 429 or 5xx, identify ourselves in User-Agent, stop at MAX_PAGES per brand.
+ *
+ * Maps a Shopify product → WhatsMyStyle `items` row:
+ * id → external_id (composite: shopify:<domain>:<product-id>)
+ * source → 'shopify:<domain>'
+ * title → title
+ * vendor → brand
+ * product_type→ category (lowercased, normalized to top/bottom/dress/etc.)
+ * tags → tags (JSON array)
+ * variants[0].price → price_cents (× 100)
+ * images[0].src → image_url
+ * handle → product_url (https://<domain>/products/<handle>)
+ *
+ * One brand per call. Idempotent — uses INSERT OR IGNORE so re-running adds
+ * only new SKUs and updates existing ones aren't trampled.
+ */
+const fetch = global.fetch || require('node-fetch');
+
+const UA = 'WhatsMyStyleBot/0.1 (catalog importer; steve@designerwallcoverings.com)';
+const PAGE_SIZE = 250;
+const MAX_PAGES_PER_BRAND = 4; // 4 × 250 = 1000 cap per brand per run
+const PAGE_DELAY_MS = 600; // be nice; ~1.6 RPS
+
+// Heuristic mapping from Shopify product_type → our category enum.
+const CATEGORY_MAP = [
+ [/^t.?shirt|tee|tank|cami|blouse|shirt|button[- ]?down|top$/i, 'top'],
+ [/sweater|knit|cardigan|pullover|jumper|hoodie/i, 'top'],
+ [/pant|trouser|jean|chino|legging|short/i, 'bottom'],
+ [/skirt|midi$|mini$/i, 'bottom'],
+ [/dress|gown|jumpsuit|romper/i, 'dress'],
+ [/coat|jacket|parka|trench|blazer|puffer|outerwear/i, 'outerwear'],
+ [/shoe|sneaker|boot|sandal|loafer|heel|flat$/i, 'shoes'],
+ [/bag|tote|purse|backpack|crossbody|clutch/i, 'bag'],
+ [/scarf|hat|belt|sunglasses|jewelry|earring|necklace|ring$|bracelet|accessor/i, 'accessory'],
+];
+function normalizeCategory(productType, tags) {
+ const text = `${productType || ''} ${(tags || []).join(' ')}`;
+ for (const [re, cat] of CATEGORY_MAP) if (re.test(text)) return cat;
+ return null;
+}
+
+async function fetchOnePage(domain, page) {
+ const url = `https://${domain}/products.json?limit=${PAGE_SIZE}&page=${page}`;
+ const r = await fetch(url, { headers: { 'User-Agent': UA, 'Accept': 'application/json' } });
+ if (r.status === 404) return { products: [], end: true, status: 404 };
+ if (r.status === 429) return { products: [], end: true, status: 429 };
+ if (!r.ok) throw new Error(`shopify ${domain} page ${page} → ${r.status}`);
+ const j = await r.json();
+ return { products: j.products || [], end: (j.products || []).length < PAGE_SIZE, status: r.status };
+}
+
+async function fetchAllPages(domain) {
+ const all = [];
+ for (let page = 1; page <= MAX_PAGES_PER_BRAND; page++) {
+ const { products, end, status } = await fetchOnePage(domain, page);
+ if (status !== 200) {
+ return { ok: false, status, products: all };
+ }
+ all.push(...products);
+ if (end) break;
+ await new Promise(r => setTimeout(r, PAGE_DELAY_MS));
+ }
+ return { ok: true, products: all };
+}
+
+function rowFromProduct(domain, brand, sustainTier, proGrade, p) {
+ const variant = (p.variants || [])[0] || {};
+ const price = Number(variant.price);
+ const price_cents = Number.isFinite(price) ? Math.round(price * 100) : null;
+ const image = (p.images || [])[0]?.src || p.image?.src || null;
+ const category = normalizeCategory(p.product_type, p.tags);
+ // Skip products with no image OR no price OR no recognized category — they
+ // can't render in the duel grid anyway.
+ if (!image || !price_cents || !category) return null;
+
+ return {
+ source: `shopify:${domain}`,
+ external_id: `shopify:${domain}:${p.id}`,
+ title: (p.title || '').trim().slice(0, 200),
+ brand: brand || p.vendor || domain,
+ category,
+ color: null,
+ pattern: null,
+ material: null,
+ price_cents,
+ currency: 'USD',
+ image_url: image,
+ product_url: `https://${domain}/products/${p.handle}`,
+ tags: JSON.stringify(p.tags || []),
+ pro_grade: proGrade,
+ _sustain: sustainTier, // not a column on items; importer maps to sustainability via brand match
+ };
+}
+
+async function importBrand(db, brandEntry) {
+ const { name, domain, sustain_tier, pro_grade } = brandEntry;
+ const t0 = Date.now();
+ const { ok, status, products } = await fetchAllPages(domain);
+
+ // Prepare a single-statement upsert. items has UNIQUE(source, external_id),
+ // so INSERT OR IGNORE is the idempotent path.
+ const insert = db.prepare(`
+ INSERT OR IGNORE INTO items
+ (source, external_id, title, brand, category, price_cents, currency, image_url, product_url, tags, pro_grade)
+ VALUES
+ (@source, @external_id, @title, @brand, @category, @price_cents, @currency, @image_url, @product_url, @tags, @pro_grade)
+ `);
+
+ let imported = 0, skipped = 0;
+ const tx = db.transaction((rows) => {
+ for (const r of rows) {
+ if (!r) { skipped++; continue; }
+ const res = insert.run(r);
+ if (res.changes > 0) imported++;
+ else skipped++;
+ }
+ });
+ const rows = (products || []).map(p => rowFromProduct(domain, name, sustain_tier, pro_grade, p));
+ tx(rows);
+
+ return {
+ brand: name, domain, ok, status,
+ fetched: (products || []).length,
+ imported,
+ skipped,
+ ms: Date.now() - t0,
+ };
+}
+
+module.exports = { importBrand, fetchAllPages, normalizeCategory };
diff --git a/server.js b/server.js
index 8b87271..a53fd26 100644
--- a/server.js
+++ b/server.js
@@ -795,6 +795,40 @@ app.put('/api/admin/config', express.json({ limit: '8kb' }), (req, res) => {
res.json({ ok: true, written });
});
+// ---- admin / catalog import (clothing APIs) -------------------------------
+// POST /api/admin/import-catalog — import all Shopify brands
+// POST /api/admin/import-catalog { kind:'shopify' } — Shopify only
+// POST /api/admin/import-catalog { kind:'shopify', id:'allbirds' } — one brand
+app.post('/api/admin/import-catalog', express.json({ limit: '8kb' }), async (req, res) => {
+ if (!adminGate(req)) return res.status(403).json({ error: 'admin only' });
+ const { kind, id, limit } = req.body || {};
+ try {
+ const { importAll, importOne } = require('./scripts/clothing-apis');
+ let results;
+ if (kind && id) results = [await importOne(db, kind, id)];
+ else results = await importAll(db, { kind, limit });
+ const totals = results.reduce((a, r) => {
+ a.imported += r.imported || 0;
+ a.fetched += r.fetched || 0;
+ if (r.ok === false || r.err) a.errors++;
+ if (r.skipped) a.skipped++;
+ return a;
+ }, { imported: 0, fetched: 0, errors: 0, skipped: 0 });
+ res.json({ ok: true, totals, results });
+ } catch (e) {
+ res.status(500).json({ error: e.message });
+ }
+});
+
+// GET /api/admin/catalog-status — quick rollup for the /admin-config page
+app.get('/api/admin/catalog-status', (req, res) => {
+ if (!adminGate(req)) return res.status(403).json({ error: 'admin only' });
+ const total = db.prepare('SELECT COUNT(*) c FROM items').get().c;
+ const embedded = db.prepare('SELECT COUNT(*) c FROM items WHERE embedding IS NOT NULL').get().c;
+ const bySource = db.prepare('SELECT source, COUNT(*) c FROM items GROUP BY source ORDER BY c DESC').all();
+ res.json({ total, embedded, unembedded: total - embedded, by_source: bySource });
+});
+
// ---- admin / debate -------------------------------------------------------
app.post('/api/admin/debate', async (req, res) => {
const { topic, context } = req.body || {};
← f20ac43 yolo tick 16: pro_grade catalog filter + production picker o
·
back to Whatsmystyle
·
yolo tick 17: active-production bar (set-decorator persisten 63402bb →