← back to Whatsmystyle
scripts/clothing-apis/index.js
188 lines
/**
* Catalog import orchestrator.
*
* Reads scripts/clothing-apis/brands.json and dispatches each entry to the
* right adapter (shopify, etsy, ebay, amazon). Idempotent — adapters use
* INSERT OR IGNORE on items(source, external_id), so re-runs add only new
* SKUs.
*
* CLI:
* node scripts/clothing-apis/index.js # import everything
* node scripts/clothing-apis/index.js shopify # only Shopify brands
* node scripts/clothing-apis/index.js shopify allbirds # one brand
*
* Programmatic (called from server admin endpoint):
* const { importAll, importOne } = require('./scripts/clothing-apis');
*/
const Database = require('better-sqlite3');
const path = require('path');
const fs = require('fs');
const BRANDS = JSON.parse(fs.readFileSync(path.join(__dirname, 'brands.json'), 'utf8'));
const shopify = require('./shopify');
const woocommerce = require('./woocommerce');
const headlessSpa = require('./headless-spa');
const awin = require('./awin');
const etsy = require('./etsy');
const ebay = require('./ebay');
async function importOne(db, kind, id) {
if (kind === 'shopify') {
const b = BRANDS.shopify_dtc.find(x => x.id === id);
if (!b) throw new Error(`shopify brand ${id} not in brands.json`);
return await shopify.importBrand(db, b);
}
if (kind === 'woocommerce') {
const b = (BRANDS.woocommerce_dtc || []).filter(x => typeof x === 'object').find(x => x.id === id);
if (!b) throw new Error(`woocommerce brand ${id} not in brands.json`);
return await woocommerce.importBrand(db, b);
}
if (kind === 'headless') {
const b = (BRANDS.headless_spa || []).find(x => x.id === id);
if (!b) throw new Error(`headless brand ${id} not in brands.json`);
return await headlessSpa.importBrand(db, b);
}
if (kind === 'awin') {
const b = (BRANDS.awin_advertisers || []).filter(x => typeof x === 'object').find(x => x.id === id);
if (!b) throw new Error(`awin advertiser ${id} not in brands.json`);
return await awin.importAdvertiser(db, b);
}
if (kind === 'etsy') {
const q = BRANDS.etsy_categories.find(x => x.id === id);
if (!q) throw new Error(`etsy query ${id} not in brands.json`);
return await etsy.importQuery(db, q);
}
if (kind === 'ebay') {
const q = BRANDS.ebay_categories.find(x => x.id === id);
if (!q) throw new Error(`ebay category ${id} not in brands.json`);
return await ebay.importCategory(db, q);
}
throw new Error(`unknown kind: ${kind}`);
}
async function importAll(db, opts = {}) {
const results = [];
const filterKind = opts.kind || null; // 'shopify' | 'etsy' | 'ebay' | null
const limit = Number.isFinite(opts.limit) ? opts.limit : Infinity;
let n = 0;
if (!filterKind || filterKind === 'shopify') {
// Tick 18: parallel fan-out. Each brand is its own source key (UNIQUE on
// source+external_id) so there's no DB row contention. SQLite handles
// concurrent transactions inside one process via better-sqlite3 (WAL or
// serialized). Cap concurrency at 6 so we don't slam the brands all at
// once (politeness + a memory ceiling on parsed pages).
const CONCURRENCY = 6;
// Tick 19: respect `disabled:true` in brands.json so the parallel fan-out
// doesn't keep retrying brands we've already proven aren't Shopify (or
// are unreachable from this network).
const eligible = BRANDS.shopify_dtc.filter(b => !b.disabled);
const brands = eligible.slice(0, Math.min(eligible.length, limit - n));
let idx = 0;
async function worker() {
while (idx < brands.length) {
const b = brands[idx++];
try {
const r = await shopify.importBrand(db, b);
results.push({ kind: 'shopify', id: b.id, ...r });
} catch (e) {
results.push({ kind: 'shopify', id: b.id, ok: false, err: e.message });
}
}
}
await Promise.all(Array.from({ length: CONCURRENCY }, () => worker()));
// Surface the disabled brands so admin sees them.
for (const b of BRANDS.shopify_dtc.filter(b => b.disabled)) {
results.push({ kind: 'shopify', id: b.id, skipped: b.disabled_reason || 'disabled' });
}
n += brands.length;
}
if (!filterKind || filterKind === 'woocommerce') {
const woo = (BRANDS.woocommerce_dtc || []).filter(x => typeof x === 'object' && !x.disabled);
for (const b of woo) {
if (n >= limit) break;
try {
const r = await woocommerce.importBrand(db, b);
results.push({ kind: 'woocommerce', id: b.id, ...r });
} catch (e) {
results.push({ kind: 'woocommerce', id: b.id, ok: false, err: e.message });
}
n++;
}
}
if (filterKind === 'headless') {
const enabled = (BRANDS.headless_spa || []).filter(x => !x.disabled);
for (const b of enabled) {
if (n >= limit) break;
try {
const r = await headlessSpa.importBrand(db, b);
results.push({ kind: 'headless', id: b.id, ...r });
} catch (e) {
results.push({ kind: 'headless', id: b.id, ok: false, err: e.message });
}
n++;
}
}
if (filterKind === 'awin') {
const enabled = (BRANDS.awin_advertisers || []).filter(x => typeof x === 'object' && !x.disabled);
for (const b of enabled) {
if (n >= limit) break;
try {
const r = await awin.importAdvertiser(db, b);
results.push({ kind: 'awin', id: b.id, ...r });
} catch (e) {
results.push({ kind: 'awin', id: b.id, ok: false, err: e.message });
}
n++;
}
}
if (!filterKind || filterKind === 'etsy') {
for (const q of BRANDS.etsy_categories) {
if (n >= limit) break;
const r = await etsy.importQuery(db, q);
results.push({ kind: 'etsy', id: q.id, ...r });
if (r.skipped) continue;
n++;
}
}
if (!filterKind || filterKind === 'ebay') {
for (const q of BRANDS.ebay_categories) {
if (n >= limit) break;
const r = await ebay.importCategory(db, q);
results.push({ kind: 'ebay', id: q.id, ...r });
if (r.skipped) continue;
n++;
}
}
return results;
}
async function main() {
const dbPath = path.join(__dirname, '..', '..', 'data', 'whatsmystyle.db');
const db = new Database(dbPath);
const [, , kind, id] = process.argv;
let results;
if (kind && id) {
results = [await importOne(db, kind, id)];
} else if (kind) {
results = await importAll(db, { kind });
} else {
results = await importAll(db, { kind: 'shopify' }); // default to free tier
}
const totals = results.reduce((acc, r) => {
acc.imported += r.imported || 0;
acc.fetched += r.fetched || 0;
if (r.ok === false || r.err) acc.errors++;
if (r.skipped) acc.skipped++;
return acc;
}, { imported: 0, fetched: 0, errors: 0, skipped: 0 });
console.log(JSON.stringify({ totals, results }, null, 2));
db.close();
}
if (require.main === module) main().catch(e => { console.error(e); process.exit(1); });
module.exports = { importAll, importOne, BRANDS };