← back to Whatsmystyle
yolo tick 18: embed-items --batch + boot-time drainer (5min/20 items) + parallel Shopify import (17 brands, 6x concurrent, 4805 new items in 17s) + drift cron scales with catalog size
b0fcb6230c1d83a42bc027b3a846e355ff274e55 · 2026-05-12 09:33:24 -0700 · SteveStudio2
Files touched
M scripts/clothing-apis/index.jsM scripts/embed-items.jsM server.js
Diff
commit b0fcb6230c1d83a42bc027b3a846e355ff274e55
Author: SteveStudio2 <stevestudio2@SteveStacStudio.lan>
Date: Tue May 12 09:33:24 2026 -0700
yolo tick 18: embed-items --batch + boot-time drainer (5min/20 items) + parallel Shopify import (17 brands, 6x concurrent, 4805 new items in 17s) + drift cron scales with catalog size
---
scripts/clothing-apis/index.js | 27 +++++++++++++++++++--------
scripts/embed-items.js | 34 ++++++++++++++++++++++++++++------
server.js | 33 ++++++++++++++++++++++++++++-----
3 files changed, 75 insertions(+), 19 deletions(-)
diff --git a/scripts/clothing-apis/index.js b/scripts/clothing-apis/index.js
index 27ce442..f8f1a10 100644
--- a/scripts/clothing-apis/index.js
+++ b/scripts/clothing-apis/index.js
@@ -49,16 +49,27 @@ async function importAll(db, opts = {}) {
let n = 0;
if (!filterKind || filterKind === 'shopify') {
- for (const b of BRANDS.shopify_dtc) {
- if (n >= limit) break;
- try {
- const r = await shopify.importBrand(db, b);
- results.push({ kind: 'shopify', id: b.id, ...r });
- } catch (e) {
- results.push({ kind: 'shopify', id: b.id, ok: false, err: e.message });
+ // Tick 18: parallel fan-out. Each brand is its own source key (UNIQUE on
+ // source+external_id) so there's no DB row contention. SQLite handles
+ // concurrent transactions inside one process via better-sqlite3 (WAL or
+ // serialized). Cap concurrency at 6 so we don't slam the brands all at
+ // once (politeness + a memory ceiling on parsed pages).
+ const CONCURRENCY = 6;
+ const brands = BRANDS.shopify_dtc.slice(0, Math.min(BRANDS.shopify_dtc.length, limit - n));
+ let idx = 0;
+ async function worker() {
+ while (idx < brands.length) {
+ const b = brands[idx++];
+ try {
+ const r = await shopify.importBrand(db, b);
+ results.push({ kind: 'shopify', id: b.id, ...r });
+ } catch (e) {
+ results.push({ kind: 'shopify', id: b.id, ok: false, err: e.message });
+ }
}
- n++;
}
+ await Promise.all(Array.from({ length: CONCURRENCY }, () => worker()));
+ n += brands.length;
}
if (!filterKind || filterKind === 'etsy') {
for (const q of BRANDS.etsy_categories) {
diff --git a/scripts/embed-items.js b/scripts/embed-items.js
index f620dd0..42dc24b 100644
--- a/scripts/embed-items.js
+++ b/scripts/embed-items.js
@@ -140,6 +140,25 @@ async function pingOllama() {
} catch { return false; }
}
+// Tick 18: batch mode for the unembedded tail. With 1,200+ catalog rows we
+// can't process them all in one pass — llava is slow (~3-5s/item). Run with
+// `--batch <N>` to only process N items in this invocation; pair with a
+// recurring caller (server cron) so the catalog fills incrementally.
+//
+// `--unembedded-only` picks rows where embedding IS NULL — what we want for
+// the post-import backfill (existing real embeddings stay; only new items
+// from the Shopify import get a vector).
+function parseArgs(argv) {
+ const args = { batch: null, unembeddedOnly: false, limit: null };
+ for (let i = 2; i < argv.length; i++) {
+ const a = argv[i];
+ if (a === '--batch') args.batch = Number(argv[++i]);
+ else if (a === '--unembedded-only') args.unembeddedOnly = true;
+ else if (!isNaN(Number(a))) args.limit = Number(a); // legacy positional
+ }
+ return args;
+}
+
async function main() {
const dbPath = path.join(__dirname, '..', 'data', 'whatsmystyle.db');
const db = new Database(dbPath);
@@ -148,17 +167,20 @@ async function main() {
console.log(JSON.stringify({ ok: false, reason: 'ollama+llava not available locally; leaving stub embeddings in place', model: MODEL, url: OLLAMA }));
return;
}
- // pick rows with stub embeddings — heuristic: hash-stub vectors are dense
- // (every dim non-zero, almost-normalized). Real ones have many zeros.
- // We re-embed everything for simplicity; idempotent.
- const limit = Number(process.argv[2] || 24);
- const rows = db.prepare('SELECT * FROM items ORDER BY id LIMIT ?').all(limit);
+
+ const { batch, unembeddedOnly, limit } = parseArgs(process.argv);
+ const cap = batch || limit || 24;
+ const sql = unembeddedOnly
+ ? 'SELECT * FROM items WHERE embedding IS NULL ORDER BY id LIMIT ?'
+ : 'SELECT * FROM items ORDER BY id LIMIT ?';
+ const rows = db.prepare(sql).all(cap);
+
const results = [];
for (const r of rows) {
try { results.push(await embedItem(db, r)); }
catch (e) { results.push({ id: r.id, err: e.message }); }
}
- console.log(JSON.stringify({ ok: true, processed: results.length, results }, null, 2));
+ console.log(JSON.stringify({ ok: true, processed: results.length, unembeddedOnly, results }, null, 2));
}
if (require.main === module) main().catch(e => { console.error(e); process.exit(1); });
diff --git a/server.js b/server.js
index 35c52fa..a2366bb 100644
--- a/server.js
+++ b/server.js
@@ -1770,18 +1770,19 @@ function drainAvatarBuild() {
}
setInterval(drainAvatarBuild, 30 * 1000);
-// Embedding drift sweep (tick 14): every 24h, scan a rolling window of 6
-// catalog items and write a row to embedding_drifts for any whose
+// Embedding drift sweep (tick 14, expanded tick 18): every 24h, scan a rolling
+// window of catalog items and write a row to embedding_drifts for any whose
// llava-projected vector deviates from the stored one by > 20% cosine.
-// Rolls through the catalog via OFFSET stored in app_config.drift_offset.
-// Skips silently when Ollama llava isn't reachable.
+// Rolls through via OFFSET stored in app_config.drift_offset.
+// Tick 18: max-items scales dynamically (Math.max(6, ceil(total/30))) so the
+// cron covers the whole catalog in ~30 days regardless of size.
async function embedDriftSchedule() {
try {
const { runDriftSweep, ollamaLlavaUp } = require('./scripts/embed-drift-check');
if (!(await ollamaLlavaUp())) return;
const offset = configValue('drift_offset', 0);
- const maxItems = 6;
const total = db.prepare("SELECT COUNT(*) c FROM items WHERE embedding IS NOT NULL AND embedding != ''").get().c || 0;
+ const maxItems = Math.max(6, Math.ceil(total / 30)); // tick 18: scale with catalog
const nextOffset = total > 0 ? (offset + maxItems) % total : 0;
const summary = await runDriftSweep({ db, maxItems, offset });
db.prepare(`INSERT INTO app_config (key, value) VALUES ('drift_offset', ?)
@@ -1796,6 +1797,28 @@ async function embedDriftSchedule() {
setTimeout(embedDriftSchedule, 5 * 60 * 1000);
setInterval(embedDriftSchedule, 24 * 60 * 60 * 1000);
+// Tick 18: unembedded-tail drainer. Fires a small batch every 5 minutes,
+// draining items where embedding IS NULL until the catalog is fully embedded.
+// Idempotent — embed-items.js skips items it can't fetch images for.
+// Uses spawn().unref() so the work runs out-of-process and the server stays
+// responsive even when llava is chewing on a slow vendor image.
+const { spawn: _spawnEmbed } = require('child_process');
+function drainUnembedded() {
+ const unembed = db.prepare("SELECT COUNT(*) c FROM items WHERE embedding IS NULL").get().c || 0;
+ if (unembed === 0) return;
+ const batch = Math.min(20, unembed);
+ const p = _spawnEmbed('node', [
+ path.join(__dirname, 'scripts', 'embed-items.js'),
+ '--batch', String(batch),
+ '--unembedded-only',
+ ], { detached: true, stdio: 'ignore' });
+ p.unref();
+}
+// Boot-time pass after 60s — let pm2 settle first.
+setTimeout(drainUnembedded, 60 * 1000);
+// Then every 5 min until the tail is drained.
+setInterval(drainUnembedded, 5 * 60 * 1000);
+
// ---- start ----------------------------------------------------------------
app.listen(PORT, '0.0.0.0', () => {
console.log(`[whatsmystyle] listening on :${PORT}`);
← 63402bb yolo tick 17: active-production bar (set-decorator persisten
·
back to Whatsmystyle
·
fix(load): route llava drainer to Mac1 Ollama (192.168.1.133 94d3a5a →