← back to Designer Wallcoverings
auto-save: 2026-08-01T11:06:15 (2 files) — DW-Agents/vendor-command-center/server.js shopify/scripts/cadence/data/activations-2026-08-01.jsonl
917e7d7dec1cb41fdf1203879031509f5ef745e7 · 2026-08-01 11:06:22 -0700 · Steve Abrams
Files touched
M DW-Agents/vendor-command-center/server.jsM shopify/scripts/cadence/data/activations-2026-08-01.jsonl
Diff
commit 917e7d7dec1cb41fdf1203879031509f5ef745e7
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Sat Aug 1 11:06:22 2026 -0700
auto-save: 2026-08-01T11:06:15 (2 files) — DW-Agents/vendor-command-center/server.js shopify/scripts/cadence/data/activations-2026-08-01.jsonl
---
DW-Agents/vendor-command-center/server.js | 64 +++++++++++++++++-----
.../cadence/data/activations-2026-08-01.jsonl | 36 ++++++++++++
2 files changed, 85 insertions(+), 15 deletions(-)
diff --git a/DW-Agents/vendor-command-center/server.js b/DW-Agents/vendor-command-center/server.js
index bee4bc4f..d75b69b3 100644
--- a/DW-Agents/vendor-command-center/server.js
+++ b/DW-Agents/vendor-command-center/server.js
@@ -2189,8 +2189,16 @@ async function computeSpecCompleteness(catTable) {
// --- Sync: Populate vendor_registry — CATALOG tables are source of truth ---
// Observable sync state for /healthz-deep — pm2 "online" can't see a sync loop that
// silently stopped firing (the executes-but-frozen failure mode the freshness canaries watch)
-const syncState = { lastStart: null, lastDone: null, lastDurationMs: null, lastError: null, runs: 0 };
+const syncState = { lastStart: null, lastDone: null, lastDurationMs: null, lastError: null, runs: 0, running: false };
async function syncVendorCounts() {
+ // Concurrency guard: on a slow DB a sweep can outlive the 5-min interval — without
+ // this, intervals stack dozens of concurrent sweeps that contend each other slower
+ // (observed on Kamatera 2026-08-01: 16,400s sweeps, one starting every 300s)
+ if (syncState.running) {
+ console.log('[Victor] Sync skipped — previous sweep still running');
+ return { success: false, skipped: true };
+ }
+ syncState.running = true;
console.log('[Victor] Syncing vendor counts — PostgreSQL catalogs = source of truth, Shopify = published...');
logActivity('sync', 'Sync started — scanning all vendor catalogs');
const startTime = Date.now();
@@ -2201,6 +2209,37 @@ async function syncVendorCounts() {
// Get all vendors with current counts for change detection
const { rows: allVendors } = await pool.query('SELECT vendor_code, vendor_name, catalog_table, catalog_count as old_cat, shopify_count as old_shop, total_products as old_total FROM vendor_registry');
+ // ONE grouped pass over shopify_products instead of 485 sequential per-vendor ILIKE
+ // queries — each of those cost ~5.5s on Kamatera (seq-scan under load), making a
+ // sweep ~16,400s there vs 60s locally. shopify_id is UNIQUE (one row per product,
+ // one vendor string per id), so summing per-vendor-string groups over the pattern
+ // matches is exact — identical semantics to the old DISTINCT-per-pattern query.
+ const { rows: shopGroups } = await pool.query(
+ `SELECT vendor,
+ COUNT(DISTINCT shopify_id) AS cnt,
+ COUNT(CASE WHEN image_url IS NOT NULL AND image_url != '' THEN 1 END) AS with_img,
+ COUNT(CASE WHEN UPPER(status) = 'DRAFT' THEN 1 END) AS draft_cnt,
+ COUNT(CASE WHEN UPPER(status) = 'ARCHIVED' THEN 1 END) AS arch_cnt
+ FROM shopify_products
+ WHERE status NOT IN ('DELETED_FROM_SHOPIFY','DUPLICATE_IMPORT')
+ GROUP BY vendor`);
+ const ilikeToRegex = (p) => new RegExp('^' + String(p)
+ .replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+ .replace(/%/g, '.*').replace(/_/g, '.') + '$', 'i');
+ const shopStatsFor = (patterns) => {
+ const regs = patterns.map(ilikeToRegex);
+ const acc = { cnt: 0, with_img: 0, draft_cnt: 0, arch_cnt: 0 };
+ for (const g of shopGroups) {
+ if (regs.some((r) => r.test(g.vendor || ''))) {
+ acc.cnt += parseInt(g.cnt) || 0;
+ acc.with_img += parseInt(g.with_img) || 0;
+ acc.draft_cnt += parseInt(g.draft_cnt) || 0;
+ acc.arch_cnt += parseInt(g.arch_cnt) || 0;
+ }
+ }
+ return acc;
+ };
+
// Detect catalog_tables shared by >1 vendor — these must be filtered to THIS
// vendor's rows or every sharer gets the whole-table count (the quadrille_house_catalog
// bug: 9 vendors all reported 1942). Discriminator column resolved per table:
@@ -2344,20 +2383,13 @@ async function syncVendorCounts() {
const patterns = VENDOR_NAME_MAP[code] || (v.vendor_name ? [v.vendor_name + '%'] : null);
if (patterns && patterns.length > 0) {
try {
- const conditions = patterns.map((_, i) => `vendor ILIKE $${i + 1}`).join(' OR ');
- const shopRes = await pool.query(
- `SELECT COUNT(DISTINCT shopify_id) as cnt,
- COUNT(CASE WHEN image_url IS NOT NULL AND image_url != '' THEN 1 END) as with_img,
- COUNT(CASE WHEN UPPER(status) = 'DRAFT' THEN 1 END) as draft_cnt,
- COUNT(CASE WHEN UPPER(status) = 'ARCHIVED' THEN 1 END) as arch_cnt
- FROM shopify_products WHERE (${conditions})
- AND status NOT IN ('DELETED_FROM_SHOPIFY','DUPLICATE_IMPORT')`,
- patterns
- );
- shopifyCount = parseInt(shopRes.rows[0].cnt) || 0;
- shopifyWithImages = parseInt(shopRes.rows[0].with_img) || 0;
- shopifyDraft = parseInt(shopRes.rows[0].draft_cnt) || 0;
- shopifyArchived = parseInt(shopRes.rows[0].arch_cnt) || 0;
+ // In-memory match against the single grouped pass (see shopGroups above) —
+ // replaces a per-vendor ILIKE query that cost ~5.5s each on Kamatera
+ const s = shopStatsFor(patterns);
+ shopifyCount = s.cnt;
+ shopifyWithImages = s.with_img;
+ shopifyDraft = s.draft_cnt;
+ shopifyArchived = s.arch_cnt;
} catch (_) {}
}
@@ -2432,6 +2464,8 @@ async function syncVendorCounts() {
console.error('[Victor] Sync error:', err.message);
logActivity('error', 'Sync failed: ' + err.message);
return { success: false, error: err.message };
+ } finally {
+ syncState.running = false;
}
}
diff --git a/shopify/scripts/cadence/data/activations-2026-08-01.jsonl b/shopify/scripts/cadence/data/activations-2026-08-01.jsonl
index 78b76403..c7c676b2 100644
--- a/shopify/scripts/cadence/data/activations-2026-08-01.jsonl
+++ b/shopify/scripts/cadence/data/activations-2026-08-01.jsonl
@@ -70,3 +70,39 @@
{"ts":"2026-08-01T11:58:27.846Z","vendor":"Pierre Frey","dw_sku":"DWPF-605365","product_id":"gid://shopify/Product/7878199738419","action":"tagged","tags":["Needs-Description","Needs-Image"]}
{"ts":"2026-08-01T11:58:28.524Z","vendor":"Pierre Frey","dw_sku":"DWPF-605366","product_id":"gid://shopify/Product/7878199902259","action":"tagged","tags":["Needs-Description","Needs-Image"]}
{"ts":"2026-08-01T11:58:29.172Z","vendor":"Pierre Frey","dw_sku":"DWPF-605367","product_id":"gid://shopify/Product/7878199935027","action":"tagged","tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:30.707Z","vendor":"Pierre Frey","dw_sku":"DWPF-605350","product_id":"gid://shopify/Product/7878199050291","action":"gate-blocked","reasons":["description too short (placeholder-grade)"],"tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:30.708Z","vendor":"Pierre Frey","dw_sku":"DWPF-605351","product_id":"gid://shopify/Product/7878199115827","action":"gate-blocked","reasons":["description too short (placeholder-grade)"],"tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:30.709Z","vendor":"Pierre Frey","dw_sku":"DWPF-605352","product_id":"gid://shopify/Product/7878199148595","action":"gate-blocked","reasons":["description too short (placeholder-grade)"],"tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:30.710Z","vendor":"Pierre Frey","dw_sku":"DWPF-605353","product_id":"gid://shopify/Product/7878199181363","action":"gate-blocked","reasons":["description too short (placeholder-grade)"],"tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:30.710Z","vendor":"Pierre Frey","dw_sku":"DWPF-605354","product_id":"gid://shopify/Product/7878199214131","action":"gate-blocked","reasons":["description too short (placeholder-grade)"],"tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:30.710Z","vendor":"Pierre Frey","dw_sku":"DWPF-605355","product_id":"gid://shopify/Product/7878199246899","action":"gate-blocked","reasons":["description too short (placeholder-grade)"],"tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:30.711Z","vendor":"Pierre Frey","dw_sku":"DWPF-605356","product_id":"gid://shopify/Product/7878199312435","action":"gate-blocked","reasons":["description too short (placeholder-grade)"],"tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:30.711Z","vendor":"Pierre Frey","dw_sku":"DWPF-605357","product_id":"gid://shopify/Product/7878199345203","action":"gate-blocked","reasons":["description too short (placeholder-grade)"],"tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:30.711Z","vendor":"Pierre Frey","dw_sku":"DWPF-605358","product_id":"gid://shopify/Product/7878199377971","action":"gate-blocked","reasons":["description too short (placeholder-grade)"],"tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:30.712Z","vendor":"Pierre Frey","dw_sku":"DWPF-605359","product_id":"gid://shopify/Product/7878199410739","action":"gate-blocked","reasons":["description too short (placeholder-grade)"],"tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:30.712Z","vendor":"Pierre Frey","dw_sku":"DWPF-605360","product_id":"gid://shopify/Product/7878199541811","action":"gate-blocked","reasons":["description too short (placeholder-grade)"],"tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:30.712Z","vendor":"Pierre Frey","dw_sku":"DWPF-605361","product_id":"gid://shopify/Product/7878199574579","action":"gate-blocked","reasons":["description too short (placeholder-grade)"],"tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:30.713Z","vendor":"Pierre Frey","dw_sku":"DWPF-605362","product_id":"gid://shopify/Product/7878199640115","action":"gate-blocked","reasons":["description too short (placeholder-grade)"],"tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:30.713Z","vendor":"Pierre Frey","dw_sku":"DWPF-605363","product_id":"gid://shopify/Product/7878199672883","action":"gate-blocked","reasons":["description too short (placeholder-grade)"],"tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:30.713Z","vendor":"Pierre Frey","dw_sku":"DWPF-605364","product_id":"gid://shopify/Product/7878199705651","action":"gate-blocked","reasons":["description too short (placeholder-grade)"],"tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:30.713Z","vendor":"Pierre Frey","dw_sku":"DWPF-605365","product_id":"gid://shopify/Product/7878199738419","action":"gate-blocked","reasons":["description too short (placeholder-grade)"],"tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:30.714Z","vendor":"Pierre Frey","dw_sku":"DWPF-605366","product_id":"gid://shopify/Product/7878199902259","action":"gate-blocked","reasons":["description too short (placeholder-grade)"],"tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:30.714Z","vendor":"Pierre Frey","dw_sku":"DWPF-605367","product_id":"gid://shopify/Product/7878199935027","action":"gate-blocked","reasons":["description too short (placeholder-grade)"],"tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:32.096Z","vendor":"Pierre Frey","dw_sku":"DWPF-605350","product_id":"gid://shopify/Product/7878199050291","action":"tagged","tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:32.801Z","vendor":"Pierre Frey","dw_sku":"DWPF-605351","product_id":"gid://shopify/Product/7878199115827","action":"tagged","tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:33.448Z","vendor":"Pierre Frey","dw_sku":"DWPF-605352","product_id":"gid://shopify/Product/7878199148595","action":"tagged","tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:34.088Z","vendor":"Pierre Frey","dw_sku":"DWPF-605353","product_id":"gid://shopify/Product/7878199181363","action":"tagged","tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:34.750Z","vendor":"Pierre Frey","dw_sku":"DWPF-605354","product_id":"gid://shopify/Product/7878199214131","action":"tagged","tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:35.388Z","vendor":"Pierre Frey","dw_sku":"DWPF-605355","product_id":"gid://shopify/Product/7878199246899","action":"tagged","tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:36.053Z","vendor":"Pierre Frey","dw_sku":"DWPF-605356","product_id":"gid://shopify/Product/7878199312435","action":"tagged","tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:36.707Z","vendor":"Pierre Frey","dw_sku":"DWPF-605357","product_id":"gid://shopify/Product/7878199345203","action":"tagged","tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:37.360Z","vendor":"Pierre Frey","dw_sku":"DWPF-605358","product_id":"gid://shopify/Product/7878199377971","action":"tagged","tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:37.998Z","vendor":"Pierre Frey","dw_sku":"DWPF-605359","product_id":"gid://shopify/Product/7878199410739","action":"tagged","tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:38.648Z","vendor":"Pierre Frey","dw_sku":"DWPF-605360","product_id":"gid://shopify/Product/7878199541811","action":"tagged","tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:39.290Z","vendor":"Pierre Frey","dw_sku":"DWPF-605361","product_id":"gid://shopify/Product/7878199574579","action":"tagged","tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:39.937Z","vendor":"Pierre Frey","dw_sku":"DWPF-605362","product_id":"gid://shopify/Product/7878199640115","action":"tagged","tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:40.590Z","vendor":"Pierre Frey","dw_sku":"DWPF-605363","product_id":"gid://shopify/Product/7878199672883","action":"tagged","tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:41.229Z","vendor":"Pierre Frey","dw_sku":"DWPF-605364","product_id":"gid://shopify/Product/7878199705651","action":"tagged","tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:41.879Z","vendor":"Pierre Frey","dw_sku":"DWPF-605365","product_id":"gid://shopify/Product/7878199738419","action":"tagged","tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:42.547Z","vendor":"Pierre Frey","dw_sku":"DWPF-605366","product_id":"gid://shopify/Product/7878199902259","action":"tagged","tags":["Needs-Description","Needs-Image"]}
+{"ts":"2026-08-01T17:58:43.182Z","vendor":"Pierre Frey","dw_sku":"DWPF-605367","product_id":"gid://shopify/Product/7878199935027","action":"tagged","tags":["Needs-Description","Needs-Image"]}
← 895cebe5 auto-save: 2026-08-01T10:36:04 (1 files) — DW-Agents/vendor-
·
back to Designer Wallcoverings
·
auto-save: 2026-08-01T11:36:26 (1 files) — DW-Agents/vendor- 63944705 →