← back to New Import Viewer
new-import-viewer: agent + web viewer for products new-to-import (sync_status=new / not on Shopify)
1a10b71191f7a76b6877c8e4ad7158a30b4ddda2 · 2026-06-09 15:46:53 -0700 · SteveStudio2
Zero-dep node+psql server over dw_unified.vendor_catalog. /api/stats, /api/vendors,
/api/new-products (vendor filter + search + 7 sort modes + pagination). Viewer has
the mandatory sort select + density slider (localStorage-persisted) and the admin
crawl date+time chip per standing rules. Read-only; surfaces what the latest vendor
crawl flagged as NEW before any Phase-8 push. Port 9790.
Files touched
A .gitignoreA package.jsonA public/index.htmlA server.js
Diff
commit 1a10b71191f7a76b6877c8e4ad7158a30b4ddda2
Author: SteveStudio2 <stevestudio2@SteveStacStudio.lan>
Date: Tue Jun 9 15:46:53 2026 -0700
new-import-viewer: agent + web viewer for products new-to-import (sync_status=new / not on Shopify)
Zero-dep node+psql server over dw_unified.vendor_catalog. /api/stats, /api/vendors,
/api/new-products (vendor filter + search + 7 sort modes + pagination). Viewer has
the mandatory sort select + density slider (localStorage-persisted) and the admin
crawl date+time chip per standing rules. Read-only; surfaces what the latest vendor
crawl flagged as NEW before any Phase-8 push. Port 9790.
---
.gitignore | 8 +++++
package.json | 8 +++++
public/index.html | 106 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
server.js | 96 +++++++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 218 insertions(+)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..1924158
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,8 @@
+node_modules/
+.env*
+tmp/
+*.log
+.DS_Store
+dist/
+build/
+.next/
diff --git a/package.json b/package.json
new file mode 100644
index 0000000..3983bdf
--- /dev/null
+++ b/package.json
@@ -0,0 +1,8 @@
+{
+ "name": "new-import-viewer",
+ "version": "1.0.0",
+ "description": "Products New to Import — agent + web viewer over dw_unified.vendor_catalog (sync_status=new / not on Shopify)",
+ "main": "server.js",
+ "scripts": { "start": "node server.js" },
+ "private": true
+}
diff --git a/public/index.html b/public/index.html
new file mode 100644
index 0000000..a12a025
--- /dev/null
+++ b/public/index.html
@@ -0,0 +1,106 @@
+<!doctype html><html lang="en"><head><meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Products New to Import · dw_unified</title>
+<style>
+:root{--cols:6;--bg:#0c0d12;--panel:#15171f;--card:#181b24;--ink:#eef0f5;--mut:#9aa0b0;--acc:#5eead4;--new:#f59e0b;--line:#262a36}
+*{box-sizing:border-box}
+body{margin:0;background:var(--bg);color:var(--ink);font:13px/1.45 -apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif}
+header{position:sticky;top:0;z-index:9;background:linear-gradient(180deg,#0c0d12,#0c0d12e8);backdrop-filter:blur(8px);
+ border-bottom:1px solid var(--line);padding:12px 16px;display:flex;gap:12px;align-items:center;flex-wrap:wrap}
+h1{font-size:15px;margin:0;font-weight:700;letter-spacing:.2px}
+.pill{background:var(--new);color:#1a1205;font-weight:700;border-radius:999px;padding:2px 10px;font-size:12px}
+.muted{color:var(--mut)}
+select,input[type=text]{background:var(--panel);color:var(--ink);border:1px solid var(--line);border-radius:8px;padding:6px 9px;font-size:13px}
+input[type=text]{min-width:180px}
+.ctl{display:flex;align-items:center;gap:7px}
+input[type=range]{accent-color:var(--acc)}
+.grid{display:grid;grid-template-columns:repeat(var(--cols),1fr);gap:12px;padding:16px}
+.card{background:var(--card);border:1px solid var(--line);border-radius:12px;overflow:hidden;display:flex;flex-direction:column}
+.card img{width:100%;aspect-ratio:1/1;object-fit:cover;object-position:center 12%;background:#0a0b0f;display:block}
+.card .b{padding:9px 10px;display:flex;flex-direction:column;gap:4px}
+.vend{font-size:11px;color:var(--acc);font-weight:700;text-transform:uppercase;letter-spacing:.4px}
+.ttl{font-weight:600;line-height:1.25}
+.sub{color:var(--mut);font-size:12px}
+.when{color:var(--mut);font-size:11px;display:flex;align-items:center;gap:4px;margin-top:2px}
+.row{display:flex;justify-content:space-between;align-items:center;gap:6px}
+.sku{font-family:ui-monospace,Menlo,monospace;font-size:11px;color:var(--mut)}
+a.open{color:var(--acc);text-decoration:none;font-size:11px;border:1px solid var(--line);border-radius:6px;padding:2px 7px}
+a.open:hover{background:var(--panel)}
+#more{display:block;margin:8px auto 40px;background:var(--panel);color:var(--ink);border:1px solid var(--line);border-radius:8px;padding:9px 18px;cursor:pointer}
+#empty{padding:60px;text-align:center;color:var(--mut)}
+</style></head>
+<body>
+<header>
+ <h1>Products New to Import</h1>
+ <span class="pill" id="total">…</span>
+ <span class="muted" id="meta"></span>
+ <div class="ctl"><label class="muted">Vendor</label>
+ <select id="vendor"><option value="">All vendors</option></select></div>
+ <div class="ctl"><input type="text" id="q" placeholder="search pattern / sku / color…"></div>
+ <div class="ctl"><label class="muted">Sort</label>
+ <select id="sort">
+ <option value="newest">Newest crawl</option>
+ <option value="oldest">Oldest crawl</option>
+ <option value="firstseen">First seen</option>
+ <option value="vendor">Vendor</option>
+ <option value="sku">SKU A→Z</option>
+ <option value="pattern">Pattern A→Z</option>
+ <option value="color">Color A→Z</option>
+ </select></div>
+ <div class="ctl"><label class="muted">Density</label>
+ <input type="range" id="density" min="3" max="10" step="1"></div>
+</header>
+<div class="grid" id="grid"></div>
+<button id="more" style="display:none">Load more</button>
+<div id="empty" style="display:none">No products match.</div>
+<script>
+const $=s=>document.querySelector(s);
+const grid=$('#grid'), state={offset:0,limit:120,count:0,loading:false};
+// persisted prefs (CLAUDE.md: sort + density persist in localStorage)
+$('#sort').value=localStorage.getItem('nip.sort')||'newest';
+const dens=localStorage.getItem('nip.density')||'6'; $('#density').value=dens;
+document.documentElement.style.setProperty('--cols',dens);
+function esc(s){return String(s==null?'':s).replace(/[&<>"']/g,c=>({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c]));}
+function card(p){
+ const title=esc(p.pattern||p.sku||'(untitled)')+(p.color?(', '+esc(p.color)):'');
+ const img=p.image?`<img loading="lazy" src="${esc(p.image)}" alt="${title}" onerror="this.style.opacity=.15">`
+ :`<div style="aspect-ratio:1/1;background:#0a0b0f;display:flex;align-items:center;justify-content:center;color:#333">no image</div>`;
+ const crawled=p.crawled?`🕓 ${esc(p.crawled)}`:'🕓 (no crawl date)';
+ const seen=p.firstSeen?` · first seen ${esc(p.firstSeen)}`:'';
+ return `<div class="card">${img}<div class="b">
+ <span class="vend">${esc(p.vendor)}${p.collection?(' · '+esc(p.collection)):''}</span>
+ <div class="ttl">${title}</div>
+ <div class="row"><span class="sub">${esc(p.type||'')}</span>
+ ${p.url?`<a class="open" href="${esc(p.url)}" target="_blank" rel="noopener noreferrer">vendor ↗</a>`:''}</div>
+ <div class="row"><span class="sku">${esc(p.sku||'')}${p.dwSku?(' · '+esc(p.dwSku)):''}</span></div>
+ <div class="when" title="crawl date+time (when this run last saw it on the vendor site)">${crawled}${seen}</div>
+ </div></div>`;
+}
+async function load(reset){
+ if(state.loading) return; state.loading=true;
+ if(reset){ state.offset=0; grid.innerHTML=''; }
+ const p=new URLSearchParams({vendor:$('#vendor').value,sort:$('#sort').value,q:$('#q').value,limit:state.limit,offset:state.offset});
+ const d=await (await fetch('/api/new-products?'+p)).json();
+ state.count=d.count; state.offset+=d.items.length;
+ grid.insertAdjacentHTML('beforeend', d.items.map(card).join(''));
+ $('#meta').textContent=`${state.offset.toLocaleString()} of ${d.count.toLocaleString()} shown`;
+ $('#empty').style.display=d.count?'none':'block';
+ $('#more').style.display=state.offset<d.count?'block':'none';
+ state.loading=false;
+}
+async function boot(){
+ const s=await (await fetch('/api/stats')).json();
+ $('#total').textContent=s.total.toLocaleString()+' new';
+ const vs=await (await fetch('/api/vendors')).json();
+ $('#vendor').insertAdjacentHTML('beforeend', vs.map(v=>`<option value="${esc(v.vendor)}">${esc(v.vendor)} (${v.count.toLocaleString()})</option>`).join(''));
+ load(true);
+}
+$('#vendor').onchange=()=>load(true);
+$('#sort').onchange=()=>{localStorage.setItem('nip.sort',$('#sort').value);load(true);};
+let t; $('#q').oninput=()=>{clearTimeout(t);t=setTimeout(()=>load(true),300);};
+$('#density').oninput=()=>{document.documentElement.style.setProperty('--cols',$('#density').value);localStorage.setItem('nip.density',$('#density').value);};
+$('#more').onclick=()=>load(false);
+addEventListener('scroll',()=>{if(innerHeight+scrollY>=document.body.offsetHeight-600)load(false);});
+boot();
+</script>
+</body></html>
diff --git a/server.js b/server.js
new file mode 100644
index 0000000..1997a7f
--- /dev/null
+++ b/server.js
@@ -0,0 +1,96 @@
+#!/usr/bin/env node
+// new-import-viewer — the "Products New to Import" agent + web viewer.
+// Surfaces every vendor_catalog row the crawl flagged as NOT yet on Shopify
+// (sync_status='new' / on_shopify=false) so Steve can review what the latest
+// vendor crawl surfaced before a Phase-8 push. Read-only against dw_unified.
+// Zero npm deps — shells out to psql (same pattern as wallco-live-viewer).
+const http = require('http');
+const fs = require('fs');
+const path = require('path');
+const { spawnSync } = require('child_process');
+
+const PORT = process.env.PORT || 9790;
+const PSQL = process.env.PSQL || '/opt/homebrew/opt/postgresql@14/bin/psql';
+const PGUSER = process.env.PGUSER || 'stevestudio2';
+const PGDB = process.env.PGDATABASE || 'dw_unified';
+const US = '\x1f'; // unit separator — safe field delim for text columns
+const RS = '\x1e'; // record separator
+
+// NEW-to-import predicate — the single source of truth for "new vs Shopify".
+const NEW_PRED = "(sync_status='new' OR ((on_shopify IS NOT TRUE) AND shopify_product_id IS NULL))";
+
+function q(sql) {
+ const r = spawnSync(PSQL, ['-U', PGUSER, '-d', PGDB, '-tA', '-F', US, '-R', RS, '-c', sql],
+ { encoding: 'utf8', timeout: 30000, maxBuffer: 1 << 28 });
+ if (r.status !== 0) throw new Error((r.stderr || 'psql failed').slice(0, 400));
+ // psql appends a trailing newline to the final record — strip per-row before split.
+ return (r.stdout || '').split(RS).map(l => l.replace(/\n+$/, '')).filter(l => l.length).map(l => l.split(US));
+}
+const esc = (s) => String(s == null ? '' : s).replace(/'/g, "''");
+const SORTS = {
+ newest: 'last_scraped_at DESC NULLS LAST, id DESC',
+ oldest: 'last_scraped_at ASC NULLS LAST, id ASC',
+ firstseen: 'first_seen_at DESC NULLS LAST, id DESC',
+ vendor: "vendor_code ASC, pattern_name ASC",
+ sku: "mfr_sku ASC",
+ pattern: "pattern_name ASC, color_name ASC",
+ color: "color_name ASC, pattern_name ASC",
+};
+
+function send(res, code, body, type) {
+ res.writeHead(code, { 'content-type': type || 'application/json', 'cache-control': 'no-store' });
+ res.end(body);
+}
+
+const server = http.createServer((req, res) => {
+ try {
+ const u = new URL(req.url, 'http://x');
+ if (u.pathname === '/' || u.pathname === '/index.html') {
+ return send(res, 200, fs.readFileSync(path.join(__dirname, 'public', 'index.html')), 'text/html; charset=utf-8');
+ }
+ if (u.pathname === '/api/stats') {
+ const r = q(`SELECT count(*), count(*) FILTER (WHERE image_url<>''), count(DISTINCT vendor_code),
+ to_char(max(last_scraped_at),'YYYY-MM-DD HH24:MI')
+ FROM vendor_catalog WHERE ${NEW_PRED}`)[0] || [];
+ return send(res, 200, JSON.stringify({ total: +r[0] || 0, withImage: +r[1] || 0, vendors: +r[2] || 0, newestCrawl: r[3] || null }));
+ }
+ if (u.pathname === '/api/vendors') {
+ const rows = q(`SELECT vendor_code, count(*),
+ to_char(max(last_scraped_at),'YYYY-MM-DD HH24:MI')
+ FROM vendor_catalog WHERE ${NEW_PRED}
+ GROUP BY vendor_code ORDER BY count(*) DESC`);
+ return send(res, 200, JSON.stringify(rows.map(r => ({ vendor: r[0], count: +r[1], crawled: r[2] }))));
+ }
+ if (u.pathname === '/api/new-products') {
+ const vendor = (u.searchParams.get('vendor') || '').replace(/[^a-z0-9_]/gi, '').slice(0, 64);
+ const sort = SORTS[u.searchParams.get('sort')] ? u.searchParams.get('sort') : 'newest';
+ const term = (u.searchParams.get('q') || '').slice(0, 80);
+ const limit = Math.min(parseInt(u.searchParams.get('limit'), 10) || 120, 500);
+ const offset = Math.max(parseInt(u.searchParams.get('offset'), 10) || 0, 0);
+ let where = `WHERE ${NEW_PRED}`;
+ if (vendor) where += ` AND vendor_code='${esc(vendor)}'`;
+ if (term) {
+ const t = `%${esc(term)}%`;
+ where += ` AND (pattern_name ILIKE '${t}' OR mfr_sku ILIKE '${t}' OR color_name ILIKE '${t}' OR dw_sku ILIKE '${t}' OR collection ILIKE '${t}')`;
+ }
+ const rows = q(`SELECT id, vendor_code, mfr_sku, dw_sku, pattern_name, color_name, collection,
+ product_type, image_url, product_url,
+ to_char(first_seen_at,'YYYY-MM-DD HH24:MI') AS first_seen,
+ to_char(last_scraped_at,'YYYY-MM-DD HH24:MI') AS crawled,
+ sync_status
+ FROM vendor_catalog ${where}
+ ORDER BY ${SORTS[sort]} LIMIT ${limit} OFFSET ${offset}`);
+ const cnt = +(q(`SELECT count(*) FROM vendor_catalog ${where}`)[0] || [0])[0];
+ const items = rows.map(r => ({
+ id: +r[0], vendor: r[1], sku: r[2], dwSku: r[3], pattern: r[4], color: r[5],
+ collection: r[6], type: r[7], image: r[8], url: r[9],
+ firstSeen: r[10], crawled: r[11], status: r[12],
+ }));
+ return send(res, 200, JSON.stringify({ count: cnt, offset, limit, items }));
+ }
+ return send(res, 404, JSON.stringify({ error: 'not found' }));
+ } catch (e) {
+ return send(res, 500, JSON.stringify({ error: e.message }));
+ }
+});
+server.listen(PORT, () => console.log(`[new-import-viewer] http://127.0.0.1:${PORT} (NEW-to-import from dw_unified.vendor_catalog)`));
(oldest)
·
back to New Import Viewer
·
fix: strip psql trailing newline from final field (stats new f6b1300 →