← back to Corvette Dashboard Viewer
server.js
194 lines
// '67 Corvette Catalog Console — viewer for TK-135 palette/descriptor data.
// Static file server + a LIVE /api/data endpoint that pulls the current AI-tagged
// catalog (shopify_color_enrichment ⋈ shopify_products) straight from the local
// dw_unified mirror. Zero-dependency: the DB shapes the JSON via json_agg and we
// shell out to psql, so no `pg` module is needed. If the DB is unreachable the
// endpoint falls back to the frozen public/data.json snapshot so the console never breaks.
const http = require('http'), https = require('https'), fs = require('fs'), path = require('path'), { execFile } = require('child_process');
const PORT = process.env.PORT || 9803;
const ROOT = path.join(__dirname, 'public');
const PGDB = process.env.PGDATABASE || 'dw_unified';
const PGHOST = process.env.PGHOST || '/tmp';
const MIME = { '.html': 'text/html', '.json': 'application/json', '.css': 'text/css', '.js': 'text/javascript', '.png': 'image/png', '.jpg': 'image/jpeg', '.svg': 'image/svg+xml' };
// Read-only live pull. `limit` is clamped to an integer before interpolation (the
// only dynamic value in the SQL) so there's no injection surface. Returns an object
// { records:[…capped…], totals:{skus,colors,vendors} } — totals are the TRUE catalog
// counts so the gauges never lie about the sample size.
function fetchLive(limit, cb) {
const n = Math.max(1, Math.min(parseInt(limit, 10) || 600, 5000));
const base = `shopify_color_enrichment e join shopify_products p on p.shopify_id = e.shopify_id
where e.colors::text ilike '%percentage%' and p.dw_sku is not null`;
const sql = `select json_build_object(
'records', (select coalesce(json_agg(row_to_json(t)),'[]') from (
select regexp_replace(e.shopify_id,'.*/','') as gid, e.title, e.vendor,
coalesce(nullif(e.color_family,''),nullif(e.dominant_color,'')) as color,
coalesce(nullif(p.dw_sku,''),nullif(p.mfr_sku,''),nullif(p.variant_sku,'')) as sku,
(select json_agg(json_build_object('hex',c->>'hex','pct',(c->>'percentage')::numeric))
from jsonb_array_elements(e.colors) c) as palette
from ${base}
order by e.enrichment_date desc nulls last
limit ${n}) t),
'totals', (select json_build_object(
'skus', count(*),
'colors', coalesce(sum(jsonb_array_length(e.colors)),0),
'vendors', count(distinct e.vendor)) from ${base})
)`;
execFile('psql', ['-h', PGHOST, '-d', PGDB, '-tAc', sql], { maxBuffer: 64 * 1024 * 1024 }, (err, stdout) => {
if (err) return cb(err);
const out = (stdout || '').trim();
if (!out) return cb(new Error('empty'));
cb(null, out);
});
}
http.createServer((req, res) => {
const u = new URL(req.url, 'http://x');
let p = decodeURIComponent(u.pathname);
if (p === '/healthz') { res.writeHead(200); return res.end('ok'); }
// Basic-auth gate (fleet standard admin/DW2024!, override via BASIC_AUTH="user:pass").
// Placed AFTER /healthz so the fleet keepalive can still probe an open 200 and never
// misreads the authed 401 as a dead process (see "auth-gated service needs /healthz").
const AUTH = process.env.BASIC_AUTH || 'admin:DW2024!';
const hdr = req.headers['authorization'] || '';
const got = hdr.startsWith('Basic ') ? Buffer.from(hdr.slice(6), 'base64').toString() : '';
if (got !== AUTH) { res.writeHead(401, { 'WWW-Authenticate': 'Basic realm="LiveSkus"' }); return res.end('auth required'); }
// Live catalog. On any DB failure, fall back to the static snapshot (wrapped to
// match the {records,totals} contract) so the front end always gets valid data.
if (p === '/api/data') {
fetchLive(u.searchParams.get('limit'), (err, json) => {
if (!err && json) { res.writeHead(200, { 'Content-Type': 'application/json', 'X-Data-Source': 'live' }); return res.end(json); }
fs.readFile(path.join(ROOT, 'data.json'), (e, buf) => {
if (e) { res.writeHead(502); return res.end('no live db and no snapshot'); }
// Guard the parse: a corrupt snapshot must not throw in this async callback
// and crash the whole process — that would turn the safety net into an outage.
let arr;
try { arr = JSON.parse(buf); } catch (pe) { res.writeHead(502); return res.end('snapshot unreadable: ' + pe.message); }
res.writeHead(200, { 'Content-Type': 'application/json', 'X-Data-Source': 'snapshot' });
res.end(JSON.stringify({ records: arr, totals: null }));
});
});
return;
}
// ---- LIVE gauge stats: real-time counts from the TK-135 apply-job progress logs.
// These numbers change every few seconds as the jobs write, so the dashboard gauges
// are live instruments (needles climb) rather than a static snapshot.
if (p === '/api/stats') {
const TK = process.env.TK135DIR || (process.env.HOME + '/Projects/Designer-Wallcoverings/shopify/scripts/data/tk135');
const cnt = f => { try { return fs.readFileSync(path.join(TK, f), 'utf8').split('\n').filter(Boolean).length; } catch (e) { return 0; } };
const color = cnt('normalize-progress.log'), style = cnt('style-proposed.progress.log');
const material = cnt('material-primary-proposed.progress.log'), collection = cnt('collection-proposed.progress.log');
const label = cnt('label-normalize-progress.log');
const descriptors = style + material + collection + label;
res.writeHead(200, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
res.end(JSON.stringify({ color, style, material, collection, label, descriptors,
writes: color + descriptors, targets: { color: 81102, descriptors: 66193 } }));
return;
}
// ---- LIVE "new items as they land" ----
// Source of truth for freshness = the TK-135 apply-job progress logs (GIDs appended
// in write-order; tail = most recently updated). We take the tail of the color-normalize
// log (the comprehensive active writer), reverse to newest-first, then join live details
// (title/vendor/handle/tags) from the mirror + the deep hex palette from palette-proposed.tsv.
if (p === '/api/new-items') {
const limit = Math.max(1, Math.min(parseInt(u.searchParams.get('limit'), 10) || 50, 200));
const TK = process.env.TK135DIR || (process.env.HOME + '/Projects/Designer-Wallcoverings/shopify/scripts/data/tk135');
// newest GIDs from the progress log tail
let gids = [], processed = 0;
try {
const lines = fs.readFileSync(path.join(TK, 'normalize-progress.log'), 'utf8').trim().split('\n').filter(Boolean);
processed = lines.length;
gids = lines.slice(-limit).reverse();
} catch (e) {}
// Defense-in-depth: a shopify_id is `gid://shopify/<Type>/<n>` or a bare number — it
// never contains a quote/semicolon/space. Whitelist to that shape so a poisoned
// progress-log line can NEVER reach the interpolated SQL below (the ''-escape stays as
// belt-and-suspenders). Fixes the stale "only limit is dynamic — no injection" claim.
gids = gids.filter(g => /^[\w:/.-]+$/.test(g));
if (!gids.length) { res.writeHead(200, { 'Content-Type': 'application/json' }); return res.end(JSON.stringify({ records: [], processed: 0 })); }
// palette map (gid-number -> [{hex,pct}]) + fresh descriptor maps my jobs applied
// (color/style/material), loaded once + cached. These are the values that ACTUALLY
// landed — truer than the stale mirror tags.
if (!global.__pal) {
global.__pal = {}; global.__desc = { color: {}, style: {}, material: {} };
try { for (const l of fs.readFileSync(path.join(TK, 'palette-proposed.tsv'), 'utf8').split('\n')) {
const f = l.split('\t'); if (f.length < 3 || f[2].startsWith('ERR')) continue;
const k = f[0].replace(/.*\//, '');
try { global.__pal[k] = JSON.parse(f[2]); } catch (e) {}
global.__desc.color[k] = (f[1] || '').replace(/^color:/i, '').trim();
} } catch (e) {}
const load = (file, key) => { try { for (const l of fs.readFileSync(path.join(TK, file), 'utf8').split('\n')) {
const f = l.split('\t'); if (f[0] && f[1]) global.__desc[key][f[0].replace(/.*\//, '')] = f[1].trim(); } } catch (e) {} };
load('style-proposed.tsv', 'style'); load('material-primary-proposed.tsv', 'material');
}
const nums = gids.map(g => g.replace(/.*\//, ''));
const inList = gids.map(g => `'${g.replace(/'/g, "''")}'`).join(',');
const sql = `select coalesce(json_agg(row_to_json(t)),'[]') from (
select regexp_replace(shopify_id,'.*/','') as gid, title, vendor, handle, image_url,
coalesce(nullif(dw_sku,''),nullif(mfr_sku,''),nullif(variant_sku,'')) as sku,
(select json_agg(v) from (select distinct trim(x) v from unnest(string_to_array(tags,',')) x
where trim(x) !~ '[:/.{}"]' and trim(x) !~ '^[0-9]' and length(trim(x)) between 2 and 22
limit 8) s) as tags
from shopify_products where shopify_id in (${inList})) t`;
execFile('psql', ['-h', PGHOST, '-d', PGDB, '-tAc', sql], { maxBuffer: 16 * 1024 * 1024 }, (err, stdout) => {
let byGid = {};
if (!err && stdout.trim()) { try { JSON.parse(stdout.trim()).forEach(r => byGid[r.gid] = r); } catch (e) {} }
const D = global.__desc;
const records = nums.map(n => {
const d = byGid[n] || {};
// fresh descriptors first, then any surviving bare mirror tags
const fresh = [D.color[n], D.style[n], D.material[n]].filter(Boolean);
const tags = [...new Set([...fresh, ...(d.tags || [])])].slice(0, 8);
return { gid: n, sku: d.sku || '', image: d.image_url || '', title: d.title || '(loading…)', vendor: d.vendor || '', handle: d.handle || '',
color: D.color[n] || '', tags, palette: global.__pal[n] || [],
href: d.handle ? `https://designer-laboratory-sandbox.myshopify.com/products/${d.handle}` : null };
});
res.writeHead(200, { 'Content-Type': 'application/json', 'X-Data-Source': 'live-progress' });
res.end(JSON.stringify({ records, processed }));
});
return;
}
// ---- image proxy for the live ticker ----
// Fetch the product image server-side so a dead Shopify CDN asset degrades to a
// transparent placeholder (same-origin 200) instead of a browser-console 404.
// SSRF-guarded: only https shopify.com CDN hosts are fetched; anything else → placeholder.
if (p === '/img') {
const PLACEHOLDER = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=', 'base64');
// Guard on res.headersSent (NOT just writableEnded): once we've written the 200 and
// started piping, a mid-body CDN stall fires ir.setTimeout -> placeholder() while the
// pipe is un-ended, and a 2nd writeHead throws ERR_HTTP_HEADERS_SENT -> process crash.
const placeholder = () => { if (res.headersSent || res.writableEnded) return; res.writeHead(200, { 'Content-Type': 'image/png', 'Cache-Control': 'no-store' }); res.end(PLACEHOLDER); };
let tu; try { tu = new URL(u.searchParams.get('u') || ''); } catch { return placeholder(); }
if (tu.protocol !== 'https:' || !/(^|\.)shopify\.com$/.test(tu.hostname)) return placeholder();
const ir = https.get(tu.href, (pres) => {
if (pres.statusCode !== 200) { pres.resume(); return placeholder(); }
res.writeHead(200, { 'Content-Type': pres.headers['content-type'] || 'image/jpeg', 'Cache-Control': 'public,max-age=3600' });
pres.pipe(res);
});
ir.on('error', placeholder);
ir.setTimeout(6000, () => {
ir.destroy();
// If the 200 already went out and piping began (stall MID-body), placeholder() no-ops
// on the headersSent guard — which would leak a half-piped, never-ended response. So
// tear the client socket down here. Otherwise (stall pre-headers) emit the placeholder.
if (res.headersSent && !res.writableEnded) res.destroy(); else placeholder();
});
return;
}
if (p === '/') p = '/index.html';
const fp = path.join(ROOT, path.normalize(p));
if (!fp.startsWith(ROOT)) { res.writeHead(403); return res.end('forbidden'); }
fs.readFile(fp, (e, buf) => {
if (e) { res.writeHead(404); return res.end('not found'); }
res.writeHead(200, { 'Content-Type': MIME[path.extname(fp)] || 'application/octet-stream' });
res.end(buf);
});
}).listen(PORT, () => console.log(`'67 Corvette Console → http://127.0.0.1:${PORT} (live /api/data ⋈ ${PGDB})`));