pivot to 'where the data comes from' dashboard (Steve: show actual RSS feeds/DB crawl, not abstract viz): per-source provenance (method/pull-count/deals/samples) + DB-crawl status card + Nav Agent dockable panel (left OR right, collapse, click-a-value-to-filter per FIELD) + no-WebGL so it scrolls & never freezes; default view -> sources dashboard (TK-10708)

c9ace0f8642f8f1157cda3bdc60e5c0918fd440d · 2026-08-19 17:09:08 -0700 · Steve Abrams
Files touched

A public/sources/index.html
M scripts/cre-news-monitor.mjs
M serve-viewers.mjs
Diff

commit c9ace0f8642f8f1157cda3bdc60e5c0918fd440d
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Wed Aug 19 17:09:08 2026 -0700

    pivot to 'where the data comes from' dashboard (Steve: show actual RSS feeds/DB crawl, not abstract viz): per-source provenance (method/pull-count/deals/samples) + DB-crawl status card + Nav Agent dockable panel (left OR right, collapse, click-a-value-to-filter per FIELD) + no-WebGL so it scrolls & never freezes; default view -> sources dashboard (TK-10708)
---
 public/sources/index.html    | 116 +++++++++++++++++++++++++++++++++++++++++++
 scripts/cre-news-monitor.mjs |  14 ++++++
 serve-viewers.mjs            |  12 ++---
 3 files changed, 135 insertions(+), 7 deletions(-)

diff --git a/public/sources/index.html b/public/sources/index.html
new file mode 100644
index 0000000..f22e8e3
--- /dev/null
+++ b/public/sources/index.html
@@ -0,0 +1,116 @@
+<!doctype html><html lang="en"><head><meta charset="utf-8">
+<meta name="viewport" content="width=device-width,initial-scale=1">
+<title>Where the data comes from — CRE pipeline · Abrams</title>
+<style>
+ :root{--navy:#0E2350;--blue:#16357A;--gold:#E8A81C;--ink:#141414;--gray:#6B7280}
+ *{box-sizing:border-box} body{margin:0;font-family:Inter,-apple-system,Helvetica,Arial,sans-serif;background:#f4f6fb;color:var(--ink)}
+ /* ---- Nav Agent dockable panel (left OR right) ---- */
+ #nav{position:fixed;top:0;bottom:0;left:0;width:250px;z-index:30;background:#0b1120;border-right:1px solid #1c2540;color:#cfe0ff;overflow-y:auto;transition:width .18s}
+ #nav.right{left:auto;right:0;border-right:none;border-left:1px solid #1c2540}
+ #nav.collapsed{width:36px;overflow:hidden}
+ #nav .nh{display:flex;justify-content:space-between;align-items:center;padding:16px 14px 6px;font-size:13px;letter-spacing:2px;color:#7fd7ff;font-weight:800}
+ #nav .nh button{background:#152037;color:#9fb4e6;border:1px solid #24304f;border-radius:6px;cursor:pointer;font-size:12px;padding:2px 8px;margin-left:5px}
+ #nav .hint{padding:0 14px 8px;font-size:10px;color:#5a6b92;letter-spacing:1px}
+ #nav .field{padding:9px 14px 4px;font-size:11px;color:#8aa;text-transform:uppercase;letter-spacing:1px;border-top:1px solid #131c33}
+ #nav .val{display:flex;justify-content:space-between;align-items:center;padding:6px 14px;cursor:pointer;font-size:13px}
+ #nav .val:hover{background:#131c33} #nav .val.on{background:#16263f;color:#7fd7ff;font-weight:600}
+ #nav .val .b{background:#0e2a3a;color:#7fd7ff;border-radius:10px;padding:1px 9px;font-size:11px}
+ #nav.collapsed .nh span:first-child,#nav.collapsed .hint,#nav.collapsed #navfields{display:none}
+ body.navL header,body.navL .wrap{margin-left:250px} body.navR header,body.navR .wrap{margin-right:250px}
+ body.navLc header,body.navLc .wrap{margin-left:36px} body.navRc header,body.navRc .wrap{margin-right:36px}
+ header{background:var(--navy);color:#fff;padding:18px 26px;transition:margin .18s}
+ header h1{margin:0;font-size:21px} header .s{color:var(--gold);font-size:12px;margin-top:3px}
+ .flow{display:flex;gap:8px;align-items:center;flex-wrap:wrap;margin-top:12px;font-size:12px}
+ .flow .n{background:#182444;border:1px solid #2a3a66;border-radius:8px;padding:8px 12px;text-align:center}
+ .flow .n b{display:block;font-size:18px;color:#7fd7ff} .flow .a{color:#E8A81C;font-size:18px}
+ .wrap{max-width:1180px;margin:20px auto;padding:0 20px;transition:margin .18s}
+ h2{font-size:15px;color:var(--navy);margin:22px 0 10px}
+ .controls{display:flex;gap:12px;align-items:center;margin-bottom:10px;font-size:13px}
+ .controls input{padding:6px 9px;border:1px solid #d6dced;border-radius:6px}
+ .clr{color:var(--blue);cursor:pointer;font-size:12px} .stat{color:var(--gray);margin-left:auto}
+ table{border-collapse:collapse;width:100%;font-size:13px;background:#fff;border-radius:10px;overflow:hidden;box-shadow:0 2px 10px rgba(0,0,0,.05)}
+ th{background:#eef1f8;text-align:left;padding:10px 12px;color:#445;cursor:pointer;white-space:nowrap}
+ td{padding:10px 12px;border-top:1px solid #eef0f6;vertical-align:top} tr:hover td{background:#fbfcff}
+ .badge{font-size:10px;font-weight:700;padding:2px 7px;border-radius:5px;color:#fff}
+ .direct{background:#0E7C4A} .socialua{background:#8A5A00} .proxy{background:#0b6fb8}
+ .bar{height:8px;background:#eef0f6;border-radius:5px;overflow:hidden;min-width:80px;margin-top:4px} .bar i{display:block;height:100%;background:#16357A}
+ a{color:var(--blue);text-decoration:none} a:hover{text-decoration:underline}
+ .samp{color:#6B7280;font-size:12px;line-height:1.5} .samp b{color:#0E7C4A}
+ .card{background:#fff;border-radius:10px;padding:16px 18px;box-shadow:0 2px 10px rgba(0,0,0,.05);margin-bottom:12px}
+ .kv{display:flex;gap:26px;flex-wrap:wrap} .kv div b{display:block;font-size:22px;color:var(--blue)} .kv div span{color:var(--gray);font-size:12px}
+ .links{margin:8px 0;font-size:13px} .links a{margin-right:14px}
+</style></head><body>
+<div id="nav"><div class="nh"><span>NAV AGENT</span><span><button id="navdock" title="dock left/right">⇄</button><button id="navcol" title="collapse">‹</button></span></div>
+ <div class="hint">FIELDS · CLICK A VALUE TO FILTER</div><div id="navfields"></div></div>
+<header><h1>Where the data comes from</h1><div class="s">CRE deal pipeline · live provenance · Abrams</div><div class="flow" id="flow"></div></header>
+<div class="wrap">
+ <div class="links" id="links"></div>
+ <h2>DATA SOURCES — RSS feeds &amp; proxies</h2>
+ <div class="controls"><input id="q" placeholder="search source…" size="18"><span class="clr" id="clr">clear filters</span><div class="stat" id="sstat"></div></div>
+ <table><thead><tr>
+  <th data-k="name">Source</th><th data-k="method">Method</th><th data-k="region">Market</th>
+  <th data-k="pulled">Items pulled</th><th data-k="deals">Deals kept</th><th>Recent from this source</th>
+ </tr></thead><tbody id="tb"></tbody></table>
+ <h2>DATABASE CRAWL — brokers (ads + flyers on their sites)</h2>
+ <div class="card" id="crawl">Loading crawl status…</div>
+</div>
+<script>
+const MB={'RSS direct':'direct','RSS (social-UA)':'socialua','Google News proxy':'proxy'};
+let SRC=null,sortK='deals',dir=-1; let F={method:'',region:''};
+const esc=s=>(s||'').replace(/[<>&]/g,c=>({'<':'&lt;','>':'&gt;','&':'&amp;'}[c]));
+// ---- Nav Agent panel ----
+const NAVFIELDS=[['method','METHOD'],['region','MARKET']];
+function buildNav(){ if(!SRC)return; let h='';
+ for(const [key,lbl] of NAVFIELDS){ const counts={}; SRC.feeds.forEach(x=>{const v=x[key]||'—';counts[v]=(counts[v]||0)+1;});
+  h+=`<div class="field">${lbl}</div>`;
+  h+=Object.entries(counts).sort((a,b)=>b[1]-a[1]).map(([v,c])=>`<div class="val${F[key]===v?' on':''}" data-k="${key}" data-v="${esc(v)}"><span>${esc(v)}</span><span class="b">${c}</span></div>`).join('');
+ }
+ const nf=document.getElementById('navfields'); nf.innerHTML=h;
+ nf.querySelectorAll('.val').forEach(el=>el.onclick=()=>{const k=el.dataset.k,v=el.dataset.v; F[k]=F[k]===v?'':v; buildNav(); renderSrc();});
+}
+// dock left/right + collapse (persisted)
+function applyDock(){ const nav=document.getElementById('nav'); const side=localStorage.navSide||'L', col=localStorage.navCol==='1';
+ nav.classList.toggle('right',side==='R'); nav.classList.toggle('collapsed',col);
+ document.body.className=(side==='L'?(col?'navLc':'navL'):(col?'navRc':'navR'));
+ document.getElementById('navcol').textContent=col?(side==='L'?'›':'‹'):(side==='L'?'‹':'›');
+}
+document.getElementById('navdock').onclick=()=>{localStorage.navSide=(localStorage.navSide==='R')?'L':'R';applyDock();};
+document.getElementById('navcol').onclick=()=>{localStorage.navCol=(localStorage.navCol==='1')?'0':'1';applyDock();};
+document.getElementById('clr').onclick=()=>{F={method:'',region:''};document.getElementById('q').value='';buildNav();renderSrc();};
+
+function renderFlow(d){document.getElementById('flow').innerHTML=
+ `<div class="n"><b>${d.feeds.length}</b>sources</div><span class="a">→</span>
+  <div class="n"><b>${d.total_items}</b>items pulled</div><span class="a">→</span>
+  <div class="n"><b>${d.total_deals}</b>CRE deals</div><span class="a">→</span>
+  <div class="n">email · flyers · viz</div>
+  <div class="n" style="background:transparent;border:none;color:#8aa">updated ${new Date(d.generated).toLocaleString()}</div>`;
+ document.getElementById('links').innerHTML=`<a href="/live.html">◂ deal viz</a> <a href="/advertising">broker ad intel ▸</a> <a href="/advertising/broker-ad-signals.csv" download>⬇ broker CSV</a>`;
+}
+function renderSrc(){ if(!SRC)return;
+ const q=document.getElementById('q').value.toLowerCase();
+ let f=SRC.feeds.filter(x=>x.name.toLowerCase().includes(q)&&(!F.method||x.method===F.method)&&(!F.region||(x.region||'—')===F.region));
+ f.sort((a,b)=>sortK==='name'||sortK==='method'||sortK==='region'?(''+a[sortK]).localeCompare(''+b[sortK])*dir:((a[sortK]||0)-(b[sortK]||0))*dir);
+ const max=Math.max(1,...SRC.feeds.map(x=>x.pulled));
+ document.getElementById('tb').innerHTML=f.map(x=>`<tr>
+  <td><b>${esc(x.name)}</b></td>
+  <td><span class="badge ${MB[x.method]}">${x.method}</span></td><td>${esc(x.region)}</td>
+  <td>${x.pulled}<div class="bar"><i style="width:${Math.round(x.pulled/max*100)}%"></i></div></td>
+  <td style="font-weight:800;color:#0E7C4A">${x.deals||'·'}</td>
+  <td class="samp">${(x.samples||[]).map(s=>`<div>${s.price?'<b>'+esc(s.price)+'</b> ':''}<a href="${s.link}" target="_blank" rel="noopener">${esc(s.title)}</a> <span style="color:#9aa">· ${esc(s.market||'')}</span></div>`).join('')||'<span style="color:#c3c9d6">—</span>'}</td>
+ </tr>`).join('');
+ const fil=(F.method||F.region)?` · filtered ${F.method||''} ${F.region||''}`:'';
+ document.getElementById('sstat').textContent=`${f.length} of ${SRC.feeds.length} sources${fil}`;
+}
+document.querySelectorAll('th[data-k]').forEach(th=>th.onclick=()=>{const k=th.dataset.k;if(sortK===k)dir*=-1;else{sortK=k;dir=(k==='name'||k==='method'||k==='region')?1:-1;}renderSrc();});
+document.getElementById('q').oninput=renderSrc;
+function loadSrc(){fetch('/sources.json?t='+Date.now()).then(r=>r.json()).then(d=>{SRC=d;renderFlow(d);buildNav();renderSrc();}).catch(()=>document.getElementById('sstat').textContent='sources load error');}
+function loadCrawl(){fetch('/advertising/broker-ad-signals.json?t='+Date.now()).then(r=>r.json()).then(d=>{
+  const done=d.filter(r=>!r.error),adv=done.filter(r=>r.paid_count>0),fly=done.filter(r=>r.flyer_count>0),ga=done.filter(r=>r.google_analytics);
+  document.getElementById('crawl').innerHTML=`<div class="kv">
+   <div><b>${d.length}</b><span>brokers crawled</span></div><div><b>${adv.length}</b><span>running paid ads</span></div>
+   <div><b>${fly.length}</b><span>with flyers/PDFs on site</span></div><div><b>${ga.length}</b><span>Google Analytics</span></div></div>
+   <div class="links"><a href="/advertising">open full broker ad-intel table ▸</a></div>
+   <div class="samp" style="margin-top:10px"><b>Advertisers:</b> ${adv.slice(0,14).map(r=>esc(r.domain)).join(' · ')||'—'}</div>`;
+}).catch(()=>document.getElementById('crawl').textContent='crawl data not available yet (running…)');}
+applyDock(); loadSrc(); loadCrawl(); setInterval(()=>{loadSrc();loadCrawl();},20000);
+</script></body></html>
diff --git a/scripts/cre-news-monitor.mjs b/scripts/cre-news-monitor.mjs
index 9dd9fe0..88042f1 100644
--- a/scripts/cre-news-monitor.mjs
+++ b/scripts/cre-news-monitor.mjs
@@ -138,3 +138,17 @@ if (!args.includes('--dry')) {
     console.log(`-> out/crenews-new-deals-${stamp}.json (${fresh.length} deals) for the alert feed.`);
   }
 }
+
+// --- SOURCE PROVENANCE: where the data comes from (per-feed pull + deal counts + samples) ---
+const bySrc = {};
+for (const f of FEEDS) bySrc[f.name] = {
+  name: f.name, method: f.mode === 'gnews' ? 'Google News proxy' : (f.ua ? 'RSS (social-UA)' : 'RSS direct'),
+  region: f.region, url: f.url, pulled: 0, deals: 0, samples: []
+};
+for (const x of all) if (bySrc[x.feed]) bySrc[x.feed].pulled++;
+for (const d of dealItems) { const b = bySrc[d.feed]; if (b) { b.deals++; if (b.samples.length < 3) b.samples.push({ title: (d.title || '').slice(0, 90), price: d.label, type: d.type, market: d.location || d.feed_region, link: d.link }); } }
+writeFileSync(join(ROOT, 'public', 'sources.json'), JSON.stringify({
+  generated: new Date().toISOString(), total_items: all.length, total_deals: dealItems.length,
+  feeds: Object.values(bySrc).sort((a, b) => b.deals - a.deals || b.pulled - a.pulled)
+}, null, 1));
+console.log(`-> public/sources.json (${FEEDS.length} feeds, provenance)`);
diff --git a/serve-viewers.mjs b/serve-viewers.mjs
index a629184..ff6b2a1 100644
--- a/serve-viewers.mjs
+++ b/serve-viewers.mjs
@@ -1,14 +1,12 @@
 import http from 'node:http'; import { readFileSync, statSync } from 'node:fs'; import { join, extname, dirname } from 'node:path'; import { fileURLToPath } from 'node:url';
 const PUB = join(dirname(fileURLToPath(import.meta.url)), 'public');
-const FLY = join(PUB, 'flyers'), ADV = join(PUB, 'advertising');
 const MIME = {'.html':'text/html','.json':'application/json','.csv':'text/csv','.js':'text/javascript','.css':'text/css','.png':'image/png'};
+const ALIAS = { '/':'/sources/index.html', '/sources':'/sources/index.html', '/live.html':'/flyers/live.html',
+  '/flyers.json':'/flyers/flyers.json', '/advertising':'/advertising/index.html', '/flyers':'/flyers/index.html' };
 http.createServer((q,s)=>{
-  let u = decodeURIComponent(q.url.split('?')[0]);
-  let base = FLY, path = u;                                   // default: the live/flyers viewer
-  if (u==='/' ) path = '/live.html';
-  else if (u.startsWith('/advertising')) { base = ADV; path = u.replace(/^\/advertising\/?/, '/') || '/index.html'; if(path==='/')path='/index.html'; }
-  let fp = join(base, path);
+  let u = decodeURIComponent(q.url.split('?')[0]); u = ALIAS[u] || u;
+  let fp = join(PUB, u);
   try { if (statSync(fp).isDirectory()) fp = join(fp,'index.html'); } catch {}
   try { const b = readFileSync(fp); s.writeHead(200,{'content-type':MIME[extname(fp)]||'text/html','cache-control':'no-store','access-control-allow-origin':'*'}); s.end(b); }
   catch(e){ s.writeHead(404); s.end('not found: '+u); }
-}).listen(9862, ()=>console.log('viewers on :9862 — / and /live.html = flyers viz, /advertising = ad intel'));
+}).listen(9862, ()=>console.log('viewers :9862 — / = sources dashboard, /live.html = viz, /advertising = ad intel'));
← c943015 harden live.html vs freeze: 35fps cap + pause-render-when-ta · back to Re Flyer Aggregator · retire the 3D WebGL viz (Steve: bad UI, freezes, scroll-lock a4e3237 →