dashboard: incorporate ALL data — load article-archive.json, show total 'articles archived' in the flow + per-source 'Archived (all)' column (click -> modal of every article from that source, all 56 sources' accumulated totals); the dashboard now reflects the full growing archive, not just this-run's pull (Steve) (TK-10708)

9abec8f26a32aa978d1eb0a3f21435685ee33335 · 2026-08-20 08:01:52 -0700 · Steve Abrams
Files touched

M public/sources/index.html
Diff

commit 9abec8f26a32aa978d1eb0a3f21435685ee33335
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Thu Aug 20 08:01:52 2026 -0700

    dashboard: incorporate ALL data — load article-archive.json, show total 'articles archived' in the flow + per-source 'Archived (all)' column (click -> modal of every article from that source, all 56 sources' accumulated totals); the dashboard now reflects the full growing archive, not just this-run's pull (Steve) (TK-10708)
---
 public/sources/index.html | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/public/sources/index.html b/public/sources/index.html
index 02a3bc6..3849787 100644
--- a/public/sources/index.html
+++ b/public/sources/index.html
@@ -115,7 +115,7 @@
 <div id="modal" class="mask"><div class="box"><div class="mh"><span id="mtitle"></span><button id="mx" title="close">✕</button></div><div id="mbody" class="mbody"></div></div></div>
 <script>
 const MB={'RSS direct':'direct','RSS (social-UA)':'socialua','Google News proxy':'proxy'};
-let SRC=null,sortK='deals',dir=-1; let F={method:'',region:''}; let VIEW=localStorage.srcView||'table';
+let SRC=null,sortK='deals',dir=-1; let F={method:'',region:''}; let VIEW=localStorage.srcView||'table'; let ARCH_BY_SRC={};
 const esc=s=>(s||'').replace(/[<>&]/g,c=>({'<':'&lt;','>':'&gt;','&':'&amp;'}[c]));
 const dt=s=>{if(!s)return'';const d=new Date(s);return isNaN(d)?(''+s).slice(0,11):d.toLocaleDateString('en-US',{month:'short',day:'numeric',year:'2-digit'});};
 let CR=null; // broker-crawl records, kept for drill-down
@@ -142,7 +142,7 @@ document.addEventListener('keydown',e=>{if(e.key==='Escape')closeModal();});
 // ---- Nav Agent panel ----
 const NAVFIELDS=[['method','METHOD'],['region','MARKET']];
 // ---- full nav-agent columns: movable (drag) + toggleable (COLUMNS) + sortable (click) ----
-const ALLCOLS=[['name','Source'],['method','Method'],['region','Market'],['pulled','Items pulled'],['deals','Deals kept'],['latest','Latest'],['recent','Recent from this source']];
+const ALLCOLS=[['name','Source'],['method','Method'],['region','Market'],['pulled','Items pulled'],['deals','Deals kept'],['archived','Archived (all)'],['latest','Latest'],['recent','Recent from this source']];
 let COLORDER=JSON.parse(localStorage.srcColOrder||'null')||ALLCOLS.map(c=>c[0]);
 let CVIS=JSON.parse(localStorage.srcColVis||'null')||ALLCOLS.reduce((o,c)=>(o[c[0]]=true,o),{});
 let COLW=JSON.parse(localStorage.srcColW||'null')||{};   // per-column resized widths (px)
@@ -175,6 +175,7 @@ function cellHTML(x,k,max){
  if(k==='region')return `<td>${esc(x.region)}</td>`;
  if(k==='pulled')return `<td class="num" data-modal="pulled" style="color:var(--blue);font-weight:700" title="click to view the items pulled">${x.pulled}<div class="bar"><i style="width:${Math.round(x.pulled/max*100)}%"></i></div></td>`;
  if(k==='deals')return `<td class="num" data-modal="deals" style="color:#0E7C4A;font-weight:800" title="click to view the deals kept">${x.deals||'·'}</td>`;
+ if(k==='archived')return `<td class="num" data-modal="archived" style="color:#6d28d9;font-weight:700" title="ALL articles ever recorded from this source (our growing archive)">${(x.archived||0).toLocaleString()}</td>`;
  if(k==='latest')return `<td style="white-space:nowrap;color:#8A5A00">${latestOf(x)?dt(new Date(latestOf(x)).toISOString()):'—'}</td>`;
  if(k==='recent')return `<td class="samp">${sortSamp(x.samples).slice(0,3).map(s=>`<div><span style="color:#8A5A00">${dt(s.date)}</span> ${s.price?'<b>'+esc(s.price)+'</b> ':''}<a href="${s.link}" target="_blank" rel="noopener" onclick="event.stopPropagation()">${esc(s.title)}</a> <span style="color:#9aa">· ${esc(s.market||'')}</span></div>`).join('')||'<span style="color:#c3c9d6">—</span>'}</td>`;
  return '<td></td>';}
@@ -210,7 +211,8 @@ function renderFlow(d){document.getElementById('flow').innerHTML=
  `<div class="n"><b>${d.feeds.length}</b>sources</div><span class="a">→</span>
   <div class="n"><b>${d.total_items}</b>items pulled</div><span class="a">→</span>
   <div class="n"><b>${d.total_deals}</b>CRE deals</div><span class="a">→</span>
-  <div class="n">email · flyers · viz</div>
+  <div class="n" style="background:#241a3a;border-color:#4a2f7a"><b style="color:#b061ff">${(d.archive_total||0).toLocaleString()}</b>articles archived</div><span class="a">→</span>
+  <div class="n"><a href="/search" style="color:#7fd7ff">🔍 search all</a></div>
   <div class="n" style="background:transparent;border:none;color:#8aa">updated ${new Date(d.generated).toLocaleString()}</div>`;
  document.getElementById('links').innerHTML=`<a href="/search" style="font-weight:700;color:#0E7C4A">🔍 Search · All Listings</a> <a href="/live.html">◂ deal viz</a> <a href="/advertising">broker ad intel ▸</a> <a href="/advertising/broker-ad-signals.csv" download>⬇ broker CSV</a>`;
 }
@@ -240,14 +242,18 @@ function renderSrc(){ if(!SRC)return; const f=feedsFiltered();
 document.getElementById('tb').addEventListener('click',e=>{ if(e.target.closest('a'))return;
  const row=e.target.closest('tr.srow'); if(!row)return; const n=row.dataset.n; const x=SRC.feeds.find(s=>s.name===n); if(!x)return;
  const num=e.target.closest('td.num');
- if(num){ if(num.dataset.modal==='pulled')openModal(n+' — items pulled',(x.pulled_list||[]).length,x.pulled_list||[],'deal');
+ if(num){ const m=num.dataset.modal;
+   if(m==='pulled')openModal(n+' — items pulled',(x.pulled_list||[]).length,x.pulled_list||[],'deal');
+   else if(m==='archived')openModal(n+' — all archived articles',(ARCH_BY_SRC[n]||[]).length,ARCH_BY_SRC[n]||[],'deal');
    else openModal(n+' — deals kept',(x.samples||[]).length,x.samples||[],'deal'); return; }
  EXP.has(n)?EXP.delete(n):EXP.add(n); renderSrc();
 });
 // CARD click: number -> modal
 document.getElementById('srccards').addEventListener('click',e=>{ if(e.target.closest('a'))return;
  const num=e.target.closest('.num'); const host=e.target.closest('[data-n]'); if(!num||!host)return; const x=SRC.feeds.find(s=>s.name===host.dataset.n); if(!x)return;
- if(num.dataset.modal==='pulled')openModal(x.name+' — items pulled',(x.pulled_list||[]).length,x.pulled_list||[],'deal');
+ const m=num.dataset.modal;
+ if(m==='pulled')openModal(x.name+' — items pulled',(x.pulled_list||[]).length,x.pulled_list||[],'deal');
+ else if(m==='archived')openModal(x.name+' — all archived articles',(ARCH_BY_SRC[x.name]||[]).length,ARCH_BY_SRC[x.name]||[],'deal');
  else openModal(x.name+' — deals kept',(x.samples||[]).length,x.samples||[],'deal'); });
 // VIEW switch + DENSITY
 document.querySelectorAll('.vsw button').forEach(b=>b.onclick=()=>{VIEW=b.dataset.v;localStorage.srcView=VIEW;document.querySelectorAll('.vsw button').forEach(z=>z.classList.toggle('on',z.dataset.v===VIEW));renderSrc();});
@@ -283,7 +289,15 @@ function buildCrawlCharts(cr){ if(!window.Chart)return; const done=cr.filter(r=>
  const buckets=[['Advertising',bAdv],['Flyers only',bFly],['GA only',bGa],['No signals',bNo]];
  mkDon('gCrawl',buckets.map(x=>x[0]),buckets.map(x=>x[1].length),['#0E7C4A','#E8A81C','#8aa0c0','#dde3f0'],i=>openModal('Brokers · '+buckets[i][0],buckets[i][1].length,buckets[i][1],'broker'));
 }
-function loadSrc(){fetch('/sources.json?t='+Date.now()).then(r=>r.json()).then(d=>{SRC=d;renderFlow(d);buildNav();renderHead();renderSrc();buildSrcCharts();}).catch(()=>document.getElementById('sstat').textContent='sources load error');}
+function loadSrc(){Promise.all([
+   fetch('/sources.json?t='+Date.now()).then(r=>r.json()),
+   fetch('/article-archive.json?t='+Date.now()).then(r=>r.ok?r.json():[]).catch(()=>[])
+ ]).then(([d,arch])=>{
+   ARCH_BY_SRC={}; for(const a of arch){(ARCH_BY_SRC[a.source]=ARCH_BY_SRC[a.source]||[]).push(a);}
+   d.feeds.forEach(f=>f.archived=(ARCH_BY_SRC[f.name]||[]).length);
+   d.archive_total=arch.length; d.archive_sources=Object.keys(ARCH_BY_SRC).length;
+   SRC=d;renderFlow(d);buildNav();renderHead();renderSrc();buildSrcCharts();
+ }).catch(()=>document.getElementById('sstat').textContent='sources load error');}
 function loadCrawl(){fetch('/advertising/broker-ad-signals.json?t='+Date.now()).then(r=>r.json()).then(d=>{ CR=d;
   const done=d.filter(r=>!r.error),adv=done.filter(r=>r.paid_count>0),fly=done.filter(r=>r.flyer_count>0),ga=done.filter(r=>r.google_analytics);
   document.getElementById('crawl').innerHTML=`<div class="kv">
← e7619ad auto-data-snapshot: 2026-08-20T08:00:08 (2 data files) — out · back to Re Flyer Aggregator · archive: extract buyer/seller from headlines (same patterns 9193ea1 →