dashboard.js: wire ct-source-ctlog (CT_SOURCE=ctlog default)

da5c6535407a72b007af221d0c1a1a9ccb15d7e4 · 2026-05-12 16:21:24 -0700 · Steve Abrams

Mirrors brand-typo-watcher refactor — extracts handleCertEvent() so both
ctlog and certstream paths share the classify+broadcast+log flow.

Dashboard PID 80709 confirmed 6 HOT events flowing in first 20s after
restart (vs zero for 2h58m on dead certstream). Browser UI at :9895 now
shows live activity.

Files touched

M dashboard.js

Diff

commit da5c6535407a72b007af221d0c1a1a9ccb15d7e4
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Tue May 12 16:21:24 2026 -0700

    dashboard.js: wire ct-source-ctlog (CT_SOURCE=ctlog default)
    
    Mirrors brand-typo-watcher refactor — extracts handleCertEvent() so both
    ctlog and certstream paths share the classify+broadcast+log flow.
    
    Dashboard PID 80709 confirmed 6 HOT events flowing in first 20s after
    restart (vs zero for 2h58m on dead certstream). Browser UI at :9895 now
    shows live activity.
---
 dashboard.js | 69 +++++++++++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 50 insertions(+), 19 deletions(-)

diff --git a/dashboard.js b/dashboard.js
index 644803d..eb24097 100644
--- a/dashboard.js
+++ b/dashboard.js
@@ -16,9 +16,16 @@ const http = require('http');
 const WebSocket = require('ws');
 const path = require('path');
 const fs = require('fs');
+const ctLogSource = require('./ct-source-ctlog');
 
 const PORT = parseInt(process.env.DASH_PORT || '9895', 10);
 const CERTSTREAM_URL = process.env.CERTSTREAM_URL || 'wss://certstream.calidog.io/';
+// CT_SOURCE=ctlog → direct CT-log polling (default, since certstream is dead)
+// CT_SOURCE=certstream → original WebSocket flow (kept as fallback)
+const CT_SOURCE = process.env.CT_SOURCE || 'ctlog';
+const CT_LOG = process.env.CT_LOG || 'argon';
+const CTLOG_POLL_MS = parseInt(process.env.CTLOG_POLL_MS || '5000', 10);
+const CTLOG_BATCH = parseInt(process.env.CTLOG_BATCH || '32', 10);
 const DATA_DIR = path.join(__dirname, 'data');
 const HITS_LOG = path.join(DATA_DIR, 'dashboard-hits.jsonl');
 
@@ -72,40 +79,64 @@ function appendHit(rec) {
   fs.appendFileSync(HITS_LOG, JSON.stringify(rec) + '\n');
 }
 
+function handleCertEvent(m) {
+  if (m.message_type !== 'certificate_update') return;
+  const all = (m.data && m.data.leaf_cert && m.data.leaf_cert.all_domains) || [];
+  for (const d of all) {
+    stats.seen++;
+    const c = classify(d);
+    if (c.brand) stats.brand++;
+    if (c.hot) stats.hot++;
+    if (c.hot || c.brand) {
+      const rec = { type: 'hit', ts: Date.now(), domain: d, hot: c.hot, brand: c.brand, source: m.data.log_source || 'certstream' };
+      broadcast(rec);
+      appendHit(rec);
+    }
+  }
+}
+
 let reconnectDelay = 1000;
-function connectCT() {
+function connectCertstream() {
+  console.log(`[${new Date().toISOString()}] CT_SOURCE=certstream — connecting to ${CERTSTREAM_URL}`);
   const cs = new WebSocket(CERTSTREAM_URL, { handshakeTimeout: 15_000 });
   cs.on('open', () => {
     console.log(`[${new Date().toISOString()}] CertStream connected`);
     reconnectDelay = 1000;
   });
   cs.on('message', (raw) => {
-    let m;
-    try { m = JSON.parse(raw.toString()); } catch { return; }
-    if (m.message_type !== 'certificate_update') return;
-    const all = (m.data && m.data.leaf_cert && m.data.leaf_cert.all_domains) || [];
-    for (const d of all) {
-      stats.seen++;
-      const c = classify(d);
-      if (c.brand) stats.brand++;
-      if (c.hot) stats.hot++;
-      // Don't flood the browser with the full firehose — only emit interesting hits.
-      // (Otherwise the browser would render ~50/sec and choke.)
-      if (c.hot || c.brand) {
-        const rec = { type: 'hit', ts: Date.now(), domain: d, hot: c.hot, brand: c.brand };
-        broadcast(rec);
-        appendHit(rec);
-      }
-    }
+    let m; try { m = JSON.parse(raw.toString()); } catch { return; }
+    handleCertEvent(m);
   });
   cs.on('close', (code) => {
     console.error(`[${new Date().toISOString()}] CT closed (${code}) — reconnect in ${reconnectDelay}ms`);
-    setTimeout(connectCT, reconnectDelay);
+    setTimeout(connectCertstream, reconnectDelay);
     reconnectDelay = Math.min(reconnectDelay * 2, 30_000);
   });
   cs.on('error', (e) => console.error(`[${new Date().toISOString()}] CT error: ${e.message}`));
 }
 
+function connectCtLog() {
+  console.log(`[${new Date().toISOString()}] CT_SOURCE=ctlog log=${CT_LOG} poll=${CTLOG_POLL_MS}ms batch=${CTLOG_BATCH}`);
+  ctLogSource.start({
+    logKey: CT_LOG,
+    pollMs: CTLOG_POLL_MS,
+    batchSize: CTLOG_BATCH,
+    onEvent: handleCertEvent,
+    onStatus: (st) => {
+      if (st.type === 'sth_err' || st.type === 'fetch_err' || st.type === 'error') {
+        console.error(`[${new Date().toISOString()}] ctlog ${st.type}: ${st.msg || JSON.stringify(st)}`);
+      } else if (st.type === 'seeded') {
+        console.log(`[${new Date().toISOString()}] ctlog seeded at tree_size=${st.position}`);
+      }
+    },
+  });
+}
+
+function connectCT() {
+  if (CT_SOURCE === 'certstream') return connectCertstream();
+  return connectCtLog();
+}
+
 setInterval(() => broadcast({ type: 'stats', ...stats }), 2000);
 
 server.listen(PORT, '127.0.0.1', () => {

← 9bc1439 brand-typo-watcher: wire ct-source-ctlog (CT_SOURCE=ctlog de · back to Domain Sniper · ct-source-ctlog: parse PRECERT_ENTRY (TBSCertificate) for SA 2c5c65f →