`dashboard.js`

201 lines
#!/usr/bin/env node
// dashboard.js — live web viewer of the global newly-registered-domain firehose.
//
// Pulls Certificate Transparency log events (every TLS cert issued worldwide)
// from certstream.calidog.io and rebroadcasts to browser clients over WebSocket.
//
// CT is the closest free PUBLIC real-time stream to what aggregators see.
// They have a paid WHOIS query feed (~$3K–50K/mo, gated to accredited registrars).
// CT lags that by minutes but covers ~95% of newly-registered domains.
//
// Run:   node dashboard.js
// Open:  http://127.0.0.1:9895

const express = require('express');
const http = require('http');
const WebSocket = require('ws');
const path = require('path');
const fs = require('fs');
const ctLogSource = require('./ct-source-ctlog');

const PORT = parseInt(process.env.DASH_PORT || '9895', 10);
const CERTSTREAM_URL = process.env.CERTSTREAM_URL || 'wss://certstream.calidog.io/';
// CT_SOURCE=ctlog → direct CT-log polling (default, since certstream is dead)
// CT_SOURCE=certstream → original WebSocket flow (kept as fallback)
const CT_SOURCE = process.env.CT_SOURCE || 'ctlog';
const CT_LOG = process.env.CT_LOG || 'argon';
const CT_LOGS = (process.env.CT_LOGS || 'argon,xenon,wyvern,sphinx,elephant,tiger').split(',').map((s) => s.trim()).filter(Boolean);
const CTLOG_POLL_MS = parseInt(process.env.CTLOG_POLL_MS || '5000', 10);
const CTLOG_BATCH = parseInt(process.env.CTLOG_BATCH || '32', 10);
const DATA_DIR = path.join(__dirname, 'data');
const HITS_LOG = path.join(DATA_DIR, 'dashboard-hits.jsonl');

const HOT_TLDS = new Set(['com', 'app', 'io', 'co', 'ai', 'dev', 'xyz', 'org', 'net']);
const MAX_STEM_LEN = 8;
const BRAND_KEYWORDS = (process.env.BRAND_KEYWORDS || [
  'designerwallcoverings', 'philipperomano', 'venturacorridor', 'venturaclaw',
  'starsofdesign', 'bubbesblock', 'wholivedthere', 'agentabrams',
  'nationalpaperhangers', 'novasuede', 'callr', 'butlr',
  'flockedwallpaper', 'grasscloth', 'glassbeaded',
].join(',')).split(',').map(s => s.trim().toLowerCase()).filter(Boolean);

function classify(d) {
  if (!d || d.startsWith('*.')) return { hot: false, brand: null };
  const dl = d.toLowerCase().replace(/^\*\./, '');
  const brand = BRAND_KEYWORDS.find(k => dl.includes(k)) || null;
  const parts = dl.split('.');
  if (parts.length < 2) return { hot: false, brand };
  const tld = parts[parts.length - 1];
  if (!HOT_TLDS.has(tld)) return { hot: false, brand };
  const stem = parts[parts.length - 2];
  if (stem.length > MAX_STEM_LEN || stem.length < 3) return { hot: false, brand };
  if (/[-0-9]/.test(stem)) return { hot: false, brand };
  if (!/[aeiou]/.test(stem) || !/[bcdfghjklmnpqrstvwxyz]/.test(stem)) return { hot: false, brand };
  return { hot: true, brand };
}

const app = express();
app.use(express.static(path.join(__dirname, 'public')));
app.get('/api/keywords', (_req, res) => res.json({ keywords: BRAND_KEYWORDS }));
app.get('/api/perlog', (_req, res) => {
  const now = Date.now();
  const out = {};
  for (const [k, v] of Object.entries(perLog)) out[k] = { ...v, stale: v.lastSeenMs ? (now - v.lastSeenMs > 60_000) : false };
  res.json({ stats, perLog: out });
});

// Liveness probe — 200 if dashboard is up AND at least one CT source has
// reported any event (tick or idle) within 90s. Otherwise 503 so pm2/launchd
// can auto-restart a wedged process. Returns JSON so the body is also useful
// for human eyeball checks. No external calls; cheap to poll.
app.get('/healthz', (_req, res) => {
  const now = Date.now();
  const entries = Object.entries(perLog).map(([name, v]) => ({
    name,
    ageSec: v.lastSeenMs ? Math.round((now - v.lastSeenMs) / 1000) : null,
  }));
  const fresh = entries.filter((e) => e.ageSec !== null && e.ageSec < 90);
  const ok = fresh.length > 0;
  const uptimeSec = Math.round((now - stats.startedAt) / 1000);
  res.status(ok ? 200 : 503).json({
    ok,
    uptimeSec,
    sourcesFresh: fresh.length,
    sourcesTotal: entries.length,
    sources: entries,
    seen: stats.seen,
    hits: stats.hot + stats.brand,
  });
});

const server = http.createServer(app);
const wss = new WebSocket.Server({ server });
const clients = new Set();

let stats = { seen: 0, hot: 0, brand: 0, startedAt: Date.now() };
const perLog = {}; // log_name -> { fetched, x509, precert, emitted, idle, lastSeenMs }

wss.on('connection', (ws) => {
  clients.add(ws);
  ws.send(JSON.stringify({ type: 'stats', ...stats }));
  ws.on('close', () => clients.delete(ws));
});

function broadcast(obj) {
  const msg = JSON.stringify(obj);
  for (const ws of clients) if (ws.readyState === 1) ws.send(msg);
}

function appendHit(rec) {
  fs.mkdirSync(DATA_DIR, { recursive: true });
  fs.appendFileSync(HITS_LOG, JSON.stringify(rec) + '\n');
}

function handleCertEvent(m) {
  if (m.message_type !== 'certificate_update') return;
  const all = (m.data && m.data.leaf_cert && m.data.leaf_cert.all_domains) || [];
  for (const d of all) {
    stats.seen++;
    const c = classify(d);
    if (c.brand) stats.brand++;
    if (c.hot) stats.hot++;
    if (c.hot || c.brand) {
      const rec = { type: 'hit', ts: Date.now(), domain: d, hot: c.hot, brand: c.brand, source: m.data.log_source || 'certstream' };
      broadcast(rec);
      appendHit(rec);
    }
  }
}

let reconnectDelay = 1000;
function connectCertstream() {
  console.log(`[${new Date().toISOString()}] CT_SOURCE=certstream — connecting to ${CERTSTREAM_URL}`);
  const cs = new WebSocket(CERTSTREAM_URL, { handshakeTimeout: 15_000 });
  cs.on('open', () => {
    console.log(`[${new Date().toISOString()}] CertStream connected`);
    reconnectDelay = 1000;
  });
  cs.on('message', (raw) => {
    let m; try { m = JSON.parse(raw.toString()); } catch { return; }
    handleCertEvent(m);
  });
  cs.on('close', (code) => {
    console.error(`[${new Date().toISOString()}] CT closed (${code}) — reconnect in ${reconnectDelay}ms`);
    setTimeout(connectCertstream, reconnectDelay);
    reconnectDelay = Math.min(reconnectDelay * 2, 30_000);
  });
  cs.on('error', (e) => console.error(`[${new Date().toISOString()}] CT error: ${e.message}`));
}

function connectCtLog() {
  const opts = CT_LOGS.length > 1
    ? { logKeys: CT_LOGS, pollMs: CTLOG_POLL_MS, batchSize: CTLOG_BATCH }
    : { logKey: CT_LOG, pollMs: CTLOG_POLL_MS, batchSize: CTLOG_BATCH };
  console.log(`[${new Date().toISOString()}] CT_SOURCE=ctlog ${CT_LOGS.length > 1 ? 'multi=[' + CT_LOGS.join(',') + ']' : 'log=' + CT_LOG} poll=${CTLOG_POLL_MS}ms batch=${CTLOG_BATCH}`);
  ctLogSource.start({
    ...opts,
    onEvent: handleCertEvent,
    onStatus: (st) => {
      if (st.type === 'sth_err' || st.type === 'fetch_err' || st.type === 'error') {
        console.error(`[${new Date().toISOString()}] ctlog ${st.type} on ${st.log || '?'}: ${st.msg || JSON.stringify(st)}`);
      } else if (st.type === 'seeded') {
        console.log(`[${new Date().toISOString()}] ctlog seeded ${st.log} at tree_size=${st.position}`);
      } else if (st.type === 'resumed') {
        console.log(`[${new Date().toISOString()}] ctlog resumed ${st.log} at cursor=${st.position}`);
      } else if (st.type === 'tick' && st.log) {
        const p = (perLog[st.log] = perLog[st.log] || { fetched: 0, x509: 0, precert: 0, emitted: 0, idle: 0, lastSeenMs: 0 });
        if (st.stats) { p.fetched = st.stats.fetched; p.x509 = st.stats.x509; p.precert = st.stats.precert; p.emitted = st.stats.emitted; }
        p.lastSeenMs = Date.now();
      } else if (st.type === 'idle' && st.log) {
        const p = (perLog[st.log] = perLog[st.log] || { fetched: 0, x509: 0, precert: 0, emitted: 0, idle: 0, lastSeenMs: 0 });
        p.idle++;
        p.lastSeenMs = Date.now();
      }
    },
  });
}

function connectCT() {
  if (CT_SOURCE === 'certstream') return connectCertstream();
  return connectCtLog();
}

setInterval(() => {
  const now = Date.now();
  const perLogPayload = {};
  for (const [k, v] of Object.entries(perLog)) {
    perLogPayload[k] = { ...v, stale: v.lastSeenMs ? (now - v.lastSeenMs > 60_000) : false };
  }
  broadcast({ type: 'stats', ...stats, perLog: perLogPayload });
}, 2000);

server.listen(PORT, '127.0.0.1', () => {
  console.log(`\n  domain-sniper :: live dashboard`);
  console.log(`  open  http://127.0.0.1:${PORT}\n`);
  console.log(`  data source: CertStream  (closest free real-time analog of an aggregator's WHOIS feed)`);
  console.log(`  brand keywords: ${BRAND_KEYWORDS.join(', ')}\n`);
  connectCT();
});

process.on('SIGINT', () => process.exit(0));