← back to Domain Sniper
ct-source-ctlog.js
382 lines
// ct-source-ctlog.js — Direct Certificate Transparency log poller.
// Bypasses ALL the broken/gated middlemen (CertStream, Cert Spotter, crt.sh,
// merklemap) by talking to the underlying CT log servers directly per RFC 6962.
//
// Two probed-working endpoints as of 2026-05-12:
// - Google Argon : https://ct.googleapis.com/logs/us1/argon2025h2/ct/v1/
// - Cloudflare Nimbus: https://ct.cloudflare.com/logs/nimbus2026/ct/v1/
//
// Each log is append-only. We poll get-sth periodically; when tree_size
// advances, we fetch get-entries for the new range. Each entry is a binary
// MerkleTreeLeaf — we parse the timestamp + entry_type + (for X509_ENTRY)
// the embedded ASN.1 cert, then extract SAN DNS names via Node 22's built-in
// crypto.X509Certificate.
//
// PRECERT_ENTRY parsing requires ASN.1 dive into the TBSCertificate — deferred.
// We log skip count so we know what we're missing.
//
// Bandwidth: ~256 entries × ~3KB each = ~750KB per poll. Default poll 5s
// means we sample but don't fully consume the firehose (world rate is much
// higher) — for brand defense, sampling is fine: any given brand appears in
// many issuances per day so we'll catch one within minutes.
const https = require('https');
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const { EventEmitter } = require('events');
const DATA_DIR = path.join(__dirname, 'data');
const CURSOR_FILE = path.join(DATA_DIR, 'ctlog-cursor.json');
const UA = 'domain-sniper/0.1 (CT log monitor)';
// From https://www.gstatic.com/ct/log_list/v3/log_list.json
// (filtered to currently-active 2026 logs as of 2026-05-12)
const LOGS = {
argon: { name: 'argon2026h1', base: 'https://ct.googleapis.com/logs/us1/argon2026h1/ct/v1' },
xenon: { name: 'xenon2026h1', base: 'https://ct.googleapis.com/logs/eu1/xenon2026h1/ct/v1' },
nimbus: { name: 'nimbus2026', base: 'https://ct.cloudflare.com/logs/nimbus2026/ct/v1' },
wyvern: { name: 'wyvern2026h1', base: 'https://wyvern.ct.digicert.com/2026h1/ct/v1' },
sphinx: { name: 'sphinx2026h1', base: 'https://sphinx.ct.digicert.com/2026h1/ct/v1' },
elephant:{ name: 'elephant2026h1', base: 'https://elephant2026h1.ct.sectigo.com/ct/v1' },
tiger: { name: 'tiger2026h1', base: 'https://tiger2026h1.ct.sectigo.com/ct/v1' },
// Older frozen log kept as a known-frozen smoke-test target
argon2025h2: { name: 'argon2025h2', base: 'https://ct.googleapis.com/logs/us1/argon2025h2/ct/v1' },
};
function loadCursors() {
try { return JSON.parse(fs.readFileSync(CURSOR_FILE, 'utf8')); }
catch { return {}; }
}
function saveCursors(c) {
fs.mkdirSync(DATA_DIR, { recursive: true });
fs.writeFileSync(CURSOR_FILE, JSON.stringify(c, null, 2));
}
function getJson(url) {
return new Promise((resolve, reject) => {
const u = new URL(url);
const req = https.request({
host: u.host, path: u.pathname + u.search, method: 'GET',
headers: { 'User-Agent': UA, 'Accept': 'application/json' },
timeout: 20_000,
}, (res) => {
let buf = '';
res.on('data', (c) => buf += c);
res.on('end', () => {
if (res.statusCode === 200) {
try { resolve(JSON.parse(buf)); }
catch (e) { reject(new Error('parse: ' + e.message)); }
} else reject(new Error('HTTP ' + res.statusCode + ': ' + buf.slice(0, 160)));
});
});
req.on('error', reject);
req.on('timeout', () => req.destroy(new Error('timeout')));
req.end();
});
}
// Parse a single MerkleTreeLeaf base64 → { entryType, timestamp, cert?: Buffer }
function parseLeaf(leafB64) {
const buf = Buffer.from(leafB64, 'base64');
if (buf.length < 12) return { error: 'short' };
const version = buf.readUInt8(0);
const leafType = buf.readUInt8(1);
if (leafType !== 0) return { error: 'leaf_type=' + leafType };
let off = 2;
const timestamp = Number(buf.readBigUInt64BE(off)); off += 8;
const entryType = buf.readUInt16BE(off); off += 2;
if (entryType === 0) {
// X509_ENTRY: 3-byte length + cert DER
if (buf.length < off + 3) return { entryType, error: 'truncated_len' };
const certLen = (buf.readUInt8(off) << 16) | buf.readUInt16BE(off + 1);
off += 3;
if (buf.length < off + certLen) return { entryType, error: 'truncated_cert' };
return { entryType, timestamp, cert: buf.subarray(off, off + certLen) };
} else if (entryType === 1) {
// PRECERT_ENTRY: 32-byte issuer key hash + 3-byte len + TBSCert
if (buf.length < off + 32 + 3) return { entryType, error: 'truncated_precert' };
off += 32;
const tbsLen = (buf.readUInt8(off) << 16) | buf.readUInt16BE(off + 1);
off += 3;
if (buf.length < off + tbsLen) return { entryType, error: 'truncated_tbs' };
const tbsOuter = buf.subarray(off, off + tbsLen);
// tbsOuter is a complete TBSCertificate DER (starts with SEQUENCE tag 0x30).
// Strip outer SEQUENCE wrapper to get the body for the SAN walker.
const top = readTlv(tbsOuter, 0);
if (!top || top.tag !== 0x30) return { entryType, error: 'bad_tbs_wrap' };
return { entryType, timestamp, tbs: top.val };
}
return { entryType, error: 'unknown_entry_type' };
}
// Extract DNS names from a DER X.509 cert. Returns array of strings (lowercase).
// Uses Node 22's crypto.X509Certificate. Returns [] on parse failure.
function extractDnsNames(certDer) {
try {
const c = new crypto.X509Certificate(certDer);
const san = c.subjectAltName || '';
// Format: "DNS:example.com, DNS:www.example.com, IP Address:..., ..."
const names = [];
for (const part of san.split(',')) {
const t = part.trim();
if (t.startsWith('DNS:')) names.push(t.slice(4).toLowerCase());
}
if (!names.length && c.subject) {
// Fall back to CN= if no SAN
const cn = c.subject.split('\n').find((l) => l.startsWith('CN='));
if (cn) names.push(cn.slice(3).toLowerCase());
}
return names;
} catch {
return [];
}
}
// --- Minimal ASN.1 DER walker (just enough to find subjectAltName in TBSCertificate) ---
//
// TBSCertificate is a TBSCertificate (no outer wrap, no signatureAlgorithm,
// no signature). We need to find:
// extensions [3] EXPLICIT Extensions OPTIONAL
// then within that SEQUENCE OF Extension, find the one with OID 2.5.29.17
// (subjectAltName = 06 03 55 1D 11), then parse its extnValue (OCTET STRING
// wrapping SEQUENCE OF GeneralName) and pick out dNSName entries (tag 0x82).
function readDerLen(buf, off) {
if (off >= buf.length) throw new Error('eof');
const b = buf.readUInt8(off);
if (b < 0x80) return { len: b, hdr: 1 };
const n = b & 0x7f;
if (n === 0 || n > 4 || off + 1 + n > buf.length) throw new Error('bad_len');
let len = 0;
for (let i = 0; i < n; i++) len = (len << 8) | buf.readUInt8(off + 1 + i);
return { len, hdr: 1 + n };
}
function readTlv(buf, off) {
if (off >= buf.length) return null;
const tag = buf.readUInt8(off);
const { len, hdr } = readDerLen(buf, off + 1);
const start = off + 1 + hdr;
return { tag, len, valStart: start, val: buf.subarray(start, start + len), next: start + len };
}
const SAN_OID = Buffer.from([0x55, 0x1d, 0x11]); // 2.5.29.17
function extractDnsNamesFromTBS(tbs) {
try {
// Walk children of TBSCertificate (top-level passed in as `tbs` body — caller has
// already unwrapped the outer SEQUENCE). Skip until we find the [3] EXPLICIT tag (0xa3).
let off = 0;
let extensionsBody = null;
while (off < tbs.length) {
const t = readTlv(tbs, off);
if (!t) break;
if (t.tag === 0xa3) {
// [3] EXPLICIT — body is itself a TLV: SEQUENCE OF Extension
const inner = readTlv(tbs, t.valStart);
if (inner && inner.tag === 0x30) extensionsBody = inner.val;
break;
}
off = t.next;
}
if (!extensionsBody) return [];
// Iterate extensions
let eOff = 0;
while (eOff < extensionsBody.length) {
const ext = readTlv(extensionsBody, eOff);
if (!ext || ext.tag !== 0x30) break;
// ext.val: SEQUENCE { oid OBJECT IDENTIFIER, critical BOOLEAN OPTIONAL, value OCTET STRING }
let inOff = 0;
const oidTlv = readTlv(ext.val, inOff);
if (!oidTlv || oidTlv.tag !== 0x06) { eOff = ext.next; continue; }
const oid = oidTlv.val;
inOff = oidTlv.next;
// Optional critical
let valTlv = readTlv(ext.val, inOff);
if (valTlv && valTlv.tag === 0x01) {
inOff = valTlv.next;
valTlv = readTlv(ext.val, inOff);
}
if (!valTlv || valTlv.tag !== 0x04) { eOff = ext.next; continue; } // OCTET STRING
if (!oid.equals(SAN_OID)) { eOff = ext.next; continue; }
// valTlv.val is OCTET STRING contents = DER-encoded SubjectAltName (SEQUENCE OF GeneralName)
const sanSeq = readTlv(valTlv.val, 0);
if (!sanSeq || sanSeq.tag !== 0x30) return [];
const names = [];
let gOff = 0;
while (gOff < sanSeq.val.length) {
const gn = readTlv(sanSeq.val, gOff);
if (!gn) break;
if (gn.tag === 0x82) names.push(gn.val.toString('ascii').toLowerCase()); // dNSName [2] IMPLICIT
gOff = gn.next;
}
return names;
}
return [];
} catch {
return [];
}
}
// Multi-log fan-out: if logKeys (array) is given, spawn one polling loop per
// log against the SAME EventEmitter — consumers receive a merged firehose.
// Returns a wrapper {stop, stats} that aggregates across logs.
function startMulti({ logKeys, pollMs = 5_000, batchSize = 32, onEvent, onStatus } = {}) {
const ee = new EventEmitter();
if (onEvent) ee.on('certificate_update', onEvent);
if (onStatus) ee.on('status', onStatus);
const subs = logKeys.map((k, i) => startSingle({
logKey: k, pollMs, batchSize,
// Stagger polls so we don't burst all logs at the same instant
initialDelayMs: i * Math.max(500, Math.floor(pollMs / logKeys.length)),
sink: ee,
}));
return {
on: (...a) => ee.on(...a),
stop: () => subs.forEach((s) => s.stop()),
stats: () => subs.map((s) => ({ log: s.logName, stats: s.stats() })),
};
}
function start(opts = {}) {
if (Array.isArray(opts.logKeys) && opts.logKeys.length > 1) return startMulti(opts);
return startSingle(opts);
}
function startSingle({
logKey = 'argon',
pollMs = 5_000,
batchSize = 32,
initialDelayMs = 0,
sink, // optional shared EventEmitter for multi-log fan-out
onEvent,
onStatus,
} = {}) {
const ee = sink || new EventEmitter();
if (onEvent) ee.on('certificate_update', onEvent);
if (onStatus) ee.on('status', onStatus);
const log = LOGS[logKey];
if (!log) throw new Error('unknown log: ' + logKey);
const cursors = loadCursors();
let position = cursors[log.name] || null; // last tree_size we processed
let stopped = false;
let consecutiveErrs = 0;
const stats = { sths: 0, fetched: 0, x509: 0, precert: 0, parseErr: 0, emitted: 0 };
ee.emit('status', { type: 'started', log: log.name, pollMs, batchSize });
if (position !== null) ee.emit('status', { type: 'resumed', log: log.name, position });
async function tick() {
if (stopped) return;
try {
// Cache-buster: STH is small + Cloudflare/CDN-fronted; force fresh.
const sth = await getJson(log.base + '/get-sth?_=' + Date.now());
stats.sths++;
const treeSize = sth.tree_size;
if (position === null) {
// First run: seed at current tree size, only sample going forward
position = treeSize;
cursors[log.name] = position;
saveCursors(cursors);
ee.emit('status', { type: 'seeded', log: log.name, position });
} else if (treeSize > position) {
// Fetch up to batchSize newest entries, even if more accrued
const start = Math.max(position, treeSize - batchSize);
const end = treeSize - 1;
try {
const r = await getJson(`${log.base}/get-entries?start=${start}&end=${end}`);
const entries = r.entries || [];
stats.fetched += entries.length;
for (let i = 0; i < entries.length; i++) {
const entryIdx = start + i;
const parsed = parseLeaf(entries[i].leaf_input);
if (parsed.error && parsed.entryType === undefined) { stats.parseErr++; continue; }
let names = [];
if (parsed.entryType === 0 && parsed.cert) {
stats.x509++;
names = extractDnsNames(parsed.cert);
} else if (parsed.entryType === 1 && parsed.tbs) {
stats.precert++;
names = extractDnsNamesFromTBS(parsed.tbs);
} else {
stats.parseErr++;
continue;
}
if (!names.length) continue;
stats.emitted++;
ee.emit('certificate_update', {
message_type: 'certificate_update',
data: {
cert_index: entryIdx,
log_source: log.name,
entry_type: parsed.entryType === 1 ? 'precert' : 'x509',
leaf_cert: {
all_domains: names,
not_before: null,
not_after: null,
issuer: null,
},
},
});
}
position = treeSize;
cursors[log.name] = position;
saveCursors(cursors);
ee.emit('status', { type: 'tick', log: log.name, treeSize, fetched: entries.length, stats: { ...stats } });
} catch (e) {
ee.emit('status', { type: 'fetch_err', log: log.name, msg: e.message });
}
} else {
// No new entries yet
ee.emit('status', { type: 'idle', log: log.name, treeSize });
}
consecutiveErrs = 0;
} catch (e) {
consecutiveErrs++;
ee.emit('status', { type: 'sth_err', log: log.name, msg: e.message, consecutive: consecutiveErrs });
}
if (!stopped) {
const wait = consecutiveErrs > 0 ? Math.min(60_000, pollMs * Math.pow(2, Math.min(consecutiveErrs, 4))) : pollMs;
setTimeout(tick, wait);
}
}
if (initialDelayMs > 0) setTimeout(tick, initialDelayMs); else tick();
return {
on: (...a) => ee.on(...a),
stop: () => { stopped = true; ee.emit('status', { type: 'stopped', log: log.name, stats }); },
stats: () => ({ ...stats }),
position: () => position,
logName: log.name,
};
}
module.exports = { start, startMulti, LOGS, parseLeaf, extractDnsNames, extractDnsNamesFromTBS };
if (require.main === module) {
const pollMs = parseInt(process.env.POLL_MS || '5000', 10);
const batchSize = parseInt(process.env.BATCH || '32', 10);
const ctLogs = process.env.CT_LOGS; // comma-separated for multi-log
const ctLog = process.env.CT_LOG || 'argon';
const opts = ctLogs
? { logKeys: ctLogs.split(',').map((s) => s.trim()), pollMs, batchSize }
: { logKey: ctLog, pollMs, batchSize };
console.log(`[${new Date().toISOString()}] ct-source-ctlog starting ${ctLogs ? 'multi=[' + ctLogs + ']' : 'log=' + ctLog} poll=${pollMs}ms batch=${batchSize}`);
const s = start({
...opts,
onEvent: (e) => {
const names = e.data.leaf_cert.all_domains;
const src = e.data.log_source || '?';
console.log(` cert#${e.data.cert_index} [${src}] ${names.slice(0, 2).join(', ')}${names.length > 2 ? ` …+${names.length - 2}` : ''}`);
},
onStatus: (st) => {
if (st.type === 'tick') console.log(` [tick ${st.log}] tree=${st.treeSize} batch=${st.fetched} x509=${st.stats.x509} pre=${st.stats.precert}`);
else console.log(` [${st.type}] ${JSON.stringify(st).slice(0, 220)}`);
},
});
process.on('SIGINT', () => { s.stop(); process.exit(0); });
}