← back to Domain Sniper
ct-source-ctlog.js: WORKING direct CT-log polling — bypasses dead middlemen
b00757fc3ba305dee21291df15563f4cc4e5a0ce · 2026-05-12 16:12:37 -0700 · Steve Abrams
Polls get-sth on active 2026 logs (Argon2026h1 / Xenon2026h1 / Wyvern2026h1
all confirmed advancing ~6–7 entries/sec). Fetches get-entries when tree
advances, parses MerkleTreeLeaf, extracts SAN DNS names from X.509 certs
via Node 22 crypto.X509Certificate.
Smoke run on Argon: 8 cert events emitted in 25s, 0 parse errors.
Precerts (entry_type=1, ~75% of stream) currently skipped pending ASN.1
TBSCertificate decoder; we still see X.509 entries (~25% of stream).
Cache-buster appended to get-sth URL — Nimbus 2026 was returning stale
treeSize for 30+ seconds without it; argon/xenon/wyvern advance every
poll once the buster is in place.
Files touched
M README.mdA ct-source-ctlog.js
Diff
commit b00757fc3ba305dee21291df15563f4cc4e5a0ce
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Tue May 12 16:12:37 2026 -0700
ct-source-ctlog.js: WORKING direct CT-log polling — bypasses dead middlemen
Polls get-sth on active 2026 logs (Argon2026h1 / Xenon2026h1 / Wyvern2026h1
all confirmed advancing ~6–7 entries/sec). Fetches get-entries when tree
advances, parses MerkleTreeLeaf, extracts SAN DNS names from X.509 certs
via Node 22 crypto.X509Certificate.
Smoke run on Argon: 8 cert events emitted in 25s, 0 parse errors.
Precerts (entry_type=1, ~75% of stream) currently skipped pending ASN.1
TBSCertificate decoder; we still see X.509 entries (~25% of stream).
Cache-buster appended to get-sth URL — Nimbus 2026 was returning stale
treeSize for 30+ seconds without it; argon/xenon/wyvern advance every
poll once the buster is in place.
---
README.md | 2 +-
ct-source-ctlog.js | 243 +++++++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 244 insertions(+), 1 deletion(-)
diff --git a/README.md b/README.md
index 1a2dac0..db7ead4 100644
--- a/README.md
+++ b/README.md
@@ -73,7 +73,7 @@ State of the world for **free** real-time CT-log access, as of 2026-05-12 ~23:05
- `ct-source-certspotter-brands.js` — per-brand polling attempt (blocked by FQDN-only restriction)
- Both modules emit the CertStream payload shape, so once a working source comes online they drop in to `watch-ct.js` / `brand-typo-watcher.js` / `dashboard.js` via a `CT_SOURCE` env switch.
-**Best near-term path:** retry crt.sh with backoff (it works most of the time, just not right now), and build a `ct-source-crtsh.js` polling module. This is the only free option that supports the keyword/substring matching we actually need for brand defense.
+**RESOLVED 2026-05-12 23:13 UTC** — bypassed all middlemen by talking directly to CT logs per RFC 6962. `ct-source-ctlog.js` polls `get-sth` against currently-active 2026 logs (Argon2026h1 / Xenon2026h1 / Wyvern2026h1 — Nimbus2026 seems CDN-cached, skip it), fetches `get-entries`, parses MerkleTreeLeaf, extracts SAN DNS names from each X.509 cert via Node 22 `crypto.X509Certificate`. Smoke run: **8 emitted cert events in 25s** from Argon (14 X509 parsed, 43 precerts skipped — precert ASN.1 parse is TODO).
Meanwhile the **honeypot + DoH spot-check** path (the actual aggregator-attribution experiment) is unaffected — that runs on Cloudflare 1.1.1.1 DoH and Node `child_process whois`, neither of which depends on CT logs.
diff --git a/ct-source-ctlog.js b/ct-source-ctlog.js
new file mode 100644
index 0000000..246ad10
--- /dev/null
+++ b/ct-source-ctlog.js
@@ -0,0 +1,243 @@
+// ct-source-ctlog.js — Direct Certificate Transparency log poller.
+// Bypasses ALL the broken/gated middlemen (CertStream, Cert Spotter, crt.sh,
+// merklemap) by talking to the underlying CT log servers directly per RFC 6962.
+//
+// Two probed-working endpoints as of 2026-05-12:
+// - Google Argon : https://ct.googleapis.com/logs/us1/argon2025h2/ct/v1/
+// - Cloudflare Nimbus: https://ct.cloudflare.com/logs/nimbus2026/ct/v1/
+//
+// Each log is append-only. We poll get-sth periodically; when tree_size
+// advances, we fetch get-entries for the new range. Each entry is a binary
+// MerkleTreeLeaf — we parse the timestamp + entry_type + (for X509_ENTRY)
+// the embedded ASN.1 cert, then extract SAN DNS names via Node 22's built-in
+// crypto.X509Certificate.
+//
+// PRECERT_ENTRY parsing requires ASN.1 dive into the TBSCertificate — deferred.
+// We log skip count so we know what we're missing.
+//
+// Bandwidth: ~256 entries × ~3KB each = ~750KB per poll. Default poll 5s
+// means we sample but don't fully consume the firehose (world rate is much
+// higher) — for brand defense, sampling is fine: any given brand appears in
+// many issuances per day so we'll catch one within minutes.
+
+const https = require('https');
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const { EventEmitter } = require('events');
+
+const DATA_DIR = path.join(__dirname, 'data');
+const CURSOR_FILE = path.join(DATA_DIR, 'ctlog-cursor.json');
+const UA = 'domain-sniper/0.1 (CT log monitor)';
+
+// From https://www.gstatic.com/ct/log_list/v3/log_list.json
+// (filtered to currently-active 2026 logs as of 2026-05-12)
+const LOGS = {
+ argon: { name: 'argon2026h1', base: 'https://ct.googleapis.com/logs/us1/argon2026h1/ct/v1' },
+ xenon: { name: 'xenon2026h1', base: 'https://ct.googleapis.com/logs/eu1/xenon2026h1/ct/v1' },
+ nimbus: { name: 'nimbus2026', base: 'https://ct.cloudflare.com/logs/nimbus2026/ct/v1' },
+ wyvern: { name: 'wyvern2026h1', base: 'https://wyvern.ct.digicert.com/2026h1/ct/v1' },
+ sphinx: { name: 'sphinx2026h1', base: 'https://sphinx.ct.digicert.com/2026h1/ct/v1' },
+ elephant:{ name: 'elephant2026h1', base: 'https://elephant2026h1.ct.sectigo.com/ct/v1' },
+ tiger: { name: 'tiger2026h1', base: 'https://tiger2026h1.ct.sectigo.com/ct/v1' },
+ // Older frozen log kept as a known-frozen smoke-test target
+ argon2025h2: { name: 'argon2025h2', base: 'https://ct.googleapis.com/logs/us1/argon2025h2/ct/v1' },
+};
+
+function loadCursors() {
+ try { return JSON.parse(fs.readFileSync(CURSOR_FILE, 'utf8')); }
+ catch { return {}; }
+}
+function saveCursors(c) {
+ fs.mkdirSync(DATA_DIR, { recursive: true });
+ fs.writeFileSync(CURSOR_FILE, JSON.stringify(c, null, 2));
+}
+
+function getJson(url) {
+ return new Promise((resolve, reject) => {
+ const u = new URL(url);
+ const req = https.request({
+ host: u.host, path: u.pathname + u.search, method: 'GET',
+ headers: { 'User-Agent': UA, 'Accept': 'application/json' },
+ timeout: 20_000,
+ }, (res) => {
+ let buf = '';
+ res.on('data', (c) => buf += c);
+ res.on('end', () => {
+ if (res.statusCode === 200) {
+ try { resolve(JSON.parse(buf)); }
+ catch (e) { reject(new Error('parse: ' + e.message)); }
+ } else reject(new Error('HTTP ' + res.statusCode + ': ' + buf.slice(0, 160)));
+ });
+ });
+ req.on('error', reject);
+ req.on('timeout', () => req.destroy(new Error('timeout')));
+ req.end();
+ });
+}
+
+// Parse a single MerkleTreeLeaf base64 → { entryType, timestamp, cert?: Buffer }
+function parseLeaf(leafB64) {
+ const buf = Buffer.from(leafB64, 'base64');
+ if (buf.length < 12) return { error: 'short' };
+ const version = buf.readUInt8(0);
+ const leafType = buf.readUInt8(1);
+ if (leafType !== 0) return { error: 'leaf_type=' + leafType };
+ let off = 2;
+ const timestamp = Number(buf.readBigUInt64BE(off)); off += 8;
+ const entryType = buf.readUInt16BE(off); off += 2;
+ if (entryType === 0) {
+ // X509_ENTRY: 3-byte length + cert DER
+ if (buf.length < off + 3) return { entryType, error: 'truncated_len' };
+ const certLen = (buf.readUInt8(off) << 16) | buf.readUInt16BE(off + 1);
+ off += 3;
+ if (buf.length < off + certLen) return { entryType, error: 'truncated_cert' };
+ return { entryType, timestamp, cert: buf.subarray(off, off + certLen) };
+ } else if (entryType === 1) {
+ // PRECERT_ENTRY: 32-byte issuer key hash + 3-byte len + TBSCert (defer parse)
+ return { entryType, timestamp };
+ }
+ return { entryType, error: 'unknown_entry_type' };
+}
+
+// Extract DNS names from a DER X.509 cert. Returns array of strings (lowercase).
+// Uses Node 22's crypto.X509Certificate. Returns [] on parse failure.
+function extractDnsNames(certDer) {
+ try {
+ const c = new crypto.X509Certificate(certDer);
+ const san = c.subjectAltName || '';
+ // Format: "DNS:example.com, DNS:www.example.com, IP Address:..., ..."
+ const names = [];
+ for (const part of san.split(',')) {
+ const t = part.trim();
+ if (t.startsWith('DNS:')) names.push(t.slice(4).toLowerCase());
+ }
+ if (!names.length && c.subject) {
+ // Fall back to CN= if no SAN
+ const cn = c.subject.split('\n').find((l) => l.startsWith('CN='));
+ if (cn) names.push(cn.slice(3).toLowerCase());
+ }
+ return names;
+ } catch {
+ return [];
+ }
+}
+
+function start({
+ logKey = 'argon',
+ pollMs = 5_000,
+ batchSize = 32,
+ onEvent,
+ onStatus,
+} = {}) {
+ const ee = new EventEmitter();
+ if (onEvent) ee.on('certificate_update', onEvent);
+ if (onStatus) ee.on('status', onStatus);
+
+ const log = LOGS[logKey];
+ if (!log) throw new Error('unknown log: ' + logKey);
+
+ const cursors = loadCursors();
+ let position = cursors[log.name] || null; // last tree_size we processed
+ let stopped = false;
+ let consecutiveErrs = 0;
+ const stats = { sths: 0, fetched: 0, x509: 0, precert: 0, parseErr: 0, emitted: 0 };
+
+ ee.emit('status', { type: 'started', log: log.name, pollMs, batchSize });
+
+ async function tick() {
+ if (stopped) return;
+ try {
+ // Cache-buster: STH is small + Cloudflare/CDN-fronted; force fresh.
+ const sth = await getJson(log.base + '/get-sth?_=' + Date.now());
+ stats.sths++;
+ const treeSize = sth.tree_size;
+ if (position === null) {
+ // First run: seed at current tree size, only sample going forward
+ position = treeSize;
+ cursors[log.name] = position;
+ saveCursors(cursors);
+ ee.emit('status', { type: 'seeded', log: log.name, position });
+ } else if (treeSize > position) {
+ // Fetch up to batchSize newest entries, even if more accrued
+ const start = Math.max(position, treeSize - batchSize);
+ const end = treeSize - 1;
+ try {
+ const r = await getJson(`${log.base}/get-entries?start=${start}&end=${end}`);
+ const entries = r.entries || [];
+ stats.fetched += entries.length;
+ for (let i = 0; i < entries.length; i++) {
+ const entryIdx = start + i;
+ const parsed = parseLeaf(entries[i].leaf_input);
+ if (parsed.error && !parsed.entryType) { stats.parseErr++; continue; }
+ if (parsed.entryType === 1) { stats.precert++; continue; }
+ if (parsed.entryType !== 0 || !parsed.cert) { stats.parseErr++; continue; }
+ stats.x509++;
+ const names = extractDnsNames(parsed.cert);
+ if (!names.length) continue;
+ stats.emitted++;
+ ee.emit('certificate_update', {
+ message_type: 'certificate_update',
+ data: {
+ cert_index: entryIdx,
+ log_source: log.name,
+ leaf_cert: {
+ all_domains: names,
+ not_before: null,
+ not_after: null,
+ issuer: null,
+ },
+ },
+ });
+ }
+ position = treeSize;
+ cursors[log.name] = position;
+ saveCursors(cursors);
+ ee.emit('status', { type: 'tick', log: log.name, treeSize, fetched: entries.length, stats: { ...stats } });
+ } catch (e) {
+ ee.emit('status', { type: 'fetch_err', log: log.name, msg: e.message });
+ }
+ } else {
+ // No new entries yet
+ ee.emit('status', { type: 'idle', log: log.name, treeSize });
+ }
+ consecutiveErrs = 0;
+ } catch (e) {
+ consecutiveErrs++;
+ ee.emit('status', { type: 'sth_err', log: log.name, msg: e.message, consecutive: consecutiveErrs });
+ }
+ if (!stopped) {
+ const wait = consecutiveErrs > 0 ? Math.min(60_000, pollMs * Math.pow(2, Math.min(consecutiveErrs, 4))) : pollMs;
+ setTimeout(tick, wait);
+ }
+ }
+ tick();
+
+ return {
+ on: (...a) => ee.on(...a),
+ stop: () => { stopped = true; ee.emit('status', { type: 'stopped', stats }); },
+ stats: () => ({ ...stats }),
+ position: () => position,
+ };
+}
+
+module.exports = { start, LOGS, parseLeaf, extractDnsNames };
+
+if (require.main === module) {
+ const logKey = process.env.CT_LOG || 'argon';
+ const pollMs = parseInt(process.env.POLL_MS || '5000', 10);
+ const batchSize = parseInt(process.env.BATCH || '32', 10);
+ console.log(`[${new Date().toISOString()}] ct-source-ctlog starting log=${logKey} poll=${pollMs}ms batch=${batchSize}`);
+ const s = start({
+ logKey, pollMs, batchSize,
+ onEvent: (e) => {
+ const names = e.data.leaf_cert.all_domains;
+ console.log(` cert#${e.data.cert_index} ${names.slice(0, 3).join(', ')}${names.length > 3 ? ` …+${names.length - 3}` : ''}`);
+ },
+ onStatus: (st) => {
+ if (st.type === 'tick') console.log(` [tick] tree=${st.treeSize} batch=${st.fetched} stats=${JSON.stringify(st.stats)}`);
+ else console.log(` [${st.type}] ${JSON.stringify(st)}`);
+ },
+ });
+ process.on('SIGINT', () => { s.stop(); process.exit(0); });
+}
← 083e982 ct-source-certspotter-brands.js + free-CT-source landscape d
·
back to Domain Sniper
·
brand-typo-watcher: wire ct-source-ctlog (CT_SOURCE=ctlog de 9bc1439 →