← back to Domain Sniper
ct-source-certspotter.js: skeleton module + standalone smoke runner
7a91f94453603e9e673b519dc40d456cf9413e4c · 2026-05-12 16:00:29 -0700 · Steve Abrams
Free tier requires domain= per request — no anonymous firehose. Module supports
per-brand polling (next tick: add brands-mode harness so 18 keywords cycle at
12min each = 90 req/hr, within 100/hr free limit). Cursor persistence + retry-
after backoff + 429/503 handling already wired. Drop-in replacement for the
CertStream WS payload shape.
Files touched
A ct-source-certspotter.js
Diff
commit 7a91f94453603e9e673b519dc40d456cf9413e4c
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Tue May 12 16:00:29 2026 -0700
ct-source-certspotter.js: skeleton module + standalone smoke runner
Free tier requires domain= per request — no anonymous firehose. Module supports
per-brand polling (next tick: add brands-mode harness so 18 keywords cycle at
12min each = 90 req/hr, within 100/hr free limit). Cursor persistence + retry-
after backoff + 429/503 handling already wired. Drop-in replacement for the
CertStream WS payload shape.
---
ct-source-certspotter.js | 164 +++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 164 insertions(+)
diff --git a/ct-source-certspotter.js b/ct-source-certspotter.js
new file mode 100644
index 0000000..38b7e98
--- /dev/null
+++ b/ct-source-certspotter.js
@@ -0,0 +1,164 @@
+// ct-source-certspotter.js — Cert Spotter REST fallback for when
+// certstream.calidog.io is dead. Polls https://api.certspotter.com/v1/issuances
+// and emits 'certificate_update' events with the same payload shape consumers
+// of CertStream already expect:
+// { message_type:'certificate_update',
+// data: { cert_index, leaf_cert: { all_domains: [...] } } }
+//
+// IMPORTANT: Cert Spotter's free tier REQUIRES domain= per request — there is
+// no anonymous firehose. This module supports two modes:
+// 1. domain=<keyword> → per-brand polling. 18 brands × 12-min cycle = 90 req/hr
+// (within 100/hr free limit). Use for BRAND defense.
+// 2. with CERTSPOTTER_API_KEY env → can omit domain (paid-tier firehose).
+//
+// Free tier: 100 req/hr without API key.
+// Cursor is the `id` of the newest issuance seen, persisted to
+// data/certspotter-cursor.json so restarts don't re-emit history.
+//
+// Usage:
+// const src = require('./ct-source-certspotter');
+// const s = src.start({ pollMs: 40_000, onEvent: e => ... });
+// s.stop();
+//
+// Or as a standalone process:
+// node ct-source-certspotter.js # logs each batch to stdout
+
+const https = require('https');
+const fs = require('fs');
+const path = require('path');
+const { EventEmitter } = require('events');
+
+const DATA_DIR = path.join(__dirname, 'data');
+const CURSOR_FILE = path.join(DATA_DIR, 'certspotter-cursor.json');
+const API_HOST = 'api.certspotter.com';
+const API_PATH = '/v1/issuances';
+const UA = 'domain-sniper/0.1 (defensive brand monitoring; contact via project README)';
+
+function loadCursor() {
+ try { return JSON.parse(fs.readFileSync(CURSOR_FILE, 'utf8')).after || null; }
+ catch { return null; }
+}
+function saveCursor(after) {
+ fs.mkdirSync(DATA_DIR, { recursive: true });
+ fs.writeFileSync(CURSOR_FILE, JSON.stringify({ after, savedAt: new Date().toISOString() }));
+}
+
+function fetchBatch(after, apiKey) {
+ return new Promise((resolve, reject) => {
+ const qs = new URLSearchParams();
+ if (after) qs.set('after', after);
+ qs.append('expand', 'dns_names');
+ qs.append('expand', 'issuer');
+ const headers = { 'User-Agent': UA, 'Accept': 'application/json' };
+ if (apiKey) headers['Authorization'] = 'Bearer ' + apiKey;
+ const req = https.request({
+ host: API_HOST,
+ path: API_PATH + '?' + qs.toString(),
+ method: 'GET',
+ headers,
+ timeout: 15_000,
+ }, (res) => {
+ let buf = '';
+ res.on('data', (c) => buf += c);
+ res.on('end', () => {
+ if (res.statusCode === 200) {
+ try { resolve({ status: 200, body: JSON.parse(buf) }); }
+ catch (e) { reject(new Error('parse: ' + e.message)); }
+ } else {
+ resolve({ status: res.statusCode, body: buf, retryAfter: res.headers['retry-after'] });
+ }
+ });
+ });
+ req.on('error', reject);
+ req.on('timeout', () => { req.destroy(new Error('timeout')); });
+ req.end();
+ });
+}
+
+function start({ pollMs = 40_000, apiKey = process.env.CERTSPOTTER_API_KEY, onEvent, onStatus } = {}) {
+ const ee = new EventEmitter();
+ if (onEvent) ee.on('certificate_update', onEvent);
+ if (onStatus) ee.on('status', onStatus);
+
+ let after = loadCursor();
+ let stopped = false;
+ let backoff = 0;
+
+ ee.emit('status', { type: 'started', after, pollMs });
+
+ async function tick() {
+ if (stopped) return;
+ try {
+ const r = await fetchBatch(after, apiKey);
+ if (r.status === 429 || r.status === 503) {
+ const wait = parseInt(r.retryAfter, 10) * 1000 || Math.min(60_000, Math.max(5000, backoff * 2 || 5000));
+ backoff = wait;
+ ee.emit('status', { type: 'throttled', status: r.status, retryInMs: wait });
+ setTimeout(tick, wait);
+ return;
+ }
+ if (r.status !== 200) {
+ ee.emit('status', { type: 'http_error', status: r.status, snippet: String(r.body).slice(0, 200) });
+ const wait = Math.min(60_000, Math.max(5000, backoff * 2 || 5000));
+ backoff = wait;
+ setTimeout(tick, wait);
+ return;
+ }
+ backoff = 0;
+ const items = Array.isArray(r.body) ? r.body : [];
+ let newCursor = after;
+ let emitted = 0;
+ for (const it of items) {
+ const domains = (it.dns_names || []).filter(Boolean);
+ if (!domains.length) continue;
+ ee.emit('certificate_update', {
+ message_type: 'certificate_update',
+ data: {
+ cert_index: it.id,
+ leaf_cert: {
+ all_domains: domains,
+ issuer: it.issuer && (it.issuer.name || it.issuer.friendly_name || null),
+ not_before: it.not_before || null,
+ not_after: it.not_after || null,
+ },
+ },
+ });
+ emitted++;
+ if (!newCursor || it.id > newCursor) newCursor = it.id;
+ }
+ if (newCursor && newCursor !== after) {
+ after = newCursor;
+ saveCursor(after);
+ }
+ ee.emit('status', { type: 'tick', count: items.length, emitted, cursor: after });
+ } catch (e) {
+ ee.emit('status', { type: 'error', msg: e.message });
+ }
+ if (!stopped) setTimeout(tick, pollMs);
+ }
+ tick();
+
+ return {
+ on: (...a) => ee.on(...a),
+ stop: () => { stopped = true; ee.emit('status', { type: 'stopped' }); },
+ getCursor: () => after,
+ };
+}
+
+module.exports = { start, loadCursor, saveCursor };
+
+// Standalone mode: run directly to smoke-test
+if (require.main === module) {
+ const pollMs = parseInt(process.env.POLL_MS || '40000', 10);
+ console.log(`[${new Date().toISOString()}] ct-source-certspotter starting (poll=${pollMs}ms)`);
+ const s = start({
+ pollMs,
+ onEvent: (e) => {
+ const domains = e.data.leaf_cert.all_domains;
+ // Print only first 3 per cert to keep output sane
+ console.log(` CERT id=${e.data.cert_index} ${domains.slice(0, 3).join(', ')}${domains.length > 3 ? ` …+${domains.length - 3}` : ''}`);
+ },
+ onStatus: (s) => console.log(` [status] ${JSON.stringify(s)}`),
+ });
+ process.on('SIGINT', () => { s.stop(); process.exit(0); });
+}
← 6760b8c note: certstream.calidog.io connect-close loop (code 1000, 0
·
back to Domain Sniper
·
ct-source-certspotter-brands.js + free-CT-source landscape d 083e982 →