`watch-drops.js`

141 lines
#!/usr/bin/env node
// watch-drops.js — daily pull of pending-delete CSVs from public drop sources.
//
// STATUS: SCAFFOLD. Refuses to fetch unless WATCH_DROPS_SOURCES is set.
//
// Why scaffolded-not-shipped:
//   1. Anti-bot walls on pool.com / dropcatch.com / snapnames.com / expireddomains.net.
//      Cloudflare / Akamai challenge pages bounce anonymous GETs.
//   2. Same hygiene rule that bans WHOIS/RDAP probing applies here: any GET against
//      a drop-house auction page is a "we're interested" signal. If we ever hit a
//      drop list and then go register one, the request stream is correlatable.
//   3. The only TOS-clean source is ICANN CZDS (Centralized Zone Data Service),
//      which requires per-TLD application + API key. Once Steve has CZDS creds
//      routed via the secrets skill, the czds adapter below activates.
//
// Adapter interface:
//   { name, source, fetch(opts) -> { ok, names: [...domain], fetchedAt, raw } }
//
// Usage:
//   node watch-drops.js list-sources         # show what's configured
//   WATCH_DROPS_SOURCES=czds node watch-drops.js pull   # pull enabled adapters
//
// Output (per enabled adapter):
//   data/drops/<source>-<YYYY-MM-DD>.jsonl   # one {ts, domain, source} per line
//   data/drops/index.json                    # tally per day / source

const fs = require('fs');
const path = require('path');
const https = require('https');

const DATA_DIR = path.join(__dirname, 'data', 'drops');
const INDEX_FILE = path.join(DATA_DIR, 'index.json');

// Adapters keyed by short name. fetch() returns names ready to dedupe.
const ADAPTERS = {
  // ICANN CZDS — daily zone-file diff against yesterday's snapshot.
  // Requires CZDS_API_KEY (routed via secrets skill once Steve approves a TLD).
  // Each .com/.net/.app/.dev zone is a tab-separated NSDNAME record file ~GB-sized;
  // diff yields adds (registrations) and deletes (drops).
  czds: {
    name: 'czds',
    requiresEnv: 'CZDS_API_KEY',
    description: 'ICANN Centralized Zone Data Service zone-diff (drops = yesterday \\ today)',
    fetch: async () => {
      const apiKey = process.env.CZDS_API_KEY;
      if (!apiKey) return { ok: false, error: 'CZDS_API_KEY not set — route via secrets skill' };
      // TODO: Implement per-TLD zone fetch + diff once a key is provisioned.
      // CZDS flow: POST /api/authenticate -> JWT -> GET /czds/downloads/links -> per-zone GET
      return { ok: false, error: 'czds adapter is a stub — implement after CZDS-TLD approval' };
    },
  },

  // Pool.com / DropCatch / SnapNames / ExpiredDomains.net are intentionally NOT
  // listed as adapters here. They have anti-bot walls AND visiting them is a
  // leak signal per the same rule that bans WHOIS probing. Don't add them
  // without explicit Steve sign-off.
};

function ensureDir(p) {
  fs.mkdirSync(p, { recursive: true });
}

function loadIndex() {
  try { return JSON.parse(fs.readFileSync(INDEX_FILE, 'utf8')); }
  catch { return { byDay: {} }; }
}

function saveIndex(ix) {
  ensureDir(DATA_DIR);
  fs.writeFileSync(INDEX_FILE, JSON.stringify(ix, null, 2));
}

function listSources() {
  console.log('\n  watch-drops :: configured adapters');
  console.log('  ' + '-'.repeat(78));
  for (const [k, a] of Object.entries(ADAPTERS)) {
    const envOk = a.requiresEnv ? (process.env[a.requiresEnv] ? '\x1b[32mYES\x1b[0m' : '\x1b[31mno\x1b[0m') : '\x1b[32mYES\x1b[0m';
    console.log(`  ${k.padEnd(12)} env-ready=${envOk}  ${a.description}`);
    if (a.requiresEnv && !process.env[a.requiresEnv]) {
      console.log(`               requires ${a.requiresEnv} (route via secrets skill: ~/Projects/secrets-manager/cli.js)`);
    }
  }
  console.log('\n  policy: drop-house URLs (pool.com / dropcatch.com / snapnames.com / expireddomains.net)');
  console.log('          are intentionally NOT registered as adapters — they are leak channels per the');
  console.log('          DoH-only rule and are anti-bot-walled. Add only with explicit Steve sign-off.\n');
}

async function pull() {
  const enabled = (process.env.WATCH_DROPS_SOURCES || '').split(',').map((s) => s.trim()).filter(Boolean);
  if (!enabled.length) {
    console.error('\n  WATCH_DROPS_SOURCES is empty — refusing to fetch.');
    console.error('  Run:  node watch-drops.js list-sources    to see what\'s available.\n');
    process.exit(2);
  }
  ensureDir(DATA_DIR);
  const ix = loadIndex();
  const day = new Date().toISOString().slice(0, 10);
  ix.byDay[day] = ix.byDay[day] || {};
  let totalNames = 0;
  for (const name of enabled) {
    const a = ADAPTERS[name];
    if (!a) {
      console.error(`  unknown adapter: ${name}`);
      continue;
    }
    console.log(`  [${name}] fetching…`);
    try {
      const r = await a.fetch({});
      if (!r.ok) {
        console.error(`  [${name}] skipped: ${r.error}`);
        ix.byDay[day][name] = { count: 0, error: r.error };
        continue;
      }
      const out = path.join(DATA_DIR, `${name}-${day}.jsonl`);
      const ts = new Date().toISOString();
      const lines = (r.names || []).map((d) => JSON.stringify({ ts, domain: d, source: name }) + '\n');
      fs.writeFileSync(out, lines.join(''));
      ix.byDay[day][name] = { count: lines.length, file: out };
      totalNames += lines.length;
      console.log(`  [${name}] wrote ${lines.length} names → ${out}`);
    } catch (e) {
      console.error(`  [${name}] error: ${e.message}`);
      ix.byDay[day][name] = { count: 0, error: e.message };
    }
  }
  saveIndex(ix);
  console.log(`\n  total names pulled today: ${totalNames}`);
}

const cmd = process.argv[2] || 'help';
if (cmd === 'list-sources') {
  listSources();
} else if (cmd === 'pull') {
  pull().catch((e) => { console.error(e); process.exit(1); });
} else {
  console.error('usage:');
  console.error('  node watch-drops.js list-sources');
  console.error('  WATCH_DROPS_SOURCES=czds[,dropcatch...] node watch-drops.js pull');
  process.exit(2);
}