`audit-noise.js`

94 lines
#!/usr/bin/env node
// audit-noise.js — surface top candidate noise patterns from brand-typo-hits.jsonl.
//
// Groups every hit by the last 2 + 3 labels of the domain (so subdomain churn
// rolls up to the registrable parent), tallies occurrences, and flags any
// suffix that is NOT already in brand-typo-watcher's NOISE_PATTERNS as a
// candidate to add. Read-only — never writes back.
//
// Usage:  node audit-noise.js [N]   (default top-30)

const fs = require('fs');
const path = require('path');

const FILE = path.join(__dirname, 'data', 'brand-typo-hits.jsonl');
const TOP_N = parseInt(process.argv[2] || '30', 10);

// Mirror the NOISE_PATTERNS from brand-typo-watcher (keep in sync if you change either).
const CURRENT_FILTER = [
  /\.haplorrhini\.com$/i,
  /\.beutl\.in$/i,
  /\.cellt\.net$/i,
];
const isFiltered = (suffix) => CURRENT_FILTER.some((r) => r.test('x.' + suffix));

if (!fs.existsSync(FILE)) {
  console.error(`no hits file at ${FILE}`);
  process.exit(2);
}

// Two parallel tallies:
//   tally2/tally3   — every hit (raw audit view, includes BARE_PUNYCODE)
//   brand2/brand3   — ONLY real brand-match hits (SUBSTRING/TYPO). These are
//                     the ones worth proposing as NOISE_PATTERNS additions —
//                     suppressing BARE_PUNYCODE rows would silence the
//                     intentional homoglyph-monitoring audit trail.
const tally2 = new Map();
const tally3 = new Map();
const brand2 = new Map();
const brand3 = new Map();
let total = 0;
let totalBrand = 0;
const lines = fs.readFileSync(FILE, 'utf8').split('\n').filter(Boolean);
for (const ln of lines) {
  let r; try { r = JSON.parse(ln); } catch { continue; }
  const d = (r.domain || '').toLowerCase().replace(/^\*\./, '');
  if (!d) continue;
  total++;
  const isBrandHit = r.matchType === 'SUBSTRING' || r.matchType === 'TYPO';
  if (isBrandHit) totalBrand++;
  const parts = d.split('.');
  if (parts.length >= 2) {
    const s = parts.slice(-2).join('.');
    tally2.set(s, (tally2.get(s) || 0) + 1);
    if (isBrandHit) brand2.set(s, (brand2.get(s) || 0) + 1);
  }
  if (parts.length >= 3) {
    const s = parts.slice(-3).join('.');
    tally3.set(s, (tally3.get(s) || 0) + 1);
    if (isBrandHit) brand3.set(s, (brand3.get(s) || 0) + 1);
  }
}

const ranked2 = [...tally2.entries()].sort((a, b) => b[1] - a[1]).slice(0, TOP_N);
const ranked3 = [...tally3.entries()].sort((a, b) => b[1] - a[1]).slice(0, TOP_N);

console.log(`\n  audit-noise :: ${lines.length} hit rows  /  ${total} domain entries  /  ${totalBrand} real brand-match hits (rest = BARE_PUNYCODE audit trail)\n`);
console.log(`  top ${TOP_N} 2-label suffixes (registrable parent):`);
console.log(`  ` + '-'.repeat(72));
for (const [s, n] of ranked2) {
  const flag = isFiltered(s) ? '\x1b[32m[filtered]\x1b[0m' : '\x1b[33m[unfiltered]\x1b[0m';
  console.log(`  ${String(n).padStart(5)}  ${s.padEnd(40)}  ${flag}`);
}

console.log(`\n  top ${TOP_N} 3-label suffixes (rolled-up subdomains):`);
console.log(`  ` + '-'.repeat(72));
for (const [s, n] of ranked3) {
  const flag = isFiltered(s) ? '\x1b[32m[filtered]\x1b[0m' : '\x1b[33m[unfiltered]\x1b[0m';
  console.log(`  ${String(n).padStart(5)}  ${s.padEnd(40)}  ${flag}`);
}

// Real candidates = BRAND-match suffixes only. BARE_PUNYCODE rows are the
// intentional homoglyph-monitor audit trail and must not be suppressed.
const ranked2Brand = [...brand2.entries()].sort((a, b) => b[1] - a[1]);
const candidates = ranked2Brand.filter(([s]) => !isFiltered(s) && brand2.get(s) >= 3);
if (candidates.length) {
  console.log(`\n  candidate NOISE_PATTERNS to add (≥3 real brand-match hits, not yet filtered):`);
  for (const [s, n] of candidates.slice(0, 10)) {
    console.log(`    /\\.${s.replace(/\./g, '\\.')}$/i,    // ${n} brand-match hits`);
  }
  console.log('');
} else {
  console.log(`\n  no candidate NOISE_PATTERNS — all brand-match suffixes either filtered or <3 hits.\n`);
}