`src/jobs/auto_dedupe_headlines.ts`

113 lines
// Nightly template-buster. When qwen3 converges to identical headlines (3+
// features sharing the same headline), regenerate the runner-ups so we end
// with unique titles. Keeps the OLDEST copy of each group.
//
// Run manually: npm run magazine:auto-dedupe
// Cron-driven:  launchd 3:33 AM daily, after auto-publish + auto-cover.
//
// Safety:
//   - Only acts on groups with count ≥ 3 (skip noise).
//   - Caps at 5 regens per run (prevents long Mac1-busy windows).
//   - Skips features in 'spiked' state.
//   - Each regen is sequential; logs as it goes.

import { Pool } from 'pg';

const pool = new Pool({ host: '/tmp', database: 'ventura_corridor' });
const PORT = process.env.PORT || '9780';
const ADMIN_USER = process.env.ADMIN_USER || 'admin';
const ADMIN_PASS = process.env.ADMIN_PASS || 'DWSecure2024!';
const MAX_REGENS_PER_RUN = 5;

async function main() {
  // Tier 1: exact-match headline groups
  const exact = await pool.query(`
    SELECT mf.headline,
           array_agg(mf.id ORDER BY mf.generated_at ASC) AS ids
    FROM magazine_features mf
    WHERE mf.headline IS NOT NULL AND mf.status != 'spiked'
    GROUP BY mf.headline
    HAVING count(*) >= 3
    ORDER BY count(*) DESC
  `);

  // Tier 2: trigram-templated headlines (e.g. "X in Y's Heart" recurring 3+ times).
  // Compute trigram counts in JS, then map back to feature IDs.
  const allHeadlines = await pool.query(`
    SELECT id, headline, generated_at FROM magazine_features
    WHERE headline IS NOT NULL AND status != 'spiked'
    ORDER BY generated_at ASC
  `);
  const STOP = new Set('a an the and or in of to for on at with by from is are be was were has have had this that these those'.split(/\s+/));
  type Pos = { id: number; ts: string };
  const triFeatures = new Map<string, Pos[]>();
  for (const row of allHeadlines.rows) {
    const tokens = String(row.headline).toLowerCase().replace(/[^a-z\s']/g, ' ').split(/\s+/).filter(t => t && !STOP.has(t) && t.length > 2);
    const triggered = new Set<string>();
    for (let i = 0; i + 2 < tokens.length; i++) {
      const tri = `${tokens[i]} ${tokens[i+1]} ${tokens[i+2]}`;
      if (triggered.has(tri)) continue;
      triggered.add(tri);
      if (!triFeatures.has(tri)) triFeatures.set(tri, []);
      triFeatures.get(tri)!.push({ id: row.id, ts: row.generated_at });
    }
  }
  const trigramGroups = [...triFeatures.entries()]
    .filter(([, arr]) => arr.length >= 3)
    .sort((a, b) => b[1].length - a[1].length);

  const exactCount = exact.rowCount ?? 0;
  if (exactCount === 0 && trigramGroups.length === 0) {
    console.log(`[auto-dedupe] no exact dupes, no trigram templates ≥3 — nothing to do`);
    await pool.end();
    return;
  }

  // Build target list: exact dupes first (priority), then trigram runner-ups.
  const targets: number[] = [];
  const seen = new Set<number>();
  const queue = (id: number) => { if (!seen.has(id) && targets.length < MAX_REGENS_PER_RUN) { seen.add(id); targets.push(id); } };

  for (const g of exact.rows) {
    const runners = (g.ids as number[]).slice(1);
    for (const id of runners) queue(id);
  }
  for (const [, arr] of trigramGroups) {
    // Skip the oldest member (the canonical use of the phrase) and queue newer ones
    const sorted = [...arr].sort((a, b) => +new Date(a.ts) - +new Date(b.ts));
    for (const p of sorted.slice(1)) queue(p.id);
  }

  console.log(`[auto-dedupe] exact-groups=${exactCount} · trigram-groups=${trigramGroups.length} · regenerating ${targets.length} runner-up(s) (cap=${MAX_REGENS_PER_RUN})`);
  if (trigramGroups.length) {
    console.log(`[auto-dedupe] top trigrams: ${trigramGroups.slice(0, 3).map(([t, arr]) => `"${t}" (${arr.length}×)`).join(', ')}`);
  }

  const auth = 'Basic ' + Buffer.from(`${ADMIN_USER}:${ADMIN_PASS}`).toString('base64');
  let ok = 0, fail = 0;
  for (const id of targets) {
    try {
      const t0 = Date.now();
      const r = await fetch(`http://127.0.0.1:${PORT}/api/magazine/${id}/regen`, {
        method: 'POST',
        headers: { 'Content-Type': 'application/json', Authorization: auth },
      });
      const took = ((Date.now() - t0) / 1000).toFixed(1);
      if (r.ok) {
        ok++;
        console.log(`[auto-dedupe] #${id} regen OK (${took}s)`);
      } else {
        fail++;
        console.log(`[auto-dedupe] #${id} regen FAIL ${r.status} (${took}s)`);
      }
    } catch (e: any) {
      fail++;
      console.log(`[auto-dedupe] #${id} regen ERROR: ${e.message}`);
    }
  }
  console.log(`[auto-dedupe] done — ${ok} regenerated, ${fail} failed`);
  await pool.end();
}

main().catch(e => { console.error('[auto-dedupe]', e); process.exit(1); });