iter 173: auto-dedupe now also catches trigram-templated headlines (3+ shared 3-word phrases) on top of exact dupes

88857c90281201aa17580ce1cfad239ef24cf488 · 2026-05-06 20:22:50 -0700 · SteveStudio2
Files touched

M src/jobs/auto_dedupe_headlines.ts
Diff

commit 88857c90281201aa17580ce1cfad239ef24cf488
Author: SteveStudio2 <stevestudio2@SteveStacStudio.lan>
Date:   Wed May 6 20:22:50 2026 -0700

    iter 173: auto-dedupe now also catches trigram-templated headlines (3+ shared 3-word phrases) on top of exact dupes
---
 src/jobs/auto_dedupe_headlines.ts | 56 +++++++++++++++++++++++++++++++--------
 1 file changed, 45 insertions(+), 11 deletions(-)

diff --git a/src/jobs/auto_dedupe_headlines.ts b/src/jobs/auto_dedupe_headlines.ts
index cbfa2f1..7d0383a 100644
--- a/src/jobs/auto_dedupe_headlines.ts
+++ b/src/jobs/auto_dedupe_headlines.ts
@@ -20,7 +20,8 @@ const ADMIN_PASS = process.env.ADMIN_PASS || 'DWSecure2024!';
 const MAX_REGENS_PER_RUN = 5;
 
 async function main() {
-  const groups = await pool.query(`
+  // Tier 1: exact-match headline groups
+  const exact = await pool.query(`
     SELECT mf.headline,
            array_agg(mf.id ORDER BY mf.generated_at ASC) AS ids
     FROM magazine_features mf
@@ -30,24 +31,57 @@ async function main() {
     ORDER BY count(*) DESC
   `);
 
-  if (groups.rowCount === 0) {
-    console.log(`[auto-dedupe] no template groups with ≥3 dupes — nothing to do`);
+  // Tier 2: trigram-templated headlines (e.g. "X in Y's Heart" recurring 3+ times).
+  // Compute trigram counts in JS, then map back to feature IDs.
+  const allHeadlines = await pool.query(`
+    SELECT id, headline, generated_at FROM magazine_features
+    WHERE headline IS NOT NULL AND status != 'spiked'
+    ORDER BY generated_at ASC
+  `);
+  const STOP = new Set('a an the and or in of to for on at with by from is are be was were has have had this that these those'.split(/\s+/));
+  type Pos = { id: number; ts: string };
+  const triFeatures = new Map<string, Pos[]>();
+  for (const row of allHeadlines.rows) {
+    const tokens = String(row.headline).toLowerCase().replace(/[^a-z\s']/g, ' ').split(/\s+/).filter(t => t && !STOP.has(t) && t.length > 2);
+    const triggered = new Set<string>();
+    for (let i = 0; i + 2 < tokens.length; i++) {
+      const tri = `${tokens[i]} ${tokens[i+1]} ${tokens[i+2]}`;
+      if (triggered.has(tri)) continue;
+      triggered.add(tri);
+      if (!triFeatures.has(tri)) triFeatures.set(tri, []);
+      triFeatures.get(tri)!.push({ id: row.id, ts: row.generated_at });
+    }
+  }
+  const trigramGroups = [...triFeatures.entries()]
+    .filter(([, arr]) => arr.length >= 3)
+    .sort((a, b) => b[1].length - a[1].length);
+
+  const exactCount = exact.rowCount ?? 0;
+  if (exactCount === 0 && trigramGroups.length === 0) {
+    console.log(`[auto-dedupe] no exact dupes, no trigram templates ≥3 — nothing to do`);
     await pool.end();
     return;
   }
 
+  // Build target list: exact dupes first (priority), then trigram runner-ups.
   const targets: number[] = [];
-  for (const g of groups.rows) {
-    // Drop the oldest (keep it), queue the rest
+  const seen = new Set<number>();
+  const queue = (id: number) => { if (!seen.has(id) && targets.length < MAX_REGENS_PER_RUN) { seen.add(id); targets.push(id); } };
+
+  for (const g of exact.rows) {
     const runners = (g.ids as number[]).slice(1);
-    for (const id of runners) {
-      if (targets.length >= MAX_REGENS_PER_RUN) break;
-      targets.push(id);
-    }
-    if (targets.length >= MAX_REGENS_PER_RUN) break;
+    for (const id of runners) queue(id);
+  }
+  for (const [, arr] of trigramGroups) {
+    // Skip the oldest member (the canonical use of the phrase) and queue newer ones
+    const sorted = [...arr].sort((a, b) => +new Date(a.ts) - +new Date(b.ts));
+    for (const p of sorted.slice(1)) queue(p.id);
   }
 
-  console.log(`[auto-dedupe] ${groups.rowCount} template group(s) found, regenerating ${targets.length} runner-up(s) (cap=${MAX_REGENS_PER_RUN})`);
+  console.log(`[auto-dedupe] exact-groups=${exactCount} · trigram-groups=${trigramGroups.length} · regenerating ${targets.length} runner-up(s) (cap=${MAX_REGENS_PER_RUN})`);
+  if (trigramGroups.length) {
+    console.log(`[auto-dedupe] top trigrams: ${trigramGroups.slice(0, 3).map(([t, arr]) => `"${t}" (${arr.length}×)`).join(', ')}`);
+  }
 
   const auth = 'Basic ' + Buffer.from(`${ADMIN_USER}:${ADMIN_PASS}`).toString('base64');
   let ok = 0, fail = 0;
← 317fdcf iter 172: /api/magazine/similar/:id — pg_trgm word_similarit · back to Ventura Corridor · iter 174: fix /api/magazine/:id/regen — no longer demotes pu 02da0a2 →