← back to Ventura Corridor

iter 167: /api/magazine/headline-stems + /duplicates.html templated-phrases panel (5× 'precision care encino', 5× 'legal precision meets', etc.)

6a0afbbd675eb90f0515a2f24c2a4352faafdd16 · 2026-05-06 19:55:15 -0700 · SteveStudio2

Files touched

• M public/duplicates.html
• M src/server/index.ts

Diff

commit 6a0afbbd675eb90f0515a2f24c2a4352faafdd16
Author: SteveStudio2 <stevestudio2@SteveStacStudio.lan>
Date:   Wed May 6 19:55:15 2026 -0700

    iter 167: /api/magazine/headline-stems + /duplicates.html templated-phrases panel (5× 'precision care encino', 5× 'legal precision meets', etc.)
---
 public/duplicates.html | 17 +++++++++++++++++
 src/server/index.ts    | 30 ++++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+)

diff --git a/public/duplicates.html b/public/duplicates.html
index b7ba643..3cc9ec8 100644
--- a/public/duplicates.html
+++ b/public/duplicates.html
@@ -61,6 +61,23 @@ async function load() {
     return;
   }
   let html = `<div class="summary"><b>${d.groups}</b> duplicate group${d.groups===1?'':'s'} affecting <b>${d.total_affected}</b> features</div>`;
+
+  // Headline-stem analysis — looser than exact-match dupes
+  const stems = await fetch('/api/magazine/headline-stems').then(r => r.json()).catch(() => ({ stems: [] }));
+  const topStems = (stems.stems || []).slice(0, 8);
+  if (topStems.length) {
+    html += `<div class="group">
+      <div class="head" style="color:var(--metal-glow)">Templated phrases</div>
+      <div class="count">3-word phrases recurring across headlines · ${stems.count} total templates · ${stems.total_headlines} headlines analyzed</div>
+      <ul style="font-family:'JetBrains Mono',monospace;font-size:13px">${topStems.map(s => `
+        <li style="grid-template-columns:60px 1fr 1fr">
+          <span class="id">${s.count}×</span>
+          <span class="biz" style="font-family:'Cormorant Garamond',serif;font-style:italic;color:var(--ink)">"${escHtml(s.phrase)}"</span>
+          <span class="meta" style="white-space:normal;line-height:1.4">${s.examples.map(e => escHtml(e.slice(0, 32))).join(' · ')}</span>
+        </li>`).join('')}</ul>
+      <div class="legend">If a phrase appears 3+ times, qwen3 has likely fallen into a template — consider regen on the latest few features that use it.</div>
+    </div>`;
+  }
   for (const g of d.rows) {
     const features = g.features;
     // Oldest by generated_at; mark it as the "keeper"
diff --git a/src/server/index.ts b/src/server/index.ts
index 4b275d3..82e5338 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -1933,6 +1933,36 @@ app.get('/api/magazine/random', async (req, res) => {
   }
 });
 
+// Headline-phrase frequency — finds 3-word phrases ("trigrams") that recur across
+// headlines. Cousin of /api/magazine/duplicates: catches templating BEFORE it
+// becomes an exact-match duplicate. Surfaces the patterns qwen3 reaches for.
+app.get('/api/magazine/headline-stems', async (_req, res) => {
+  try {
+    const r = await query(`SELECT headline FROM magazine_features WHERE headline IS NOT NULL`);
+    const STOP = new Set('a an the and or in of to for on at with by from is are be was were has have had this that these those'.split(/\s+/));
+    const counts: Record<string, number> = {};
+    const examples: Record<string, string[]> = {};
+    for (const row of r.rows) {
+      const h = String(row.headline);
+      const tokens = h.toLowerCase().replace(/[^a-z\s']/g, ' ').split(/\s+/).filter(t => t && !STOP.has(t) && t.length > 2);
+      for (let i = 0; i + 2 < tokens.length; i++) {
+        const tri = `${tokens[i]} ${tokens[i+1]} ${tokens[i+2]}`;
+        counts[tri] = (counts[tri] || 0) + 1;
+        if (!examples[tri]) examples[tri] = [];
+        if (examples[tri].length < 3 && !examples[tri].includes(h)) examples[tri].push(h);
+      }
+    }
+    const ranked = Object.entries(counts)
+      .filter(([, n]) => n >= 2)
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, 30)
+      .map(([phrase, count]) => ({ phrase, count, examples: examples[phrase] }));
+    res.json({ count: ranked.length, total_headlines: r.rowCount, stems: ranked });
+  } catch (e: any) {
+    res.status(500).json({ error: e.message });
+  }
+});
+
 // Duplicate-headline detector — when qwen3 converges to the same headline pattern
 // (e.g. "Precision Care in Encino's Heart" 3×), the editorial assistant is
 // templating. Surfacing duplicates lets Steve regen the runner-ups for variety.

← 1efb250 iter 166: /duplicates.html admin viewer — keeps oldest, clic  ·  back to Ventura Corridor  ·  iter 168: auto-dedupe headlines nightly job + 3:33 AM launch 59418a9 →