← back to Ventura Corridor
iter 168: auto-dedupe headlines nightly job + 3:33 AM launchd — regens up to 5 runner-ups per night, keeps oldest of each templated group
59418a9a58aed30d66fae104f5539eb2db0a938b · 2026-05-06 19:56:08 -0700 · SteveStudio2
Files touched
M package.jsonA src/jobs/auto_dedupe_headlines.ts
Diff
commit 59418a9a58aed30d66fae104f5539eb2db0a938b
Author: SteveStudio2 <stevestudio2@SteveStacStudio.lan>
Date: Wed May 6 19:56:08 2026 -0700
iter 168: auto-dedupe headlines nightly job + 3:33 AM launchd — regens up to 5 runner-ups per night, keeps oldest of each templated group
---
package.json | 3 +-
src/jobs/auto_dedupe_headlines.ts | 78 +++++++++++++++++++++++++++++++++++++++
2 files changed, 80 insertions(+), 1 deletion(-)
diff --git a/package.json b/package.json
index 2ff0279..edd9c8f 100644
--- a/package.json
+++ b/package.json
@@ -33,7 +33,8 @@
"magazine:ig-dryrun": "tsx src/jobs/ig_autopost_dryrun.ts",
"magazine:standup": "tsx src/jobs/daily_standup_audio.ts",
"magazine:auto-cover": "tsx src/jobs/auto_cover.ts",
- "magazine:coverage-snapshot": "tsx src/jobs/coverage_snapshot.ts"
+ "magazine:coverage-snapshot": "tsx src/jobs/coverage_snapshot.ts",
+ "magazine:auto-dedupe": "tsx src/jobs/auto_dedupe_headlines.ts"
},
"dependencies": {
"@types/multer": "^2.1.0",
diff --git a/src/jobs/auto_dedupe_headlines.ts b/src/jobs/auto_dedupe_headlines.ts
new file mode 100644
index 0000000..cbfa2f1
--- /dev/null
+++ b/src/jobs/auto_dedupe_headlines.ts
@@ -0,0 +1,78 @@
+// Nightly template-buster. When qwen3 converges to identical headlines (3+
+// features sharing the same headline), regenerate the runner-ups so we end
+// with unique titles. Keeps the OLDEST copy of each group.
+//
+// Run manually: npm run magazine:auto-dedupe
+// Cron-driven: launchd 3:33 AM daily, after auto-publish + auto-cover.
+//
+// Safety:
+// - Only acts on groups with count ≥ 3 (skip noise).
+// - Caps at 5 regens per run (prevents long Mac1-busy windows).
+// - Skips features in 'spiked' state.
+// - Each regen is sequential; logs as it goes.
+
+import { Pool } from 'pg';
+
+const pool = new Pool({ host: '/tmp', database: 'ventura_corridor' });
+const PORT = process.env.PORT || '9780';
+const ADMIN_USER = process.env.ADMIN_USER || 'admin';
+const ADMIN_PASS = process.env.ADMIN_PASS || 'DWSecure2024!';
+const MAX_REGENS_PER_RUN = 5;
+
+async function main() {
+ const groups = await pool.query(`
+ SELECT mf.headline,
+ array_agg(mf.id ORDER BY mf.generated_at ASC) AS ids
+ FROM magazine_features mf
+ WHERE mf.headline IS NOT NULL AND mf.status != 'spiked'
+ GROUP BY mf.headline
+ HAVING count(*) >= 3
+ ORDER BY count(*) DESC
+ `);
+
+ if (groups.rowCount === 0) {
+ console.log(`[auto-dedupe] no template groups with ≥3 dupes — nothing to do`);
+ await pool.end();
+ return;
+ }
+
+ const targets: number[] = [];
+ for (const g of groups.rows) {
+ // Drop the oldest (keep it), queue the rest
+ const runners = (g.ids as number[]).slice(1);
+ for (const id of runners) {
+ if (targets.length >= MAX_REGENS_PER_RUN) break;
+ targets.push(id);
+ }
+ if (targets.length >= MAX_REGENS_PER_RUN) break;
+ }
+
+ console.log(`[auto-dedupe] ${groups.rowCount} template group(s) found, regenerating ${targets.length} runner-up(s) (cap=${MAX_REGENS_PER_RUN})`);
+
+ const auth = 'Basic ' + Buffer.from(`${ADMIN_USER}:${ADMIN_PASS}`).toString('base64');
+ let ok = 0, fail = 0;
+ for (const id of targets) {
+ try {
+ const t0 = Date.now();
+ const r = await fetch(`http://127.0.0.1:${PORT}/api/magazine/${id}/regen`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json', Authorization: auth },
+ });
+ const took = ((Date.now() - t0) / 1000).toFixed(1);
+ if (r.ok) {
+ ok++;
+ console.log(`[auto-dedupe] #${id} regen OK (${took}s)`);
+ } else {
+ fail++;
+ console.log(`[auto-dedupe] #${id} regen FAIL ${r.status} (${took}s)`);
+ }
+ } catch (e: any) {
+ fail++;
+ console.log(`[auto-dedupe] #${id} regen ERROR: ${e.message}`);
+ }
+ }
+ console.log(`[auto-dedupe] done — ${ok} regenerated, ${fail} failed`);
+ await pool.end();
+}
+
+main().catch(e => { console.error('[auto-dedupe]', e); process.exit(1); });
← 6a0afbb iter 167: /api/magazine/headline-stems + /duplicates.html te
·
back to Ventura Corridor
·
iter 169: gen prompt now blocks the 6 most-overused headline bfb00de →