`src/enrich/extract_headlines.ts`

113 lines
/**
 * Headline extractor.
 *
 * Reads each cached front-page HTML and pulls the first usable headline for
 * the H1 Mural surface. Preference order:
 *   1. The first non-empty <h1> (text content, stripped of tags)
 *   2. The <title> tag if no h1 (with the brand name suffix trimmed off)
 *   3. <meta name="description"> as a last resort
 *
 * Headlines longer than 200 chars are truncated; headlines shorter than 4
 * chars are rejected. The source ('h1', 'title', 'meta_description') is
 * recorded so the viewer can render different sources differently.
 *
 * Pure regex over cached HTML. No HTTP, no LLM.
 */
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import { query, pool } from '../../db/pool.ts';

interface AuditRow {
  business_id: number;
  raw_html_path: string;
}

const H1_RE = /<h1\b[^>]*>([\s\S]*?)<\/h1>/i;
const TITLE_RE = /<title\b[^>]*>([\s\S]*?)<\/title>/i;
const META_DESC_RE = /<meta\s+(?:[^>]*\s)?name=['"]description['"][^>]*content=['"]([^'"]+)['"][^>]*>/i;
const META_DESC_RE2 = /<meta\s+(?:[^>]*\s)?content=['"]([^'"]+)['"][^>]*name=['"]description['"][^>]*>/i;

function stripTags(s: string): string {
  return s.replace(/<[^>]+>/g, '').replace(/&nbsp;/g, ' ').replace(/&amp;/g, '&').replace(/&#39;/g, "'").replace(/&quot;/g, '"').replace(/&apos;/g, "'").replace(/\s+/g, ' ').trim();
}

function extract(html: string): { text: string; source: string } | null {
  const h1 = html.match(H1_RE);
  if (h1) {
    const t = stripTags(h1[1]);
    if (t.length >= 4 && t.length <= 200) return { text: t, source: 'h1' };
    if (t.length > 200) return { text: t.slice(0, 200).replace(/\s+\S*$/, '') + '…', source: 'h1' };
  }
  const title = html.match(TITLE_RE);
  if (title) {
    let t = stripTags(title[1]);
    // Trim common "| Brand Name" / " — Brand" / " - Brand" suffixes if present
    t = t.replace(/\s*[\|·•—–-]\s*[^|·•—–-]+$/, '').trim();
    if (t.length >= 4 && t.length <= 200) return { text: t, source: 'title' };
  }
  const desc = html.match(META_DESC_RE) || html.match(META_DESC_RE2);
  if (desc) {
    const t = stripTags(desc[1]);
    if (t.length >= 4) return { text: t.length > 200 ? t.slice(0, 200).replace(/\s+\S*$/, '') + '…' : t, source: 'meta_description' };
  }
  return null;
}

async function main() {
  const r = await query<AuditRow>(`
    SELECT DISTINCT ON (a.business_id) a.business_id, a.raw_html_path
    FROM front_page_audits a
    WHERE a.raw_html_path IS NOT NULL
    ORDER BY a.business_id, a.audited_at DESC
  `);

  console.log(`[headlines] mining ${r.rowCount} HTML files…`);

  let withH1 = 0, withTitle = 0, withDesc = 0, none = 0, errors = 0, processed = 0;

  for (const row of r.rows) {
    const rel = row.raw_html_path.startsWith('data/') ? row.raw_html_path : 'data/' + row.raw_html_path;
    let html: string;
    try {
      html = readFileSync(join(process.cwd(), rel), 'utf8');
    } catch {
      errors++;
      continue;
    }
    const result = extract(html);
    if (!result) {
      none++;
      continue;
    }
    if (result.source === 'h1') withH1++;
    else if (result.source === 'title') withTitle++;
    else withDesc++;
    processed++;

    await query(
      `
      INSERT INTO business_enrichment (business_id, headline, headline_source, headline_extracted_at)
      VALUES ($1, $2, $3, now())
      ON CONFLICT (business_id) DO UPDATE
      SET headline = EXCLUDED.headline,
          headline_source = EXCLUDED.headline_source,
          headline_extracted_at = now()
      `,
      [row.business_id, result.text, result.source]
    );
  }

  console.log(`[headlines] processed ${processed} headlines:`);
  console.log(`  from <h1>             : ${withH1}`);
  console.log(`  from <title>          : ${withTitle}`);
  console.log(`  from meta description : ${withDesc}`);
  console.log(`  no headline found     : ${none}`);
  console.log(`  read errors           : ${errors}`);
  await pool.end();
}

main().catch((e) => {
  console.error('[headlines]', e);
  process.exit(1);
});