`scripts/enrich-entities-wikipedia.ts`

106 lines
#!/usr/bin/env -S npx tsx
/**
 * Wikipedia summary enrichment.
 *
 * For each entity that already has wiki_url but is missing bio_short,
 * pulls the page summary via Wikipedia's REST API and stores the extract.
 * Tier-C source.
 *
 * Usage:
 *   npx tsx scripts/enrich-entities-wikipedia.ts
 *   npx tsx scripts/enrich-entities-wikipedia.ts --slug paul-r-williams
 *   npx tsx scripts/enrich-entities-wikipedia.ts --force   # overwrite existing bio
 *   npx tsx scripts/enrich-entities-wikipedia.ts --dry
 */
import { Pool } from 'pg';

const pool = new Pool({
  host: process.env.PGHOST ?? '/tmp',
  database: process.env.PGDATABASE ?? 'stayclaim',
  user: process.env.PGUSER ?? process.env.USER,
  max: 4,
});

function pageTitleFromUrl(url: string): string | null {
  try {
    const u = new URL(url);
    if (!u.hostname.endsWith('wikipedia.org')) return null;
    // /wiki/Foo_Bar  →  Foo_Bar
    const m = u.pathname.match(/^\/wiki\/(.+)$/);
    return m ? decodeURIComponent(m[1]!) : null;
  } catch {
    return null;
  }
}

async function fetchSummary(title: string): Promise<{ extract: string } | null> {
  const url = `https://en.wikipedia.org/api/rest_v1/page/summary/${encodeURIComponent(title)}`;
  const res = await fetch(url, {
    headers: {
      'User-Agent': 'pastdoor/0.1 (https://pastdoor.com; data-enrichment)',
      Accept: 'application/json',
    },
  });
  if (!res.ok) return null;
  const j = (await res.json()) as { extract?: string; type?: string };
  if (!j.extract) return null;
  return { extract: j.extract };
}

async function main() {
  const args = process.argv.slice(2);
  const slugArg = args.find((a, i) => args[i - 1] === '--slug');
  const dry = args.includes('--dry');
  const force = args.includes('--force');

  const conds: string[] = ['wiki_url IS NOT NULL'];
  const params: unknown[] = [];
  if (!force) conds.push("(bio_short IS NULL OR bio_short = '')");
  if (slugArg) {
    params.push(slugArg);
    conds.push(`slug = $${params.length}`);
  }
  const sql = `SELECT id, slug, display_name, wiki_url, bio_short
               FROM entity
               WHERE ${conds.join(' AND ')}
               ORDER BY display_name`;
  const { rows } = await pool.query(sql, params);

  let updated = 0;
  let nohit = 0;
  for (const e of rows) {
    const title = pageTitleFromUrl(e.wiki_url);
    if (!title) {
      console.log(`[${e.slug}] no usable Wikipedia title in ${e.wiki_url}`);
      continue;
    }
    let summary;
    try {
      summary = await fetchSummary(title);
    } catch (err) {
      console.warn(`[${e.slug}] fetch error: ${(err as Error).message}`);
      continue;
    }
    if (!summary) {
      nohit++;
      console.log(`[${e.slug}] no Wikipedia summary`);
      await new Promise(r => setTimeout(r, 400));
      continue;
    }
    // Trim to a tight editorial blurb — first 1-2 sentences, max ~280 chars.
    const tight = summary.extract.split(/(?<=\.)\s+/).slice(0, 2).join(' ').slice(0, 280);
    if (dry) {
      console.log(`[${e.slug}] DRY would set bio_short:`, tight);
    } else {
      await pool.query('UPDATE entity SET bio_short = $1 WHERE id = $2', [tight, e.id]);
      console.log(`[${e.slug}] updated`);
      updated++;
    }
    await new Promise(r => setTimeout(r, 600));
  }
  console.log(JSON.stringify({ checked: rows.length, updated, no_summary: nohit, dry }, null, 2));
  await pool.end();
}

main().catch(e => { console.error(e); process.exit(1); });