`scripts/import-apify-govdeals.ts`

31 lines
// Import GovDeals listings scraped by the Apify actor into GovArbitrage.
// Reads an already-finished dataset (free) — does NOT trigger a new paid run.
//
// Usage: APIFY_TOKEN=... npx tsx scripts/import-apify-govdeals.ts [datasetId]
//   - datasetId arg or APIFY_GOVDEALS_DATASET env → read that dataset
//   - otherwise → read the actor's most recent SUCCEEDED run

import { fetchApifyDataset, fetchApifyLastRun } from "../src/importers/apify-govdeals";
import { ingestMany } from "../src/importers/ingest";
import { prisma } from "../src/lib/db";

async function main() {
  const datasetId = process.argv[2] || process.env.APIFY_GOVDEALS_DATASET;
  const rows = datasetId ? await fetchApifyDataset(datasetId) : await fetchApifyLastRun();
  console.log(`Fetched ${rows.length} GovDeals listings via Apify (${datasetId ? `dataset ${datasetId}` : "last run"}). Ingesting…`);

  const results = await ingestMany(rows, { useAI: false });
  const created = results.filter((r) => r.ok && r.created).length;
  const updated = results.filter((r) => r.ok && !r.created).length;
  const failed = results.filter((r) => !r.ok);
  console.log(`Done: ${created} created, ${updated} updated, ${failed.length} failed.`);
  failed.slice(0, 5).forEach((f) => console.log(`  ✗ ${f.title}: ${f.error}`));
  await prisma.$disconnect();
}

main().catch(async (e) => {
  console.error("[import-apify-govdeals] ERROR:", e.message);
  await prisma.$disconnect();
  process.exit(1);
});