`scripts/import-apify-govdeals.ts`

37 lines
// Import GovDeals listings scraped by the Apify actor into GovArbitrage.
// Reads an already-finished dataset (free) — does NOT trigger a new paid run.
//
// Usage: APIFY_TOKEN=... npx tsx scripts/import-apify-govdeals.ts [datasetId]
//   - datasetId arg or APIFY_GOVDEALS_DATASET env → read that dataset
//   - otherwise → read the actor's most recent SUCCEEDED run

import { fetchApifyDataset, fetchApifyLastRun } from "../src/importers/apify-govdeals";
import { ingestMany } from "../src/importers/ingest";
import { prisma } from "../src/lib/db";

async function main() {
  const datasetId = process.argv[2] || process.env.APIFY_GOVDEALS_DATASET;
  const rows = datasetId ? await fetchApifyDataset(datasetId) : await fetchApifyLastRun();
  console.log(`Fetched ${rows.length} GovDeals listings via Apify (${datasetId ? `dataset ${datasetId}` : "last run"}). Ingesting…`);

  const results = await ingestMany(rows, { useAI: false });
  const created = results.filter((r) => r.ok && r.created).length;
  const updated = results.filter((r) => r.ok && !r.created).length;
  const failed = results.filter((r) => !r.ok);
  console.log(`Done: ${created} created, ${updated} updated, ${failed.length} failed.`);
  failed.slice(0, 5).forEach((f) => console.log(`  ✗ ${f.title}: ${f.error}`));
  // Fail the run (non-zero exit) so a scheduler sees red when the dataset was
  // empty or every row failed — not a false "success".
  if (rows.length === 0 || (results.length > 0 && failed.length === results.length)) {
    console.error(`[import-apify-govdeals] RUN FAILED: fetched ${rows.length}, ${failed.length}/${results.length} ingests failed.`);
    process.exitCode = 1;
  }
  await prisma.$disconnect();
}

main().catch(async (e) => {
  console.error("[import-apify-govdeals] ERROR:", e.message);
  await prisma.$disconnect();
  process.exit(1);
});