`src/importers/apify-govdeals.ts`

149 lines
import type { RawListing } from "./ingest";

// GovDeals via the Apify actor `parseforge/govdeals-scraper` — bypasses GovDeals'
// Akamai bot-wall (Apify handles it). We READ finished datasets (free); running
// the actor for FRESH data costs Apify compute (~$0.0037/listing) and is gated.
//
// Env: APIFY_TOKEN (required). Optionally APIFY_GOVDEALS_ACTOR (default the known
// actor) and APIFY_GOVDEALS_DATASET (a specific dataset id to read).

const API = "https://api.apify.com/v2";
const DEFAULT_ACTOR = process.env.APIFY_GOVDEALS_ACTOR || "SZdBCF9FpwOzvByvM"; // parseforge/govdeals-scraper

interface ApifyItem {
  title?: string;
  url?: string;
  imageUrl?: string;
  photos?: (string | { url?: string })[];
  accountId?: number;
  assetId?: number;
  auctionId?: number;
  category?: string;
  parentCategory?: string;
  make?: string | null;
  model?: string | null;
  condition?: string | null;
  quantity?: number;
  weight?: number | null;
  description?: string;
  locationCity?: string;
  locationState?: string;
  locationZip?: string;
  currentBid?: number;
  bidCount?: number;
  auctionEnd?: string;
  auctionEndUtc?: string;
  specialInstructions?: string;
  removalInstructions?: string;
  paymentInstructions?: string;
}

const CONDITION_MAP: Record<string, RawListing["condition"]> = {
  N: "NEW",
  U: "USED_GOOD",
  R: "LIKE_NEW",
  F: "USED_FAIR",
  S: "FOR_PARTS",
  P: "FOR_PARTS",
};

const stripHtml = (s?: string) =>
  (s || "").replace(/<[^>]*>/g, " ").replace(/&nbsp;/g, " ").replace(/\s+/g, " ").trim() || undefined;

function photoUrls(item: ApifyItem): string[] {
  if (Array.isArray(item.photos) && item.photos.length) {
    return item.photos.map((p) => (typeof p === "string" ? p : p?.url || "")).filter(Boolean).slice(0, 8);
  }
  return item.imageUrl ? [item.imageUrl] : [];
}

function mapItem(it: ApifyItem): RawListing | null {
  const title = it.title?.trim();
  if (!title || it.assetId == null) return null;
  return {
    source: "GOVDEALS",
    sourceAuctionId: `${it.accountId ?? "x"}-${it.assetId}`,
    sourceUrl: it.url || "https://www.govdeals.com",
    title,
    description: stripHtml(it.description),
    category: it.category || it.parentCategory || undefined,
    manufacturer: it.make || undefined,
    model: it.model || undefined,
    condition: CONDITION_MAP[(it.condition || "").trim().toUpperCase()] || "UNKNOWN",
    quantity: it.quantity && it.quantity > 0 ? it.quantity : 1,
    weightLbs: it.weight ?? undefined,
    locationCity: it.locationCity,
    locationState: it.locationState,
    locationZip: it.locationZip,
    currentBid: it.currentBid ?? 0,
    bidCount: it.bidCount ?? 0,
    closingAt: it.auctionEndUtc || it.auctionEnd || null,
    imageUrls: photoUrls(it),
    auctionTerms: [it.specialInstructions, it.removalInstructions, it.paymentInstructions]
      .map(stripHtml)
      .filter(Boolean)
      .join(" ") || undefined,
  };
}

async function apifyJson(path: string): Promise<unknown> {
  const token = process.env.APIFY_TOKEN;
  if (!token) throw new Error("APIFY_TOKEN not set");
  const sep = path.includes("?") ? "&" : "?";
  const res = await fetch(`${API}${path}${sep}token=${token}&clean=true`);
  if (!res.ok) throw new Error(`Apify ${res.status}: ${(await res.text()).slice(0, 160)}`);
  return res.json();
}

/** Read an already-scraped dataset by id (free). */
export async function fetchApifyDataset(datasetId: string, limit = 200): Promise<RawListing[]> {
  const items = (await apifyJson(`/datasets/${datasetId}/items?limit=${limit}`)) as ApifyItem[];
  return items.map(mapItem).filter((x): x is RawListing => x !== null);
}

/** Read the most recent SUCCEEDED run's dataset for the GovDeals actor (free). */
export async function fetchApifyLastRun(limit = 200): Promise<RawListing[]> {
  const items = (await apifyJson(
    `/acts/${DEFAULT_ACTOR}/runs/last/dataset/items?status=SUCCEEDED&limit=${limit}`,
  )) as ApifyItem[];
  return items.map(mapItem).filter((x): x is RawListing => x !== null);
}

/**
 * Trigger a NEW GovDeals scrape run (PAID — ~$0.0037/listing) and return the
 * scraped rows plus the run's actual USD cost. Steve-authorized for the daily
 * 100-listing job. Polls up to ~8 min for completion.
 */
export async function triggerApifyRun(
  maxItems = 100,
): Promise<{ rows: RawListing[]; costUsd: number; runId: string }> {
  const token = process.env.APIFY_TOKEN;
  if (!token) throw new Error("APIFY_TOKEN not set");

  const startRes = await fetch(`${API}/acts/${DEFAULT_ACTOR}/runs?token=${token}`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({ maxItems, sortOrder: "asc" }),
  });
  if (!startRes.ok) throw new Error(`Apify run start ${startRes.status}: ${(await startRes.text()).slice(0, 160)}`);
  const run = ((await startRes.json()) as { data: { id: string; status: string; defaultDatasetId: string } }).data;

  const deadline = Date.now() + 8 * 60 * 1000;
  let status = run.status;
  let datasetId = run.defaultDatasetId;
  let costUsd = 0;
  while (Date.now() < deadline) {
    await new Promise((r) => setTimeout(r, 5000));
    const res = await fetch(`${API}/actor-runs/${run.id}?token=${token}`);
    const s = ((await res.json()) as { data: { status: string; defaultDatasetId: string; usageTotalUsd?: number } }).data;
    status = s.status;
    datasetId = s.defaultDatasetId;
    costUsd = s.usageTotalUsd ?? 0;
    if (["SUCCEEDED", "FAILED", "ABORTED", "TIMED-OUT"].includes(status)) break;
  }
  if (status !== "SUCCEEDED") throw new Error(`Apify run ${run.id} ended ${status}`);

  const rows = await fetchApifyDataset(datasetId, maxItems);
  return { rows, costUsd, runId: run.id };
}