`src/importers/govplanet-free.ts`

228 lines
import type { RawListing } from "./ingest";

// FREE GovPlanet importer. $0, no browser at runtime.
//
// GovPlanet runs the IronPlanet / Ritchie Bros commerce stack (Java, `.ips`
// URLs). It is NOT on the Liquidity Services `maestro.lqdt1.com` API (verified:
// businessIds GP/GOVPLANET/IP/IRONPLANET all return 0 results). Its category
// landing pages are JS-hydrated, BUT the item search page embeds a per-item
// `quickviews.push({...})` JSON object with every field we need — served to
// plain curl (HTTP 200, no bot wall). So the crack is: fetch the search HTML,
// extract each quickviews object with a brace/string-aware scanner, JSON.parse.
//
// Verified 2026-07-10. Endpoints (each returns ~60 items):
//   https://www.govplanet.com/buy-now
//   https://www.govplanet.com/searchResults.ips?keyword=<kw>   (empty kw = all)
// quickviews object fields:
//   equipId, description (title), price/convPrice (USD), currency ("USD"),
//   eumeLocation, itemPageUri, photoThumb, timeLeft (relative), bidCnt, features
// Prices are already USD (US government surplus) — no FX conversion needed.
// Native pagination is AJAX-only, so we broaden coverage across the two base
// endpoints plus a set of high-value category keywords, deduped by equipId.

const BASE = "https://www.govplanet.com";
const UA =
  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " +
  "(KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36";

// High-value US-resale categories to broaden coverage beyond the ~60-item
// default page (GovPlanet's on-page pagination is AJAX-only).
const KEYWORDS = [
  "", // all
  "generator",
  "truck",
  "trailer",
  "forklift",
  "excavator",
  "loader",
  "trailer",
  "tractor",
  "compressor",
  "welder",
  "hmmwv",
  "humvee",
];

function stripTags(s: string): string {
  return s
    .replace(/<[^>]*>/g, " ")
    .replace(/&amp;/g, "&")
    .replace(/&#39;/g, "'")
    .replace(/&quot;/g, '"')
    .replace(/&nbsp;/g, " ")
    .replace(/\s+/g, " ")
    .trim();
}

function toNumber(s: string | undefined): number {
  if (!s) return 0;
  const m = /([\d,]+(?:\.\d+)?)/.exec(s.replace(/[^0-9.,]/g, ""));
  return m ? Number(m[1].replace(/,/g, "")) : 0;
}

// Parse a relative "timeLeft" like "2 days", "4 hrs", "15 mins" -> ISO close.
// GovPlanet often renders "&nbsp;" (blank) for offer-only items -> null.
function parseRelativeClose(raw: string | undefined): string | null {
  if (!raw) return null;
  const txt = stripTags(raw).toLowerCase();
  if (!txt || /ended|closed/.test(txt)) return null;
  const units: [RegExp, number][] = [
    [/(\d+)\s*day/, 86400_000],
    [/(\d+)\s*h(?:ou)?r/, 3600_000],
    [/(\d+)\s*min/, 60_000],
    [/(\d+)\s*sec/, 1000],
  ];
  let deltaMs = 0;
  let matched = false;
  for (const [re, ms] of units) {
    const m = re.exec(txt);
    if (m) {
      deltaMs += Number(m[1]) * ms;
      matched = true;
    }
  }
  if (!matched) return null;
  return new Date(Date.now() + deltaMs).toISOString();
}

/**
 * Extract every `quickviews.push({...})` object from a GovPlanet search page.
 * Uses a string/brace-aware scanner because values contain unescaped `)` and
 * `{` inside strings (e.g. "(2,424 mi away)"), which a greedy regex would break.
 */
function extractQuickviews(html: string): Record<string, unknown>[] {
  const out: Record<string, unknown>[] = [];
  const marker = "quickviews.push(";
  let idx = html.indexOf(marker);
  while (idx !== -1) {
    let i = idx + marker.length;
    // Expect the object to start at '{'.
    while (i < html.length && html[i] !== "{") i++;
    if (i >= html.length) break;
    const start = i;
    let depth = 0;
    let inStr = false;
    let quote = "";
    for (; i < html.length; i++) {
      const ch = html[i];
      if (inStr) {
        if (ch === "\\") {
          i++; // skip escaped char
          continue;
        }
        if (ch === quote) inStr = false;
      } else {
        if (ch === '"' || ch === "'") {
          inStr = true;
          quote = ch;
        } else if (ch === "{") depth++;
        else if (ch === "}") {
          depth--;
          if (depth === 0) {
            i++;
            break;
          }
        }
      }
    }
    const objStr = html.slice(start, i);
    try {
      out.push(JSON.parse(objStr));
    } catch {
      // Some pages use single-quoted values in a few fields; best-effort skip.
    }
    idx = html.indexOf(marker, i);
  }
  return out;
}

function qvToListing(qv: Record<string, unknown>): RawListing | null {
  const id = String(qv.equipId || qv.itemId || "").trim();
  const title = stripTags(String(qv.description || "")).trim();
  if (!id || !title) return null;

  const priceHtml = String(qv.price || qv.convPrice || "");
  const currentBid = toNumber(priceHtml);

  const uri = String(qv.itemPageUri || `/item/${id}`);
  const sourceUrl = uri.startsWith("http") ? uri : `${BASE}${uri}`;

  const state = String(qv.eumeLocation || "").trim() || undefined;
  const img = qv.photoThumb ? String(qv.photoThumb) : undefined;
  const bidCount = toNumber(String(qv.bidCnt || "")) || 0;
  const closingAt = parseRelativeClose(qv.timeLeft ? String(qv.timeLeft) : undefined);
  const features = stripTags(String(qv.features || "")) || undefined;

  return {
    source: "GOVPLANET",
    sourceAuctionId: id,
    sourceUrl,
    title,
    description: features,
    locationState: state,
    currentBid,
    bidCount,
    closingAt,
    imageUrls: img ? [img] : [],
  };
}

async function fetchSearch(keyword: string): Promise<string> {
  const url = keyword
    ? `${BASE}/searchResults.ips?keyword=${encodeURIComponent(keyword)}`
    : `${BASE}/buy-now`;
  const res = await fetch(url, {
    headers: {
      "User-Agent": UA,
      Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
      "Accept-Language": "en-US,en;q=0.9",
    },
  });
  if (!res.ok) throw new Error(`GovPlanet "${keyword || "buy-now"}" -> HTTP ${res.status}`);
  return res.text();
}

export interface FetchGovPlanetOpts {
  limit?: number;
}

/**
 * Fetch up to `limit` active GovPlanet items by scanning the buy-now page and a
 * set of high-value category keyword searches, deduped by equipId. All prices
 * are USD. $0 — plain HTTPS, no browser, no API key.
 */
export async function fetchGovPlanetFree(
  opts: FetchGovPlanetOpts = {},
): Promise<RawListing[]> {
  const limit = opts.limit ?? 100;
  const seen = new Set<string>();
  const rows: RawListing[] = [];

  // Always start with the dedicated buy-now page, then keyword searches.
  const queries = ["", ...KEYWORDS];
  const uniqueQueries = [...new Set(queries)];

  for (const kw of uniqueQueries) {
    if (rows.length >= limit) break;
    let html: string;
    try {
      html = await fetchSearch(kw);
    } catch (e) {
      console.warn(`[govplanet-free] ${(e as Error).message}`);
      continue;
    }
    const qvs = extractQuickviews(html);
    for (const qv of qvs) {
      const listing = qvToListing(qv);
      if (!listing) continue;
      if (seen.has(listing.sourceAuctionId)) continue;
      seen.add(listing.sourceAuctionId);
      rows.push(listing);
      if (rows.length >= limit) break;
    }
    await new Promise((r) => setTimeout(r, 350));
  }

  return rows.slice(0, limit);
}