`src/importers/publicsurplus-free.ts`

212 lines
import type { RawListing } from "./ingest";

// FREE Public Surplus importer. $0, no browser at runtime.
//
// Public Surplus is server-rendered (Java/Spring `/sms/…` app). The category
// listing pages return fully-rendered auction cards to plain curl (HTTP 200,
// no bot wall), so the crack is HTML parsing of the `.auction-item` cards.
//
// Verified 2026-07-10:
//   list  -> GET /sms/browse/cataucs?catid={C}&page={P}   (25 cards/page, 0-idx)
//   item  -> /sms/auction/view?auc={id}
//   card (per `div.auction-item[id="{id}catGrid"]`):
//     id    -> id="{id}catGrid"
//     title -> <a href="/sms/auction/view?auc={id}" title="#{id} - {TITLE}">
//     state -> <span class="auction-item-state"> CA </span>
//     price -> <b id="val_{id}catGrid"> $35.00 </b>
//     time  -> <span id="timeLeftValue{id}catGrid" ...> 4 mins </span>  (RELATIVE)
//     image -> https://d37qv0n5b4mbzm.cloudfront.net/.../thumb-b/{id}/{doc}
//
// Close time is relative ("4 mins" / "2 days" / "1 hour"), not absolute, so we
// compute closingAt = now + parsed delta. Categories enumerated from the home
// page (catid 1..29). No API key, no browser.

const BASE = "https://www.publicsurplus.com";
const UA =
  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " +
  "(KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36";

// Category ids present on the browse home (2026-07-10). Enumerated rather than
// hardcoded 1..N because 7 is absent; kept as a static list for stability.
const CATEGORY_IDS = [
  1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22,
  23, 24, 25, 26, 27, 28, 29,
];

function decode(s: string | undefined): string {
  if (!s) return "";
  return s
    .replace(/&amp;/g, "&")
    .replace(/&#39;/g, "'")
    .replace(/&#039;/g, "'")
    .replace(/&quot;/g, '"')
    .replace(/&nbsp;/g, " ")
    .replace(/&lt;/g, "<")
    .replace(/&gt;/g, ">")
    .replace(/\s+/g, " ")
    .trim();
}

function first(re: RegExp, s: string): string | undefined {
  const m = re.exec(s);
  return m ? m[1] : undefined;
}

// Parse relative "Time Left" like "4 mins", "2 days", "1 hour", "3 hrs",
// "5 days, 2 hours" -> ISO close time (now + delta). Returns null if unparseable
// or already ended.
function parseRelativeClose(raw: string | undefined): string | null {
  if (!raw) return null;
  const txt = raw.toLowerCase();
  if (/ended|closed|over/.test(txt)) return null;
  const units: [RegExp, number][] = [
    [/(\d+)\s*day/, 86400_000],
    [/(\d+)\s*h(?:ou)?r/, 3600_000],
    [/(\d+)\s*min/, 60_000],
    [/(\d+)\s*sec/, 1000],
  ];
  let deltaMs = 0;
  let matched = false;
  for (const [re, ms] of units) {
    const m = re.exec(txt);
    if (m) {
      deltaMs += Number(m[1]) * ms;
      matched = true;
    }
  }
  if (!matched) return null;
  return new Date(Date.now() + deltaMs).toISOString();
}

function parseCard(block: string, id: string): RawListing | null {
  // Title: prefer the <a title="#id - TITLE"> attribute (full), strip the
  // leading "#id - " prefix.
  let title = decode(
    first(
      new RegExp(`/sms/auction/view\\?auc=${id}"[^>]*title="([^"]+)"`),
      block,
    ),
  );
  title = title.replace(new RegExp(`^#?${id}\\s*[-–]\\s*`), "").trim();
  if (!title) {
    // fallback: visible anchor text
    title = decode(
      first(
        new RegExp(`/sms/auction/view\\?auc=${id}"[^>]*>([\\s\\S]*?)</a>`),
        block,
      ),
    ).replace(new RegExp(`^#?${id}\\s*[-–]\\s*`), "");
  }
  if (!title || title === "...") return null;

  const priceStr = first(
    new RegExp(`id="val_${id}catGrid"[^>]*>\\s*\\$?([\\d,]+(?:\\.\\d+)?)`),
    block,
  );
  const currentBid = priceStr ? Number(priceStr.replace(/,/g, "")) : 0;

  const timeRaw = decode(
    first(
      new RegExp(`id="timeLeftValue${id}catGrid"[^>]*>([\\s\\S]*?)</span>`),
      block,
    ),
  );
  const closingAt = parseRelativeClose(timeRaw);

  const state = decode(
    first(/class="auction-item-state"[^>]*>([\s\S]*?)<\/span>/, block),
  );

  const img = first(
    /(https:\/\/[a-z0-9]+\.cloudfront\.net\/sms\/docviewer\/[^\s"']+)/,
    block,
  );

  return {
    source: "PUBLIC_SURPLUS",
    sourceAuctionId: id,
    sourceUrl: `${BASE}/sms/auction/view?auc=${id}`,
    title,
    locationState: state && /^[A-Z]{2}$/.test(state) ? state : undefined,
    currentBid,
    bidCount: 0, // not shown on the list card; enriched later if needed
    closingAt,
    imageUrls: img ? [img] : [],
  };
}

function parsePage(html: string): RawListing[] {
  const out: RawListing[] = [];
  // Each card is `<div class="auction-item" id="{id}catGrid">`. Split on that
  // boundary and parse each block against its own id.
  const re = /id="(\d+)catGrid"/g;
  const ids: { id: string; idx: number }[] = [];
  let m: RegExpExecArray | null;
  while ((m = re.exec(html))) ids.push({ id: m[1], idx: m.index });
  for (let i = 0; i < ids.length; i++) {
    const start = ids[i].idx;
    const end = i + 1 < ids.length ? ids[i + 1].idx : Math.min(html.length, start + 4000);
    const block = html.slice(start, end);
    const card = parseCard(block, ids[i].id);
    if (card) out.push(card);
  }
  return out;
}

async function fetchList(catid: number, page: number): Promise<string> {
  const url = `${BASE}/sms/browse/cataucs?catid=${catid}&page=${page}`;
  const res = await fetch(url, {
    headers: {
      "User-Agent": UA,
      Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
      "Accept-Language": "en-US,en;q=0.9",
    },
  });
  if (!res.ok) throw new Error(`PublicSurplus cataucs c=${catid} p=${page} -> HTTP ${res.status}`);
  return res.text();
}

export interface FetchPublicSurplusOpts {
  limit?: number;
  pagesPerCategory?: number;
}

/**
 * Fetch up to `limit` active Public Surplus listings by walking categories and
 * paginating each. Dedupes by auction id. $0 — plain HTTPS, no browser/API key.
 */
export async function fetchPublicSurplusFree(
  opts: FetchPublicSurplusOpts = {},
): Promise<RawListing[]> {
  const limit = opts.limit ?? 100;
  const pagesPerCategory = opts.pagesPerCategory ?? 4;
  const seen = new Set<string>();
  const rows: RawListing[] = [];

  outer: for (const catid of CATEGORY_IDS) {
    for (let page = 0; page < pagesPerCategory; page++) {
      let html: string;
      try {
        html = await fetchList(catid, page);
      } catch (e) {
        console.warn(`[publicsurplus-free] ${(e as Error).message}`);
        break; // next category
      }
      const cards = parsePage(html);
      if (cards.length === 0) break; // past the last page of this category
      let added = 0;
      for (const c of cards) {
        if (seen.has(c.sourceAuctionId)) continue;
        seen.add(c.sourceAuctionId);
        rows.push(c);
        added++;
        if (rows.length >= limit) break outer;
      }
      if (added === 0) break; // looped back — next category
      await new Promise((r) => setTimeout(r, 300 + (page % 3) * 120));
    }
  }

  return rows.slice(0, limit);
}