← back to Govarbitrage

src/importers/municibid-free.ts

247 lines

import type { RawListing } from "./ingest";

// FREE Municibid importer. $0, no browser at runtime.
//
// Municibid is a classic server-side-rendered ASP.NET site (jQuery + SignalR
// for live bid pushes) — there is NO JSON search/listings API to replay. The
// listing cards arrive inside the initial /browse HTML document, and plain
// curl gets HTTP 200 (no Cloudflare wall). So the crack is straight HTML
// parsing of the `.browse-item` cards, paginated via `?page=N`.
//
// Verified 2026-07-10 via openclaw real-Chrome capture:
//   - performance.getEntriesByType('resource') showed ZERO listings API —
//     only analytics (Google/Clarity/Customer.io), FontAwesome, wisepops, and
//     SignalR (/signalr/* on `listinghub` = live bid-price push, not data).
//   - The "gist.build" lead from an earlier pass was a red herring: it's the
//     Customer.io "Gist" support-chat widget, unrelated to auctions.
//
// Card shape (per `div.browse-item[data-listingid]`):
//   id      -> data-listingid
//   title   -> desktop <h2 class="text-card"><a>FULL TITLE</a>
//   url     -> /Listing/Details/{id}/{slug}
//   bid     -> <span class="awe-rt-CurrentPrice ...">$<span class="NumberPart">235.00</span>
//   bids    -> <span class="awe-rt-AcceptedListingActionCount" ...>8</span>
//   close   -> <span data-epoch="ending" data-action-time="MM/DD/YYYY HH:MM:SS">
//   image   -> storagemunicibidpro.blob.core.windows.net/assets/media/{uuid}_thumbcrop.jpg
//   loc     -> <p class="card-subtitle">City , ST | Seller Agency</p>

const BASE = "https://municibid.com";
const UA =
  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " +
  "(KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36";

const STATE_ABBR = new Set([
  "AL","AK","AZ","AR","CA","CO","CT","DE","FL","GA","HI","ID","IL","IN","IA","KS",
  "KY","LA","ME","MD","MA","MI","MN","MS","MO","MT","NE","NV","NH","NJ","NM","NY",
  "NC","ND","OH","OK","OR","PA","RI","SC","SD","TN","TX","UT","VT","VA","WA","WV",
  "WI","WY","DC","PR",
]);

function decode(s: string | undefined): string {
  if (!s) return "";
  return s
    .replace(/&amp;/g, "&")
    .replace(/&#39;/g, "'")
    .replace(/&#039;/g, "'")
    .replace(/&quot;/g, '"')
    .replace(/&nbsp;/g, " ")
    .replace(/&lt;/g, "<")
    .replace(/&gt;/g, ">")
    .replace(/\s+/g, " ")
    .trim();
}

function first(re: RegExp, s: string): string | undefined {
  const m = re.exec(s);
  return m ? m[1] : undefined;
}

// Parse "MM/DD/YYYY HH:MM:SS" (US Eastern, Municibid's server zone) to ISO.
// We treat it as local wall-clock; exactness to the minute isn't required by
// the research pipeline, and a naive Date parse of this format is stable.
function parseCloseTime(raw: string | undefined): string | null {
  if (!raw) return null;
  const m = /(\d{1,2})\/(\d{1,2})\/(\d{4})\s+(\d{1,2}):(\d{2}):(\d{2})/.exec(raw);
  if (!m) return null;
  const [, mo, d, y, h, mi, s] = m;
  const dt = new Date(
    Number(y),
    Number(mo) - 1,
    Number(d),
    Number(h),
    Number(mi),
    Number(s),
  );
  return isNaN(dt.getTime()) ? null : dt.toISOString();
}

function parseCard(block: string): RawListing | null {
  const id = first(/data-listingid="(\d+)"/, block);
  if (!id) return null;

  // Full (untruncated) title from the desktop `.text-card` <h2><a>…</a>.
  // Fall back to the slug in the detail URL, or the truncated card-title.
  const detailRe = new RegExp(
    `/Listing/Details/${id}/([A-Za-z0-9%\\-_.]+)`,
  );
  const slug = first(detailRe, block);
  let title = first(
    /class="text-card"[^>]*>\s*<a[^>]*>([\s\S]*?)<\/a>/,
    block,
  );
  title = decode(title);
  if (!title && slug) {
    title = decode(decodeURIComponent(slug).replace(/[-_]+/g, " "));
  }
  if (!title) {
    title = decode(
      first(/class="card-title[^"]*"[^>]*>\s*<a[^>]*>([\s\S]*?)<\/a>/, block),
    ).replace(/\.\.\.$/, "");
  }
  if (!title) return null;

  const url = slug
    ? `${BASE}/Listing/Details/${id}/${slug}`
    : `${BASE}/Listing/Details/${id}`;

  // Current bid: $<span class="NumberPart">235.00</span>
  const priceStr = first(
    /awe-rt-CurrentPrice[^>]*>\s*\$?\s*<span class="NumberPart">([\d,]+(?:\.\d+)?)<\/span>/,
    block,
  );
  const currentBid = priceStr ? Number(priceStr.replace(/,/g, "")) : 0;

  // Bid count.
  const bidStr = first(
    /awe-rt-AcceptedListingActionCount[^>]*>\s*(\d+)\s*</,
    block,
  );
  const bidCount = bidStr ? Number(bidStr) : 0;

  // Absolute close time.
  const closingAt = parseCloseTime(
    first(/data-epoch="ending"[^>]*data-action-time="([^"]+)"/, block) ||
      first(/data-action-time="([^"]+)"[^>]*data-epoch="ending"/, block),
  );

  // Image (thumbcrop → strip to a larger asset when possible).
  let img = first(
    /(https:\/\/storagemunicibidpro\.blob\.core\.windows\.net\/assets\/media\/[A-Za-z0-9-]+_thumbcrop\.jpg)/,
    block,
  );
  const imageUrls = img ? [img] : [];

  // Location + seller from card-subtitle: "City , ST | Seller Agency".
  const subtitle = decode(
    first(/class="card-subtitle[^"]*"[^>]*>([\s\S]*?)<\/p>/, block),
  );
  let locationCity: string | undefined;
  let locationState: string | undefined;
  let seller: string | undefined;
  if (subtitle) {
    const [locPart, ...rest] = subtitle.split("|");
    seller = rest.join("|").trim() || undefined;
    const lm = /^(.*?)[,\s]+([A-Z]{2})\s*$/.exec(locPart.trim());
    if (lm && STATE_ABBR.has(lm[2])) {
      locationCity = lm[1].replace(/,\s*$/, "").trim() || undefined;
      locationState = lm[2];
    } else {
      locationCity = locPart.trim() || undefined;
    }
  }

  const descParts = [
    seller ? `Seller: ${seller}` : "",
    locationCity || locationState
      ? `Location: ${[locationCity, locationState].filter(Boolean).join(", ")}`
      : "",
  ].filter(Boolean);

  return {
    source: "MUNICIBID",
    sourceAuctionId: id,
    sourceUrl: url,
    title,
    description: descParts.join(" · ") || undefined,
    // NB: `seller` is the government agency, NOT the product maker — keep it in
    // description only. Leaving `manufacturer` unset lets the research pipeline
    // extract the real brand from the title.
    locationCity,
    locationState,
    currentBid,
    bidCount,
    closingAt,
    imageUrls,
  };
}

async function fetchPage(page: number): Promise<string> {
  const url = page <= 1 ? `${BASE}/browse` : `${BASE}/browse?page=${page}`;
  const res = await fetch(url, {
    headers: {
      "User-Agent": UA,
      Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
      "Accept-Language": "en-US,en;q=0.9",
    },
  });
  if (!res.ok) throw new Error(`Municibid /browse page ${page} -> HTTP ${res.status}`);
  return res.text();
}

function parsePage(html: string): RawListing[] {
  // Split into per-card blocks on the data-listingid boundary. Each card's
  // markup (mobile + desktop) lives between one data-listingid and the next.
  const parts = html.split(/(?=<div class="row browse-item)/);
  const out: RawListing[] = [];
  for (const part of parts) {
    if (!/data-listingid="\d+"/.test(part)) continue;
    const card = parseCard(part);
    if (card) out.push(card);
  }
  return out;
}

export interface FetchMunicibidOpts {
  limit?: number;
  maxPages?: number;
}

/**
 * Fetch up to `limit` active Municibid listings by paginating /browse?page=N.
 * Dedupes by listing id. $0 — plain HTTPS, no browser, no API key.
 */
export async function fetchMunicibidFree(
  opts: FetchMunicibidOpts = {},
): Promise<RawListing[]> {
  const limit = opts.limit ?? 100;
  const maxPages = opts.maxPages ?? 40;
  const seen = new Set<string>();
  const rows: RawListing[] = [];

  for (let page = 1; page <= maxPages && rows.length < limit; page++) {
    let html: string;
    try {
      html = await fetchPage(page);
    } catch (e) {
      console.warn(`[municibid-free] page ${page} fetch failed: ${(e as Error).message}`);
      break;
    }
    const cards = parsePage(html);
    if (cards.length === 0) break; // ran past the last page
    let added = 0;
    for (const c of cards) {
      if (seen.has(c.sourceAuctionId)) continue;
      seen.add(c.sourceAuctionId);
      rows.push(c);
      added++;
      if (rows.length >= limit) break;
    }
    // If a full page yielded no NEW ids, we've looped back to the start.
    if (added === 0) break;
    // Be polite: small jitter between page fetches.
    await new Promise((r) => setTimeout(r, 350 + Math.floor(page % 3) * 120));
  }

  return rows.slice(0, limit);
}