← back to Govarbitrage

src/importers/govdeals-free.ts

128 lines

import type { RawListing } from "./ingest";

// FREE GovDeals importer — hits GovDeals' own backend search API directly
// (maestro.lqdt1.com/search/list), the same call the site's Angular app makes.
// No Apify, no scraping, no login, $0. The x-api-key is GovDeals' PUBLIC client
// key (shipped in their public JS bundle), overridable via GOVDEALS_MAESTRO_KEY.
//
// The same endpoint serves sibling Liquidity Services marketplaces by businessId:
//   GD = GovDeals · GI = GoIndustry DoveBid · NI = Network Int'l.

const MAESTRO = "https://maestro.lqdt1.com/search/list";
const API_KEY = process.env.GOVDEALS_MAESTRO_KEY || "af93060f-337e-428c-87b8-c74b5837d6cd";
const PHOTO_BASE = "https://webassets.lqdt1.com/assets/photos";

interface Asset {
  accountId?: number;
  assetId?: number;
  assetShortDescription?: string;
  assetLongDescription?: string | null;
  makebrand?: string | null;
  model?: string | null;
  modelYear?: string | null;
  categoryDescription?: string | null;
  currentBid?: number | null;
  bidCount?: number | null;
  assetAuctionEndDate?: string | null;
  locationCity?: string;
  locationState?: string;
  locationZip?: string;
  photo?: string | null;
}

const uuid = () => globalThis.crypto.randomUUID();

// Liquidity Services marketplaces served by the same maestro API, keyed by businessId.
export const MARKETS: Record<string, { source: string; base: string }> = {
  GD: { source: "GOVDEALS", base: "https://www.govdeals.com" },
  GI: { source: "GOINDUSTRY", base: "https://www.go-dove.com" },
  NI: { source: "NETWORK_INTL", base: "https://www.networkintl.com" },
};

function mapAsset(a: Asset, market: { source: string; base: string }): RawListing | null {
  if (!a.assetShortDescription || a.assetId == null) return null;
  return {
    source: market.source,
    sourceAuctionId: `${a.accountId ?? "x"}-${a.assetId}`,
    sourceUrl: `${market.base}/en/asset/${a.accountId}/${a.assetId}`,
    title: a.assetShortDescription.trim(),
    description: a.assetLongDescription || undefined,
    category: a.categoryDescription || undefined,
    manufacturer: a.makebrand || undefined,
    model: a.model || undefined,
    condition: "UNKNOWN",
    quantity: 1,
    locationCity: a.locationCity,
    locationState: a.locationState,
    locationZip: a.locationZip,
    currentBid: Number(a.currentBid) || 0,
    bidCount: Number(a.bidCount) || 0,
    closingAt: a.assetAuctionEndDate || null,
    imageUrls: a.photo ? [`${PHOTO_BASE}/${a.accountId}/${a.photo}`] : [],
  };
}

async function fetchPage(businessId: string, page: number, displayRows: number): Promise<Asset[]> {
  const body = {
    categoryIds: "",
    businessId,
    searchText: "*",
    isQAL: false,
    locationId: null,
    model: "",
    makebrand: "",
    auctionTypeId: null,
    page,
    displayRows,
    sortField: "bestfit",
    sortOrder: "",
    sessionId: uuid(),
    requestType: "search",
    responseStyle: "fullResponse",
    facets: [
      "categoryName", "auctionTypeID", "condition", "saleEventName", "sellerDisplayName",
      "product_pricecents", "isReserveMet", "hasBuyNowPrice", "isReserveNotMet", "sellerType",
      "warehouseId", "region", "currencyTypeCode", "tierId",
    ],
    facetsFilter: [],
    timeType: "newListings",
    sellerTypeId: null,
    accountIds: [],
  };
  const res = await fetch(MAESTRO, {
    method: "POST",
    headers: {
      "x-api-key": API_KEY,
      "Content-Type": "application/json",
      audience: "Ecom",
      "x-user-id": "-1",
      "x-api-correlation-id": uuid(),
    },
    body: JSON.stringify(body),
  });
  if (!res.ok) throw new Error(`maestro ${res.status}: ${(await res.text()).slice(0, 160)}`);
  const j = (await res.json()) as { assetSearchResults?: Asset[] };
  return j.assetSearchResults ?? [];
}

/** Fetch newest GovDeals (or GI/NI) listings, paging until `limit`. Free, $0. */
export async function fetchGovdealsFree(
  opts: { limit?: number; businessId?: string } = {},
): Promise<RawListing[]> {
  const businessId = (opts.businessId || "GD").toUpperCase();
  const market = MARKETS[businessId] ?? MARKETS.GD;
  const limit = opts.limit ?? 120;
  const perPage = 120;
  const out: RawListing[] = [];
  for (let page = 1; out.length < limit && page <= 20; page++) {
    const assets = await fetchPage(businessId, page, perPage);
    if (!assets.length) break;
    for (const a of assets) {
      const m = mapAsset(a, market);
      if (m) out.push(m);
    }
    if (assets.length < perPage) break;
  }
  return out.slice(0, limit);
}