`scripts/liveness-sweep.ts`

168 lines
// Fast dead-listing removal. Keeps the live grid honest so Steve never chases
// an item that's already gone. Two tiers:
//
//   Tier 1 (instant, $0, no network): any ACTIVE listing whose closingAt is in
//     the past → ENDED. This clears the vast majority the moment they close.
//     Run this often (e.g. every 10 min) — it's a single indexed UPDATE.
//
//   Tier 2 (network verify, rate-limited): re-fetch a batch of ACTIVE listings
//     that have NO closingAt (make-offer items, e.g. GovPlanet) or are stalest
//     by livenessCheckedAt. Definitive "gone" signals (HTTP 404/410, or explicit
//     "no longer available / has ended / not found" text) → REMOVED. Anything
//     else is treated as still-live (never remove on a transient 5xx/timeout).
//
// Run: npx tsx scripts/liveness-sweep.ts [tier2Batch]   (default 40)

import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { dirname, join } from "node:path";

function loadEnv() {
  const root = join(dirname(fileURLToPath(import.meta.url)), "..");
  try {
    const raw = readFileSync(join(root, ".env"), "utf8");
    for (const line of raw.split("\n")) {
      const m = line.match(/^\s*([A-Z0-9_]+)\s*=\s*(.*)\s*$/i);
      if (!m) continue;
      const key = m[1];
      let val = m[2].trim();
      if ((val.startsWith('"') && val.endsWith('"')) || (val.startsWith("'") && val.endsWith("'"))) {
        val = val.slice(1, -1);
      }
      if (process.env[key] === undefined) process.env[key] = val;
    }
  } catch {
    /* ambient env */
  }
}
loadEnv();

const { prisma } = await import("../src/lib/db");

const UA =
  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " +
  "(KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36";

// Text markers that DEFINITIVELY mean the lot is gone. Kept conservative to
// avoid false removals.
const GONE_MARKERS = [
  "no longer available",
  "this auction has ended",
  "auction has ended",
  "listing has ended",
  "item not found",
  "page not found",
  "the resource cannot be found",
  "no longer active",
  "has been removed",
  "sale has closed",
];

// Signals that a still-LIVE listing is rendered. A gone-marker found anywhere in
// the body (nav/footer/"related items" widget/cookie banner) is NOT trusted when
// any of these are present, so an unrelated ended-lot string can't falsely
// remove a live listing. Only a marker in the page <title> is trusted outright.
const LIVE_SIGNALS = [
  "current bid",
  "place bid",
  "time remaining",
  "time left",
  "bid history",
  "add to watchlist",
  "buy it now",
];

async function tier1ExpireByClock(): Promise<number> {
  const res = await prisma.listing.updateMany({
    where: { listingStatus: "ACTIVE", closingAt: { lt: new Date() } },
    data: { listingStatus: "ENDED", endedAt: new Date() },
  });
  return res.count;
}

interface VerifyResult { gone: boolean; live: boolean }

async function verifyOne(url: string): Promise<VerifyResult> {
  try {
    const res = await fetch(url, {
      headers: { "User-Agent": UA, Accept: "text/html,*/*" },
      redirect: "follow",
      // Bound each request so one host that accepts the connection but never
      // finishes the body can't block the whole sequential batch indefinitely.
      signal: AbortSignal.timeout(10_000),
    });
    if (res.status === 404 || res.status === 410) return { gone: true, live: false };
    if (!res.ok) return { gone: false, live: false }; // transient — don't touch
    const html = await res.text();
    const body = html.toLowerCase();
    const title = (html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] ?? "").toLowerCase();
    // A gone-marker in the <title> is authoritative for THIS page.
    if (GONE_MARKERS.some((m) => title.includes(m))) return { gone: true, live: false };
    // A body-only marker is trusted ONLY when the page shows no live-listing
    // signals — otherwise an unrelated ended-lot string in a sidebar/footer/
    // related-items widget would falsely remove a still-live listing.
    const bodyGone = GONE_MARKERS.some((m) => body.includes(m));
    const looksLive = LIVE_SIGNALS.some((s) => body.includes(s));
    if (bodyGone && !looksLive) return { gone: true, live: false };
    return { gone: false, live: true };
  } catch {
    return { gone: false, live: false }; // network error / timeout — don't remove
  }
}

async function tier2VerifyBatch(batch: number): Promise<{ checked: number; removed: number }> {
  // Prioritize items with no clock (can't expire via Tier 1) then stalest checks.
  const candidates = await prisma.listing.findMany({
    where: { listingStatus: "ACTIVE", sourceUrl: { not: null } },
    orderBy: [{ closingAt: { sort: "asc", nulls: "first" } }, { livenessCheckedAt: { sort: "asc", nulls: "first" } }],
    take: batch,
    select: { id: true, sourceUrl: true, title: true },
  });

  let removed = 0;
  for (const l of candidates) {
    if (!l.sourceUrl) continue;
    const { gone, live } = await verifyOne(l.sourceUrl);
    const now = new Date();
    if (gone) {
      await prisma.listing.update({
        where: { id: l.id },
        data: { listingStatus: "REMOVED", endedAt: now, livenessCheckedAt: now },
      });
      await prisma.listingEvent
        .create({ data: { listingId: l.id, type: "STATUS_CHANGE", message: "Removed: no longer available on source" } })
        .catch(() => {});
      removed++;
    } else if (live) {
      await prisma.listing.update({
        where: { id: l.id },
        data: { livenessCheckedAt: now, lastSeenAt: now },
      });
    } else {
      // transient — just record we looked, don't change status
      await prisma.listing.update({ where: { id: l.id }, data: { livenessCheckedAt: now } });
    }
    await new Promise((r) => setTimeout(r, 200)); // be polite
  }
  return { checked: candidates.length, removed };
}

async function main() {
  const batch = Number(process.argv[2] || process.env.LIVENESS_TIER2_BATCH || 40);
  const t0 = Date.now();
  const expired = await tier1ExpireByClock();
  const { checked, removed } = await tier2VerifyBatch(batch);
  const ms = Date.now() - t0;
  console.log(
    `[${new Date().toISOString()}] liveness: tier1 expired=${expired}, tier2 checked=${checked} removed=${removed} (${ms}ms). Cost: $0.`,
  );
  await prisma.$disconnect();
}

main()
  .then(() => process.exit(0))
  .catch((e) => {
    console.error(`[liveness-sweep] ${new Date().toISOString()} ERROR:`, e.message);
    process.exit(1);
  });