`scripts/liveness-sweep.ts`

142 lines
// Fast dead-listing removal. Keeps the live grid honest so Steve never chases
// an item that's already gone. Two tiers:
//
//   Tier 1 (instant, $0, no network): any ACTIVE listing whose closingAt is in
//     the past → ENDED. This clears the vast majority the moment they close.
//     Run this often (e.g. every 10 min) — it's a single indexed UPDATE.
//
//   Tier 2 (network verify, rate-limited): re-fetch a batch of ACTIVE listings
//     that have NO closingAt (make-offer items, e.g. GovPlanet) or are stalest
//     by livenessCheckedAt. Definitive "gone" signals (HTTP 404/410, or explicit
//     "no longer available / has ended / not found" text) → REMOVED. Anything
//     else is treated as still-live (never remove on a transient 5xx/timeout).
//
// Run: npx tsx scripts/liveness-sweep.ts [tier2Batch]   (default 40)

import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { dirname, join } from "node:path";

function loadEnv() {
  const root = join(dirname(fileURLToPath(import.meta.url)), "..");
  try {
    const raw = readFileSync(join(root, ".env"), "utf8");
    for (const line of raw.split("\n")) {
      const m = line.match(/^\s*([A-Z0-9_]+)\s*=\s*(.*)\s*$/i);
      if (!m) continue;
      const key = m[1];
      let val = m[2].trim();
      if ((val.startsWith('"') && val.endsWith('"')) || (val.startsWith("'") && val.endsWith("'"))) {
        val = val.slice(1, -1);
      }
      if (process.env[key] === undefined) process.env[key] = val;
    }
  } catch {
    /* ambient env */
  }
}
loadEnv();

const { prisma } = await import("../src/lib/db");

const UA =
  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " +
  "(KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36";

// Text markers that DEFINITIVELY mean the lot is gone. Kept conservative to
// avoid false removals.
const GONE_MARKERS = [
  "no longer available",
  "this auction has ended",
  "auction has ended",
  "listing has ended",
  "item not found",
  "page not found",
  "the resource cannot be found",
  "no longer active",
  "has been removed",
  "sale has closed",
];

async function tier1ExpireByClock(): Promise<number> {
  const res = await prisma.listing.updateMany({
    where: { listingStatus: "ACTIVE", closingAt: { lt: new Date() } },
    data: { listingStatus: "ENDED", endedAt: new Date() },
  });
  return res.count;
}

interface VerifyResult { gone: boolean; live: boolean }

async function verifyOne(url: string): Promise<VerifyResult> {
  try {
    const res = await fetch(url, {
      headers: { "User-Agent": UA, Accept: "text/html,*/*" },
      redirect: "follow",
    });
    if (res.status === 404 || res.status === 410) return { gone: true, live: false };
    if (!res.ok) return { gone: false, live: false }; // transient — don't touch
    const body = (await res.text()).toLowerCase();
    if (GONE_MARKERS.some((m) => body.includes(m))) return { gone: true, live: false };
    return { gone: false, live: true };
  } catch {
    return { gone: false, live: false }; // network error — don't remove
  }
}

async function tier2VerifyBatch(batch: number): Promise<{ checked: number; removed: number }> {
  // Prioritize items with no clock (can't expire via Tier 1) then stalest checks.
  const candidates = await prisma.listing.findMany({
    where: { listingStatus: "ACTIVE", sourceUrl: { not: null } },
    orderBy: [{ closingAt: { sort: "asc", nulls: "first" } }, { livenessCheckedAt: { sort: "asc", nulls: "first" } }],
    take: batch,
    select: { id: true, sourceUrl: true, title: true },
  });

  let removed = 0;
  for (const l of candidates) {
    if (!l.sourceUrl) continue;
    const { gone, live } = await verifyOne(l.sourceUrl);
    const now = new Date();
    if (gone) {
      await prisma.listing.update({
        where: { id: l.id },
        data: { listingStatus: "REMOVED", endedAt: now, livenessCheckedAt: now },
      });
      await prisma.listingEvent
        .create({ data: { listingId: l.id, type: "STATUS_CHANGE", message: "Removed: no longer available on source" } })
        .catch(() => {});
      removed++;
    } else if (live) {
      await prisma.listing.update({
        where: { id: l.id },
        data: { livenessCheckedAt: now, lastSeenAt: now },
      });
    } else {
      // transient — just record we looked, don't change status
      await prisma.listing.update({ where: { id: l.id }, data: { livenessCheckedAt: now } });
    }
    await new Promise((r) => setTimeout(r, 200)); // be polite
  }
  return { checked: candidates.length, removed };
}

async function main() {
  const batch = Number(process.argv[2] || process.env.LIVENESS_TIER2_BATCH || 40);
  const t0 = Date.now();
  const expired = await tier1ExpireByClock();
  const { checked, removed } = await tier2VerifyBatch(batch);
  const ms = Date.now() - t0;
  console.log(
    `[${new Date().toISOString()}] liveness: tier1 expired=${expired}, tier2 checked=${checked} removed=${removed} (${ms}ms). Cost: $0.`,
  );
  await prisma.$disconnect();
}

main()
  .then(() => process.exit(0))
  .catch((e) => {
    console.error(`[liveness-sweep] ${new Date().toISOString()} ERROR:`, e.message);
    process.exit(1);
  });