`apps/admin/src/app/ingestion/IngestionLive.tsx`

148 lines
"use client";

import { useEffect, useRef, useState } from "react";

type Src = { key: string; runs: number; published: number; failed_runs: number; err_records: number; last_age_s: number | null };
// staleness relative to the ~2.5h sweep cadence: >4h (or never) = likely dead, >2.5h = missed a sweep
const RED_S = 14400, AMBER_S = 9000;
const staleClass = (s: number | null) => (s == null || s > RED_S ? "text-red-600 font-semibold" : s > AMBER_S ? "text-amber-600" : "text-neutral-500");
type Stats = {
  ts: string;
  byStatus: { status: string; n: number }[];
  throughput: { total?: number; m1?: number; m5?: number; m30?: number; m60?: number };
  newHomes: { n30?: number | null; n60?: number | null };
  runs: { r10?: number; r60?: number; newest_age_s?: number };
  health: { err_runs?: number; runs?: number; err_count?: number };
  staged: { total?: number; s30?: number };
  topSources: Src[];
};

function Stat({ label, value, sub }: { label: string; value: React.ReactNode; sub?: React.ReactNode }) {
  return (
    <div className="rounded-xl border border-neutral-200 bg-white p-4 shadow-sm">
      <div className="text-xs uppercase tracking-wide text-neutral-500">{label}</div>
      <div className="mt-1 text-3xl font-bold tabular-nums">{value}</div>
      {sub && <div className="mt-0.5 text-xs text-neutral-400">{sub}</div>}
    </div>
  );
}

// tiny dependency-free SVG sparkline
function Spark({ data }: { data: number[] }) {
  const w = 220, h = 40;
  if (data.length < 2) return <div className="text-xs text-neutral-400">gathering trend…</div>;
  const max = Math.max(...data, 1);
  const pts = data.map((v, i) => `${(i / (data.length - 1)) * w},${h - (v / max) * (h - 4) - 2}`).join(" ");
  const last = data[data.length - 1] ?? 0;
  return (
    <svg width={w} height={h} className="overflow-visible">
      <polyline points={pts} fill="none" stroke="#0d9488" strokeWidth="2" strokeLinejoin="round" strokeLinecap="round" />
      <circle cx={w} cy={h - (last / max) * (h - 4) - 2} r="3" fill="#0d9488" />
    </svg>
  );
}

const age = (s?: number) =>
  s == null || s < 0 ? "—" : s < 90 ? `${s}s` : s < 5400 ? `${Math.round(s / 60)}m` : `${(s / 3600).toFixed(1)}h`;

export default function IngestionLive() {
  const [s, setS] = useState<Stats | null>(null);
  const [err, setErr] = useState(false);
  const [pulse, setPulse] = useState(false);
  const hist = useRef<number[]>([]);
  const [, tick] = useState(0);

  useEffect(() => {
    let alive = true;
    const load = async () => {
      try {
        const r = await fetch("/api/ingestion-stats", { cache: "no-store" });
        if (!r.ok) throw new Error(String(r.status));
        const j: Stats = await r.json();
        if (!alive) return;
        hist.current = [...hist.current, j.throughput.m1 ?? 0].slice(-40);
        setS(j); setErr(false); setPulse(true); setTimeout(() => alive && setPulse(false), 600); tick((n) => n + 1);
      } catch { if (alive) setErr(true); }
    };
    load();
    const id = setInterval(load, 8000);
    return () => { alive = false; clearInterval(id); };
  }, []);

  if (!s) return <div className="mt-6 text-sm text-neutral-500">{err ? "Stats unavailable." : "Loading live stats…"}</div>;

  const ageS = s.runs.newest_age_s ?? 999999;
  const fresh = ageS < 21600;
  const errRuns = s.health.err_runs ?? 0;
  const errRecords = s.health.err_count ?? 0; // per-RECORD drops (homes that errored mid-run) — the honest signal

  return (
    <div className="mt-4">
      <div className="flex flex-wrap items-center gap-2 text-sm">
        <span className={`inline-block h-2.5 w-2.5 rounded-full ${fresh ? "bg-emerald-500" : "bg-red-500"} ${pulse ? "animate-ping" : ""}`} />
        <span className={fresh ? "text-emerald-700" : "text-red-700"}>{fresh ? "LIVE — ingesting" : "STALE — no recent runs"}</span>
        <span className="text-neutral-400">· newest SourceRun {age(ageS)} ago · refreshed {new Date(s.ts).toLocaleTimeString()}</span>
        <span
          title="Homes that errored out mid-run (per-record parse/validate failures) in the last 60 min. Runs still succeed; these homes are silently dropped."
          className={`ml-auto rounded-full px-2 py-0.5 text-xs ${errRecords > 0 ? "bg-amber-100 text-amber-800" : "bg-emerald-100 text-emerald-800"}`}
        >
          {errRuns > 0 && `${errRuns} failed runs · `}{errRecords.toLocaleString()} homes dropped (60m)
        </span>
      </div>

      <div className="mt-4 grid gap-4 sm:grid-cols-2 lg:grid-cols-4">
        <Stat label="Published homes" value={(s.throughput.total ?? 0).toLocaleString()} sub={s.byStatus.map((b) => `${b.status} ${b.n.toLocaleString()}`).join(" · ")} />
        <Stat
          label="Homes / min (live)"
          value={(s.throughput.m1 ?? 0).toLocaleString()}
          sub={<Spark data={hist.current} />}
        />
        <Stat label="Refreshed / 30 min" value={(s.throughput.m30 ?? 0).toLocaleString()} sub={`${(s.throughput.m60 ?? 0).toLocaleString()} in the last hour`} />
        <Stat
          label="New homes / 30 min"
          value={s.newHomes.n30 == null ? "—" : (s.newHomes.n30 ?? 0).toLocaleString()}
          sub={s.newHomes.n60 == null ? "createdAt n/a" : `${(s.newHomes.n60 ?? 0).toLocaleString()} newly found / hr`}
        />
      </div>

      <div className="mt-4 grid gap-4 sm:grid-cols-2 lg:grid-cols-4">
        <Stat label="SourceRuns / 10 min" value={(s.runs.r10 ?? 0).toLocaleString()} sub={`${s.runs.r60 ?? 0} in the last hour`} />
        <Stat label="Records staged / 30 min" value={(s.staged.s30 ?? 0).toLocaleString()} sub={`${(s.staged.total ?? 0).toLocaleString()} staged total`} />
      </div>

      {s.topSources.length > 0 && (
        <div className="mt-6">
          <h2 className="text-sm font-semibold text-neutral-600">
            Sources — all active <span className="font-normal text-neutral-400">({s.topSources.filter((t) => t.last_age_s == null || t.last_age_s > RED_S).length} stale/absent · stats over last 60 min)</span>
          </h2>
          <table className="mt-2 w-full max-w-2xl text-sm">
            <thead>
              <tr className="text-left text-xs uppercase text-neutral-400">
                <th className="py-1 font-medium">Source</th>
                <th className="py-1 text-right font-medium">Runs</th>
                <th className="py-1 text-right font-medium">Published</th>
                <th className="py-1 text-right font-medium" title="Homes dropped to per-record errors (runs still succeed)">Dropped</th>
                <th className="py-1 text-right font-medium">Last run</th>
              </tr>
            </thead>
            <tbody>
              {s.topSources.map((t) => (
                <tr key={t.key} className="border-t border-neutral-100">
                  <td className="py-1 font-mono text-xs">{t.key}</td>
                  <td className="py-1 text-right tabular-nums">{t.runs}</td>
                  <td className="py-1 text-right tabular-nums">{t.published.toLocaleString()}</td>
                  <td className={`py-1 text-right tabular-nums ${t.err_records > 0 ? "text-amber-600 font-semibold" : "text-neutral-400"}`} title={t.failed_runs > 0 ? `${t.failed_runs} failed runs` : ""}>{t.err_records.toLocaleString()}{t.failed_runs > 0 ? " ⚠" : ""}</td>
                  <td className={`py-1 text-right tabular-nums ${staleClass(t.last_age_s)}`}>
                    {t.last_age_s == null ? "never run" : `${age(t.last_age_s)} ago`}
                  </td>
                </tr>
              ))}
            </tbody>
          </table>
        </div>
      )}
      <p className="mt-6 text-xs text-neutral-400">Auto-refreshes every 8s · served from the Kamatera-local Postgres the ingestion loop writes.</p>
    </div>
  );
}