[object Object]

← back to Homesonspec

admin/ingestion: LEFT JOIN active sources — surface stale/absent sources (Cody-gated: silent-source blind spot)

04910e045bad3b94038b7b4f8cc4b85559bd31e9 · 2026-07-28 09:24:28 -0700 · Steve Abrams

Files touched

Diff

commit 04910e045bad3b94038b7b4f8cc4b85559bd31e9
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Tue Jul 28 09:24:28 2026 -0700

    admin/ingestion: LEFT JOIN active sources — surface stale/absent sources (Cody-gated: silent-source blind spot)
---
 apps/admin/src/app/api/ingestion-stats/route.ts | 19 ++++++++++++-------
 apps/admin/src/app/ingestion/IngestionLive.tsx  | 13 ++++++++++---
 2 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/apps/admin/src/app/api/ingestion-stats/route.ts b/apps/admin/src/app/api/ingestion-stats/route.ts
index 609153f..87f980a 100644
--- a/apps/admin/src/app/api/ingestion-stats/route.ts
+++ b/apps/admin/src/app/api/ingestion-stats/route.ts
@@ -49,15 +49,20 @@ export async function GET() {
     ),
     // per-source: recent runs, homes published, errors, and how long since its last run
     prisma.$queryRawUnsafe<any[]>(
+      // LEFT JOIN from active sources so a source that STOPPED running entirely still
+      // appears (as a stale/absent row) instead of silently vanishing — the catastrophic
+      // failure this table exists to catch (Cody-gated fix, 2026-07-28). Stalest first.
       `SELECT sr."key" AS key,
-         count(*)::int AS runs,
-         COALESCE(sum(run."published"),0)::int AS published,
-         count(*) FILTER (WHERE run."ok" = false)::int AS failed_runs,
-         COALESCE(sum(run."errorCount"),0)::int AS err_records,
+         count(run."id") FILTER (WHERE run."startedAt" > now() - interval '60 min')::int AS runs,
+         COALESCE(sum(run."published") FILTER (WHERE run."startedAt" > now() - interval '60 min'),0)::int AS published,
+         count(run."id") FILTER (WHERE run."ok" = false AND run."startedAt" > now() - interval '60 min')::int AS failed_runs,
+         COALESCE(sum(run."errorCount") FILTER (WHERE run."startedAt" > now() - interval '60 min'),0)::int AS err_records,
          EXTRACT(EPOCH FROM (now() - max(run."startedAt")))::int AS last_age_s
-       FROM "SourceRun" run JOIN "SourceRegistry" sr ON sr."id" = run."sourceId"
-       WHERE run."startedAt" > now() - interval '60 min'
-       GROUP BY sr."key" ORDER BY err_records DESC, runs DESC LIMIT 10`,
+       FROM "SourceRegistry" sr
+       LEFT JOIN "SourceRun" run ON run."sourceId" = sr."id"
+       WHERE sr."active" = true
+       GROUP BY sr."key"
+       ORDER BY last_age_s DESC NULLS FIRST LIMIT 14`,
     ).catch(() => [] as any[]),
   ]);
 
diff --git a/apps/admin/src/app/ingestion/IngestionLive.tsx b/apps/admin/src/app/ingestion/IngestionLive.tsx
index 91849a9..a988221 100644
--- a/apps/admin/src/app/ingestion/IngestionLive.tsx
+++ b/apps/admin/src/app/ingestion/IngestionLive.tsx
@@ -2,7 +2,10 @@
 
 import { useEffect, useRef, useState } from "react";
 
-type Src = { key: string; runs: number; published: number; failed_runs: number; err_records: number; last_age_s: number };
+type Src = { key: string; runs: number; published: number; failed_runs: number; err_records: number; last_age_s: number | null };
+// staleness relative to the ~2.5h sweep cadence: >4h (or never) = likely dead, >2.5h = missed a sweep
+const RED_S = 14400, AMBER_S = 9000;
+const staleClass = (s: number | null) => (s == null || s > RED_S ? "text-red-600 font-semibold" : s > AMBER_S ? "text-amber-600" : "text-neutral-500");
 type Stats = {
   ts: string;
   byStatus: { status: string; n: number }[];
@@ -109,7 +112,9 @@ export default function IngestionLive() {
 
       {s.topSources.length > 0 && (
         <div className="mt-6">
-          <h2 className="text-sm font-semibold text-neutral-600">Sources — last 60 min</h2>
+          <h2 className="text-sm font-semibold text-neutral-600">
+            Sources — all active <span className="font-normal text-neutral-400">({s.topSources.filter((t) => t.last_age_s == null || t.last_age_s > RED_S).length} stale/absent · stats over last 60 min)</span>
+          </h2>
           <table className="mt-2 w-full max-w-2xl text-sm">
             <thead>
               <tr className="text-left text-xs uppercase text-neutral-400">
@@ -127,7 +132,9 @@ export default function IngestionLive() {
                   <td className="py-1 text-right tabular-nums">{t.runs}</td>
                   <td className="py-1 text-right tabular-nums">{t.published.toLocaleString()}</td>
                   <td className={`py-1 text-right tabular-nums ${t.err_records > 0 ? "text-amber-600 font-semibold" : "text-neutral-400"}`} title={t.failed_runs > 0 ? `${t.failed_runs} failed runs` : ""}>{t.err_records.toLocaleString()}{t.failed_runs > 0 ? " ⚠" : ""}</td>
-                  <td className="py-1 text-right tabular-nums text-neutral-500">{age(t.last_age_s)} ago</td>
+                  <td className={`py-1 text-right tabular-nums ${staleClass(t.last_age_s)}`}>
+                    {t.last_age_s == null ? "never run" : `${age(t.last_age_s)} ago`}
+                  </td>
                 </tr>
               ))}
             </tbody>

← 80fb677 feat: HomesOnSpec data-feed API (:9799) — share the developm  ·  back to Homesonspec  ·  auto-save: 2026-07-28T09:29:05 (1 files) — apps/workers/var/ 8873056 →