[object Object]

← back to La Socrata Ingester

yoloforever c3: close Cody-c1 canary gaps — CSLB tracking + rolling-window bleed detection

f79485046aa9899b4e9d20d42ea63e9ce44e2098 · 2026-08-11 15:00:48 -0700 · steve

(1) CSLB now recorded in la_ingest_state (cslb_master) so the source-staleness check
covers it (was the untracked blind spot). (2) Rolling history (canary-history.jsonl)
+ BLEED status: flags a slow net decline vs earliest-observed count that the ratchet
floor misses. Verified: no false-bleed on stable data; verdict still correctly FAILs
on gis_zoning's real 502.

Files touched

Diff

commit f79485046aa9899b4e9d20d42ea63e9ce44e2098
Author: steve <steve@designerwallcoverings.com>
Date:   Tue Aug 11 15:00:48 2026 -0700

    yoloforever c3: close Cody-c1 canary gaps — CSLB tracking + rolling-window bleed detection
    
    (1) CSLB now recorded in la_ingest_state (cslb_master) so the source-staleness check
    covers it (was the untracked blind spot). (2) Rolling history (canary-history.jsonl)
    + BLEED status: flags a slow net decline vs earliest-observed count that the ratchet
    floor misses. Verified: no false-bleed on stable data; verdict still correctly FAILs
    on gis_zoning's real 502.
---
 .gitignore                  |  1 +
 scripts/canary-freshness.js | 21 +++++++++++++++++++--
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/.gitignore b/.gitignore
index b07f084..8b410b7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,3 +7,4 @@ tmp/
 .DS_Store
 dist/
 build/
+data/canary-history.jsonl
diff --git a/scripts/canary-freshness.js b/scripts/canary-freshness.js
index 955166f..3ac227d 100644
--- a/scripts/canary-freshness.js
+++ b/scripts/canary-freshness.js
@@ -30,6 +30,15 @@ async function main() {
   const firstRun = Object.keys(baseline).length === 0;
   const results = [];
 
+  // Rolling history for SLOW-BLEED detection (Cody c1 #2): the ratchet floor catches a
+  // big drop but hides a slow net decline. Compare each table's count to its EARLIEST
+  // observation (widest window); flag BLEED on a sustained shrink the floor would miss.
+  const HIST = path.resolve(__dirname, '../data/canary-history.jsonl');
+  const history = fs.existsSync(HIST) ? fs.readFileSync(HIST, 'utf8').trim().split('\n').filter(Boolean).map(l => JSON.parse(l)) : [];
+  const oldest = history[0]?.counts || {};
+  const nowTs = (await q(`SELECT now() n`)).rows[0].n;
+  const curCounts = {};
+
   for (const [table, cfg] of Object.entries(MANIFEST)) {
     const whereSql = cfg.where ? `WHERE ${cfg.where}` : '';
     const count = Number((await q(`SELECT count(*) c FROM ${table} ${whereSql}`)).rows[0].c);
@@ -42,8 +51,11 @@ async function main() {
 
     const base = baseline[table] ?? count;
     const floor = Math.floor(base * DROP_TOLERANCE);
+    curCounts[table] = count;
+    const ref = oldest[table]; // earliest observed count
     let status = 'OK', reason = '';
     if (count < floor) { status = 'LOW'; reason = `count ${count.toLocaleString()} < floor ${floor.toLocaleString()} (baseline ${base.toLocaleString()})`; }
+    else if (ref != null && history.length >= 1 && count < Math.floor(ref * 0.99)) { status = 'BLEED'; reason = `count ${count.toLocaleString()} < earliest observed ${Number(ref).toLocaleString()} (slow decline the floor misses)`; }
     else if (ageDays != null && ageDays > cfg.maxAgeDays) { status = 'STALE'; reason = `newest record ${ageDays}d old (> ${cfg.maxAgeDays}d)`; }
 
     results.push({ table, count, baseline: base, ageDays, freshest, status, reason });
@@ -65,18 +77,23 @@ async function main() {
 
   fs.mkdirSync(path.dirname(BASE), { recursive: true });
   fs.writeFileSync(BASE, JSON.stringify(baseline, null, 2));
+  // append this run to rolling history (keep last 120 runs)
+  fs.appendFileSync(HIST, JSON.stringify({ ts: nowTs, counts: curCounts }) + '\n');
+  const trimmed = fs.readFileSync(HIST, 'utf8').trim().split('\n').filter(Boolean);
+  if (trimmed.length > 120) fs.writeFileSync(HIST, trimmed.slice(-120).join('\n') + '\n');
   fs.mkdirSync(OUT, { recursive: true });
   const anyErr = sources.some(s => s.status === 'ERRORED');
   const anyStaleSrc = sources.some(s => s.status === 'STALE');
+  const anyBleed = results.some(r => r.status === 'BLEED');
   const worst = (results.some(r => r.status === 'LOW') || anyErr) ? 'FAIL'
-    : (results.some(r => r.status === 'STALE') || anyStaleSrc) ? 'STALE' : 'OK';
+    : (results.some(r => r.status === 'STALE') || anyStaleSrc || anyBleed) ? 'STALE' : 'OK';
   fs.writeFileSync(path.join(OUT, 'latest.json'), JSON.stringify({ verdict: worst, firstRun, results, sources }, null, 2));
 
   // report
   console.log(`\nLA data canary — ${firstRun ? 'BASELINE established' : 'verdict ' + worst}\n`);
   console.log('  ROW COUNTS + freshness:');
   for (const r of results) {
-    const mark = r.status === 'OK' ? '✔' : r.status === 'STALE' ? '⏳' : '✖';
+    const mark = r.status === 'OK' ? '✔' : r.status === 'STALE' ? '⏳' : r.status === 'BLEED' ? '↓' : '✖';
     const fresh = r.ageDays != null ? `  fresh ${r.ageDays}d` : '';
     console.log(`  ${mark} ${r.table.padEnd(30)} ${r.count.toLocaleString().padStart(12)}${fresh}${r.reason ? '  — ' + r.reason : ''}`);
   }

← b3681eb yoloforever c2 FIX (Cody): median_value + explicit scope lab  ·  back to La Socrata Ingester  ·  yoloforever c3 FIX (Cody): green-able canary + real CSLB loa 0377c1a →