← back to Commercialrealestate
scripts/crcp-drift-canary.js
465 lines
#!/usr/bin/env node
/**
* crcp-drift-canary.js — READ-ONLY origin-vs-live drift monitor for CRCP.
*
* WHY (front-end half): crcp.agentabrams.com is a Kamatera-local copy synced by
* MANUAL rsync. It silently drifted 9 days stale (Jul 7->16 2026) because nobody
* NOTICED — the manual deploy wasn't the failure, the INVISIBILITY was. DTD
* verdict D (2026-07-16): keep deploys manual (push-frontend-kamatera.sh) + this
* canary. Compares sha256 of each front-end asset (public/*.html,*.js,*.css) on
* the origin (this box) vs live (Kamatera).
*
* WHY (data half, added TK-12033, 2026-09-22): the LISTING DATA (data/ranked.json,
* data/listings.json) is NOT covered by the front-end check above, and is
* deliberately excluded from BOTH deploy scripts (push-frontend-kamatera.sh,
* push-full-kamatera.sh — see their SCOPE comments) — Kamatera has "its own
* snapshots" that nothing was actually keeping fresh. The Mac2 rotating cron
* (com.steve.crcp-listings-rotating -> refresh-listings-multi.js -> npm run
* analyze) regenerates data/ranked.json + data/listings.json locally every day,
* but nothing carried that to prod, so prod listing data FROZE since Sep 15
* while this canary reported IN_SYNC (it only ever hashed public/*.{html,js,css}).
* That was a textbook false-green (CLAUDE.md TK-11431 amendment 1 — an
* unmeasured input is never PASS). This half closes that hole: it compares
* local vs remote data/ranked.json + data/listings.json by content hash + the
* REMOTE file's own mtime, and FAILs when prod is materially older AND
* different from local ("prod frozen"). Fix when this fires:
* bash deploy/push-data-kamatera.sh
*
* Emits BOTH the legacy domain-word `status` (IN_SYNC/DRIFTED/UNKNOWN, kept for
* humans/back-compat) AND a top-level `verdict` in the fleet-health-rollup
* PASS/WARN/FAIL vocabulary (CLAUDE.md TK-11431 rule: an unrecognized verdict
* word is silently read as GREEN — never ship a health signal without this).
*
* WORSENING-ONLY alerting for both halves (steady-state / recovery / first-run
* are silent, EXCEPT a first-run FAIL on the data check fires once immediately
* — TK-11431 amendment 2's "day-one" rule: don't grandfather in the exact
* condition this canary exists to catch). Touches nothing on prod — read-only
* ssh (sha256sum / stat / node JSON.parse, no data transferred). Always writes
* data/latest.json so dw-canary-meta-watchdog sees it ran.
*
* Self-test: `node scripts/crcp-drift-canary.js --test` runs the data-freshness
* logic against local fixture directories (no ssh, no prod contact) and proves
* it goes WARN/FAIL on injected staleness/drift and PASS when in sync. This is
* the negative test CLAUDE.md TK-11431 amendment 3 requires before a health
* check ships — a positive-only test proves nothing about a detector.
*
* Testability seam (never used by the scheduled launchd job — the plist does
* not and must not set this): CRCP_DATA_TEST_LOCAL_DIR points the "remote"
* data read at a local directory instead of ssh'ing Kamatera, for exercising
* the real end-to-end dispatch path in a test without touching prod.
*/
const { execSync } = require('child_process');
const fs = require('fs');
const os = require('os');
const path = require('path');
const crypto = require('crypto');
const ROOT = path.join(__dirname, '..');
const PUB = path.join(ROOT, 'public');
const LOCAL_DATA_DIR = path.join(ROOT, 'data');
const HOST = process.env.CRCP_DATA_HOST || 'root@45.61.58.125';
const REMOTE = '/root/public-projects/commercialrealestate/public';
const REMOTE_DATA_DIR = process.env.CRCP_DATA_REMOTE_DIR || '/root/public-projects/commercialrealestate/data';
const STATE = path.join(ROOT, 'data', 'crcp-drift-state.json');
const DATA_STATE = path.join(ROOT, 'data', 'crcp-data-freshness-state.json');
const LATEST = path.join(ROOT, 'data', 'latest.json');
const EXT = /\.(html|js|css)$/;
// Data-freshness files + tuning. Refresh cadence is daily (com.steve.crcp-listings-rotating,
// 5:30am) so a ~30h margin absorbs one missed/late run before calling it a "freeze".
//
// KNOWN LIMITATION (named on purpose, not silently assumed — Cody-panel finding,
// TK-12033): remoteAgeH is computed as (Date.now() on THIS machine) - (remote file's
// own mtime, as reported by Kamatera's filesystem). If Mac2 and Kamatera's clocks
// drift apart, staleness reads wrong in whichever direction the skew points. Both are
// cloud/local boxes on NTP and this has never been observed to matter at a 30h margin,
// so this is deliberately NOT compensated for — but if this canary ever reports an
// age that doesn't match reality, check `date` on both boxes before distrusting the logic.
const DATA_FILES = ['ranked.json', 'listings.json'];
const STALE_SOURCE_HOURS = Number(process.env.CRCP_DATA_STALE_SOURCE_HOURS || 30);
const FREEZE_HOURS = Number(process.env.CRCP_DATA_FREEZE_HOURS || 30);
const RANK = { PASS: 0, WARN: 1, FAIL: 2 };
function worseOf(a, b) { return RANK[b] > RANK[a] ? b : a; }
function sha(buf) { return crypto.createHash('sha256').update(buf).digest('hex'); }
// ---------------------------------------------------------------------------
// Front-end (public/*.html,*.js,*.css) drift check — unchanged behavior.
// ---------------------------------------------------------------------------
function localHashes() {
const out = {};
for (const f of fs.readdirSync(PUB)) {
if (!EXT.test(f)) continue;
out[f] = sha(fs.readFileSync(path.join(PUB, f)));
}
return out;
}
function remoteHashes() {
// single read-only ssh call; sha256sum every html/js/css in the remote public dir
const cmd = `ssh -o ConnectTimeout=10 -o BatchMode=yes ${HOST} ` +
`"cd ${REMOTE} 2>/dev/null && for f in *.html *.js *.css; do [ -f \\"\\$f\\" ] && sha256sum \\"\\$f\\"; done"`;
const raw = execSync(cmd, { encoding: 'utf8', timeout: 30000 });
const out = {};
for (const line of raw.trim().split('\n')) {
const m = line.match(/^([0-9a-f]{64})\s+(.+)$/);
if (m) out[m[2]] = m[1];
}
return out;
}
// ---------------------------------------------------------------------------
// Data-freshness (data/ranked.json, data/listings.json) check — new.
// ---------------------------------------------------------------------------
function countRows(name, parsed) {
try {
if (name === 'ranked.json') return Array.isArray(parsed.ranked) ? parsed.ranked.length : null;
if (name === 'listings.json') return Array.isArray(parsed.listings) ? parsed.listings.length : null;
} catch (_) { /* fall through */ }
return null;
}
// Reads DATA_FILES from a plain local directory. Used for BOTH the real local
// side and (via the CRCP_DATA_TEST_LOCAL_DIR seam / the self-test) a fixture
// standing in for "remote" — same code path, so the comparison logic is
// exercised identically in production and in the negative test.
function readDataDir(dir) {
const out = {};
for (const name of DATA_FILES) {
const p = path.join(dir, name);
try {
const st = fs.statSync(p);
const buf = fs.readFileSync(p);
const hash = sha(buf);
let count = null;
try { count = countRows(name, JSON.parse(buf.toString('utf8'))); } catch (_) { /* count is best-effort */ }
out[name] = { present: true, mtimeMs: st.mtimeMs, size: st.size, sha256: hash, count };
} catch (e) {
out[name] = { present: false, error: String(e.message).split('\n')[0] };
}
}
return out;
}
// Remote equivalent of readDataDir(), run ON Kamatera via ssh so no multi-MB
// file is ever transferred — only the resulting small JSON summary comes back.
function remoteDataDir(host, dir) {
const remoteScript = `
const fs=require('fs');const crypto=require('crypto');const path=require('path');
const DIR=process.argv[1];
const files=['ranked.json','listings.json'];
const out={};
for(const name of files){
const p=path.join(DIR,name);
try{
const st=fs.statSync(p);
const buf=fs.readFileSync(p);
const hash=crypto.createHash('sha256').update(buf).digest('hex');
let count=null;
try{
const j=JSON.parse(buf.toString('utf8'));
if(name==='ranked.json') count=Array.isArray(j.ranked)?j.ranked.length:null;
if(name==='listings.json') count=Array.isArray(j.listings)?j.listings.length:null;
}catch(e){}
out[name]={present:true,mtimeMs:st.mtimeMs,size:st.size,sha256:hash,count};
}catch(e){out[name]={present:false,error:String(e.message).split('\\n')[0]};}
}
process.stdout.write(JSON.stringify(out));
`.trim();
const b64 = Buffer.from(remoteScript, 'utf8').toString('base64');
const cmd = `ssh -o ConnectTimeout=15 -o BatchMode=yes ${host} ` +
`"node -e \\"eval(Buffer.from('${b64}','base64').toString())\\" '${dir}'"`;
const raw = execSync(cmd, { encoding: 'utf8', timeout: 60000, maxBuffer: 4 * 1024 * 1024 });
return JSON.parse(raw.trim());
}
// Dispatch: real ssh unless the test-only env seam is set (never set by the
// scheduled launchd job — see the header note).
function getRemoteDataInfo() {
if (process.env.CRCP_DATA_TEST_LOCAL_DIR) return readDataDir(process.env.CRCP_DATA_TEST_LOCAL_DIR);
return remoteDataDir(HOST, REMOTE_DATA_DIR);
}
// Pure function: local + remote per-file info -> a verdict. No I/O, so the
// self-test can call it directly against fabricated info objects too.
function evaluateDataFreshness(local, remote, now = Date.now()) {
const files = {};
let overall = 'PASS';
for (const name of DATA_FILES) {
const l = local[name];
const r = remote[name];
let verdict, reason;
if (!l || !l.present) {
verdict = 'WARN'; reason = 'local file missing/unreadable — NOT MEASURED';
} else if (!r || !r.present) {
verdict = 'WARN'; reason = 'remote file missing/unreadable — NOT MEASURED';
} else if (l.sha256 === r.sha256) {
const localAgeH = (now - l.mtimeMs) / 3600000;
if (localAgeH > STALE_SOURCE_HOURS) {
verdict = 'WARN'; reason = `content matches, but the LOCAL source itself is ${localAgeH.toFixed(1)}h old (is the rotating cron running?)`;
} else {
verdict = 'PASS'; reason = 'content matches, fresh';
}
} else {
const remoteAgeH = (now - r.mtimeMs) / 3600000;
if (remoteAgeH > FREEZE_HOURS) {
verdict = 'FAIL'; reason = `prod ${name} is ${remoteAgeH.toFixed(1)}h old and differs from local — PROD FROZEN. Fix: bash deploy/push-data-kamatera.sh`;
} else {
verdict = 'WARN'; reason = `prod ${name} differs from local (pushed ${remoteAgeH.toFixed(1)}h ago) — deploy due soon`;
}
}
files[name] = {
verdict, reason,
local: l ? { present: l.present, mtimeMs: l.mtimeMs, size: l.size, count: l.count } : null,
remote: r ? { present: r.present, mtimeMs: r.mtimeMs, size: r.size, count: r.count } : null,
};
overall = worseOf(overall, verdict);
}
return { verdict: overall, files };
}
function mapFrontendVerdict(status, worsening) {
if (status === 'UNKNOWN') return 'WARN';
if (status === 'IN_SYNC') return 'PASS';
return worsening ? 'FAIL' : 'WARN'; // DRIFTED
}
function postCncp(title, note) {
try {
execSync(`curl -s -m 8 -X POST http://127.0.0.1:3333/api/parking-lot ` +
`-H 'Content-Type: application/json' ` +
`-d ${JSON.stringify(JSON.stringify({ title, note, source: 'crcp-drift-canary' }))}`,
{ stdio: 'ignore' });
} catch (_) { /* CNCP best-effort */ }
}
function writeLatest(o) {
try { fs.mkdirSync(path.dirname(LATEST), { recursive: true }); } catch (_) {}
fs.writeFileSync(LATEST, JSON.stringify({ canary: 'crcp-drift', ...o }, null, 2));
}
// ---------------------------------------------------------------------------
// main
// ---------------------------------------------------------------------------
function main() {
let live, err = null;
const local = localHashes();
try { live = remoteHashes(); } catch (e) { err = String(e.message).split('\n')[0]; }
const now = new Date().toISOString();
// --- front-end half ---
let feStatus, feCount = null, feDrifted = [], feWorsening = false;
if (err) {
feStatus = 'UNKNOWN';
} else {
const names = new Set([...Object.keys(local), ...Object.keys(live)]);
for (const n of names) {
if (local[n] !== live[n]) {
feDrifted.push({ file: n, state: !live[n] ? 'missing-on-live' : !local[n] ? 'extra-on-live' : 'differs' });
}
}
feCount = feDrifted.length;
const prev = fs.existsSync(STATE) ? JSON.parse(fs.readFileSync(STATE, 'utf8')) : null;
const baseline = prev ? prev.count : null;
feWorsening = baseline !== null && feCount > baseline;
feStatus = feCount === 0 ? 'IN_SYNC' : 'DRIFTED';
fs.writeFileSync(STATE, JSON.stringify({ count: feCount, ts: now, drifted: feDrifted }, null, 2));
console.log(`[crcp-drift-canary] frontend: ${feStatus} — ${feCount} file(s) drifted` +
(baseline !== null ? ` (was ${baseline})` : ' (first run — baseline set, silent)'));
if (feWorsening) {
const msg = `CRCP live is ${feCount} front-end file(s) behind origin (was ${baseline}). ` +
`Files: ${feDrifted.map(d => d.file + ':' + d.state).join(', ')}. ` +
`Fix: bash deploy/push-frontend-kamatera.sh`;
console.log('[crcp-drift-canary] frontend WORSENING — alerting: ' + msg);
postCncp('CRCP deploy drift', msg);
}
}
if (err) console.log(`[crcp-drift-canary] frontend: UNKNOWN — ${err}`);
// --- data-freshness half ---
let dataResult;
let localData, remoteData, dataErr = null;
localData = readDataDir(LOCAL_DATA_DIR);
try { remoteData = getRemoteDataInfo(); } catch (e) { dataErr = String(e.message).split('\n')[0]; }
if (dataErr) {
dataResult = { verdict: 'WARN', files: {}, error: dataErr, unmeasured: true };
console.log(`[crcp-drift-canary] data: WARN (NOT MEASURED) — ${dataErr}`);
} else {
dataResult = evaluateDataFreshness(localData, remoteData);
for (const [name, f] of Object.entries(dataResult.files)) {
console.log(`[crcp-drift-canary] data ${name}: ${f.verdict} — ${f.reason}`);
}
}
const prevData = fs.existsSync(DATA_STATE) ? JSON.parse(fs.readFileSync(DATA_STATE, 'utf8')) : null;
const prevDataVerdict = prevData ? prevData.verdict : null;
const dataWorsening = prevDataVerdict !== null && RANK[dataResult.verdict] > RANK[prevDataVerdict];
const dataFirstRunFail = prevDataVerdict === null && dataResult.verdict === 'FAIL';
fs.writeFileSync(DATA_STATE, JSON.stringify({ verdict: dataResult.verdict, ts: now }, null, 2));
if (dataWorsening || dataFirstRunFail) {
const reasons = Object.entries(dataResult.files || {})
.filter(([, f]) => f.verdict !== 'PASS')
.map(([name, f]) => `${name}: ${f.reason}`)
.join(' | ') || (dataResult.error || 'unmeasured');
const msg = `CRCP prod listing data freshness is ${dataResult.verdict}` +
(prevDataVerdict ? ` (was ${prevDataVerdict})` : ' (first run)') + `. ${reasons}`;
console.log('[crcp-drift-canary] data ' + (dataFirstRunFail ? 'FIRST-RUN FAIL' : 'WORSENING') + ' — alerting: ' + msg);
postCncp('CRCP prod data frozen/stale', msg);
}
const overallVerdict = worseOf(mapFrontendVerdict(feStatus, feWorsening), dataResult.verdict);
writeLatest({
status: feStatus, // legacy domain word (front-end), kept for back-compat
verdict: overallVerdict, // fleet-health-rollup vocabulary: PASS | WARN | FAIL
ts: now,
frontend: { status: feStatus, driftCount: feCount, drifted: feDrifted, worsening: feWorsening, error: err },
data: dataResult,
});
console.log(`[crcp-drift-canary] OVERALL: ${overallVerdict}`);
}
// ---------------------------------------------------------------------------
// self-test: proves the data-freshness detector actually goes WARN/FAIL on
// injected staleness, and PASS when clean. Run with --test. Never called by
// the scheduled launchd job.
// ---------------------------------------------------------------------------
function makeFixtureDir(rankedCount, listingsCount) {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'crcp-canary-test-'));
fs.writeFileSync(path.join(dir, 'ranked.json'), JSON.stringify({
meta: {}, ranked: Array.from({ length: rankedCount }, (_, i) => ({ id: 'r' + i })),
}));
fs.writeFileSync(path.join(dir, 'listings.json'), JSON.stringify({
meta: {}, listings: Array.from({ length: listingsCount }, (_, i) => ({ id: 'l' + i })),
}));
return dir;
}
function setMtimeHoursAgo(dir, hours) {
const t = new Date(Date.now() - hours * 3600000);
for (const name of DATA_FILES) fs.utimesSync(path.join(dir, name), t, t);
}
function rmrf(dir) { try { fs.rmSync(dir, { recursive: true, force: true }); } catch (_) {} }
function runSelfTest() {
let failures = 0;
const assert = (cond, label) => {
if (cond) { console.log(` ok ${label}`); } else { console.log(` FAIL ${label}`); failures++; }
};
console.log('[crcp-drift-canary --test] scenario A: prod frozen (differs + remote >30h old) -> expect FAIL');
{
const localDir = makeFixtureDir(10, 20); // fresh local, current mtime
const remoteDir = makeFixtureDir(8, 20); // DIFFERENT content (ranked count 8 vs 10)
setMtimeHoursAgo(remoteDir, 40); // and stale on prod
const local = readDataDir(localDir);
const remote = readDataDir(remoteDir);
const result = evaluateDataFreshness(local, remote);
assert(result.verdict === 'FAIL', `pure evaluateDataFreshness() verdict FAIL (got ${result.verdict})`);
assert(result.files['ranked.json'].verdict === 'FAIL', `ranked.json flagged FAIL (got ${result.files['ranked.json'].verdict})`);
// Also exercise the REAL dispatch path (getRemoteDataInfo -> the env seam)
// end-to-end, not just the pure function, so the wiring itself is proven.
process.env.CRCP_DATA_TEST_LOCAL_DIR = remoteDir;
let dispatchResult;
try {
const remoteViaDispatch = getRemoteDataInfo();
dispatchResult = evaluateDataFreshness(local, remoteViaDispatch);
} finally {
delete process.env.CRCP_DATA_TEST_LOCAL_DIR;
}
assert(dispatchResult.verdict === 'FAIL', `end-to-end dispatch (env seam) verdict FAIL (got ${dispatchResult.verdict})`);
rmrf(localDir); rmrf(remoteDir);
}
console.log('[crcp-drift-canary --test] scenario B: recently pushed but differs (remote <30h old) -> expect WARN');
{
const localDir = makeFixtureDir(12, 20);
const remoteDir = makeFixtureDir(11, 20);
setMtimeHoursAgo(remoteDir, 2);
const result = evaluateDataFreshness(readDataDir(localDir), readDataDir(remoteDir));
assert(result.verdict === 'WARN', `verdict WARN (got ${result.verdict})`);
rmrf(localDir); rmrf(remoteDir);
}
console.log('[crcp-drift-canary --test] scenario C: in sync, fresh -> expect PASS');
{
const localDir = makeFixtureDir(15, 25);
const remoteDir = makeFixtureDir(15, 25); // identical content -> identical sha256
const result = evaluateDataFreshness(readDataDir(localDir), readDataDir(remoteDir));
assert(result.verdict === 'PASS', `verdict PASS (got ${result.verdict})`);
rmrf(localDir); rmrf(remoteDir);
}
console.log('[crcp-drift-canary --test] scenario D: in sync but LOCAL source stale (cron dead) -> expect WARN, not PASS');
{
const localDir = makeFixtureDir(9, 9);
setMtimeHoursAgo(localDir, 48);
const remoteDir = makeFixtureDir(9, 9);
setMtimeHoursAgo(remoteDir, 48);
const result = evaluateDataFreshness(readDataDir(localDir), readDataDir(remoteDir));
assert(result.verdict === 'WARN', `verdict WARN on stale-but-matching source (got ${result.verdict})`);
rmrf(localDir); rmrf(remoteDir);
}
console.log('[crcp-drift-canary --test] scenario E: remote unreadable -> expect WARN (never a false PASS)');
{
const localDir = makeFixtureDir(5, 5);
const result = evaluateDataFreshness(readDataDir(localDir), {});
assert(result.verdict === 'WARN', `verdict WARN on missing remote (got ${result.verdict})`);
rmrf(localDir);
}
console.log('[crcp-drift-canary --test] scenario F: LOCAL file missing/unreadable -> expect WARN (never a false PASS)');
{
// Asymmetric to scenario E on purpose (Cody-panel finding, TK-12033): a local read
// failure (permissions, mid-write truncation from `npm run analyze`) must be caught
// just as honestly as a remote one — readDataDir() hits the same !present branch,
// but nothing had actually exercised the LOCAL side of it until now.
const remoteDir = makeFixtureDir(7, 7);
const local = readDataDir(path.join(os.tmpdir(), 'crcp-canary-does-not-exist-' + Date.now()));
const result = evaluateDataFreshness(local, readDataDir(remoteDir));
assert(result.verdict === 'WARN', `verdict WARN on missing local (got ${result.verdict})`);
assert(local['ranked.json'].present === false, 'local ranked.json correctly reports present:false');
rmrf(remoteDir);
}
console.log('[crcp-drift-canary --test] scenario G: real ssh failure (main()\'s dataErr path) -> must throw, never silently return empty/PASS-shaped data');
{
// Scenario E only exercises evaluateDataFreshness()'s pure "remote object empty"
// branch. The env seam (CRCP_DATA_TEST_LOCAL_DIR) bypasses ssh entirely via
// readDataDir(), which swallows per-file errors and can't reproduce a real ssh
// failure. So THIS scenario calls the actual ssh-based remoteDataDir() against an
// unresolvable host — the exact function main() wraps in try/catch to produce
// `dataResult = { verdict: 'WARN', ..., unmeasured: true }` (see main(), the
// `dataErr` branch) — proving that path is a real throw, not a silent pass-through
// that could accidentally read as PASS. Fast: DNS resolution failure, no network
// wait (confirmed ~0.1s, no 15s ConnectTimeout hit).
let threw = false, msg = '';
try {
remoteDataDir('root@this-host-does-not-exist.invalid.test', '/tmp');
} catch (e) { threw = true; msg = String(e.message).split('\n')[0]; }
assert(threw, `remoteDataDir() throws on an unreachable host (main()'s dataErr catch depends on this) — ${msg}`);
}
console.log(failures === 0 ? '[crcp-drift-canary --test] ALL PASS' : `[crcp-drift-canary --test] ${failures} FAILURE(S)`);
process.exit(failures === 0 ? 0 : 1);
}
if (require.main === module) {
if (process.argv.includes('--test')) runSelfTest();
else main();
}
module.exports = { evaluateDataFreshness, readDataDir, getRemoteDataInfo };