← back to Commercialrealestate
CRCP alerts FIX-THEN-SHIP (contrarian+DTD-B): enforce per-subscriber cadence + watermarks, log untagged listings, gov seen-state sidecar (no main-state bloat), Browserbase cost logging
24bc8a156d0b7420901e3fc97488fff0e13a4816 · 2026-07-12 12:16:51 -0700 · Steve
Files touched
M .gitignoreM scripts/email-alerts.jsM scripts/refresh-listings-rotating.sh
Diff
commit 24bc8a156d0b7420901e3fc97488fff0e13a4816
Author: Steve <steve@designerwallcoverings.com>
Date: Sun Jul 12 12:16:51 2026 -0700
CRCP alerts FIX-THEN-SHIP (contrarian+DTD-B): enforce per-subscriber cadence + watermarks, log untagged listings, gov seen-state sidecar (no main-state bloat), Browserbase cost logging
---
.gitignore | 1 +
scripts/email-alerts.js | 116 +++++++++++++++++++++++++----------
scripts/refresh-listings-rotating.sh | 9 +++
3 files changed, 95 insertions(+), 31 deletions(-)
diff --git a/.gitignore b/.gitignore
index 6261140..35630b3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,3 +16,4 @@ data/assessor.sqlite-wal
data/dre-cache.json
data/alerts-out/
+data/gov-seen-state.json
diff --git a/scripts/email-alerts.js b/scripts/email-alerts.js
index ffef581..b230705 100644
--- a/scripts/email-alerts.js
+++ b/scripts/email-alerts.js
@@ -104,7 +104,37 @@ function loadConfig() {
console.log(`seeded default config → ${CONFIG_PATH}`);
return seed;
}
-function loadState() { const s = loadJSON(STATE_PATH, { last_run: null, seen: { listings: [], brokers: [] } }); s.seen = s.seen || {}; s.seen.listings = s.seen.listings || []; s.seen.brokers = s.seen.brokers || []; s.seen.gov = s.seen.gov || {}; return s; }
+// Per-subscriber state (fixes the cadence contract: each subscriber has their own
+// seen-watermark + last_sent, so a weekly subscriber isn't spammed daily AND sees
+// items added since THEIR last send — not since some other subscriber's send).
+const GOV_STATE_PATH = path.join(DATA, 'gov-seen-state.json'); // sidecar: gov license IDs kept OUT of the main state file so it can't balloon
+function loadState() {
+ const s = loadJSON(STATE_PATH, { last_run: null, subs: {} });
+ s.subs = s.subs || {};
+ // one-time migration: legacy global seen.{listings,brokers} → seed nothing extra; subs seed lazily from it
+ s._legacySeen = s.seen && (s.seen.listings || s.seen.brokers) ? { listings: new Set(s.seen.listings || []), brokers: new Set(s.seen.brokers || []) } : null;
+ delete s.seen;
+ return s;
+}
+function subState(state, email) {
+ if (!state.subs[email]) {
+ // seed a new subscriber's watermark from the legacy global baseline (so we don't blast the whole catalog on first run)
+ state.subs[email] = {
+ last_sent: null,
+ seen_listings: state._legacySeen ? [...state._legacySeen.listings] : [],
+ seen_brokers: state._legacySeen ? [...state._legacySeen.brokers] : [],
+ };
+ }
+ return state.subs[email];
+}
+// cadence → min interval before the next send is "due"
+const CADENCE_MS = { instant: 0, daily: 20 * 3600e3, weekly: 6.5 * 24 * 3600e3, monthly: 27 * 24 * 3600e3 };
+function isDue(sub, ss, nowMs) {
+ if (!ss.last_sent) return true;
+ const gap = CADENCE_MS[sub.cadence] ?? CADENCE_MS.daily;
+ return (nowMs - Date.parse(ss.last_sent)) >= gap;
+}
+function loadGovSeen() { return loadJSON(GOV_STATE_PATH, {}); } // { email: { metro: [license_numbers] } }
// Gov-agent segment (DTD verdict C, opt-in): read data/gov/<metro>.json for the
// metros any opt-in subscriber wants, return the NEW licensees vs seen.gov[metro].
@@ -214,58 +244,82 @@ function georgeSend({ to, subject, html, from }) {
fs.mkdirSync(OUT, { recursive: true });
const cfg = loadConfig();
const state = loadState();
- const seenL = new Set(state.seen.listings || []);
- const seenB = new Set(state.seen.brokers || []);
+ const govSeen = loadGovSeen();
+ const nowMs = Date.now();
const listings = loadListings();
const brokers = loadBrokers();
- const newListings = listings.filter(l => !seenL.has(l.id));
- const newBrokers = brokers.filter(b => !seenB.has(b.id));
- // Gov-agents segment (opt-in, DTD verdict C): only diff metros some subscriber wants.
- const govSubs = cfg.subscribers.filter(s => (s.segments || []).includes('gov-agents'));
- const govMetros = new Set();
- for (const s of govSubs) { if (s.metros.includes('*')) { Object.keys(state.seen.gov).forEach(m => govMetros.add(m)); ['los-angeles','miami','texas','nyc','las-vegas'].forEach(m => govMetros.add(m)); } else s.metros.forEach(m => govMetros.add(m)); }
- const govAll = govSubs.length ? loadGovForMetros([...govMetros]) : {};
- const govNewByMetro = {};
- for (const [m, arr] of Object.entries(govAll)) {
- const seenG = new Set(state.seen.gov[m] || []);
- const fresh = arr.filter(a => !seenG.has(a.license_number));
- if (fresh.length) govNewByMetro[m] = fresh;
- }
+ // Fix (contrarian #3): untagged listings were silently dropped from every digest.
+ // Now counted + logged so the gap is visible and fixable (add cities to METRO_CITIES).
+ const untagged = listings.filter(l => !l.metro).length;
+ if (untagged) console.warn(`⚠ ${untagged}/${listings.length} listings UNTAGGED (city not in METRO_CITIES) — not deliverable to any digest. Add their cities to METRO_CITIES to fix.`);
- console.log(`snapshot: ${listings.length} listings (${newListings.length} new), ${brokers.length} brokers (${newBrokers.length} new)`);
- console.log(`mode: ${SEND ? 'SEND (gated)' : 'DRY-RUN (no send, state unchanged unless --commit-state)'}`);
+ console.log(`snapshot: ${listings.length} listings, ${brokers.length} brokers`);
+ console.log(`mode: ${SEND ? 'SEND (gated)' : 'DRY-RUN'}`);
const date = new Date().toISOString().slice(0, 10);
const results = [];
- let sendFailures = 0;
+ let sendFailures = 0, dirtyGov = false;
+
for (const sub of cfg.subscribers) {
+ const ss = subState(state, sub.email);
+ // Fix (contrarian #1): enforce cadence. A non-due subscriber is skipped on send.
+ if (SEND && !isDue(sub, ss, nowMs)) { console.log(` ${sub.email}: not due (${sub.cadence}, last ${ss.last_sent}) — skip`); continue; }
+
+ // per-subscriber diff (their own watermark → correct deltas regardless of others' cadence)
+ const seenL = new Set(ss.seen_listings), seenB = new Set(ss.seen_brokers);
+ const newListings = listings.filter(l => !seenL.has(l.id));
+ const newBrokers = brokers.filter(b => !seenB.has(b.id));
+
+ // opt-in gov-agents, per-subscriber, from the sidecar file
+ let govNewByMetro = {}, govCurrentByMetro = {};
+ if ((sub.segments || []).includes('gov-agents')) {
+ const metros = sub.metros.includes('*') ? ['los-angeles','miami','texas','nyc','las-vegas','miami-beach','palm-beach','silicon-valley'] : sub.metros;
+ govCurrentByMetro = loadGovForMetros(metros);
+ const mineSeen = govSeen[sub.email] || {};
+ for (const [m, arr] of Object.entries(govCurrentByMetro)) {
+ const sg = new Set(mineSeen[m] || []);
+ const fresh = arr.filter(a => !sg.has(a.license_number));
+ if (fresh.length) govNewByMetro[m] = fresh;
+ }
+ }
+
const { html, listingCount, brokerCount, govCount } = buildDigestHTML(sub, newListings, newBrokers, govNewByMetro);
if (!listingCount && !brokerCount && !govCount) { console.log(` ${sub.email}: nothing new — skip`); continue; }
const subject = `CRCP: ${listingCount} new listing${listingCount!==1?'s':''}${brokerCount?` · ${brokerCount} new broker${brokerCount!==1?'s':''}`:''}${govCount?` · ${govCount} new agent${govCount!==1?'s':''}`:''}`;
const file = path.join(OUT, `${date}-${sub.email.replace(/[^a-z0-9]/gi,'_')}.html`);
fs.writeFileSync(file, html);
- results.push({ email: sub.email, subject, listingCount, brokerCount, file });
- console.log(` ${sub.email}: ${listingCount} listings + ${brokerCount} brokers → ${path.relative(ROOT, file)}`);
+ results.push({ email: sub.email, subject, listingCount, brokerCount, govCount, file });
+ console.log(` ${sub.email}: ${listingCount} listings + ${brokerCount} brokers${govCount?` + ${govCount} agents`:''} → ${path.relative(ROOT, file)}`);
+ let sent = false;
if (SEND) {
- try { await georgeSend({ to: sub.email, subject, html, from: cfg.from }); console.log(` ✉ sent via George`); }
+ try { await georgeSend({ to: sub.email, subject, html, from: cfg.from }); console.log(` ✉ sent via George`); sent = true; }
catch (e) { sendFailures++; console.error(` ⚠ send failed: ${e.message}`); }
}
+
+ // advance THIS subscriber's watermark only when actually sent (or baseline seed) — news-loss guard, per-subscriber
+ if (COMMIT_STATE && (!SEND || sent)) {
+ ss.seen_listings = listings.map(l => l.id);
+ ss.seen_brokers = brokers.map(b => b.id);
+ if (SEND) ss.last_sent = new Date().toISOString();
+ if (Object.keys(govCurrentByMetro).length) {
+ govSeen[sub.email] = govSeen[sub.email] || {};
+ for (const [m, arr] of Object.entries(govCurrentByMetro)) govSeen[sub.email][m] = arr.map(a => a.license_number);
+ dirtyGov = true;
+ }
+ }
}
- // In SEND mode, only advance state if every send succeeded — otherwise the news
- // would be silently consumed and lost on the next run. Baseline seed always advances.
- const advance = COMMIT_STATE && !(SEND && sendFailures > 0);
- if (SEND && sendFailures > 0) console.error(`⚠ ${sendFailures} send failure(s) — state NOT advanced; news preserved for retry.`);
- if (advance) {
- state.seen.listings = listings.map(l => l.id);
- state.seen.brokers = brokers.map(b => b.id);
- for (const [m, arr] of Object.entries(govAll)) state.seen.gov[m] = arr.map(a => a.license_number);
+ if (SEND && sendFailures) console.error(`⚠ ${sendFailures} send failure(s) — those subscribers' watermarks NOT advanced; news preserved for retry.`);
+
+ if (COMMIT_STATE) {
state.last_run = new Date().toISOString();
+ delete state._legacySeen;
fs.writeFileSync(STATE_PATH, JSON.stringify(state, null, 2));
- console.log(`state advanced → ${path.relative(ROOT, STATE_PATH)} (${SEND ? 'post-send' : 'baseline seed'})`);
+ if (dirtyGov) fs.writeFileSync(GOV_STATE_PATH, JSON.stringify(govSeen, null, 2)); // gov IDs kept OUT of the main state file
+ console.log(`state advanced → ${path.relative(ROOT, STATE_PATH)}${dirtyGov?' + gov sidecar':''}`);
} else {
console.log('state NOT advanced (dry-run). Re-run with --send to send + advance, or --commit-state to seed baseline.');
}
diff --git a/scripts/refresh-listings-rotating.sh b/scripts/refresh-listings-rotating.sh
index 75fced1..f3ce609 100755
--- a/scripts/refresh-listings-rotating.sh
+++ b/scripts/refresh-listings-rotating.sh
@@ -24,4 +24,13 @@ for m in "${TODAY[@]}"; do
echo "=== $m ==="
node scripts/refresh-listings-multi.js "$m" 2>&1 | grep -E "captured|ADDED|FATAL" | sed "s/^/ [$m] /"
done
+
+# Cost visibility (global CLAUDE.md "always show costs"): log the paid Browserbase
+# sessions to the cost-tracker ledger (~1 short session per metro, ~$0.03 each).
+COST_METROS="${TODAY[*]}" COST_N="$SLICE" node -e '
+const fs=require("fs"); const n=+process.env.COST_N||0; const rate=0.03;
+const entry={ts:new Date().toISOString(),api:"browserbase",vendor:"Browserbase",app:"crcp-listings-rotating",note:"metros: "+process.env.COST_METROS,units:[{qty:n,unit:"session",rate,subtotal:+(n*rate).toFixed(4)}],cost_usd:+(n*rate).toFixed(4)};
+fs.appendFileSync(process.env.HOME+"/.claude/cost-ledger.jsonl",JSON.stringify(entry)+"\n");
+console.log(" cost: ~$"+(n*rate).toFixed(2)+" ("+n+" browserbase session(s)) logged to ledger");
+'
echo "$(date '+%F %T') rotating refresh done: ${TODAY[*]}"
← 06ae110 CRCP listings: all-metro Crexi sweep complete — 2,791 listin
·
back to Commercialrealestate
·
auto-save: 2026-07-12T12:18:09 (3 files) — data/condos-redfi f83266a →