`scripts/run-residential-refresh.sh`

92 lines
#!/bin/bash
# run-residential-refresh.sh — LOCAL ($0) refresh of LA County residential inventory for
# crcp.agentabrams.com/mls.html. Runs the Redfin gis-csv scrapers via LOCAL Chrome (CC_LOCAL=1,
# no Browserbase spend), rebuilds the SFR snapshot, and pushes DATA-ONLY (the two JSON snapshots)
# to the Kamatera prod box. No code is deployed and no pm2 reload is needed — the /api/residential
# and /api/condos snapshot fallbacks readFileSync the JSON on every request, so a fresh file is live
# on the next request.
#
# Cost: $0 (local Chrome on a residential IP). Scheduled 4x/day (06:00/12:00/18:00/00:00) by
# ~/Library/LaunchAgents/com.steve.crcp-residential-refresh.plist.
set -uo pipefail
cd "$HOME/Projects/commercialrealestate" || exit 1
# launchd runs with a minimal PATH (no /opt/homebrew/bin) — export it so `node` resolves.
export PATH="/usr/local/bin:/opt/homebrew/bin:$HOME/.npm-global/bin:$PATH"
export NODE_PATH="$HOME/.claude/skills/browserbase/node_modules"
export CC_LOCAL=1
export CHROME_PATH="/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
# CC_FRESH=1 clears the SFR resume ledger so every scheduled run RE-SWEEPS all regions (refreshes
# current prices + catches new listings). Without it the ledger skips already-swept regions and the
# refresh is a no-op. NOTE: both scrapers are upsert-only (no delete), so off-market listings linger
# until a listing id stops resolving — acceptable for a 4x/day price/inventory refresh.
export CC_FRESH=1

DEPLOY_HOST=45.61.58.125
DEPLOY_PATH=/root/public-projects/commercialrealestate
TS=$(date '+%Y-%m-%d %H:%M:%S')
LOG=tmp/residential-refresh.log
mkdir -p tmp
echo "===== [$TS] residential refresh (LOCAL, \$0) =====" >> "$LOG"

# Alert steve-office (George/Gmail) when a scheduled refresh fails silently — a bad scrape that the
# sanity-gate blocks, or an rsync-to-prod failure. Reuses the fleet's canonical george-send.sh.
ALERT_TO="${RESIDENTIAL_REFRESH_TO:-steve@designerwallcoverings.com}"
send_alert() { # $1=subject  $2=html-body
  [ -f "$HOME/.claude/skills/_shared/george-send.sh" ] || { echo "[$(date '+%H:%M:%S')] no george-send.sh — alert skipped" >> "$LOG"; return; }
  . "$HOME/.claude/skills/_shared/george-send.sh"
  local resp; resp="$(george_send steve-office "$ALERT_TO" "$1" "$2")"
  echo "$resp" | grep -q '"success":true' && echo "[$(date '+%H:%M:%S')] alert emailed to $ALERT_TO" >> "$LOG" \
    || echo "[$(date '+%H:%M:%S')] alert email FAILED: $resp" >> "$LOG"
}

# Capture the sweep-start BEFORE scraping. mark-off-market flags any active listing whose last_seen
# is older than this (i.e. it didn't reappear in this sweep = pulled off the market).
MARK_SINCE_TS=$(node -e 'console.log(new Date().toISOString())')
echo "[$(date '+%H:%M:%S')] sweep start (MARK_SINCE) = $MARK_SINCE_TS" >> "$LOG"

# 1) Scrape condos (full sweep, local Chrome). Upserts cre.condo (last_seen=now on each seen listing).
echo "[$(date '+%H:%M:%S')] condos..." >> "$LOG"
node scripts/fetch-condos-redfin.js >> "$LOG" 2>&1
CONDO_RC=$?

# 2) Scrape SFR (full sweep, local Chrome). Upserts cre.sfr (last_seen=now on each seen listing).
echo "[$(date '+%H:%M:%S')] sfr..." >> "$LOG"
node scripts/fetch-sfr-redfin.js >> "$LOG" 2>&1
SFR_RC=$?

# 3) Mark listings that stopped appearing as off_market + label SOLD (vs closed_sale) / WITHDRAWN.
#    Has its own >40%-would-flag safety abort, so a partial/throttled sweep can't mass-flag the market.
echo "[$(date '+%H:%M:%S')] mark off-market..." >> "$LOG"
MARK_SINCE="$MARK_SINCE_TS" node scripts/mark-off-market.js >> "$LOG" 2>&1

# 4) Export BOTH snapshots from the DB AFTER marking (carries status/sold/withdrawn + prices).
echo "[$(date '+%H:%M:%S')] export snapshots..." >> "$LOG"
node scripts/export-residential-snapshots.js >> "$LOG" 2>&1
# Rebuild the map points (residential colored by status) from the freshly-marked DB.
node scripts/build-map-points.js >> "$LOG" 2>&1 || echo "[$(date '+%H:%M:%S')] build-map-points failed (non-fatal)" >> "$LOG"
EXP_RC=$?

# 4) Guard: never push a truncated snapshot. Require both files to be sane before rsync.
SFR_N=$(node -e 'try{console.log(require("./data/sfr-redfin.json").sfr.length)}catch(e){console.log(0)}')
CONDO_N=$(node -e 'try{const c=require("./data/condos-redfin.json");console.log((c.condos||c).length)}catch(e){console.log(0)}')
echo "[$(date '+%H:%M:%S')] snapshot counts — sfr=$SFR_N condo=$CONDO_N (rc: condo=$CONDO_RC sfr=$SFR_RC exp=$EXP_RC)" >> "$LOG"

if [ "$SFR_N" -lt 5000 ] || [ "$CONDO_N" -lt 500 ]; then
  echo "[$(date '+%H:%M:%S')] ABORT push — snapshot below sanity floor (sfr<5000 or condo<500). Keeping prod as-is." >> "$LOG"
  send_alert "⚠ CRCP residential refresh ABORTED — bad scrape ($(date '+%Y-%m-%d %H:%M'))" \
    "<div style=\"font-family:-apple-system,Helvetica,sans-serif;color:#222\"><h3 style=\"color:#b23b3b\">⚠ Residential refresh aborted — prod left untouched</h3><div>The scheduled Redfin scrape came back short, so the sanity-gate <b>blocked the push</b> to crcp.agentabrams.com (last good data is still live).</div><div style=\"margin:10px 0;padding:8px 12px;border:1px solid #eee;border-radius:6px\">SFR scraped: <b>$SFR_N</b> (floor 5000)<br>Condos scraped: <b>$CONDO_N</b> (floor 500)<br>rc: condo=$CONDO_RC sfr=$SFR_RC exp=$EXP_RC</div><div style=\"font-size:13px;color:#444\">Likely Redfin rate-limiting the local IP. Check <code>tmp/residential-refresh.log</code>. Next scheduled run may recover on its own; if it keeps aborting, dial the cron back to 2x/day.</div></div>"
  exit 2
fi

# 5) Push DATA-ONLY to prod (two files). No code, no reload.
echo "[$(date '+%H:%M:%S')] rsync snapshots -> prod..." >> "$LOG"
rsync -az data/sfr-redfin.json data/condos-redfin.json data/map-points.json \
  "root@${DEPLOY_HOST}:${DEPLOY_PATH}/data/" >> "$LOG" 2>&1
RSYNC_RC=$?
echo "[$(date '+%H:%M:%S')] DONE — rsync rc=$RSYNC_RC · cost \$0 (local) · live: https://crcp.agentabrams.com/mls.html" >> "$LOG"
if [ "$RSYNC_RC" -ne 0 ]; then
  send_alert "⚠ CRCP residential refresh — rsync to prod FAILED ($(date '+%Y-%m-%d %H:%M'))" \
    "<div style=\"font-family:-apple-system,Helvetica,sans-serif;color:#222\"><h3 style=\"color:#b23b3b\">⚠ Fresh data scraped but not pushed</h3><div>The local scrape succeeded (SFR <b>$SFR_N</b>, condos <b>$CONDO_N</b>) but the rsync to Kamatera <b>failed</b> (rc=$RSYNC_RC). The live site is showing the previous snapshot.</div><div style=\"font-size:13px;color:#444;margin-top:8px\">Check SSH/network to root@$DEPLOY_HOST and <code>tmp/residential-refresh.log</code>.</div></div>"
fi
exit $RSYNC_RC