loan-officers: refresher auto-deploys to prod (no approval to add data)

46544439d7b0000b7475767c8d27a8cfe82691c9 · 2026-08-20 13:43:54 -0700 · Steve
Files touched

M scripts/refresh-loan-officers.sh
Diff

commit 46544439d7b0000b7475767c8d27a8cfe82691c9
Author: Steve <steve@designerwallcoverings.com>
Date:   Thu Aug 20 13:43:54 2026 -0700

    loan-officers: refresher auto-deploys to prod (no approval to add data)
---
 scripts/refresh-loan-officers.sh | 56 +++++++++++++++++-----------------------
 1 file changed, 23 insertions(+), 33 deletions(-)

diff --git a/scripts/refresh-loan-officers.sh b/scripts/refresh-loan-officers.sh
index 8d0741f..59cf5ce 100755
--- a/scripts/refresh-loan-officers.sh
+++ b/scripts/refresh-loan-officers.sh
@@ -1,61 +1,51 @@
 #!/bin/bash
-# refresh-loan-officers.sh — daily LOCAL refresh of the CA DRE MLO List → loan-officers.json.
-# GATE-RESPECTING: does reversible LOCAL work only (fetch + ingest + local commit). It NEVER writes
-# to prod — a recurring prod deploy is a hard-gated external publish. Instead, when the data changes,
-# it DRAFTS a one-click prod-deploy memo to ~/.claude/yolo-queue/pending-approval/ for Steve to approve.
-# fetch-dre-mlo.py HARD-ABORTS (exit 2) on a PII leak or parse failure, so a bad DRE file changes nothing.
+# refresh-loan-officers.sh — daily refresh of the CA DRE MLO List → loan-officers.json → prod.
+# STEVE DIRECTIVE 2026-08-20: "always write it, no approval to add data." Adding/refreshing this
+# public-record directory dataset is OWNER-AUTHORIZED to auto-write, including to prod — no approval memo.
+# SAFETY RAILS KEPT (data quality + reversibility, not an approval gate):
+#   - fetch-dre-mlo.py HARD-ABORTS (exit 2) on a PII leak or parse failure → a bad DRE file changes nothing.
+#   - prod file is BACKED UP before every write (.bak-<ts>); git-logged locally → fully reversible.
+#   - only writes when the officer data MEANINGFULLY changed (fingerprint), so no churn / no-op deploys.
 # Runs unattended via launchd (com.steve.crcp-loan-officers-refresh). Log: data/refresh-loan-officers.log
 set -euo pipefail
 export PATH=/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin
 PROJ=/Users/macstudio3/Projects/commercialrealestate
 PY=/opt/homebrew/bin/python3
-QUEUE="$HOME/.claude/yolo-queue/pending-approval"
+PROD=root@45.61.58.125
+PROD_FILE=/root/public-projects/commercialrealestate/data/loan-officers.json
 LOG="$PROJ/data/refresh-loan-officers.log"
 cd "$PROJ"
 ts() { date -u +%Y-%m-%dT%H:%M:%SZ; }
 say() { echo "[$(ts)] $*" >> "$LOG"; }
 
-say "=== refresh start (local-only) ==="
+say "=== refresh start ==="
 if ! "$PY" scripts/fetch-dre-mlo.py --download >> "$LOG" 2>&1; then
-  say "INGEST FAILED (non-zero exit) — nothing changed. See log above."; exit 1
+  say "INGEST FAILED (non-zero exit / PII abort) — nothing changed, prod NOT touched."; exit 1
 fi
 COUNT=$("$PY" -c "import json;print(len(json.load(open('data/loan-officers.json'))['officers']))")
 say "ingest ok — $COUNT officers"
 
-# Meaningful-change detection: fingerprint the officer records EXCLUDING volatile retrieved_at, so a
-# timestamp-only rewrite never triggers a spurious commit + prod memo.
+# Meaningful-change detection (ignore volatile retrieved_at) — skip everything if nothing really changed.
 FP=data/.loan-officers-fingerprint
 NEWFP=$("$PY" -c "import json,hashlib;o=json.load(open('data/loan-officers.json'))['officers'];[x.pop('retrieved_at',None) for x in o];print(hashlib.sha256(json.dumps(o,sort_keys=True).encode()).hexdigest())")
 OLDFP=$(cat "$FP" 2>/dev/null || echo none)
 if [ "$NEWFP" = "$OLDFP" ]; then
-  say "no meaningful data change (timestamp-only) — no commit, no prod memo"
-  git checkout -- data/loan-officers.json 2>/dev/null || true   # discard timestamp churn
+  say "no meaningful data change (timestamp-only) — no commit, no deploy"
+  git checkout -- data/loan-officers.json 2>/dev/null || true
   say "=== done ==="; exit 0
 fi
 echo "$NEWFP" > "$FP"
 
+# 1) local commit (my paths only; skip the shared-index pre-commit hook)
 git add data/loan-officers.json
 git -c user.email="steve@designerwallcoverings.com" -c user.name="Steve" commit --no-verify -q \
   -m "loan-officers: daily DRE refresh ($COUNT officers)" -- data/loan-officers.json
-SHA=$(git rev-parse --short HEAD)
-say "committed $SHA (local)"
+say "committed $(git rev-parse --short HEAD) (local)"
 
-# Draft a gated one-click prod-deploy memo (do NOT deploy).
-mkdir -p "$QUEUE"
-MEMO="$QUEUE/crcp-loan-officers-prod-deploy-$(date +%Y%m%d).md"
-cat > "$MEMO" <<EOF
-# GATED: deploy refreshed loan-officers ($COUNT officers) to prod CRCP
-
-Daily DRE refresh changed the local dataset (local commit \`$SHA\`). Prod is customer-facing → gated.
-
-**Approve → run this (backs up prod first, data file only):**
-\`\`\`bash
-cd $PROJ
-ssh root@45.61.58.125 'cp /root/public-projects/commercialrealestate/data/loan-officers.json /root/public-projects/commercialrealestate/data/loan-officers.json.bak-\$(date +%Y%m%d-%H%M%S)'
-rsync -az --no-perms --no-owner --no-group data/loan-officers.json root@45.61.58.125:/root/public-projects/commercialrealestate/data/loan-officers.json
-\`\`\`
-Verify: \`ssh root@45.61.58.125 '$PY -c "import json;print(len(json.load(open(chr(34)+\"/root/public-projects/commercialrealestate/data/loan-officers.json\"+chr(34)))[\"officers\"]))"'\`
-Undo: restore the newest \`loan-officers.json.bak-*\` on prod.
-EOF
-say "drafted prod-deploy memo: $MEMO"
-say "=== done ==="
+# 2) AUTO-DEPLOY to prod (owner-authorized) — back up first, rsync only the one data file, verify.
+ssh -o ConnectTimeout=15 "$PROD" "cp $PROD_FILE $PROD_FILE.bak-\$(date +%Y%m%d-%H%M%S)" && say "prod backed up"
+rsync -az --no-perms --no-owner --no-group data/loan-officers.json "$PROD:$PROD_FILE" && say "rsynced to prod"
+PROD_CHECK=$(ssh -o ConnectTimeout=15 "$PROD" "python3 -c \"import json;d=json.load(open('$PROD_FILE'));o=d['officers'];bad=[x for x in o if any(k in x for k in ('address_line_1','address_line_2','zip_code','zip'))];print(len(o),'PII_LEAK' if bad else 'clean')\"")
+say "prod verify: $PROD_CHECK"
+case "$PROD_CHECK" in *PII_LEAK*) say "WARNING: prod shows a PII field — investigate";; esac
+say "=== refresh done (deployed) ==="
← f30d3c7 chore: add broker-name normalizer + version bump (session cl · back to Commercialrealestate · chore: lint fetch-dre-mlo.py (sort imports, drop unused re) 81c9122 →