refresh-clients.sh: read CSVs from project data/sources (launchd can't read ~/Downloads — TCC), nullglob rsync

ad37a602d937588487632a44c1f618de4bb6095b · 2026-07-16 11:06:44 -0700 · steve

Files touched

M .gitignore
M scripts/refresh-clients.sh

Diff

commit ad37a602d937588487632a44c1f618de4bb6095b
Author: steve <steve@designerwallcoverings.com>
Date:   Thu Jul 16 11:06:44 2026 -0700

    refresh-clients.sh: read CSVs from project data/sources (launchd can't read ~/Downloads — TCC), nullglob rsync
---
 .gitignore                 |  1 +
 scripts/refresh-clients.sh | 21 ++++++++++++---------
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/.gitignore b/.gitignore
index e48dee7..f5aa3a2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -42,3 +42,4 @@ data/calendars-activation.json
 data/calendars-overrides.json
 public/data/clients-fm.json
 public/data/prospects-*.json
+data/sources/
diff --git a/scripts/refresh-clients.sh b/scripts/refresh-clients.sh
index be7cbf6..5013a9b 100755
--- a/scripts/refresh-clients.sh
+++ b/scripts/refresh-clients.sh
@@ -12,7 +12,9 @@ export PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:$PAT
 PROJ="$HOME/Projects/marketing-command-center"
 LOG=/tmp/mcc-clients-refresh.log
 LOCKDIR=/tmp/mcc-clients-refresh.lockd
-DL="$HOME/Downloads"
+# Source CSVs live IN the project (data/sources) — launchd can't read ~/Downloads
+# (macOS TCC privacy protection).
+SRC="$HOME/Projects/marketing-command-center/data/sources"
 exec >>"$LOG" 2>&1
 
 # single instance — mkdir is atomic + portable (macOS has no flock)
@@ -27,12 +29,12 @@ node "$HOME/Projects/filemaker-mcp/scripts/fetch-clients.js" || echo "!! FM extr
 
 # 2. Google-Places CSVs → contact-email enrichment (skips any source not present)
 crawl(){ local src; src=$(ls $1 2>/dev/null | head -1); [ -n "$src" ] && python3 scripts/crawl-csv-emails.py "$src" --slug "$2" --workers 14 || echo "-- skip $2 (no source csv)"; }
-crawl "$DL/*2000LAArchitects*dataset_crawler-google-places*"            la-architects
-crawl "$DL/*2000_arch.id.plus.LosAngeles*dataset_crawler-google-places*" la-arch-interior-design
-crawl "$DL/*1138architects-SF*dataset_crawler-google-places*"           sf-architects
-crawl "$DL/*nyc*dataset_crawler-google-places*"                         nyc-architects
-crawl "$DL/*Santa_Barbara*dataset_crawler-google-places*"              santa-barbara
-crawl "$DL/dataset_crawler-google-places_2026-07-10_07-01-32-993*"      places-batch-1
+crawl "$SRC/*2000LAArchitects*dataset_crawler-google-places*"            la-architects
+crawl "$SRC/*2000_arch.id.plus.LosAngeles*dataset_crawler-google-places*" la-arch-interior-design
+crawl "$SRC/*1138architects-SF*dataset_crawler-google-places*"           sf-architects
+crawl "$SRC/*nyc*dataset_crawler-google-places*"                         nyc-architects
+crawl "$SRC/*Santa_Barbara*dataset_crawler-google-places*"              santa-barbara
+crawl "$SRC/dataset_crawler-google-places_2026-07-10_07-01-32-993* $SRC/*_07-01-32-993*" places-batch-1
 
 # 3. Manifest the panel reads
 echo "-- manifest"
@@ -41,7 +43,8 @@ node scripts/build-clients-manifest.js
 # 4. Push refreshed data to the live MCC (behind Basic auth; not customer-facing)
 echo "-- rsync data → Kamatera"
 RP="root@45.61.58.125:/root/DW-Agents/marketing-command-center/public/data/"
-rsync -az public/data/clients-fm.json public/data/prospects-*.json public/data/clients-manifest.json "$RP" \
-  && echo "-- data pushed" || echo "!! rsync to Kamatera failed"
+shopt -s nullglob   # so an empty prospects-*.json glob drops out instead of failing rsync
+FILES=(public/data/clients-fm.json public/data/prospects-*.json public/data/clients-manifest.json)
+rsync -az "${FILES[@]}" "$RP" && echo "-- data pushed (${#FILES[@]} files)" || echo "!! rsync to Kamatera failed"
 
 echo "===== $(date '+%F %T') mcc-clients-refresh DONE ====="

← 87dc441 calendars: decouple launched layer so it stays live where bu · back to Marketing Command Center · clients: browser+local-model contact enrichment (direct Inst e0ae6f5 →