`scripts/refresh-clients.sh`

59 lines
#!/bin/bash
# refresh-clients.sh — fleet-managed refresh of the Clients & Prospects datasets
# that back the MCC "Clients" panel. Re-pulls FileMaker clients, re-crawls the
# Google-Places CSVs for contact emails (local, free), rebuilds the manifest,
# and pushes the refreshed data to the live MCC on Kamatera.
#
# Run by launchd (com.steve.mcc-clients-refresh) weekly + on-demand kickstart.
# Idempotent + single-instance (flock). Nothing here uploads to Constant Contact.
set -uo pipefail
# launchd runs with a minimal PATH — pin node/python/homebrew locations.
export PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:$PATH"
PROJ="$HOME/Projects/marketing-command-center"
LOG=/tmp/mcc-clients-refresh.log
LOCKDIR=/tmp/mcc-clients-refresh.lockd
# Source CSVs live IN the project (data/sources) — launchd can't read ~/Downloads
# (macOS TCC privacy protection).
SRC="$HOME/Projects/marketing-command-center/data/sources"
exec >>"$LOG" 2>&1

# single instance — mkdir is atomic + portable (macOS has no flock)
if ! mkdir "$LOCKDIR" 2>/dev/null; then echo "$(date '+%F %T') already running — skip"; exit 0; fi
trap 'rmdir "$LOCKDIR" 2>/dev/null' EXIT
echo "===== $(date '+%F %T') mcc-clients-refresh START ====="
cd "$PROJ" || exit 1

# 1. FileMaker clients (43k) → public/data/clients-fm.json
echo "-- FM clients"
node "$HOME/Projects/filemaker-mcp/scripts/fetch-clients.js" || echo "!! FM extract failed (keeping prior clients-fm.json)"

# 2. Google-Places CSVs → contact-email enrichment (skips any source not present)
crawl(){ local src; src=$(ls $1 2>/dev/null | head -1); [ -n "$src" ] && python3 scripts/crawl-csv-emails.py "$src" --slug "$2" --workers 14 || echo "-- skip $2 (no source csv)"; }
crawl "$SRC/*2000LAArchitects*dataset_crawler-google-places*"            la-architects
crawl "$SRC/*2000_arch.id.plus.LosAngeles*dataset_crawler-google-places*" la-arch-interior-design
crawl "$SRC/*1138architects-SF*dataset_crawler-google-places*"           sf-architects
crawl "$SRC/*nyc*dataset_crawler-google-places*"                         nyc-architects
crawl "$SRC/*Santa_Barbara*dataset_crawler-google-places*"              santa-barbara
crawl "$SRC/dataset_crawler-google-places_2026-07-10_07-01-32-993* $SRC/*_07-01-32-993*" places-batch-1

# 2b. Deep browser + local-model enrichment — direct Instagram/LinkedIn links +
#     email gap-fill on JS-rendered sites the HTTP pass can't read. Local ($0).
echo "-- browser + local-model enrichment"
shopt -s nullglob
for pj in public/data/prospects-*.json; do
  node scripts/enrich-contacts-browser.mjs "$pj" --workers 6 --model qwen2.5:latest || echo "!! enrich failed: $pj"
done

# 3. Manifest the panel reads
echo "-- manifest"
node scripts/build-clients-manifest.js

# 4. Push refreshed data to the live MCC (behind Basic auth; not customer-facing)
echo "-- rsync data → Kamatera"
RP="root@45.61.58.125:/root/DW-Agents/marketing-command-center/public/data/"
shopt -s nullglob   # so an empty prospects-*.json glob drops out instead of failing rsync
FILES=(public/data/clients-fm.json public/data/prospects-*.json public/data/clients-manifest.json)
rsync -az "${FILES[@]}" "$RP" && echo "-- data pushed (${#FILES[@]} files)" || echo "!! rsync to Kamatera failed"

echo "===== $(date '+%F %T') mcc-clients-refresh DONE ====="