`run_pipeline.sh`

72 lines
#!/usr/bin/env bash
# run_pipeline.sh — one full LIVE cycle: refresh current permits, recover contractors,
# merge into a growing master, refresh stats, commit if anything changed. $0, idempotent.
# Safe to run on a loop — the LADBS detail fetch is cached + rate-limited.
set -euo pipefail
cd "$(dirname "$0")"

# overlap guard — never let two cycles run at once (mkdir is atomic)
LOCK="tmp/.cycle.lock"
mkdir -p tmp
if ! mkdir "$LOCK" 2>/dev/null; then
  echo "another cycle is running (lock held) — exiting."; exit 0
fi
trap 'rmdir "$LOCK" 2>/dev/null || true' EXIT

echo "=== [$(date '+%Y-%m-%d %H:%M')] LIVE pipeline cycle ==="

# 1. pull current permits (last 365d, >$1M commercial/new) — updates ~daily upstream
python3 pull_permits_live.py --min-valuation 1000000 --days 365 --out permits_live.csv

# 2. recover live contractor + CSLB license per permit (cached -> only new permits fetch)
python3 enrich_live_contractor.py --in permits_live.csv --out permits_live_enriched.csv --delay 1.0

# 3. merge into the accumulating master (dedupe by permit_nbr; keep first-seen date)
python3 - <<'PY'
import csv, os
MASTER="master_live.csv"
new=list(csv.DictReader(open("permits_live_enriched.csv",newline="",encoding="utf-8")))
if not new: raise SystemExit("no enriched rows")
cols=list(new[0].keys())
if "first_seen" not in cols: cols=cols+["first_seen"]
seen={}
if os.path.exists(MASTER):
    for r in csv.DictReader(open(MASTER,newline="",encoding="utf-8")):
        seen[r["permit_nbr"]]=r
added=0
for r in new:
    k=r["permit_nbr"]
    if k in seen:  # keep original first_seen
        r["first_seen"]=seen[k].get("first_seen","")
    else:
        r["first_seen"]=r.get("issue_date","")
        added+=1
    seen[k]=r
# STABLE sort (valuation desc, then permit_nbr) so unchanged data -> byte-identical file
rows=sorted(seen.values(), key=lambda r: (-float(r.get("valuation") or 0), r.get("permit_nbr","")))
with open(MASTER,"w",newline="",encoding="utf-8") as f:
    w=csv.DictWriter(f,fieldnames=cols); w.writeheader()
    for r in rows: w.writerow({c:r.get(c,"") for c in cols})
print(f"  master: {len(rows)} total permits (+{added} new this cycle)")
# stats file carries ONLY stable totals (no volatile per-cycle counter) so no-op
# cycles produce no diff and the git guard below correctly skips them
open("live_stats.txt","w").write(
    f"master_live permits: {len(rows)}\n"
    f"total value: ${sum(float(r.get('valuation') or 0) for r in rows):,.0f}\n")
PY

# 4. regenerate the ranked live board (the loop's analytical product)
python3 live_board.py

# 5. commit if the tracked artifacts changed (master + stats + board force-tracked; data is the point)
git add -f master_live.csv live_stats.txt LIVE-BOARD.md 2>/dev/null || true
git add -A
if ! git diff --cached --quiet; then
  git commit -q -m "live cycle: refresh permits + contractors, merge master ($(date '+%Y-%m-%d %H:%M'))" \
    --author="Steve Abrams <steve@designerwallcoverings.com>"
  echo "  committed."
else
  echo "  no changes to commit."
fi
echo "=== cycle done (cost: \$0) ==="