`scripts/refine-loop.sh`

26 lines
#!/bin/bash
# TK-10708  "Refine all night long" (Steve). Each cycle: pull every feed -> record EVERY article into the
# growing archive + deals into the store -> pull CA public-record deeds -> rebuild the flyers site -> commit.
# LOCAL only (no auto-deploy — that stays Steve-gated). 90-min spacing so Google News rate-limits recover.
export PATH=/opt/homebrew/bin:/opt/homebrew/opt/postgresql@14/bin:/usr/bin:/bin:/usr/sbin:/sbin
cd "$HOME/Projects/re-flyer-aggregator" || exit 1
LOG=data/refine-loop.log
echo "=== refine-loop START $(date) ===" >> "$LOG"
n=0
while true; do
  n=$((n+1))
  echo "--- cycle $n $(date) ---" >> "$LOG"
  node scripts/cre-news-monitor.mjs   >> "$LOG" 2>&1   # records every article -> article-archive.json (no cap)
  node scripts/ingest-rentv.mjs        >> "$LOG" 2>&1   # RENTV direct (Steve rentv.agentabrams.com API)
  node scripts/ingest-public-deals.mjs >> "$LOG" 2>&1  # CA deeds/DTT
  node scripts/build-flyers-site.mjs  >> "$LOG" 2>&1   # rebuild flyers.json + index
  node scripts/build-property-index.mjs >> "$LOG" 2>&1  # per-property index (news+RENTV)
  node scripts/ingest-usre-parcels.mjs  >> "$LOG" 2>&1  # scale-up: enrich w/ usre commercial parcels (re-usre TK-10708) — MUST run AFTER build-property-index
  git add data/ public/ scripts/ >/dev/null 2>&1
  git commit -q -m "refine-loop cycle $n: +articles/deals $(date '+%m-%d %H:%M')" >/dev/null 2>&1
  arch=$(node -e 'try{console.log(require("./data/article-archive.json").length)}catch(e){console.log(0)}' 2>/dev/null)
  store=$(node -e 'try{console.log(require("./data/deal-store.json").length)}catch(e){console.log(0)}' 2>/dev/null)
  echo "  cycle $n done $(date) — archive=$arch, store=$store" >> "$LOG"
  sleep 5400   # 90 min — spacing to dodge Google News rate-limits
done