`drain.sh`

68 lines
#!/bin/zsh
# Daily auto-drain of the 5-field auto-fixable worklist.
# Budget-aware (claims the day's leftover Shopify variant headroom via budget.cjs),
# idempotent + resumable (skips anything already in the result JSON), DELETE-nothing.
# Keep polling for new work; the separate display pass shares this job. Singleton-guarded.
set -u
export PATH="/opt/homebrew/opt/postgresql@14/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
SK="$HOME/Projects/dw-five-field-step0"
LOG="$SK/drain.log"
LOCKDIR="$SK/.drain.lock"
EXE="$SK/bulk-fivefield-exec.py"
RESULT="$SK/out/bulk-fivefield-result.json"
LABEL="com.steve.dw-fivefield-drain"
ts(){ date '+%Y-%m-%d %H:%M:%S'; }

# singleton (macOS has no flock) — atomic mkdir lock, stale after 2h
if ! mkdir "$LOCKDIR" 2>/dev/null; then
  if [ -d "$LOCKDIR" ] && [ "$(find "$LOCKDIR" -maxdepth 0 -mmin +120 2>/dev/null)" ]; then
    rmdir "$LOCKDIR" 2>/dev/null; mkdir "$LOCKDIR" 2>/dev/null || { echo "[$(ts)] locked, skip" >>"$LOG"; exit 0; }
  else
    echo "[$(ts)] already running, skip" >>"$LOG"; exit 0
  fi
fi
trap 'rmdir "$LOCKDIR" 2>/dev/null' EXIT

# Count current product/repair identities against the same audit as the executor.
# Never subtract historical event counts from current worklist row counts.
COUNTS=$(python3 "$SK/worklist_progress.py" --audit "$RESULT" --counts) || {
  echo "[$(ts)] cannot read current repair progress — refusing an unverified run" >>"$LOG"; exit 1
}
read TOTAL DONE REMAINING <<< "$COUNTS"
echo "[$(ts)] drain start — done=$DONE / total=$TOTAL" >>"$LOG"

if [ "$REMAINING" -eq 0 ]; then
  echo "[$(ts)] variant worklist complete ($DONE/$TOTAL); executor may still run the separately enabled display-variant pass" >>"$LOG"
fi

# run one SMALL per-slot batch so the day's 500 'backlog' budget spreads evenly
# across the 24 hourly slots (~21/slot) instead of bursting at once. The shared
# budget.cjs 'backlog' category still hard-caps the daily total at 500.
SLOT_MAX="${DW_BACKLOG_SLOT_MAX:-30}"

# END-OF-DAY RECLAIM (DTD 2/2 verdict C, 2026-07-02): from 21:00 budget.cjs lets
# 'backlog' sweep the day's unspent global remainder (~60/day historically) that
# would otherwise expire at midnight. Ask for more in the 21/22/23 slots so the
# sweep isn't capped by the request size; the ledger still bounds every grant.
if [ "$(date +%H)" -ge 21 ]; then
  SLOT_MAX="${DW_BACKLOG_RECLAIM_SLOT_MAX:-150}"
fi

# display_variant legacy tag backfill (Option A, DTD 3/3 2026-06-24).
# GATED OFF by default: the executor only runs the tag pass when env
# DISPLAY_VARIANT_BACKFILL=1 is present. tagsAdd is NOT a variant create -> it
# does NOT debit the variant budget. Own per-slot cap (DISPLAY_VARIANT_SLOT_MAX,
# default 30) + same inter-call sleep; runs AFTER the variant drain each slot.
# To GO LIVE: add DISPLAY_VARIANT_BACKFILL=1 to this job's environment (the
# launchd plist EnvironmentVariables block) and reload the plist. Until then the
# scheduled run logs "backfill GATED OFF" and changes no tags.
echo "[$(ts)] display_variant backfill = ${DISPLAY_VARIANT_BACKFILL:-0} (slot_max=${DISPLAY_VARIANT_SLOT_MAX:-30})" >>"$LOG"
python3 "$EXE" --max "$SLOT_MAX" >>"$LOG" 2>&1
RC=$?
COUNTS2=$(python3 "$SK/worklist_progress.py" --audit "$RESULT" --counts) || {
  echo "[$(ts)] drain end rc=$RC — progress unavailable" >>"$LOG"; exit 1
}
read TOTAL2 DONE2 REMAINING2 <<< "$COUNTS2"
echo "[$(ts)] drain end rc=$RC — now done=$DONE2 / total=$TOTAL2 remaining=$REMAINING2 (this run +$((DONE2-DONE)))" >>"$LOG"
exit "$RC"