`scripts/harvest_trade_openclaw.sh`

113 lines
#!/bin/bash
# Daily trade-portal harvest THROUGH the persistent authed openclaw real-Chrome session.
#
# DURABLE replacement for harvest_trade.js: that script launched BUNDLED puppeteer, which
# SDG's anti-bot wall fingerprint-flags on every unattended run -> NEEDS_REAUTH set daily
# (TK-10400 / root cause behind TK-00058). openclaw drives Steve's real Chrome (persistent
# 'openclaw' profile, CDP :18800), which the wall lets through, and the session persists
# across daily runs until it naturally expires — so the cadence keeps flowing.
#
# Same output contract as harvest_trade.js (downstream parse_trade.py -> load_trade.sql ->
# reconcile.sql is UNCHANGED): data/sanderson_<cat>_raw.txt + data/trade_summary.json,
# and the same NEEDS_REAUTH marker on auth loss.
#
# Env: CAP_WALLPAPER / CAP_FABRIC override the load-more iteration caps (for a quick smoke test).
set -uo pipefail
export PATH="$HOME/.npm-global/bin:/opt/homebrew/bin:$PATH"   # openclaw CLI + node under cron
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
DATA="$ROOT/data"; SC="$ROOT/scripts"
BASE="https://trade.sandersondesigngroup.com/us"
CAP_WALLPAPER="${CAP_WALLPAPER:-120}"
CAP_FABRIC="${CAP_FABRIC:-260}"
log(){ echo "$1"; echo "$(date -u +%FT%TZ) $1" >> "$DATA/trade_harvest.log"; }

# 0. openclaw browser must be up (persistent authed profile)
if ! openclaw browser status 2>/dev/null | grep -q "running: true"; then
  log "AUTH/BROWSER: openclaw browser not running -> NEEDS_REAUTH (start via scripts/reauth.sh)"
  echo "$(date -u +%FT%TZ) openclaw browser down -> openclaw reauth" > "$DATA/NEEDS_REAUTH"
  exit 3
fi

# 1. Open a FRESH light tab for the run. Reusing a pre-loaded PLP tab (e.g. fabric
#    #limit=3120) makes document.body.innerText huge, and the openclaw CDP gateway
#    times out (~20s) reading it -> a FALSE "auth lost". A fresh tab starts light;
#    openclaw_harvest_cat.sh then navigates it per category and loads incrementally.
list_sand(){ openclaw browser tabs --json 2>/dev/null | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>{try{const j=JSON.parse(s);for(const t of (j.tabs||[]))if(t.type==="page"&&/trade\.sandersondesigngroup\.com/.test(t.url||""))console.log(t.targetId);}catch(e){}})'; }
RUN_START="$(date +%s)"
before="$(list_sand)"
openclaw browser open "$BASE/" >/dev/null 2>&1; sleep 6
# Compute the new-tab diff ONCE (two independent diffs race: the delayed tab could appear
# between them, making FRESH_TAB=1 while SAND is a pre-existing tab -> we'd close a tab we
# never opened). Single source of truth: new_tabs.
new_tabs="$(comm -13 <(echo "$before" | sort) <(list_sand | sort))"
SAND="$(echo "$new_tabs" | grep -v '^$' | head -1)"
if [ -n "$SAND" ]; then FRESH_TAB=1; else SAND="$(list_sand | head -1)"; FRESH_TAB=0; fi   # fallback: reuse
[ -z "$SAND" ] && { log "AUTH: no Sanderson tab and open failed -> NEEDS_REAUTH"; echo "$(date -u +%FT%TZ) no sanderson tab -> openclaw reauth" > "$DATA/NEEDS_REAUTH"; exit 3; }
log "using openclaw target $SAND (fresh=$FRESH_TAB)"

# 2. auth check — cheap selector query (NOT a full innerText scan, which times out on
#    a heavy PLP). Retries once, and treats a timeout/empty read as UNKNOWN -> proceeds
#    to harvest rather than false-failing; the harvest ('Viewing X of Y' vs login text)
#    is the real auth proof, and only a definite 'NO' writes NEEDS_REAUTH.
authcheck(){ openclaw browser evaluate --timeout 30000 --target-id "$SAND" --fn "() => { const q=document.querySelector('a[href*=\"logout\" i],a[href*=\"signout\" i],a[href*=\"account\" i],[class*=\"account\" i]'); const h=(document.body.innerText||'').slice(0,3000).toLowerCase(); return (q||h.includes('my account')||h.includes('quick order')||h.includes('logout'))?'YES':'NO'; }" 2>/dev/null | tr -d '[:space:]"'; }
AUTH="$(authcheck)"; [ "$AUTH" != "YES" ] && { sleep 3; AUTH="$(authcheck)"; }
if [ "$AUTH" = "NO" ]; then
  log "AUTH LOST -> NEEDS_REAUTH written; skipping trade harvest (rerun scripts/reauth.sh openclaw login)"
  echo "$(date -u +%FT%TZ) trade portal session expired -> openclaw reauth" > "$DATA/NEEDS_REAUTH"
  exit 3
fi
[ "$AUTH" = "YES" ] && rm -f "$DATA/NEEDS_REAUTH" || log "AUTH UNKNOWN (evaluate empty/timeout) — proceeding; harvest is the real test"

# 3. harvest each category through the authed tab (existing per-cat primitive)
for cat in wallpaper fabric; do
  cap_var="CAP_${cat^^}"; cap="${!cap_var}"
  log "harvesting $cat via openclaw (cap=$cap)..."
  bash "$SC/openclaw_harvest_cat.sh" "$SAND" "$cat" "$cap" >> "$DATA/trade_harvest.log" 2>&1 \
    || log "$cat harvest returned nonzero (partial raw kept — dump guard preserves last good data)"
done

# 3b. FRESHNESS GATE (Cody HOLE 3/4): the dump guard in openclaw_harvest_cat.sh only
# overwrites the raw file when the new dump is BIGGER. So an expired session (small login
# page) leaves the PRIOR run's large raw file untouched — and a naive summary would read
# that stale file, look healthy, and silently reload old prices. Trust a category ONLY if
# its raw file was actually written THIS run (mtime >= RUN_START). This is auth-signal-
# independent, so it also covers the unverified "public teaser PLP" case.
mt(){ stat -f %m "$1" 2>/dev/null || echo 0; }
FRESH=0
for cat in wallpaper fabric; do
  f="$DATA/sanderson_${cat}_raw.txt"
  if [ -f "$f" ] && [ "$(mt "$f")" -ge "$RUN_START" ]; then FRESH=$((FRESH+1)); else log "STALE: $cat raw not updated this run (dump rejected / auth loss) — will NOT be trusted"; fi
done
if [ "$FRESH" -eq 0 ]; then
  log "NO category harvested fresh data this run -> NEEDS_REAUTH; stale data NOT reloaded downstream"
  echo "$(date -u +%FT%TZ) no fresh harvest (both raw files stale) -> openclaw reauth" > "$DATA/NEEDS_REAUTH"
  [ "${FRESH_TAB:-0}" = "1" ] && openclaw browser close "$SAND" >/dev/null 2>&1
  exit 3
fi

# 4. summary from the raw dumps (Viewing X of Y)
node -e '
const fs=require("fs");const DATA=process.argv[1];const out={};
for(const cat of ["wallpaper","fabric"]){
  let shown=0,total=0;
  try{const t=fs.readFileSync(DATA+"/sanderson_"+cat+"_raw.txt","utf8");const m=t.match(/Viewing\s+(\d+)\s+of\s+(\d+)/i);if(m){shown=+m[1];total=+m[2];}}catch(e){}
  out[cat]={shown,total};
}
fs.writeFileSync(DATA+"/trade_summary.json",JSON.stringify(out,null,2));
// Data-driven auth-loss detection: a logged-out session shows NO "Viewing X of Y" on
// any product page. If BOTH categories harvested 0, treat it as a real reauth need
// (backstop for the flaky live auth-evaluate).
if(out.wallpaper.total===0 && out.fabric.total===0){
  fs.writeFileSync(DATA+"/NEEDS_REAUTH", new Date().toISOString()+" 0 products across both categories -> likely logged out; rerun scripts/reauth.sh openclaw login\n");
  console.log("TRADE HARVEST: 0/0 both cats -> NEEDS_REAUTH written");
  process.exit(3);
}
console.log("TRADE HARVEST OK "+JSON.stringify(out));
' "$DATA"
HARVEST_RC=$?

# 5. close the fresh tab we opened (don't leak a tab per daily run); leave a reused tab alone
[ "${FRESH_TAB:-0}" = "1" ] && openclaw browser close "$SAND" >/dev/null 2>&1 && log "closed fresh openclaw tab $SAND"
log "openclaw trade harvest complete (rc=${HARVEST_RC:-0})"
exit "${HARVEST_RC:-0}"