`scripts/openclaw_harvest_brand_cat.sh`

42 lines
#!/bin/bash
# Harvest one BRAND+category trade PLP THROUGH the authed openclaw real-Chrome tab.
# Generalization of openclaw_harvest_cat.sh (TK-10877) — parameterized by brand PLP URL.
# Usage: openclaw_harvest_brand_cat.sh <target-id> <plp-url> <raw-out-file> [max-iters]
# Dumps full innerText to <raw-out-file> after every few batches (innerText is cumulative,
# so latest dump = most-complete; survives a tab crash). Same fresh-light-tab / incremental
# load-more pattern that avoids the CDP 20s innerText timeout.
SAND="$1"; PLP_URL="$2"; RAW="$3"; CAP="${4:-40}"
THROTTLE="${THROTTLE:-2}"
STABLE_MAX="${STABLE_MAX:-5}"
EV(){ openclaw browser evaluate --timeout 60000 --target-id "$SAND" --fn "$1" 2>/dev/null; }

openclaw browser navigate --target-id "$SAND" "$PLP_URL" >/dev/null 2>&1
sleep 6

# Prices + stock lazy-load per card as it enters the viewport. Scroll the whole list in
# steps (dwell for the fetch) so EVERY card hydrates its $SSP/$Trade before we snapshot.
hydrate(){
  EV "async () => { const h=document.body.scrollHeight; const steps=Math.min(40,Math.max(12,Math.ceil(h/1200))); for(let s=1;s<=steps;s++){ window.scrollTo(0,h*s/steps); await new Promise(r=>setTimeout(r,600)); } window.scrollTo(0,0); await new Promise(r=>setTimeout(r,400)); return 'ok'; }" >/dev/null 2>&1
}

dump(){
  hydrate
  EV "() => document.body.innerText" | node -e 'let s="";process.stdin.on("data",d=>s+=d);process.stdin.on("end",()=>{try{process.stdout.write(JSON.parse(s));}catch(e){process.stdout.write(s);}})' > "$RAW.tmp"
  newb=$(wc -c < "$RAW.tmp" | tr -d ' '); oldb=$( [ -f "$RAW" ] && wc -c < "$RAW" | tr -d ' ' || echo 0 )
  if [ "$newb" -gt 200 ] && [ "$newb" -ge "$oldb" ]; then mv "$RAW.tmp" "$RAW"; echo "  dumped $newb bytes -> $RAW"; else echo "  skip dump (new=$newb old=$oldb, kept existing)"; rm -f "$RAW.tmp"; fi
}

last=-1; stable=0
for i in $(seq 1 "$CAP"); do
  info=$(EV "() => { const m=document.body.innerText.match(/Viewing\s+(\d+)\s+of\s+(\d+)/i); const shown=m?+m[1]:-1, total=m?+m[2]:-1; if(shown>=0 && shown<total){const b=Array.from(document.querySelectorAll('button,a')).find(x=>/^load more/i.test((x.textContent||'').trim())&&x.offsetParent!==null); if(b)b.click(); else window.scrollTo(0,document.body.scrollHeight);} return shown+'/'+total; }" | tr -d '[:space:]"')
  shown=${info%/*}; total=${info#*/}
  echo "iter $i: $info"
  if [ -z "$total" ] || [ "$shown" = "-1" ]; then sleep "$THROTTLE"; continue; fi
  if [ "$shown" -ge "$total" ] 2>/dev/null; then echo "COMPLETE"; dump; break; fi
  if [ "$shown" = "$last" ]; then stable=$((stable+1)); [ $stable -ge $STABLE_MAX ] && { echo "STABLE-STOP at $shown/$total"; dump; break; }; else stable=0; last=$shown; fi
  [ $((i % 5)) -eq 0 ] && dump
  sleep "$THROTTLE"
done
dump
echo "HARVEST-DONE $RAW"