`scripts/openclaw_harvest_cat.sh`

38 lines
#!/bin/bash
# Harvest one trade PLP category THROUGH the authed openclaw real-Chrome tab.
# Usage: openclaw_harvest_cat.sh <target-id> <category> <max-iters>
# Dumps full innerText to data/sanderson_<category>_raw.txt after every few batches
# (innerText is cumulative, so latest dump = most-complete; survives a tab crash).
SAND="$1"; CAT="$2"; CAP="${3:-40}"
THROTTLE="${THROTTLE:-2}"      # seconds between load-more clicks
STABLE_MAX="${STABLE_MAX:-4}"  # consecutive no-progress reads before giving up
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
RAW="$ROOT/data/sanderson_${CAT}_raw.txt"
EV(){ openclaw browser evaluate --timeout 60000 --target-id "$SAND" --fn "$1" 2>/dev/null; }

# reset to clean page 1
openclaw browser navigate --target-id "$SAND" "https://trade.sandersondesigngroup.com/us/products/${CAT}" >/dev/null 2>&1
sleep 5

dump(){
  # write full innerText to raw file (strip surrounding JSON quotes, unescape)
  EV "() => document.body.innerText" | node -e 'let s="";process.stdin.on("data",d=>s+=d);process.stdin.on("end",()=>{try{process.stdout.write(JSON.parse(s));}catch(e){process.stdout.write(s);}})' > "$RAW.tmp"
  # guard: only replace the raw file if the new dump is bigger (never overwrite good data with an empty/timed-out read)
  newb=$(wc -c < "$RAW.tmp" | tr -d ' '); oldb=$( [ -f "$RAW" ] && wc -c < "$RAW" | tr -d ' ' || echo 0 )
  if [ "$newb" -gt 200 ] && [ "$newb" -ge "$oldb" ]; then mv "$RAW.tmp" "$RAW"; echo "  dumped $newb bytes -> $RAW"; else echo "  skip dump (new=$newb old=$oldb, kept existing)"; rm -f "$RAW.tmp"; fi
}

last=-1; stable=0
for i in $(seq 1 "$CAP"); do
  info=$(EV "() => { const m=document.body.innerText.match(/Viewing\s+(\d+)\s+of\s+(\d+)/i); const shown=m?+m[1]:-1, total=m?+m[2]:-1; if(shown>=0 && shown<total){const b=Array.from(document.querySelectorAll('button,a')).find(x=>/^load more/i.test((x.textContent||'').trim())&&x.offsetParent!==null); if(b)b.click(); else window.scrollTo(0,document.body.scrollHeight);} return shown+'/'+total; }" | tr -d '[:space:]"')
  shown=${info%/*}; total=${info#*/}
  echo "iter $i: $info"
  # skip bad reads (don't count toward stable)
  if [ -z "$total" ] || [ "$shown" = "-1" ]; then sleep "$THROTTLE"; continue; fi
  if [ "$shown" -ge "$total" ] 2>/dev/null; then echo "COMPLETE"; dump; break; fi
  if [ "$shown" = "$last" ]; then stable=$((stable+1)); [ $stable -ge $STABLE_MAX ] && { echo "STABLE-STOP at $shown/$total"; dump; break; }; else stable=0; last=$shown; fi
  [ $((i % 5)) -eq 0 ] && dump   # periodic crash-safe snapshot
  sleep "$THROTTLE"
done
echo "HARVEST-DONE $CAT"