Add detached poll-and-download bg job

8e805d9967c54979c8c5b8a951d4f3e22c69f14e · 2026-08-06 15:59:29 -0700 · Steve
Files touched

A poll-and-download.sh
Diff

commit 8e805d9967c54979c8c5b8a951d4f3e22c69f14e
Author: Steve <steve@designerwallcoverings.com>
Date:   Thu Aug 6 15:59:29 2026 -0700

    Add detached poll-and-download bg job
---
 poll-and-download.sh | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)

diff --git a/poll-and-download.sh b/poll-and-download.sh
new file mode 100755
index 0000000..e62cbbf
--- /dev/null
+++ b/poll-and-download.sh
@@ -0,0 +1,64 @@
+#!/bin/bash
+# Detached poll-until-key-then-download job for USPTO TRTYRAP.
+# Polls the ODP key-reveal page via openclaw; when a key (or Generate button)
+# appears, captures/generates the key -> .apikey (chmod 600), then runs the
+# downloader. Logs everything to poll.log. Caps at ~2h so it can't run forever.
+set -uo pipefail
+cd "$(dirname "$0")"
+LOG=poll.log
+KEY_URL="https://data.uspto.gov/apikey/key-reveal"
+MAX=60          # iterations
+SLEEP=120       # seconds between polls
+log(){ echo "[$(date '+%H:%M:%S')] $*" >> "$LOG"; }
+
+extract_state(){
+  openclaw browser open "$KEY_URL" >/dev/null 2>&1; sleep 5
+  local tid
+  tid=$(openclaw browser tabs 2>/dev/null | grep -iB1 "Manage API\|apikey\|key-reveal\|key-warning" | grep "id:" | head -1 | awk '{print $2}')
+  openclaw browser evaluate ${tid:+--target-id "$tid"} --fn "() => {
+    const t=document.body.innerText||'';
+    const cand=[...document.querySelectorAll('input,code,pre,span,td')].map(e=>(e.value||e.textContent||'').trim()).find(v=>/^[A-Za-z0-9_-]{28,60}\$/.test(v));
+    const gen=[...document.querySelectorAll('a,button')].find(b=>/generate|create.*key|new api key/i.test(b.textContent));
+    let state='BLOCKED';
+    if(cand) state='KEY_PRESENT';
+    else if(gen) state='CAN_GENERATE';
+    else if(/not currently verified with ID\.me/i.test(t)) state='NOT_VERIFIED';
+    else if(/sign in|registration requirement/i.test(t)) state='SIGNED_OUT';
+    return JSON.stringify({state, key:cand||''});
+  }" 2>/dev/null | tail -1
+}
+
+click_generate(){
+  local tid
+  tid=$(openclaw browser tabs 2>/dev/null | grep -iB1 "Manage API\|apikey\|key-reveal" | grep "id:" | head -1 | awk '{print $2}')
+  openclaw browser evaluate ${tid:+--target-id "$tid"} --fn "() => { const b=[...document.querySelectorAll('a,button')].find(x=>/generate|create.*key|new api key/i.test(x.textContent)); if(b){b.click();return 'gen-clicked'} return 'no-gen' }" >/dev/null 2>&1
+  sleep 6
+}
+
+log "=== poll-and-download started (pid $$) ==="
+for i in $(seq 1 $MAX); do
+  raw=$(extract_state)
+  state=$(echo "$raw" | sed -E 's/.*"state":"([^"]*)".*/\1/')
+  log "poll $i/$MAX -> $state"
+  if [ "$state" = "CAN_GENERATE" ]; then
+    log "clicking Generate API key"; click_generate; raw=$(extract_state); state=$(echo "$raw" | sed -E 's/.*"state":"([^"]*)".*/\1/')
+    log "post-generate -> $state"
+  fi
+  if [ "$state" = "KEY_PRESENT" ]; then
+    key=$(echo "$raw" | python3 -c "import json,sys;print(json.load(sys.stdin).get('key',''))" 2>/dev/null)
+    if [ -n "$key" ]; then
+      printf '%s' "$key" > .apikey; chmod 600 .apikey
+      log "KEY captured (…${key: -4}); starting download"
+      break
+    fi
+  fi
+  sleep $SLEEP
+done
+
+if [ -s .apikey ]; then
+  log "=== running download_backfill.py ==="
+  ./.venv/bin/python download_backfill.py >> "$LOG" 2>&1
+  log "=== download finished (exit $?) ==="
+else
+  log "=== gave up: no key after $MAX polls (still not signed-in/verified) ==="
+fi
← ba701cf USPTO TRTYRAP bulk-download scaffold (API path, WAF findings · back to Uspto Trtyrap · Fix bg-job state parser (double-decode openclaw output) fd59c5c →