← back to Uspto Trtyrap
USPTO TRTYRAP bulk-download scaffold (API path, WAF findings)
ba701cf82319563b2acf01716d6ba0bc4718a04a · 2026-08-06 15:21:46 -0700 · Steve
Files touched
A .gitignoreA README.mdA dl_one.pyA download_backfill.pyA fetch_list.py
Diff
commit ba701cf82319563b2acf01716d6ba0bc4718a04a
Author: Steve <steve@designerwallcoverings.com>
Date: Thu Aug 6 15:21:46 2026 -0700
USPTO TRTYRAP bulk-download scaffold (API path, WAF findings)
---
.gitignore | 9 +++++
README.md | 21 ++++++++++
dl_one.py | 16 ++++++++
download_backfill.py | 107 +++++++++++++++++++++++++++++++++++++++++++++++++++
fetch_list.py | 29 ++++++++++++++
5 files changed, 182 insertions(+)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..533ec5e
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,9 @@
+.venv/
+.cookie
+files/
+*.zip
+list.json
+filenames.txt
+__pycache__/
+*.log
+.DS_Store
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..8d03d4e
--- /dev/null
+++ b/README.md
@@ -0,0 +1,21 @@
+# USPTO TRTYRAP bulk download
+
+**Dataset:** TRTYRAP — Trademark Full Text XML Data (No Images), Annual Applications,
+backfile APR 1884 – DEC 31 2025. 177 files (~15–20 GB). Fields: word mark, serial #,
+registration #, filing/registration dates, goods & services, classifications, status
+codes, design search codes, pseudo marks.
+
+## Why the API (not scraping)
+The raw file bytes (`data.uspto.gov/files/TRTYRAP/*.zip`, CloudFront-signed) sit behind
+**AWS WAF bot-control** that serves the SPA shell to plain curl AND curl_cffi Chrome-TLS
+impersonation. Only the real browser — or the **official Bulk Data API with `X-API-KEY`** —
+gets the files. The API key requires a one-time **ID.me** verification of the USPTO.gov
+account (account.uspto.gov/profile#idverification).
+
+## Run
+```
+export USPTO_API_KEY=... # or write it to .apikey (gitignored)
+.venv/bin/python download_backfill.py
+```
+Resume-safe (skips completed files, verifies PK zip magic). Endpoint shapes in
+`download_backfill.py` are best-effort per ODP docs; confirm against the live key.
diff --git a/dl_one.py b/dl_one.py
new file mode 100644
index 0000000..655bffa
--- /dev/null
+++ b/dl_one.py
@@ -0,0 +1,16 @@
+from curl_cffi import requests
+import json,sys
+cookie=open('.cookie').read().strip()
+base="https://data.uspto.gov/ui/datasets/products/files/TRTYRAP/"
+name="apc18840407-20251231-1.zip"
+H={"Accept":"application/json","Referer":"https://data.uspto.gov/bulkdata/datasets/trtyrap","Cookie":cookie}
+r=requests.get(base+name, headers=H, impersonate="chrome", timeout=60)
+print("sign step:", r.status_code, r.headers.get("content-type"), "len", len(r.content))
+try: loc=r.json().get("Location")
+except Exception: print("no json:", r.text[:80]); sys.exit(1)
+print("got signed URL:", bool(loc))
+# now download the signed URL with chrome impersonation
+r2=requests.get(loc, headers={"Referer":"https://data.uspto.gov/","Cookie":cookie}, impersonate="chrome", timeout=300, stream=True)
+print("file step:", r2.status_code, r2.headers.get("content-type"), "clen", r2.headers.get("content-length"))
+first=next(r2.iter_content(chunk_size=8), b"")
+print("magic:", first[:4].hex(), "(504b0304 = PK zip)")
diff --git a/download_backfill.py b/download_backfill.py
new file mode 100644
index 0000000..f4d9bf1
--- /dev/null
+++ b/download_backfill.py
@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""
+USPTO ODP Bulk Data downloader for TRTYRAP
+(Trademark Full Text XML Data (No Images) - Annual Applications, 1884-2025 backfile).
+
+Uses the OFFICIAL Open Data Portal Bulk Data API with an X-API-KEY
+(obtained after ID.me verification at account.uspto.gov). This is the
+sanctioned path -- it sidesteps the AWS WAF bot-control that blocks
+curl / curl_cffi on the raw file bytes.
+
+Key resolution order:
+ 1) env USPTO_API_KEY
+ 2) file .apikey in this dir (gitignored)
+
+Run: .venv/bin/python download_backfill.py
+Resume-safe: skips files already fully downloaded (size match).
+"""
+import os, sys, json, time, pathlib
+from curl_cffi import requests
+
+HERE = pathlib.Path(__file__).parent
+OUT = HERE / "files"; OUT.mkdir(exist_ok=True)
+PRODUCT = "TRTYRAP"
+FROM, TO = "1884-04-07", "2025-12-31"
+
+def get_key():
+ k = os.environ.get("USPTO_API_KEY")
+ if not k and (HERE / ".apikey").exists():
+ k = (HERE / ".apikey").read_text().strip()
+ if not k:
+ sys.exit("No API key. Set USPTO_API_KEY or write it to .apikey")
+ return k
+
+def api_get(url, key, **kw):
+ h = {"X-API-KEY": key, "Accept": "application/json"}
+ return requests.get(url, headers=h, impersonate="chrome", timeout=kw.pop("timeout", 60), **kw)
+
+def list_files(key):
+ # Primary ODP bulk-data product endpoint (returns file bag w/ download URIs).
+ candidates = [
+ f"https://data.uspto.gov/api/v1/datasets/products/{PRODUCT}?fileDataFromDate={FROM}&fileDataToDate={TO}",
+ f"https://data.uspto.gov/api/v1/datasets/products/{PRODUCT}",
+ ]
+ for url in candidates:
+ r = api_get(url, key)
+ if r.status_code == 200 and "json" in (r.headers.get("content-type") or ""):
+ d = r.json()
+ files = extract_files(d)
+ if files:
+ return files
+ print(f" [list] {url} -> {r.status_code} {r.headers.get('content-type')}")
+ sys.exit("Could not list files from the API -- inspect the endpoint shape with the real key.")
+
+def extract_files(o, out=None):
+ out = [] if out is None else out
+ if isinstance(o, dict):
+ fn = o.get("fileName") or o.get("productFileName")
+ uri = o.get("fileDownloadURI") or o.get("fileDownloadUri") or o.get("downloadUrl")
+ if fn and fn.endswith(".zip"):
+ out.append({"name": fn, "uri": uri, "size": o.get("fileSize") or o.get("fileSizeBytes")})
+ for v in o.values(): extract_files(v, out)
+ elif isinstance(o, list):
+ for x in o: extract_files(x, out)
+ return out
+
+def download(f, key):
+ name, uri, size = f["name"], f["uri"], f.get("size")
+ dest = OUT / name
+ if dest.exists() and size and dest.stat().st_size == int(size):
+ return dest.stat().st_size, "skip"
+ if not uri:
+ uri = f"https://data.uspto.gov/api/v1/datasets/products/{PRODUCT}/files/{name}"
+ h = {"X-API-KEY": key, "Accept": "application/zip,application/octet-stream,*/*"}
+ with requests.get(uri, headers=h, impersonate="chrome", timeout=1800, stream=True) as r:
+ if r.status_code != 200 or "html" in (r.headers.get("content-type") or ""):
+ return 0, f"FAIL {r.status_code} {r.headers.get('content-type')}"
+ tmp = dest.with_suffix(".part")
+ n = 0
+ with open(tmp, "wb") as fh:
+ for chunk in r.iter_content(chunk_size=1 << 20):
+ fh.write(chunk); n += len(chunk)
+ # verify zip magic
+ with open(tmp, "rb") as fh:
+ if fh.read(4) != b"PK\x03\x04":
+ tmp.unlink(missing_ok=True)
+ return 0, "FAIL not-a-zip"
+ tmp.rename(dest)
+ return n, "ok"
+
+def main():
+ key = get_key()
+ print(f"Listing {PRODUCT} files via ODP API...")
+ files = list_files(key)
+ total = len(files)
+ tot_bytes = sum(int(f["size"]) for f in files if f.get("size"))
+ print(f"{total} files, ~{tot_bytes/1e9:.1f} GB total\n")
+ done_b = 0
+ for i, f in enumerate(files, 1):
+ t0 = time.time()
+ n, status = download(f, key)
+ done_b += n
+ print(f"[{i}/{total}] {f['name']:40s} {status:8s} {n/1e6:8.1f} MB "
+ f"({done_b/1e9:5.2f} GB cum, {time.time()-t0:4.1f}s)")
+ print(f"\nDone. {sum(1 for p in OUT.glob('*.zip'))} zips in {OUT}")
+
+if __name__ == "__main__":
+ main()
diff --git a/fetch_list.py b/fetch_list.py
new file mode 100644
index 0000000..79471fa
--- /dev/null
+++ b/fetch_list.py
@@ -0,0 +1,29 @@
+from curl_cffi import requests
+import json, sys
+cookie = open('.cookie').read().strip()
+H = {"Accept":"application/json","Referer":"https://data.uspto.gov/bulkdata/datasets/trtyrap?fileDataFromDate=1884-04-07&fileDataToDate=2025-12-31","Cookie":cookie}
+url = "https://data.uspto.gov/ui/datasets/products/trtyrap?includeFiles=true&fileDataFromDate=1884-04-07&fileDataToDate=2025-12-31"
+r = requests.get(url, headers=H, impersonate="chrome", timeout=60)
+print("status", r.status_code, "ctype", r.headers.get("content-type"), "len", len(r.content))
+ct = r.headers.get("content-type","")
+if "json" not in ct:
+ print("HEAD:", r.text[:100]); sys.exit(1)
+d = r.json()
+def files(o):
+ a=[]
+ if isinstance(o,dict):
+ for k,v in o.items():
+ if k.lower().endswith("filename") and isinstance(v,str) and v.endswith(".zip"): a.append(v)
+ else: a+=files(v)
+ elif isinstance(o,list):
+ for x in o: a+=files(x)
+ return a
+fns=sorted(set(files(d)))
+open("filenames.txt","w").write("\n".join(fns))
+print("SAVED", len(fns), "filenames")
+print("first:", fns[:2], "last:", fns[-2:])
+# also capture total size if present
+s=json.dumps(d)
+import re
+sizes=re.findall(r'"fileSize"\s*:\s*(\d+)', s)
+if sizes: print("total size GB:", round(sum(int(x) for x in sizes)/1e9,2))
(oldest)
·
back to Uspto Trtyrap
·
Add detached poll-and-download bg job 8e805d9 →