`download_backfill.py`

108 lines
#!/usr/bin/env python3
"""
USPTO ODP Bulk Data downloader for TRTYRAP
(Trademark Full Text XML Data (No Images) - Annual Applications, 1884-2025 backfile).

Uses the OFFICIAL Open Data Portal Bulk Data API with an X-API-KEY
(obtained after ID.me verification at account.uspto.gov). This is the
sanctioned path -- it sidesteps the AWS WAF bot-control that blocks
curl / curl_cffi on the raw file bytes.

Key resolution order:
  1) env USPTO_API_KEY
  2) file .apikey in this dir (gitignored)

Run:  .venv/bin/python download_backfill.py
Resume-safe: skips files already fully downloaded (size match).
"""
import os, sys, json, time, pathlib
from curl_cffi import requests

HERE = pathlib.Path(__file__).parent
OUT = HERE / "files"; OUT.mkdir(exist_ok=True)
PRODUCT = "TRTYRAP"
FROM, TO = "1884-04-07", "2025-12-31"

def get_key():
    k = os.environ.get("USPTO_API_KEY")
    if not k and (HERE / ".apikey").exists():
        k = (HERE / ".apikey").read_text().strip()
    if not k:
        sys.exit("No API key. Set USPTO_API_KEY or write it to .apikey")
    return k

def api_get(url, key, **kw):
    h = {"X-API-KEY": key, "Accept": "application/json"}
    return requests.get(url, headers=h, impersonate="chrome", timeout=kw.pop("timeout", 60), **kw)

def list_files(key):
    # Primary ODP bulk-data product endpoint (returns file bag w/ download URIs).
    candidates = [
        f"https://api.uspto.gov/api/v1/datasets/products/{PRODUCT}?fileDataFromDate={FROM}&fileDataToDate={TO}",
        f"https://api.uspto.gov/api/v1/datasets/products/{PRODUCT}",
    ]
    for url in candidates:
        r = api_get(url, key)
        if r.status_code == 200 and "json" in (r.headers.get("content-type") or ""):
            d = r.json()
            files = extract_files(d)
            if files:
                return files
        print(f"  [list] {url} -> {r.status_code} {r.headers.get('content-type')}")
    sys.exit("Could not list files from the API -- inspect the endpoint shape with the real key.")

def extract_files(o, out=None):
    out = [] if out is None else out
    if isinstance(o, dict):
        fn = o.get("fileName") or o.get("productFileName")
        uri = o.get("fileDownloadURI") or o.get("fileDownloadUri") or o.get("downloadUrl")
        if fn and fn.endswith(".zip"):
            out.append({"name": fn, "uri": uri, "size": o.get("fileSize") or o.get("fileSizeBytes")})
        for v in o.values(): extract_files(v, out)
    elif isinstance(o, list):
        for x in o: extract_files(x, out)
    return out

def download(f, key):
    name, uri, size = f["name"], f["uri"], f.get("size")
    dest = OUT / name
    if dest.exists() and size and dest.stat().st_size == int(size):
        return dest.stat().st_size, "skip"
    if not uri:
        uri = f"https://api.uspto.gov/api/v1/datasets/products/{PRODUCT}/files/{name}"
    h = {"X-API-KEY": key, "Accept": "application/zip,application/octet-stream,*/*"}
    with requests.get(uri, headers=h, impersonate="chrome", timeout=1800, stream=True) as r:
        if r.status_code != 200 or "html" in (r.headers.get("content-type") or ""):
            return 0, f"FAIL {r.status_code} {r.headers.get('content-type')}"
        tmp = dest.with_suffix(".part")
        n = 0
        with open(tmp, "wb") as fh:
            for chunk in r.iter_content(chunk_size=1 << 20):
                fh.write(chunk); n += len(chunk)
        # verify zip magic
        with open(tmp, "rb") as fh:
            if fh.read(4) != b"PK\x03\x04":
                tmp.unlink(missing_ok=True)
                return 0, "FAIL not-a-zip"
        tmp.rename(dest)
        return n, "ok"

def main():
    key = get_key()
    print(f"Listing {PRODUCT} files via ODP API...")
    files = list_files(key)
    total = len(files)
    tot_bytes = sum(int(f["size"]) for f in files if f.get("size"))
    print(f"{total} files, ~{tot_bytes/1e9:.1f} GB total\n")
    done_b = 0
    for i, f in enumerate(files, 1):
        t0 = time.time()
        n, status = download(f, key)
        done_b += n
        print(f"[{i}/{total}] {f['name']:40s} {status:8s} {n/1e6:8.1f} MB  "
              f"({done_b/1e9:5.2f} GB cum, {time.time()-t0:4.1f}s)")
    print(f"\nDone. {sum(1 for p in OUT.glob('*.zip'))} zips in {OUT}")

if __name__ == "__main__":
    main()