`scripts/fetch-tm-register.mjs`

98 lines
#!/usr/bin/env node
// Fetch USPTO bulk trademark data into tmp_tm_cache/ (local-only, rsync-excluded).
//
// WHY bulk, not an API: USPTO retired TESS (2023); the new tmsearch.uspto.gov has
// NO public real-time name-search API, and the official ODP/TSDR JSON APIs need an
// ID.me-linked key and look up by serial#, not by mark name. The free, $0, no-rate-
// limit path is the published bulk register, which we index locally once.
//
// SOURCE (2026): the legacy host bulkdata.uspto.gov is RETIRED. Trademark bulk
// data now lives on the USPTO Open Data Portal:
//   dataset page : https://data.uspto.gov/bulkdata/datasets/trtdxfap
//   product      : "Trademark Daily XML File (TDXF) applications" (trtdxfap)
// Each file is XML with <case-file> records (mark text, status, Nice class).
//
// REALITY (verified 2026-08-06): the ODP is behind an AWS WAF anti-bot challenge
// AND its Bulk-Search/Download (BSD) API requires an ID.me-linked USPTO API key
// (X-API-KEY). So a plain headless fetch is BLOCKED. Three real ways in:
//   (A) --file <path>   : ingest a file YOU downloaded in a real browser (or via
//                         the openclaw/portal-driver real-Chrome flow). $0, no key.
//   (B) --odp <fileName>: hit the BSD download API with USPTO_ODP_API_KEY (env/.env)
//                         — needs Steve's ID.me-linked key. $0 once keyed.
//   (C) --url <httpsUrl>: fetch a direct file URL (if you have an unwalled one).
// Files land in tmp_tm_cache/ ; then run build-tm-index.mjs + build-tm-clearance.mjs.
import { mkdir, writeFile, copyFile } from 'node:fs/promises';
import { fileURLToPath } from 'node:url';
import { join, basename } from 'node:path';
import { readFileSync } from 'node:fs';

const ROOT = fileURLToPath(new URL('..', import.meta.url));
const CACHE = join(ROOT, 'tmp_tm_cache');
const DATASET = 'https://data.uspto.gov/bulkdata/datasets/trtdxfap';
const BSD_DOWNLOAD = 'https://data.uspto.gov/api/v1/datasets/products/files'; // BSD download base (key-gated)

const args = process.argv.slice(2);
const get = (flag) => { const i = args.indexOf(flag); return i >= 0 ? args[i + 1] : null; };
function envVal(name) {
  if (process.env[name]) return process.env[name];
  try { const m = readFileSync(join(ROOT, '.env'), 'utf8').match(new RegExp('^' + name + '=(.+)$', 'm')); if (m) return m[1].trim(); } catch {}
  return null;
}

async function main() {
  await mkdir(CACHE, { recursive: true });
  if (args.includes('--list') || args.length === 0) {
    console.log('USPTO trademark bulk register — Open Data Portal (data.uspto.gov):');
    console.log('  Dataset : ' + DATASET + '  (product trtdxfap)');
    console.log('  NOTE: ODP is behind an AWS WAF + the BSD API needs an ID.me-linked key.');
    console.log('        A plain headless fetch is BLOCKED. Choose one:');
    console.log('   (A) node scripts/fetch-tm-register.mjs --file /path/to/downloaded-trtdxf.zip   # $0, no key');
    console.log('   (B) USPTO_ODP_API_KEY=… node scripts/fetch-tm-register.mjs --odp <fileName>    # $0, needs key');
    console.log('   (C) node scripts/fetch-tm-register.mjs --url <direct-https-url>');
    console.log('Then: node scripts/build-tm-index.mjs && node scripts/build-tm-clearance.mjs');
    console.log('Until a REAL file is ingested the index stays a SEED SAMPLE (complete:false) and');
    console.log('NOTHING is sellable — "only legally clear" holds by default.');
    return;
  }

  // (A) ingest an already-downloaded file (browser / openclaw)
  const localFile = get('--file');
  if (localFile) {
    const out = join(CACHE, basename(localFile));
    await copyFile(localFile, out);
    console.log('Ingested local register file -> ' + out + '  (cost: $0)');
    console.log('Next: node scripts/build-tm-index.mjs && node scripts/build-tm-clearance.mjs');
    return;
  }

  // (B) BSD download API with an ID.me-linked key
  const odpFile = get('--odp');
  if (odpFile) {
    const key = envVal('USPTO_ODP_API_KEY');
    if (!key) { console.error('Missing USPTO_ODP_API_KEY (ID.me-linked). See --list option B.'); process.exit(2); }
    const target = `${BSD_DOWNLOAD}/${encodeURIComponent(odpFile)}/download`;
    console.log('Cost: $0 (keyed ODP). Fetching', target, '...');
    const r = await fetch(target, { headers: { 'X-API-KEY': key } });
    if (!r.ok) { console.error('ODP fetch failed:', r.status, r.statusText, '(WAF or bad key/filename?)'); process.exit(1); }
    const buf = Buffer.from(await r.arrayBuffer());
    const out = join(CACHE, basename(odpFile));
    await writeFile(out, buf);
    console.log(`Saved ${(buf.length / 1e6).toFixed(1)} MB -> ${out}`);
    console.log('Next: node scripts/build-tm-index.mjs && node scripts/build-tm-clearance.mjs');
    return;
  }

  // (C) direct URL
  const target = get('--url');
  if (!target || !/^https:\/\//.test(target)) { console.error('Refusing: --url must be an https URL.'); process.exit(2); }
  console.log('Cost: $0. Fetching', target, '...');
  const r = await fetch(target);
  if (!r.ok) { console.error('fetch failed:', r.status, r.statusText, '(the ODP host is WAF-gated; use --file or --odp instead)'); process.exit(1); }
  const buf = Buffer.from(await r.arrayBuffer());
  const out = join(CACHE, basename(new URL(target).pathname) || 'register.xml');
  await writeFile(out, buf);
  console.log(`Saved ${(buf.length / 1e6).toFixed(1)} MB -> ${out}`);
  console.log('Next: node scripts/build-tm-index.mjs && node scripts/build-tm-clearance.mjs');
}
main().catch(e => { console.error(e); process.exit(1); });