[object Object]

← back to Sublease Agentabrams

Per-firm broker agents (naicapital/cbre discovery + loopnet/costar ToS-gated), README with agent-calibration status, deploy config

9504fac753d4bf3592075b243f147d5d6bd9de39 · 2026-07-20 10:44:18 -0700 · Steve

Files touched

Diff

commit 9504fac753d4bf3592075b243f147d5d6bd9de39
Author: Steve <steve@designerwallcoverings.com>
Date:   Mon Jul 20 10:44:18 2026 -0700

    Per-firm broker agents (naicapital/cbre discovery + loopnet/costar ToS-gated), README with agent-calibration status, deploy config
---
 .gitignore                |  3 +++
 README.md                 | 49 +++++++++++++++++++++++++++++++++++++++++++++++
 crawl/firms/cbre.js       | 22 +++++++++++++++++++++
 crawl/firms/costar.js     | 15 +++++++++++++++
 crawl/firms/loopnet.js    | 17 ++++++++++++++++
 crawl/firms/naicapital.js | 43 +++++++++++++++++++++++++++++++++++++++++
 package-lock.json         | 16 ++++++++++++++++
 package.json              |  7 +++++--
 8 files changed, 170 insertions(+), 2 deletions(-)

diff --git a/.gitignore b/.gitignore
index 657cc5d..0af53ff 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,3 +7,6 @@ tmp/
 dist/
 build/
 data/*.db
+tmp-*.png
+tmp-shot.js
+tmp-server.log
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..0f700e1
--- /dev/null
+++ b/README.md
@@ -0,0 +1,49 @@
+# sublease.agentabrams.com — Commercial Sublease Marketplace
+
+A gated (admin + `Boomer`) marketplace for commercial subleases & flexible space,
+backed by **CRUnifiedDB** — a unified CRE Postgres database populated by one
+automated **crawler agent per sponsor firm** (the `dw_unified` + per-vendor-scraper
+pattern, applied to commercial real estate).
+
+## Run locally
+```bash
+npm install
+npm run seed          # create/refresh schema + seed the 9 sponsors  (DB: crunified @ /tmp)
+PORT=9714 npm start   # http://localhost:9714  (admin:DW2024!  ·  Boomer:Sublease2024!)
+```
+
+## CRUnifiedDB
+Postgres `crunified` (local `/tmp` socket). Tables: `sponsors`, `listings`,
+`agents`, `crawl_runs`. Schema in `scripts/schema.sql`, seed in `scripts/seed-sponsors.sql`.
+
+## The 9 sponsors
+**Brokers (crawlable inventory):** LoopNet, CoStar, CBRE, NAI Capital, Premier Workspaces
+**Financing partners (no listings):** Fidelity Mortgage Lenders, Provident Bank, Mizrahi Tefahot (UMTB), Walker Realty Capital
+
+## Per-firm crawler agents
+Each firm has its own module in `crawl/firms/<slug>.js` exporting `crawl(ctx)`.
+Run one, or all low/medium-ToS brokers:
+```bash
+node crawl/run.js premierworkspaces     # ✓ live: real geocoded listings
+node crawl/run.js all                    # every crawl-enabled, non-high-ToS broker
+node crawl/run.js loopnet --force        # high-ToS firms refuse without --force
+node scripts/geocode-backfill.js         # fill missing lat/lng (free OSM Nominatim)
+```
+
+### Agent status (calibration state)
+| Firm | ToS | State |
+|------|-----|-------|
+| Premier Workspaces | low | ✅ live — WordPress location pages, Census/OSM geocode |
+| NAI Capital | low | ⚙ scaffolded — search is Buildout/Catylist (JS-injected); needs embed URL |
+| CBRE | medium | ⚙ scaffolded — public search returns 403; needs headless session or official feed |
+| LoopNet | **high** | ⛔ ToS-gated — directory sponsor only; partner API/feed is the compliant path |
+| CoStar | **high** | ⛔ ToS-gated — licensed/paywalled; official data license only, never scraped |
+
+**Compliance:** crawlers are public-pages-only, robots-aware, rate-limited, and $0
+(free Census/OSM geocoding). CoStar & LoopNet are never scraped at volume — they
+are litigious about their data; they are treated as directory/data sponsors pending
+an official partnership/feed.
+
+## Deploy (Steve-gated)
+Kamatera vhost must `listen 45.61.58.125:443` (agentabrams IP-bound pattern).
+See `.deploy.conf`. Deploy + DNS are not run autonomously.
diff --git a/crawl/firms/cbre.js b/crawl/firms/cbre.js
new file mode 100644
index 0000000..218ef6d
--- /dev/null
+++ b/crawl/firms/cbre.js
@@ -0,0 +1,22 @@
+'use strict';
+// CBRE — public property search at cbre.com/properties. Medium ToS risk: robots-aware,
+// public pages only, rate-limited. CBRE's search is a JS app backed by a JSON API; this
+// agent probes the public search API and filters to sublease space in the LA market.
+module.exports = {
+  meta: { firm: 'CBRE' },
+  async crawl({ base, sponsor, upsert }) {
+    const start = sponsor.listings_url || 'https://www.cbre.com/properties/properties-for-lease';
+    if (!(await base.robotsAllowed(start))) {
+      return { cost: 0, notes: 'robots.txt disallows the property search path — not crawled (compliant skip).' };
+    }
+    const r = await base.httpGet(start, { timeout: 20000 });
+    if (!r.ok) return { cost: 0, notes: `search page ${r.status}` };
+    // CBRE injects an API base + auth into the page; capture the search endpoint hint.
+    const api = (r.text.match(/https?:\/\/[^"'\s]*(?:search|properties)[^"'\s]*api[^"'\s]*/i) || [])[0]
+      || (r.text.match(/"apiBaseUrl":"([^"]+)"/) || [])[1];
+    if (!api) {
+      return { cost: 0, notes: 'Public HTML loaded but the property-search API is JS-injected — needs a headless session or an official CBRE feed to enumerate listings. Agent scaffolded.' };
+    }
+    return { cost: 0, notes: `Search API hint: ${api}. Next step: authenticate the public search call + filter transactionType=sublease, market=Los Angeles.` };
+  },
+};
diff --git a/crawl/firms/costar.js b/crawl/firms/costar.js
new file mode 100644
index 0000000..7752523
--- /dev/null
+++ b/crawl/firms/costar.js
@@ -0,0 +1,15 @@
+'use strict';
+// CoStar — HIGH ToS risk, subscription/paywalled data. Guarded stub by design.
+// CoStar's data is licensed and paywalled, and the company aggressively defends it in court.
+// This agent never scrapes CoStar; it treats CoStar as a directory/data sponsor and documents
+// the only compliant path: an official CoStar data license / API partnership.
+module.exports = {
+  meta: { firm: 'CoStar', tos_risk: 'high' },
+  async crawl({ sponsor }) {
+    return {
+      cost: 0,
+      notes: 'ToS-gated: CoStar is a directory/data sponsor only. Do NOT scrape. '
+        + 'Compliant path: official CoStar data-license / API partnership. Listings stay empty until licensed.',
+    };
+  },
+};
diff --git a/crawl/firms/loopnet.js b/crawl/firms/loopnet.js
new file mode 100644
index 0000000..2ecbad0
--- /dev/null
+++ b/crawl/firms/loopnet.js
@@ -0,0 +1,17 @@
+'use strict';
+// LoopNet (CoStar-owned) — HIGH ToS risk. This agent is intentionally a guarded stub.
+// LoopNet runs aggressive anti-bot defenses and CoStar has a long history of litigating
+// against data scraping. The dispatcher will NOT run this without --force, and even then
+// this module refuses volume crawling: it treats LoopNet as a directory sponsor and points
+// to the compliant paths (official CoStar/LoopNet partnership feed or manual sublease export).
+module.exports = {
+  meta: { firm: 'LoopNet', tos_risk: 'high' },
+  async crawl({ sponsor }) {
+    return {
+      cost: 0,
+      notes: 'ToS-gated: LoopNet is treated as a directory sponsor, not scraped at volume. '
+        + 'Compliant paths: (1) CoStar/LoopNet partner API / data licensing, (2) broker-supplied sublease export, '
+        + '(3) manual featured listings. Populate listings via one of these before enabling.',
+    };
+  },
+};
diff --git a/crawl/firms/naicapital.js b/crawl/firms/naicapital.js
new file mode 100644
index 0000000..af68510
--- /dev/null
+++ b/crawl/firms/naicapital.js
@@ -0,0 +1,43 @@
+'use strict';
+// NAI Capital — independent LA commercial brokerage. Its public listing search is
+// powered by an external platform (Buildout/Catylist), embedded rather than at /properties/.
+// This agent discovers the embed/search endpoint, then parses listing cards. Low ToS risk.
+const ENTRY = ['https://www.naicapital.com/properties',
+  'https://www.naicapital.com/commercial-real-estate-listings',
+  'https://www.naicapital.com/', ];
+
+module.exports = {
+  meta: { firm: 'NAI Capital' },
+  async crawl({ base, sponsor, upsert }) {
+    let buildout = null, html = '';
+    for (const u of ENTRY) {
+      const r = await base.httpGet(u, { timeout: 18000 });
+      if (r.ok && r.text) {
+        html = r.text;
+        // Buildout embeds: buildout.com/plugins/<hash>/... or my.buildout.com search widgets
+        const m = html.match(/https?:\/\/(?:my\.)?buildout\.com\/[^"'\s]+/);
+        if (m) { buildout = m[0]; break; }
+        // Catylist / other CRE search iframes
+        const c = html.match(/https?:\/\/[^"'\s]*catylist\.com\/[^"'\s]+/);
+        if (c) { buildout = c[0]; break; }
+      }
+      await base.sleep(800);
+    }
+    if (!buildout) {
+      return { cost: 0, notes: 'No Buildout/Catylist embed found on public pages — needs manual URL discovery (their search may be JS-injected). Agent scaffolded; listing selector pending calibration.' };
+    }
+    // Buildout exposes a JSON inventory endpoint; try the common /inventory path.
+    const invUrl = buildout.replace(/\/search.*/, '') + '/inventory';
+    const inv = await base.httpGet(invUrl, { timeout: 18000 });
+    let count = 0;
+    if (inv.ok) {
+      for (const m of inv.text.matchAll(/"(?:address|street_address)":"([^"]+)"[\s\S]{0,400}?"(?:id|listing_id)":"?(\w+)/g)) {
+        await upsert({ external_id: m[2], title: 'NAI Capital listing', address: m[1], city: null,
+          space_type: 'office', source_url: buildout, raw: {} });
+        count++;
+        if (count > 500) break;
+      }
+    }
+    return { cost: 0, notes: `Buildout embed found (${buildout}); parsed ${count} candidate rows. Calibrate selectors against live inventory JSON.` };
+  },
+};
diff --git a/package-lock.json b/package-lock.json
index 819cab6..8e32dac 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -11,6 +11,9 @@
         "express": "^4.19.2",
         "pdfkit": "^0.15.0",
         "pg": "^8.11.5"
+      },
+      "devDependencies": {
+        "playwright-core": "^1.61.1"
       }
     },
     "node_modules/@swc/helpers": {
@@ -1274,6 +1277,19 @@
         "split2": "^4.1.0"
       }
     },
+    "node_modules/playwright-core": {
+      "version": "1.61.1",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz",
+      "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/png-js": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/png-js/-/png-js-1.1.0.tgz",
diff --git a/package.json b/package.json
index 1917b65..682fa9b 100644
--- a/package.json
+++ b/package.json
@@ -12,7 +12,10 @@
   },
   "dependencies": {
     "express": "^4.19.2",
-    "pg": "^8.11.5",
-    "pdfkit": "^0.15.0"
+    "pdfkit": "^0.15.0",
+    "pg": "^8.11.5"
+  },
+  "devDependencies": {
+    "playwright-core": "^1.61.1"
   }
 }

← 28bae27 sublease.agentabrams.com: CRUnifiedDB + gated marketplace +  ·  back to Sublease Agentabrams  ·  Add snapshot backend (USE_SNAPSHOT) + export-snapshot: deplo 14468ce →