← back to Rentv 2026
RENTV: pull archive with LOCAL MODELS (qwen3:14b on Mac2+Mac1, $0) — TK-10250
b881b5e3c479764c30b91a279933d911584d750e · 2026-08-05 13:38:57 -0700 · Steve Abrams
Steve: 'use local models to pull data!!'. Replaces regex heuristics with local Ollama
structured extraction — txn/property type, city/state, $ amount, buyer/seller/broker parties,
size, clean summary. lib/llm-extract.mjs calls Ollama (format:json, think:false, temp:0) with a
regex fallback so an LLM timeout never drops an article. pull-archive.mjs runs a worker pool —
one worker per Ollama endpoint (Mac2 localhost + Mac1 192.168.1.133) — splitting the crawl across
both machines for ~2x throughput. Stores raw story text + extractor/model provenance; resumable,
lockfile-guarded. Verified: 2-machine split, rich extraction (parties + summary), 0 errors.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Files touched
A scripts/lib/llm-extract.mjsM scripts/pull-archive.mjs
Diff
commit b881b5e3c479764c30b91a279933d911584d750e
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Wed Aug 5 13:38:57 2026 -0700
RENTV: pull archive with LOCAL MODELS (qwen3:14b on Mac2+Mac1, $0) — TK-10250
Steve: 'use local models to pull data!!'. Replaces regex heuristics with local Ollama
structured extraction — txn/property type, city/state, $ amount, buyer/seller/broker parties,
size, clean summary. lib/llm-extract.mjs calls Ollama (format:json, think:false, temp:0) with a
regex fallback so an LLM timeout never drops an article. pull-archive.mjs runs a worker pool —
one worker per Ollama endpoint (Mac2 localhost + Mac1 192.168.1.133) — splitting the crawl across
both machines for ~2x throughput. Stores raw story text + extractor/model provenance; resumable,
lockfile-guarded. Verified: 2-machine split, rich extraction (parties + summary), 0 errors.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
scripts/lib/llm-extract.mjs | 71 +++++++++++++++++++++
scripts/pull-archive.mjs | 146 +++++++++++++++++++++++++++++---------------
2 files changed, 169 insertions(+), 48 deletions(-)
diff --git a/scripts/lib/llm-extract.mjs b/scripts/lib/llm-extract.mjs
new file mode 100644
index 00000000..af97d2cb
--- /dev/null
+++ b/scripts/lib/llm-extract.mjs
@@ -0,0 +1,71 @@
+// llm-extract.mjs — LOCAL-MODEL structured extraction for rentv.com articles (TK-10250, Steve
+// 2026-08-05: "use local models to pull data!!"). Sends the article headline + body to a local
+// Ollama model (qwen3:14b on Mac2 + Mac1) and gets back clean structured CRE fields — better than
+// the regex heuristics (real summary, buyer/seller/broker parties, robust type/amount/location).
+// $0 (local), split across both machines by the caller for throughput.
+//
+// Endpoints default to Mac2 localhost + Mac1 tailnet/LAN; model default qwen3:14b. Forces
+// format:json + think:false + temperature:0 for deterministic, thinking-free JSON.
+export const OLLAMA_ENDPOINTS = ['http://localhost:11434', 'http://192.168.1.133:11434'];
+export const DEFAULT_MODEL = 'qwen3:14b';
+
+const PROP_TYPES = 'Office, Industrial, Retail, Multifamily, Hospitality, Medical/Life Science, Mixed-Use, Land, Self-Storage';
+const TXN_TYPES = 'Sale, Lease, Financing, Development';
+
+function buildPrompt(title, story) {
+ return `You extract structured data from a commercial real estate (CRE) news article. Return ONLY a strict JSON object, no prose.
+
+Fields:
+- txn_type: one of [${TXN_TYPES}] — the deal action.
+- property_type: one of [${PROP_TYPES}] — the asset class.
+- city: the ASSET's city (not a firm HQ), or null.
+- state: 2-letter US state of the asset, or null.
+- amount: deal dollar value as a plain integer (e.g. 51800000), or null if none stated.
+- buyer: the party acquiring/borrowing/leasing-in (firm or person), or null.
+- seller: the party selling/lending/leasing-out, or null.
+- broker: the brokerage firm(s) that arranged the deal, or null.
+- size: the asset size as text (e.g. "236 units", "50,000 SF", "12 acres"), or null.
+- summary: one clean factual sentence summarizing the deal.
+
+Article headline: "${title}"
+Article body: ${story}
+
+JSON:`;
+}
+
+// Extract from one article via one Ollama endpoint. Returns the parsed object or null on any failure
+// (timeout / bad JSON / model error) so the caller can fall back to the regex parser.
+export async function llmExtract(title, story, { endpoint = OLLAMA_ENDPOINTS[0], model = DEFAULT_MODEL, timeoutMs = 30000 } = {}) {
+ try {
+ const r = await fetch(endpoint + '/api/generate', {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ model, prompt: buildPrompt(title, String(story).slice(0, 1800)), format: 'json', stream: false, think: false, options: { temperature: 0, num_ctx: 4096 } }),
+ signal: AbortSignal.timeout(timeoutMs),
+ });
+ if (!r.ok) return null;
+ const j = await r.json();
+ let obj;
+ try { obj = JSON.parse(j.response); } catch { return null; }
+ if (!obj || typeof obj !== 'object') return null;
+ // normalize
+ const num = (v) => { if (v == null) return null; const n = +String(v).replace(/[^0-9.]/g, ''); return isFinite(n) && n > 0 ? Math.round(n) : null; };
+ const amount = num(obj.amount);
+ const clean = (v) => { const s = v == null ? '' : String(v).trim(); return s && !/^(null|n\/a|none|unknown)$/i.test(s) ? s : null; };
+ return {
+ txn_type: clean(obj.txn_type), property_type: clean(obj.property_type),
+ city: clean(obj.city), state: clean(obj.state) ? String(obj.state).toUpperCase().slice(0, 2) : null,
+ amount, amount_label: amount ? (amount >= 1e9 ? `$${(amount / 1e9).toFixed(2)}B` : amount >= 1e6 ? `$${(amount / 1e6).toFixed(1)}M` : `$${amount.toLocaleString()}`) : null,
+ buyer: clean(obj.buyer), seller: clean(obj.seller), broker: clean(obj.broker),
+ size_label: clean(obj.size), summary: clean(obj.summary),
+ };
+ } catch { return null; }
+}
+
+// Verify an endpoint has the model loaded + responding (used at crawl startup).
+export async function endpointUp(endpoint, model = DEFAULT_MODEL) {
+ try {
+ const r = await fetch(endpoint + '/api/generate', { method: 'POST', body: JSON.stringify({ model, prompt: 'ok', stream: false, think: false, options: { num_predict: 2 } }), signal: AbortSignal.timeout(8000) });
+ return r.ok;
+ } catch { return false; }
+}
diff --git a/scripts/pull-archive.mjs b/scripts/pull-archive.mjs
index af64ce16..e283ed31 100644
--- a/scripts/pull-archive.mjs
+++ b/scripts/pull-archive.mjs
@@ -1,46 +1,51 @@
#!/usr/bin/env node
-// pull-archive.mjs — backfill the FULL rentv.com article archive into the corpus (TK-10247,
-// Steve 2026-08-05: "load all rentv.com articles… to analyze how to properly post news events").
+// pull-archive.mjs — backfill the FULL rentv.com article archive into the corpus, extracting each
+// article with a LOCAL MODEL (TK-10250; Steve 2026-08-05: "load all rentv.com articles… use local
+// models to pull data!!"). qwen3:14b on Mac2 + Mac1 (Ollama) does the structured CRE extraction —
+// txn/property type, city/state, $ amount, buyer/seller/broker parties, size, a clean summary —
+// far better than regex heuristics. $0 (local). Work is SPLIT across both machines via a worker
+// pool (one worker per Ollama endpoint) for ~2x throughput. Regex parse is the FALLBACK so an LLM
+// timeout never drops an article.
//
-// rentv.com articles are /content/homepage/mainnews/news/{id} with sequential integer ids. IDs
-// below ~20000 302→cart.php (gone). From ~20000→newest they're live: real articles interleaved
-// with a byte-identical empty boilerplate stub. parse-article.mjs returns null for the stub, so
-// we keep only real articles. $0 plain-fetch, polite jittered delay, iso-8859-1 decode.
+// rentv.com articles are /content/homepage/mainnews/news/{id}, sequential ids. <~20000 302→cart.php
+// (gone). ~20000→newest are live: real articles interleaved with a byte-identical empty stub
+// (parseHeadline→'' → skipped). iso-8859-1 decode. RESUMABLE: appends data/articles-corpus.jsonl,
+// tracks a descending cursor in data/archive-crawl-state.json; already-ingested ids are skipped.
//
-// RESUMABLE: appends to data/articles-corpus.jsonl (one article per line) and tracks a descending
-// cursor in data/archive-crawl-state.json. Re-run with --resume (default) to continue; already-
-// ingested ids are skipped. Runs for a while (14k+ ids) — launch in the background.
-//
-// Usage: node scripts/pull-archive.mjs [--max 34700] [--min 20000] [--delay 450] [--limit N] [--fresh]
-import { createReadStream, existsSync, appendFileSync, writeFileSync, readFileSync, renameSync } from 'node:fs';
+// Usage: node scripts/pull-archive.mjs [--max 34290] [--min 20000] [--delay 120] [--limit N]
+// [--fresh] [--model qwen3:14b] [--endpoints a,b] [--no-llm]
+import { createReadStream, existsSync, appendFileSync, writeFileSync, readFileSync, renameSync, unlinkSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { createInterface } from 'node:readline';
-import { parseArticle } from './lib/parse-article.mjs';
+import { parseArticle, parseHeadline, bodyText } from './lib/parse-article.mjs';
+import { llmExtract, endpointUp, OLLAMA_ENDPOINTS, DEFAULT_MODEL } from './lib/llm-extract.mjs';
const HERE = dirname(fileURLToPath(import.meta.url));
const DATA = join(HERE, '..', 'data');
const CORPUS = join(DATA, 'articles-corpus.jsonl');
const STATE = join(DATA, 'archive-crawl-state.json');
+const LOCK = join(DATA, '.archive-crawl.lock');
const arg = (k, d) => { const i = process.argv.indexOf(k); return i > -1 ? (process.argv[i + 1] ?? true) : d; };
-const MAX = +arg('--max', 34700);
+const MAX = +arg('--max', 34290);
const MIN = +arg('--min', 20000);
-const DELAY = +arg('--delay', 450);
+const DELAY = +arg('--delay', 120); // polite gap between an endpoint's fetches (LLM call dominates anyway)
const LIMIT = +arg('--limit', 0) || Infinity;
const FRESH = process.argv.includes('--fresh');
+const USE_LLM = !process.argv.includes('--no-llm');
+const MODEL = arg('--model', DEFAULT_MODEL);
+const ENDPOINTS = (arg('--endpoints', '') ? String(arg('--endpoints', '')).split(',') : OLLAMA_ENDPOINTS).filter(Boolean);
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
-const jitter = () => DELAY + Math.floor(Math.random() * 350);
async function fetchArticle(id) {
const url = `https://www.rentv.com/content/homepage/mainnews/news/${id}`;
const r = await fetch(url, { headers: { 'User-Agent': 'Mozilla/5.0 RENTV-archive' }, redirect: 'manual', signal: AbortSignal.timeout(20000) });
- if (r.status !== 200) return { status: r.status, html: null }; // 302 = gone
+ if (r.status !== 200) return { status: r.status, html: null };
const b = Buffer.from(await r.arrayBuffer());
let s = b.toString('utf8');
if ((s.match(/�/g) || []).length > 5) s = b.toString('latin1');
return { status: 200, html: s };
}
-
async function loadIngestedIds() {
const seen = new Set();
if (!existsSync(CORPUS)) return seen;
@@ -54,44 +59,89 @@ async function loadIngestedIds() {
const loadState = () => { try { return JSON.parse(readFileSync(STATE, 'utf8')); } catch { return null; } };
const saveState = (s) => { const tmp = STATE + '.tmp'; writeFileSync(tmp, JSON.stringify(s, null, 1)); renameSync(tmp, STATE); };
+// Build one corpus record for an id: regex parse (structure + fallback) MERGED with the local-model
+// extraction (authoritative for the fields it fills). Returns null if the page isn't a real article.
+async function buildRecord(id, html, endpoint) {
+ const base = parseArticle(id, html); // regex — real-article gate + fallback fields
+ if (!base) return null;
+ const story = (() => { const b = bodyText(html); const i = b.indexOf(base.title); return (i >= 0 ? b.slice(i + base.title.length) : b).replace(/^\s*\d{1,2}\/\d{1,2}\/\d{2,4}\s*/, '').trim().slice(0, 1800); })();
+ let rec = { ...base, story, extractor: 'regex' };
+ if (USE_LLM) {
+ const llm = await llmExtract(base.title, story, { endpoint, model: MODEL });
+ if (llm) {
+ rec = {
+ ...base,
+ txn_type: llm.txn_type || base.txn_type, property_type: llm.property_type || base.property_type,
+ city: llm.city || base.city, state: llm.state || base.state,
+ amount: llm.amount ?? base.amount, amount_label: llm.amount_label || base.amount_label,
+ size_label: llm.size_label || base.size_label,
+ buyer: llm.buyer || null, seller: llm.seller || null, broker: llm.broker || null,
+ summary: llm.summary || base.summary,
+ story, extractor: 'llm', model: MODEL,
+ };
+ }
+ }
+ return rec;
+}
+
async function main() {
+ if (existsSync(LOCK)) { try { const age = Date.now() - JSON.parse(readFileSync(LOCK, 'utf8')).at; if (age < 6 * 3600e3) { console.log('another archive crawl holds the lock — skipping.'); return; } } catch { /* reclaim */ } }
+ writeFileSync(LOCK, JSON.stringify({ pid: process.pid, at: Date.now() }));
+ process.on('exit', () => { try { unlinkSync(LOCK); } catch { /* gone */ } });
+
if (FRESH && existsSync(CORPUS)) writeFileSync(CORPUS, '');
const seen = FRESH ? new Set() : await loadIngestedIds();
const prev = FRESH ? null : loadState();
- let cursor = prev && prev.cursor != null ? Math.min(prev.cursor, MAX) : MAX; // descending
- const stat = prev && !FRESH ? prev
- : { max: MAX, min: MIN, cursor, real: 0, empty: 0, gone: 0, errors: 0, started_at: new Date().toISOString() };
- stat.max = Math.max(stat.max || MAX, MAX);
+ let nextId = prev && prev.cursor != null ? Math.min(prev.cursor, MAX) : MAX; // shared descending cursor
+ const stat = prev && !FRESH ? prev : { max: MAX, min: MIN, cursor: nextId, real: 0, empty: 0, gone: 0, errors: 0, llm: 0, regex_fallback: 0, started_at: new Date().toISOString() };
+ // ensure counters exist when resuming an older state file (pre-LLM runs lack these keys)
+ for (const k of ['real', 'empty', 'gone', 'errors', 'llm', 'regex_fallback']) if (typeof stat[k] !== 'number') stat[k] = 0;
+
+ // Verify endpoints (drop any that are down so we don't stall a worker on a dead machine).
+ let live = ENDPOINTS;
+ if (USE_LLM) {
+ const checks = await Promise.all(ENDPOINTS.map(async (e) => ({ e, up: await endpointUp(e, MODEL) })));
+ live = checks.filter((c) => c.up).map((c) => c.e);
+ console.log(`local-model pull: ${MODEL} on ${live.length}/${ENDPOINTS.length} endpoints [${live.join(', ')}]`);
+ if (!live.length) { console.error('no Ollama endpoint is up — aborting (or use --no-llm for regex).'); return; }
+ } else { live = ['(regex-only)']; console.log('regex-only mode (--no-llm)'); }
- console.log(`archive backfill: ids ${cursor}→${MIN} · ${seen.size} already in corpus · delay ~${DELAY}ms · $0 local`);
- let processed = 0, consecErr = 0;
+ console.log(`archive backfill: ids ${nextId}→${MIN} · ${seen.size} already in corpus · ${live.length} worker(s) · $0 local`);
+ let processed = 0, stop = false;
const t0 = Date.now();
- for (let id = cursor; id >= MIN; id--) {
- stat.cursor = id;
- if (seen.has(String(id))) continue;
- if (processed >= LIMIT) break;
- processed++;
- let res;
- try { res = await fetchArticle(id); consecErr = 0; }
- catch (e) { stat.errors++; consecErr++; if (consecErr >= 12) { console.error(`\n12 consecutive fetch errors — network down? stopping (resumable at ${id}).`); break; } await sleep(1500); continue; }
- if (res.status === 302 || res.status === 301) { stat.gone++; }
- else if (res.status === 200 && res.html) {
- const a = res.html ? parseArticle(id, res.html) : null;
- if (a) { appendFileSync(CORPUS, JSON.stringify(a) + '\n'); seen.add(String(id)); stat.real++; }
- else stat.empty++;
- } else stat.empty++;
- if (processed % 25 === 0) {
- stat.updated_at = new Date().toISOString();
- saveState(stat);
- const rate = processed / ((Date.now() - t0) / 1000);
- process.stdout.write(`\r at id ${id} real:${stat.real} empty:${stat.empty} gone:${stat.gone} (${rate.toFixed(1)}/s, ${processed} this run) `);
+ const persist = () => { stat.cursor = nextId; stat.updated_at = new Date().toISOString(); saveState(stat); };
+
+ // One worker per endpoint; all share nextId + seen (single-threaded event loop → no races).
+ async function worker(endpoint) {
+ while (!stop) {
+ const id = nextId--;
+ if (id < MIN) break;
+ stat.cursor = Math.min(stat.cursor, id);
+ if (seen.has(String(id))) continue;
+ if (processed >= LIMIT) { stop = true; break; }
+ processed++;
+ let res;
+ try { res = await fetchArticle(id); }
+ catch { stat.errors++; await sleep(1000); continue; }
+ if (res.status === 302 || res.status === 301) { stat.gone++; }
+ else if (res.status === 200 && res.html) {
+ if (!parseHeadline(res.html)) { stat.empty++; }
+ else {
+ const rec = await buildRecord(id, res.html, endpoint);
+ if (rec) { appendFileSync(CORPUS, JSON.stringify(rec) + '\n'); seen.add(String(id)); stat.real++; if (rec.extractor === 'llm') stat.llm++; else stat.regex_fallback++; }
+ else stat.empty++;
+ }
+ } else stat.empty++;
+ if (stat.real % 20 === 0) { persist(); const rate = processed / ((Date.now() - t0) / 1000); process.stdout.write(`\r at id ${nextId} real:${stat.real} (llm:${stat.llm} fallback:${stat.regex_fallback}) empty:${stat.empty} gone:${stat.gone} ${rate.toFixed(2)}/s `); }
+ await sleep(DELAY);
}
- await sleep(jitter());
}
- stat.updated_at = new Date().toISOString();
- stat.done = stat.cursor <= MIN;
- saveState(stat);
- console.log(`\n${stat.done ? 'DONE' : 'PAUSED'}: ${stat.real} real articles in corpus · empty:${stat.empty} · gone:${stat.gone} · errors:${stat.errors}`);
+ await Promise.all(live.map((e) => USE_LLM ? worker(e) : worker(null)));
+
+ stat.cursor = Math.max(MIN, nextId + 1);
+ stat.done = nextId < MIN;
+ persist();
+ console.log(`\n${stat.done ? 'DONE' : 'PAUSED'}: ${stat.real} articles (llm:${stat.llm} regex-fallback:${stat.regex_fallback}) · empty:${stat.empty} · gone:${stat.gone} · errors:${stat.errors}`);
if (!stat.done) console.log(` resume: node scripts/pull-archive.mjs (continues at id ${stat.cursor})`);
}
main().catch((e) => { console.error('archive crawl failed:', e.message); process.exit(1); });
← 0432aa44 RENTV: move Property Closings to backend-only + editable ove
·
back to Rentv 2026
·
auto-save: 2026-08-05T13:42:25 (8 files) — data/deals-regist b39b469c →