← back to Rentv 2026
fix(archive): dedup parse-article's STALE parser fork → import the hardened deal-parse (single source of truth). parse-article.mjs kept its own OLD copies of parseAmount/txnOf/typeOf/classify/parseSize/summarize — the news-archive pipeline (pull-archive→parseArticle) was silently running WITHOUT every deal-parse fix. Proven live-divergent: $153/sf→153 (now null), $5 billionaire→$5B (now 5), 104k sf→lost (now 104000), summarize '$1. 625' (now '$1.625'). Deleted the 6 duplicates, import+re-export from deal-parse. +5 tests (parseHeadline/parseDate/bodyText/parseArticle + a dedup-lock), 70/0.
59a60b8803a7e027db44d3ca01085c3a3a9df5d6 · 2026-08-06 18:27:19 -0700 · Steve
Files touched
M scripts/lib/parse-article.mjsA test/deals/parse-article.test.mjs
Diff
commit 59a60b8803a7e027db44d3ca01085c3a3a9df5d6
Author: Steve <steve@designerwallcoverings.com>
Date: Thu Aug 6 18:27:19 2026 -0700
fix(archive): dedup parse-article's STALE parser fork → import the hardened deal-parse (single source of truth). parse-article.mjs kept its own OLD copies of parseAmount/txnOf/typeOf/classify/parseSize/summarize — the news-archive pipeline (pull-archive→parseArticle) was silently running WITHOUT every deal-parse fix. Proven live-divergent: $153/sf→153 (now null), $5 billionaire→$5B (now 5), 104k sf→lost (now 104000), summarize '$1. 625' (now '$1.625'). Deleted the 6 duplicates, import+re-export from deal-parse. +5 tests (parseHeadline/parseDate/bodyText/parseArticle + a dedup-lock), 70/0.
---
scripts/lib/parse-article.mjs | 62 +++++----------------------------------
test/deals/parse-article.test.mjs | 45 ++++++++++++++++++++++++++++
2 files changed, 53 insertions(+), 54 deletions(-)
diff --git a/scripts/lib/parse-article.mjs b/scripts/lib/parse-article.mjs
index 5ac36870..dca60b52 100644
--- a/scripts/lib/parse-article.mjs
+++ b/scripts/lib/parse-article.mjs
@@ -5,6 +5,12 @@
//
// rentv.com pages are iso-8859-1; the caller must decode (utf8 → latin1 fallback) before parsing.
import { parseLocation } from './parse-location.mjs';
+// The deal-field parsers are the SINGLE SOURCE OF TRUTH in deal-parse.mjs (unit-tested + hardened).
+// This module used to keep its own stale copies — a duplicate fork that missed every deal-parse fix
+// (per-sf skip, magnitude-aware size, summarize whitespace/decimal, billionaire guard). Import them so
+// the ARCHIVE pipeline (pull-archive → parseArticle) parses identically to the live-deals pipeline.
+import { parseAmount, txnOf, typeOf, classify, parseSize, summarize } from './deal-parse.mjs';
+export { parseAmount, txnOf, typeOf, classify, parseSize, summarize };
// A real article page carries the headline in <td class="header">…<br>. The empty boilerplate
// stub (byte-identical ~36395 bytes) has no such header → headline '' → treated as not-an-article.
@@ -40,60 +46,8 @@ export function bodyText(html) {
.replace(/<[^>]+>/g, ' ').replace(/ | /g, ' ').replace(/\s+/g, ' ').trim();
}
-// $51.8 mil / $1.2 bil / $985,000 → normalized dollars (from pull-deals.mjs)
-export function parseAmount(text) {
- const m = text.match(/\$\s?([\d,]+(?:\.\d+)?)\s?(bil|billion|mil|million|k)?/i);
- if (!m) return { amount: null, amount_label: null };
- let n = parseFloat(m[1].replace(/,/g, ''));
- const unit = (m[2] || '').toLowerCase();
- if (/bil/.test(unit)) n *= 1e9; else if (/mil/.test(unit)) n *= 1e6; else if (unit === 'k') n *= 1e3;
- const label = n >= 1e9 ? `$${(n / 1e9).toFixed(2)}B` : n >= 1e6 ? `$${(n / 1e6).toFixed(1)}M` : `$${n.toLocaleString()}`;
- return { amount: Math.round(n), amount_label: label };
-}
-export function txnOf(t) {
- return /refinanc|\brefi\b|obtains? (a )?(new )?loan|senior loan|recap|bridge loan/.test(t) ? 'Financing'
- : /\bsold\b|sells|\bsale\b|acquir|buys|purchas|trades|closes on|pays|spends|snaps up|picks up|fetches|changes hands|nets \$|lands \$|works out to/.test(t) ? 'Sale'
- : /leas|tenant|renew|signs? (a|new)|inks? a/.test(t) ? 'Lease'
- : /break(s)? ground|develop|deliver|top(s|ped) out|complet|construction|to build|unveils?|plan(s|ned) to|underway|rises|proposes?/.test(t) ? 'Development'
- : null;
-}
-export function typeOf(t) {
- return /office|\bhq\b|high-?rise|tower/.test(t) ? 'Office'
- : /industrial|warehouse|logistics|distribution|\bflex\b/.test(t) ? 'Industrial'
- : /retail|shopping|mall|grocery|storefront|strip (center|mall)/.test(t) ? 'Retail'
- : /hotel|hospitality|resort|motel/.test(t) ? 'Hospitality'
- : /medical|life science|\blab\b|biotech/.test(t) ? 'Medical/Life Science'
- : /mixed-?use/.test(t) ? 'Mixed-Use'
- : /self-?storage/.test(t) ? 'Self-Storage'
- : /multifamily|apartment|\bres\b|residential communit/.test(t) ? 'Multifamily'
- : /\bland\b|\bsite\b|\bacres?\b/.test(t) ? 'Land'
- : null;
-}
-export function classify(title, body) {
- const tl = (title || '').toLowerCase(), bl = (body || '').toLowerCase();
- const txn = txnOf(tl) || txnOf(bl) || 'Deal';
- const type = typeOf(tl) || typeOf(bl) || (/\bunit(s)?\b/.test(bl) ? 'Multifamily' : 'Commercial');
- return { txn, type };
-}
-export function parseSize(text) {
- const units = text.match(/([\d,]+)[- ]unit/i);
- const sf = text.match(/([\d,]+)\s?(?:sf|sq\.?\s?ft|square[ -]feet|square[ -]foot)/i);
- const acres = text.match(/([\d,.]+)[- ]acre/i);
- const parts = [];
- if (units) parts.push(`${units[1]} units`);
- if (sf) parts.push(`${sf[1]} SF`);
- if (acres) parts.push(`${acres[1]} acres`);
- return { size_label: parts.join(' · ') || null,
- units: units ? +units[1].replace(/,/g, '') : null, sqft: sf ? +sf[1].replace(/,/g, '') : null };
-}
-export function summarize(title, body) {
- let b = body;
- const ti = b.indexOf(title);
- if (ti >= 0) b = b.slice(ti + title.length);
- b = b.replace(/^\s*\d{1,2}\/\d{1,2}\/\d{2,4}\s*/, '').trim();
- const sentences = b.match(/[^.!?]+[.!?]+/g) || [b];
- return sentences.slice(0, 2).join(' ').trim().slice(0, 320);
-}
+// (parseAmount / txnOf / typeOf / classify / parseSize / summarize are imported from deal-parse.mjs above
+// — the stale duplicate copies that used to live here were deleted; deal-parse is the one source of truth.)
/**
* Parse a full article page into a structured corpus record. Returns null if the page is not a
diff --git a/test/deals/parse-article.test.mjs b/test/deals/parse-article.test.mjs
new file mode 100644
index 00000000..b6b1cb8e
--- /dev/null
+++ b/test/deals/parse-article.test.mjs
@@ -0,0 +1,45 @@
+// Tests for scripts/lib/parse-article.mjs — the rentv.com article-HTML parsers that feed the news
+// ARCHIVE pipeline (pull-archive.mjs). Covers the UNIQUE HTML parsers (parseHeadline/parseDate/bodyText)
+// AND locks the dedup: parseArticle must delegate to the hardened deal-parse.mjs parsers (not a stale fork).
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { parseHeadline, parseDate, bodyText, parseArticle } from '../../scripts/lib/parse-article.mjs';
+
+const article = (headline, rest) => `<td class="header">${headline}<br> ${rest}</td>`;
+
+test('parseHeadline: pulls the header cell, rejects the site-title / stub', () => {
+ assert.equal(parseHeadline(article('Colrich Buys 104k sf Office in San Diego, CA', 'x')), 'Colrich Buys 104k sf Office in San Diego, CA');
+ assert.equal(parseHeadline('<html>no header here</html>'), ''); // boilerplate stub → not an article
+ assert.equal(parseHeadline(article('RENTV.com Commercial Real Estate', 'x')), ''); // generic site title leak
+ assert.equal(parseHeadline(article('Short', 'x')), ''); // < 8 chars → rejected
+});
+
+test('parseDate: m/d/yy(yy) after the header → ISO, with 2-digit-year windowing', () => {
+ assert.equal(parseDate(article('A Real Headline Here', '08/06/2026 body')).date, '2026-08-06');
+ assert.equal(parseDate(article('A Real Headline Here', '8/6/26 body')).date, '2026-08-06'); // 26 → 2026
+ assert.equal(parseDate(article('A Real Headline Here', '8/6/95 body')).date, '1995-08-06'); // 95 → 1995 (>=70)
+ assert.equal(parseDate('<html>no date</html>').date, null);
+ assert.equal(parseDate(article('A Real Headline Here', '13/45/2026 body')).date, null); // impossible → null
+});
+
+test('bodyText: strips tags + scripts/styles, collapses whitespace', () => {
+ assert.equal(bodyText('<style>x{}</style><p>Hello <b>world</b></p><script>evil()</script>'), 'Hello world');
+});
+
+test('parseArticle: full record + DELEGATES to the hardened deal-parse (dedup lock)', () => {
+ const a = parseArticle('99999', article('Colrich Buys 104k sf Office in San Diego, CA', '08/06/2026 The firm acquired it for $52 mil.'));
+ assert.equal(a.title, 'Colrich Buys 104k sf Office in San Diego, CA');
+ assert.equal(a.date, '2026-08-06');
+ assert.equal(a.amount, 52000000);
+ assert.equal(a.sqft, 104000); // magnitude-aware "104k sf" — proves the hardened parseSize is used
+ assert.equal(a.property_type, 'Office');
+ assert.equal(parseArticle('1', '<html>stub</html>'), null); // no headline → not an article
+});
+
+// Dedup regression guard: the archive parsers must be the SAME objects as deal-parse's (no stale fork).
+test('parse-article re-exports the hardened deal-parse parsers (no stale duplicate)', async () => {
+ const art = await import('../../scripts/lib/parse-article.mjs');
+ const deal = await import('../../scripts/lib/deal-parse.mjs');
+ for (const fn of ['parseAmount', 'txnOf', 'typeOf', 'classify', 'parseSize', 'summarize'])
+ assert.equal(art[fn], deal[fn], `${fn} must be the deal-parse instance, not a fork`);
+});
← 01984a25 auto-data-snapshot: 2026-08-06T18:06:44 (7 data files) — dat
·
back to Rentv 2026
·
auto-data-snapshot: 2026-08-06T18:37:41 (7 data files) — dat db9cc1f3 →