[object Object]

← back to Rentv

search-index: HomesOnSpec LIVE from Kamatera (InventoryHome, ~40k spec homes) — TK-10253

75b7680c4da72179115004b2312a150bcbee14e0 · 2026-08-05 13:59:38 -0700 · Steve Abrams

Steve: 'make sure live on server'. homesOnSpec source now SSHes Kamatera + psql-pulls the live
homesonspec.InventoryHome (non-demo, newest 40k of 69k) into the unified index. base64-wrapped
remote script + unit-separator (0x1f) field delimiter (a psql -F E'\t' does NOT work as a CLI arg —
first pass produced 40k empty rows); full ssh/psql paths so it works under launchd. Verified: 99.99%
city, price on 39.6k, url on all 40k. Index now 65.7k records across 4 builds (rentv+crcp+usre+homesonspec).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Files touched

Diff

commit 75b7680c4da72179115004b2312a150bcbee14e0
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Wed Aug 5 13:59:38 2026 -0700

    search-index: HomesOnSpec LIVE from Kamatera (InventoryHome, ~40k spec homes) — TK-10253
    
    Steve: 'make sure live on server'. homesOnSpec source now SSHes Kamatera + psql-pulls the live
    homesonspec.InventoryHome (non-demo, newest 40k of 69k) into the unified index. base64-wrapped
    remote script + unit-separator (0x1f) field delimiter (a psql -F E'\t' does NOT work as a CLI arg —
    first pass produced 40k empty rows); full ssh/psql paths so it works under launchd. Verified: 99.99%
    city, price on 39.6k, url on all 40k. Index now 65.7k records across 4 builds (rentv+crcp+usre+homesonspec).
    
    Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 scripts/build-search-index.mjs | 35 ++++++++++++++++++++++++++++-------
 1 file changed, 28 insertions(+), 7 deletions(-)

diff --git a/scripts/build-search-index.mjs b/scripts/build-search-index.mjs
index 73f33155..de74fa80 100644
--- a/scripts/build-search-index.mjs
+++ b/scripts/build-search-index.mjs
@@ -77,14 +77,35 @@ function usreDeals() {
     });
   } catch (e) { console.error('  usre source skipped:', String(e.message).split('\n')[0]); return []; }
 }
+// Resolve ssh explicitly (launchd minimal PATH), like PSQL.
+const SSH = (() => { for (const p of [process.env.SSH_BIN, '/usr/bin/ssh', 'ssh'].filter(Boolean)) { try { if (p === 'ssh' || existsSync(p)) return p; } catch { /* next */ } } return 'ssh'; })();
+const KAM = process.env.KAMATERA_HOST || 'root@45.61.58.125';
 function homesOnSpec() {
-  // pluggable slot — prod is Kamatera; index a local export if one exists.
-  for (const f of ['homes.json', 'homesonspec.json', 'listings.json']) {
-    const p = join(PROJ, 'homesonspec', 'data', f);
-    if (existsSync(p)) { const arr = jload(p, []); const a = Array.isArray(arr) ? arr : (arr.homes || arr.listings || []);
-      return a.map((x, i) => ({ build: 'homesonspec', type: 'listing', id: 'hos-' + (x.id || i), title: x.address || x.title || 'Spec home', subtitle: [x.status, x.beds ? x.beds + 'bd' : null].filter(Boolean).join(' · '), date: x.created_at || x.listed_date || null, city: x.city || null, region: x.state || null, amount: +x.price || null, amount_label: money(x.price), parties: x.builder || null, url: x.url || null, text: blob(x.address, x.city, x.state, x.builder, x.status) })); }
-  }
-  return [];
+  // LIVE from Kamatera prod (Steve 2026-08-05: "make sure live on server"). HomesOnSpec inventory
+  // lives in the Kamatera-local Postgres `homesonspec`.`InventoryHome` (~69k spec homes). We SSH +
+  // psql it using the app's own DATABASE_URL, base64-wrapping the remote script to avoid quote
+  // mangling. Non-demo homes, newest first. Guarded — a Kamatera hiccup skips the source, never fatal.
+  try {
+    // Field separator = ASCII unit-separator (0x1f) via bash $'\x1f' — survives base64→bash→psql and
+    // never appears in address/name data (a plain tab via -F E'\t' does NOT work as a psql CLI arg).
+    const remote = `cd /root/Projects/homesonspec && DBURL=$(grep -hoE 'postgresql://[^ "]+' .env | head -1) && ` +
+      `psql "$DBURL" -tA -F $'\\x1f' -c 'SELECT id,street,city,state,zip,price,beds,"bathsTotal",sqft,"homeType","constructionStatus","sourceUrl","publishedAt","createdAt" ` +
+      `FROM "InventoryHome" WHERE "isDemo"=false ORDER BY "publishedAt" DESC NULLS LAST LIMIT 40000'`;
+    const b64 = Buffer.from(remote).toString('base64');
+    const out = execSync(`${SSH} -o ConnectTimeout=20 -o BatchMode=yes ${KAM} "echo ${b64} | base64 -d | bash"`, { encoding: 'utf8', timeout: 90000, maxBuffer: 64 * 1024 * 1024 });
+    const SEP = String.fromCharCode(31);
+    return out.trim().split('\n').filter(Boolean).map((line) => {
+      const [id, street, city, state, zip, price, beds, baths, sqft, homeType, cstatus, url, pub, created] = line.split(SEP);
+      return {
+        build: 'homesonspec', type: 'listing', id: 'hos-' + id,
+        title: [street, city, state].filter(Boolean).join(', ') || 'Spec home',
+        subtitle: [homeType, beds ? beds + 'bd' : null, baths ? baths + 'ba' : null, sqft ? (+sqft).toLocaleString() + ' SF' : null, cstatus].filter(Boolean).join(' · '),
+        date: ((pub || created || '').slice(0, 10)) || null, city: city || null, region: state || null,
+        amount: +price || null, amount_label: money(price), parties: null, url: url || null,
+        text: blob(street, city, state, zip, homeType, cstatus),
+      };
+    });
+  } catch (e) { console.error('  homesOnSpec (Kamatera) source skipped:', String(e.message).split('\n')[0]); return []; }
 }
 
 const SOURCES = [rentvArticles, rentvNews, crcpDeals, crcpEdgar, usreDeals, homesOnSpec];

← fc57490b Audience perf: precompute per-contact search index once on l  ·  back to Rentv  ·  pr-intelligence dashboard: every data point hrefs deeper — b 5c464c00 →