`scripts/ingest-rentv.mjs`

48 lines
#!/usr/bin/env node
// TK-10708  Add RENTV as a DIRECT source (Steve). Pulls Steve's own rentv.agentabrams.com API —
// structured live deals (txn_type/property_type/city/amount/size/address) + news — far richer than the
// gnews proxy (~35 headline-only). $0 (Steve's own site). Merges into deal-store + article-archive, deduped.
//
// Usage: node scripts/ingest-rentv.mjs

import { readFileSync, writeFileSync, existsSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname, join } from 'node:path';

const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
const BASE = process.env.RENTV_BASE || 'https://rentv.agentabrams.com';
const AUTH = 'Basic ' + Buffer.from('admin:DW2024!').toString('base64');
const get = async p => { try { const r = await fetch(BASE + p, { headers: { Authorization: AUTH }, signal: AbortSignal.timeout(14000) }); return r.ok ? await r.json() : []; } catch (e) { console.error('  RENTV', p, 'error:', e.message); return []; } };
const arr = j => Array.isArray(j) ? j : (j.deals || j.news || j.items || j.data || []);
const now = new Date().toISOString();
const abs = u => !u ? null : u.startsWith('http') ? u : 'https://www.rentv.com' + u;

const deals = arr(await get('/api/deals'));
const news = arr(await get('/api/news'));
const recs = [];
for (const d of deals) {
  const mkt = [d.city, d.state].filter(Boolean).join(', ') || d.state || 'CA';
  recs.push({ type: d.txn_type || 'Sale', price: d.amount || null, label: d.amount_label || null, market: mkt, title: d.title, source: 'RENTV', link: abs(d.url), date: now, property_type: d.property_type || null, address: d.address || null, size_label: d.size_label || null, units: d.units || null, sqft: d.sqft || null, occupancy_pct: d.occupancy_pct || null, buyer: null, seller: null });
}
for (const n of news) recs.push({ type: 'News', price: null, label: null, market: n.cat || 'CA', title: n.title, source: 'RENTV', link: abs(n.url), date: now, buyer: null, seller: null });

// --- merge into the rolling deal store (deals) + article archive (all), dedup by price|title-slice ---
const sig = d => `${d.price}|${(d.title || '').toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim().slice(0, 40)}`;
let added = 0, archAdded = 0;
const STORE = join(ROOT, 'data', 'deal-store.json');
let store = existsSync(STORE) ? JSON.parse(readFileSync(STORE, 'utf8')) : [];
const have = new Set(store.map(sig));
for (const r of recs.filter(r => r.type !== 'News')) { const s = sig(r); if (!have.has(s)) { have.add(s); store.push({ ...r, added: now }); added++; } }
store.sort((a, b) => (b.date || '').localeCompare(a.date || '') || (b.price || 0) - (a.price || 0));
writeFileSync(STORE, JSON.stringify(store, null, 0));

const ARCH = join(ROOT, 'data', 'article-archive.json');
let arch = existsSync(ARCH) ? JSON.parse(readFileSync(ARCH, 'utf8')) : [];
const asig = a => (a.title || '').toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim().slice(0, 60);
const ahave = new Set(arch.map(asig));
for (const r of recs) { const s = asig(r); if (s && !ahave.has(s)) { ahave.add(s); arch.push({ title: r.title, link: r.link, date: r.date, price: r.label || null, type: r.type, market: r.market, source: 'RENTV', buyer: null, seller: null, first_seen: now }); archAdded++; } }
arch.sort((a, b) => (Date.parse(b.date) || 0) - (Date.parse(a.date) || 0));
writeFileSync(ARCH, JSON.stringify(arch, null, 0));

console.log(`RENTV direct: ${deals.length} deals + ${news.length} news from ${BASE} -> store +${added}, archive +${archAdded}.`);