`scripts/enrich-provenance.js`

114 lines
// enrich-provenance.js — Restore the broker-of-record as each deal's source of truth.
//
// DOCTRINE (docs/SOURCING.md): every deal carries its real listing broker firm + agent; the
// aggregator (CREXi/Redfin/Zillow/CoStar/LoopNet) is a breadcrumb to the broker, never the truth.
// This joins the already-captured listing brokers (data/raw/broker-blocks.json, from the CREXi
// /assets/<id>/brokers pull) back onto data/ranked.json — an edge the ranking step dropped, leaving
// every deal mislabeled firm:"Crexi". Idempotent, $0, local. Re-run safe.
//
// Adds per deal:  broker_firm, broker_agent, broker_agents[], source_of_truth, source_host
//   source_of_truth ∈ { 'firm-direct', 'broker-of-record', 'aggregator' }
//
// Usage: node scripts/enrich-provenance.js
'use strict';
const fs = require('fs');
const path = require('path');
const ROOT = path.join(__dirname, '..');
// Append-only broker-of-record history recorder (TK-10139). Records EVERY update to the broker/firm of
// record per address so we can surface "Last Broker/Firm of Record". Non-fatal: a recorder failure
// (missing native module, locked DB) must never break enrichment.
let recordBrokerOfRecord = null;
try { ({ recordBrokerOfRecord } = require('./broker-of-record-recorder')); } catch (_) { recordBrokerOfRecord = null; }

// Pure marketplaces/portals Steve named to distrust. A listing served from a *brokerage's* own
// domain (Compass, RE/MAX, an independent broker site) is firm-direct, NOT an aggregator.
const AGGREGATORS = ['crexi', 'redfin', 'zillow', 'costar', 'loopnet', 'realtor', 'myelisting'];
const hostOf = (url) => { try { return new URL(url).hostname.replace(/^www\./, ''); } catch { return null; } };
const aggFromHost = (h) => h && AGGREGATORS.find(a => h.includes(a)) || null;

// A broker's brokerage may be a string ("RE/MAX Luxe") or an object ({name}). Normalize.
const firmName = (b) => {
  const f = b && b.brokerage;
  const name = typeof f === 'string' ? f : (f && f.name);
  const s = String(name || '').trim();
  return (!s || /^crexi$/i.test(s)) ? null : s;   // "Crexi" is the marketplace, not a real firm
};
const agentName = (b) => [b && b.firstName, b && b.lastName].filter(Boolean).join(' ').trim() || null;
// Deep-link to the broker's OWN site (the canonical source per doctrine). Object-form brokerage
// blocks carry .website; normalize to an absolute URL.
const firmUrl = (b) => {
  const f = b && b.brokerage;
  let w = (f && typeof f === 'object' && f.website) ? String(f.website).trim() : '';
  if (!w) return null;
  if (!/^https?:\/\//i.test(w)) { if (!/\./.test(w)) return null; w = 'https://' + w; }
  try { new URL(w); return w; } catch { return null; }
};

(function main() {
  const rankedPath = path.join(ROOT, 'data', 'ranked.json');
  const ranked = JSON.parse(fs.readFileSync(rankedPath, 'utf8'));
  const deals = ranked.ranked || [];

  // assetId → [brokers]
  const blocksPath = path.join(ROOT, 'data', 'raw', 'broker-blocks.json');
  const byAsset = new Map();
  if (fs.existsSync(blocksPath)) {
    for (const row of JSON.parse(fs.readFileSync(blocksPath, 'utf8'))) {
      if (Array.isArray(row.brokers) && row.brokers.length) byAsset.set(String(row.id), row.brokers);
    }
  }

  const stat = { total: deals.length, firmDirect: 0, brokerOfRecord: 0, aggregator: 0, firmResolved: 0 };
  const bySource = {};

  for (const d of deals) {
    const host = hostOf(d.source);
    d.source_host = host || null;
    const agg = aggFromHost(host);

    // Attach real listing broker(s) from the captured broker block.
    const assetId = String(d.id || '').replace(/^crx/, '');
    const brokers = byAsset.get(assetId) || [];
    const agents = [...new Set(brokers.map(agentName).filter(Boolean))];
    const firms = [...new Set(brokers.map(firmName).filter(Boolean))];
    const urls = [...new Set(brokers.map(firmUrl).filter(Boolean))];
    if (agents.length) { d.broker_agents = agents; d.broker_agent = agents[0]; }
    if (firms.length) { d.broker_firm = firms.length > 1 ? firms.join(' · ') : firms[0]; stat.firmResolved++; }
    if (urls.length) { d.broker_url = urls[0]; stat.deepLinked = (stat.deepLinked || 0) + 1; }

    // Classify source-of-truth. firm-direct = came from a real brokerage's own site/API, either via
    // a verified firm scraper (firm_key) or a non-aggregator source host (the broker's own domain).
    const firmKeyDirect = d.firm_key && !AGGREGATORS.includes(String(d.firm_key).toLowerCase());
    const hostDirect = host && !agg;   // sourced from a broker/firm domain, not a marketplace
    if (firmKeyDirect || hostDirect) { d.source_of_truth = 'firm-direct'; stat.firmDirect++; }
    else if (d.broker_firm) { d.source_of_truth = 'broker-of-record'; stat.brokerOfRecord++; }
    else { d.source_of_truth = 'aggregator'; stat.aggregator++; }

    const key = agg || (firmKeyDirect ? (d.firm_key || 'firm') : (host || 'unknown'));
    bySource[key] = (bySource[key] || 0) + 1;

    // Record any UPDATE to this address's broker/firm of record (append-only). Non-fatal.
    if (recordBrokerOfRecord) {
      try {
        const r = recordBrokerOfRecord({
          address: d.address, city: d.city, ain: d.ain || null,
          broker_firm: d.broker_firm || null, broker_agent: d.broker_agent || null,
          source_of_truth: d.source_of_truth || null, source_host: d.source_host || null,
        });
        if (r && r.recorded) stat.borRecorded = (stat.borRecorded || 0) + 1;
      } catch (_) { /* recorder failure must not break enrichment */ }
    }
  }

  ranked.provenance = { enriched_at: new Date().toISOString(), ...stat, bySource };
  fs.writeFileSync(rankedPath, JSON.stringify(ranked, null, 0));

  console.log(JSON.stringify({ ...stat, bySource }, null, 2));
  const pct = (n) => ((n / stat.total) * 100).toFixed(1) + '%';
  process.stderr.write(
    `\nProvenance: ${stat.firmDirect} firm-direct (${pct(stat.firmDirect)}) · ` +
    `${stat.brokerOfRecord} broker-of-record (${pct(stat.brokerOfRecord)}) · ` +
    `${stat.aggregator} aggregator-only (${pct(stat.aggregator)})\n` +
    `Real listing broker firm resolved on ${stat.firmResolved}/${stat.total} deals ($0, local).\n`);
})();