`scripts/fetch-edgar-reit-deals.js`

101 lines
// fetch-edgar-reit-deals.js — RESELLABLE institutional deal-flow from SEC EDGAR (public, $0, no key).
//
// Doctrine (docs/SOURCING.md): resell only public-record data. Public real-estate companies + REITs are
// LEGALLY REQUIRED to disclose material property acquisitions to the SEC — free, public, resellable, and
// it catches the big institutional LLC/entity deals that dodge the county-assessor reassessment (the gap
// in derive-sale-prices). Strategy: (1) enumerate REITs by SIC 6798, (2) EDGAR full-text-search for
// acquisition 8-Ks in a window, (3) keep hits whose CIK is a REIT, (4) emit a linked filing feed.
// The $ + property extraction from each 8-K exhibit is a follow-on step (LLM-parse) — this ships the
// harness + metadata feed. SEC rule: descriptive User-Agent, <=10 req/s.
//   node scripts/fetch-edgar-reit-deals.js [days]     (default 120)
'use strict';
const fs = require('fs');
const path = require('path');
const UA = 'DesignerWallcoverings-research steve@designerwallcoverings.com';
const DAYS = +(process.argv[2] || 120);
const ROOT = path.join(__dirname, '..');
const sleep = ms => new Promise(r => setTimeout(r, ms));
const iso = d => d.toISOString().slice(0, 10);

async function get(url, asJson, tries = 3) {          // retry-then-skip: SEC 500s intermittently
  for (let i = 0; i < tries; i++) {
    try {
      const r = await fetch(url, { headers: { 'User-Agent': UA, 'Accept-Encoding': 'gzip, deflate' } });
      await sleep(140);                                // stay under SEC's 10 req/s
      if ((r.status === 500 || r.status === 429) && i < tries - 1) { await sleep(700); continue; }
      if (!r.ok) throw new Error(url + ' -> ' + r.status);
      return asJson ? r.json() : r.text();
    } catch (e) { if (i < tries - 1) { await sleep(700); continue; } throw e; }
  }
}

// 1. Enumerate REIT CIKs (SIC 6798) from browse-edgar atom. The atom's company names are a SEC bug
// ("ARRAY(0x...)"), but <cik> is clean — we only need the CIK SET for filtering; real names come from
// the FTS display_names. Source: SEC EDGAR company search, SIC=6798 (public).
async function reitCiks() {
  const set = new Set();
  for (let start = 0; start < 1500; start += 100) {
    let atom; try { atom = await get(`https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&SIC=6798&type=&dateb=&owner=include&count=100&start=${start}&output=atom`); }
    catch (e) { break; }
    const ciks = atom.match(/<cik>(\d+)<\/cik>/gi) || [];
    if (!ciks.length) break;
    for (const c of ciks) { const m = c.match(/(\d+)/); if (m) set.add(String(+m[1])); }
    process.stderr.write(`  REIT enum: ${set.size} so far\r`);
  }
  return set;
}

// 2. EDGAR full-text search for acquisition 8-Ks in the window.
async function acquisitionFilings(startdt, enddt) {
  const phrases = ['%22completed+the+acquisition%22', '%22acquired+the+property%22', '%22purchase+of+the+property%22', '%22completed+the+purchase%22'];
  const hits = [];
  for (const q of phrases) {
    let from = 0;
    for (let page = 0; page < 10; page++) {   // FTS caps ~100/page, 10 pages plenty for a window
      let d; try { d = await get(`https://efts.sec.gov/LATEST/search-index?q=${q}&forms=8-K&startdt=${startdt}&enddt=${enddt}&from=${from}`, true); }
      catch (e) { process.stderr.write(`  FTS skip (${q}): ${e.message.slice(-8)}\n`); break; }   // one bad phrase never kills the run
      const arr = (d.hits && d.hits.hits) || [];
      if (!arr.length) break;
      for (const x of arr) {
        const s = x._source || {};
        const cik = String(+(((s.display_names || [''])[0].match(/CIK\s*(\d+)/) || [])[1] || 0));
        hits.push({ id: x._id, cik, display: (s.display_names || [])[0] || '', date: s.file_date, form: (s.root_forms || s.file_type || '') });
      }
      from += arr.length;
    }
  }
  return hits;
}

(async () => {
  const end = new Date(process.env.GEN_TS ? Date.parse(process.env.GEN_TS) : Date.parse('2026-07-31'));
  const start = new Date(end.getTime() - DAYS * 86400000);
  process.stderr.write(`EDGAR REIT deal-flow: ${iso(start)} .. ${iso(end)}\n`);

  const reits = await reitCiks();
  process.stderr.write(`\nREITs (SIC 6798): ${reits.size}\n`);
  const all = await acquisitionFilings(iso(start), iso(end));
  // dedupe by filing id, keep REIT-only
  const seen = new Set(), deals = [];
  for (const h of all) {
    if (seen.has(h.id)) continue; seen.add(h.id);
    if (!reits.has(h.cik)) continue;
    const acc = h.id.split(':')[0].replace(/-/g, ''); const file = h.id.split(':')[1] || '';
    // Cody-gate fix (2026-07-31): FTS matches earnings releases/supplements that merely MENTION a past
    // acquisition. Those aren't acquisition filings — drop them so the feed isn't padded with them.
    if (/earning|supplement|quarterly|[-_]er[-_.]|q[1-4]x?20\d\d|x?er\d|bodrep/i.test(file)) continue;
    deals.push({
      company: (h.display.split('  (')[0] || h.display).trim(), cik: h.cik, ticker: (h.display.match(/\(([A-Z.\-]{1,6})\)/) || [])[1] || null,
      form: h.form, filed: h.date,
      filing_url: `https://www.sec.gov/Archives/edgar/data/${h.cik}/${acc}/${file}`,
      source: 'SEC EDGAR 8-K (public, resellable)', price: null, property: null, extract_status: 'pending-llm-parse'
    });
  }
  deals.sort((a, b) => (b.filed || '').localeCompare(a.filed || ''));
  const out = { generated: process.env.GEN_TS || iso(end), source: 'SEC EDGAR full-text search (SIC 6798 REITs, acquisition 8-Ks)', window_days: DAYS, reit_universe: reits.size, count: deals.length, deals };
  fs.writeFileSync(path.join(ROOT, 'data', 'edgar-reit-deals.json'), JSON.stringify(out, null, 2));
  process.stderr.write(`\nREIT acquisition filings (last ${DAYS}d): ${deals.length}\n`);
  deals.slice(0, 8).forEach(d => process.stderr.write(`  ${d.filed}  ${(d.company || '').slice(0, 40).padEnd(40)} ${d.ticker || ''}\n`));
  console.log(JSON.stringify({ reit_universe: reits.size, deals: deals.length, cost: '$0 (SEC public data)' }));
})().catch(e => { console.error('FATAL', e.message); process.exit(1); });