`scripts/property-flyers.mjs`

82 lines
#!/usr/bin/env node
// TK-10708  PROPERTY-FLYER FINDER. For each CRE broker, crawl the homepage AND its listing/
// for-sale/for-lease pages, harvest the PDF marketing one-sheets for SPECIFIC PROPERTIES
// (address / "For Sale" / Offering Memorandum), and drop the junk (lease apps, proxies,
// codes of conduct, investor decks). Output = a browsable property-flyer gallery. $0, read-only.
//
// Usage: node scripts/property-flyers.mjs [--n 120] [--pages 6] [--delay 900]

import { execFileSync } from 'node:child_process';
import { writeFileSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname, join } from 'node:path';

const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
const args = process.argv.slice(2);
const arg = (f, d) => { const i = args.indexOf(f); return i >= 0 ? args[i + 1] : d; };
const N = parseInt(arg('--n', '120'), 10), PAGES = parseInt(arg('--pages', '6'), 10), DELAY = parseInt(arg('--delay', '800'), 10);
const OUTDIR = join(ROOT, 'public', 'flyers-found');
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36';
const sleep = ms => new Promise(r => setTimeout(r, ms));

// a PDF is a PROPERTY flyer if it looks like a specific-property one-sheet, and NOT a legal/corp doc.
const PROP = /(\d{2,5}[-_%\s]+\w+[-_%\s]+(ave|st|blvd|dr|rd|way|street|road|drive|lane|court|place|hwy|highway|pkwy|circle)|offering[-_ ]?mem|\bOM[-_.]|[-_]om[-_.]|for[-_ ]?(sale|lease)|flyer|brochure|marketing[-_ ]?pack|property[-_ ]?(brief|package|flyer)|listing[-_ ]?flyer|deal[-_ ]?sheet|\bpsf\b|sf[-_ ]?available)/i;
const JUNK = /(application|privacy|terms|\btou\b|conduct|w-?9|credit[-_ ]?app|policy|disclosure|\bnda\b|agreement|vendor|\bguide\b|proxy|supplemental|investor[-_ ]?(deck|present)|annual[-_ ]?(report|transp)|10-?k|whitepaper|newsletter|code[-_ ]?of|tax[-_ ]?info|profile|strategic[-_ ]?deck|commitment|handbook|w9)/i;
// links worth following to find listings
const FOLLOW = /(listing|propert|for-?sale|for-?lease|available|inventory|portfolio|investment|space|our-?deals|closed)/i;

const dec = s => { try { return decodeURIComponent(decodeURIComponent(s)); } catch { try { return decodeURIComponent(s); } catch { return s; } } };
const propTitle = (url) => {
  let f = dec((url.split('/').pop() || '')).replace(/\.pdf.*$/i, '')
    .replace(/^[0-9a-f]{6,}[_-]+/i, '')          // strip leading Wix/CMS hash id
    .replace(/[_]+/g, ' ').replace(/-+/g, ' ').replace(/\s+/g, ' ').trim();
  return f.replace(/\b\w/g, c => c.toUpperCase()).slice(0, 70) || url;
};
const dom = u => { try { return new URL(u.startsWith('http') ? u : 'https://' + u).hostname.replace(/^www\./, ''); } catch { return null; } };

// broker targets — CA commercial firms w/ a website (big-first)
const rows = execFileSync('psql', ['usre', '-t', '-A', '-F', '\t', '-c',
  `SELECT coalesce(name,''), website FROM firm WHERE asset_class='commercial' AND hq_state='CA' AND website IS NOT NULL AND website<>'' ORDER BY agent_count DESC NULLS LAST LIMIT ${N * 2}`],
  { encoding: 'utf8' }).trim().split('\n').filter(Boolean).map(l => l.split('\t'));
const seenDom = new Set(), targets = [];
for (const [name, web] of rows) { const d = dom(web); if (d && !seenDom.has(d)) { seenDom.add(d); targets.push({ name, domain: d }); } if (targets.length >= N) break; }

async function get(url) {
  try { const r = await fetch(url, { headers: { 'User-Agent': UA }, redirect: 'follow', signal: AbortSignal.timeout(12000) }); return r.ok ? await r.text() : ''; } catch { return ''; }
}
const abs = (href, base) => { try { return new URL(href, base).href; } catch { return null; } };
function pdfsIn(html, base) {
  const out = new Set(), re = /href\s*=\s*["']([^"']+\.pdf[^"']*)["']/gi; let m;
  while ((m = re.exec(html))) { const u = abs(m[1], base); if (u && PROP.test(u) && !JUNK.test(u)) out.add(u); }
  return [...out];
}
function listingLinks(html, base, host) {
  const out = new Set(), re = /href\s*=\s*["']([^"']+)["']/gi; let m;
  while ((m = re.exec(html))) { const u = abs(m[1], base); if (!u) continue; try { if (new URL(u).hostname.replace(/^www\./, '') !== host) continue; } catch { continue; } if (FOLLOW.test(u) && !/\.pdf|\.jpg|\.png/i.test(u)) out.add(u.split('#')[0]); }
  return [...out].slice(0, PAGES);
}

async function crawlBroker(t) {
  const base = 'https://' + t.domain + '/';
  const home = await get(base); if (!home) return [];
  const found = new Map();   // url -> flyer
  const add = (u, pageUrl) => { if (!found.has(u)) found.set(u, { firm: t.name, domain: t.domain, property: propTitle(u), pdf: u, from: pageUrl, found_at: new Date().toISOString() }); };
  pdfsIn(home, base).forEach(u => add(u, base));
  const links = listingLinks(home, base, t.domain);
  for (const lk of links) { const h = await get(lk); if (h) pdfsIn(h, lk).forEach(u => add(u, lk)); await sleep(120); }
  return [...found.values()];
}

// pooled crawl
async function pool(items, conc, fn) { const out = []; let i = 0; const w = async () => { while (i < items.length) { const it = items[i++]; out.push(...await fn(it)); process.stdout.write('.'); await sleep(DELAY); } }; await Promise.all(Array.from({ length: Math.min(conc, items.length) }, w)); return out; }

console.log(`Property-flyer crawl: ${targets.length} CA brokers × up to ${PAGES} listing pages…`);
const flyers = await pool(targets, 6, crawlBroker);
// dedup by pdf url
const seen = new Set(), out = [];
for (const f of flyers) { if (seen.has(f.pdf)) continue; seen.add(f.pdf); out.push(f); }
out.sort((a, b) => a.firm.localeCompare(b.firm) || a.property.localeCompare(b.property));
writeFileSync(join(OUTDIR, 'property-flyers.json'), JSON.stringify(out, null, 0));
console.log(`\nFound ${out.length} PROPERTY flyers across ${new Set(out.map(f => f.domain)).size} brokers -> public/flyers-found/property-flyers.json`);
for (const f of out.slice(0, 20)) console.log(`  ${f.firm.slice(0, 26).padEnd(26)} ${f.property.slice(0, 46)}`);