`scripts/build-normalized-flyers.mjs`

133 lines
#!/usr/bin/env node
// TK-10708  Build data/flyers-normalized.json — the NORMALIZED cross-source flyer index.
// CONSOLIDATED into re-flyer-aggregator (2026-08-25, decision B): this was the standalone
// ~/Projects/re-flyers scaffold; it only ever read THIS project's own outputs + local PG,
// so it now lives here as a script. The standalone project is retired.
//
// Reads the EXISTING, already-produced flyer/marketing-asset outputs from this project
// (does NOT re-scrape anything) and normalizes them into one common schema, deduped.
// $0 local, read-only against this project's files + local PG.
//
// Common schema (per TK-10708):
//   { source_build, listing_id, title, address, flyer_url, generated_at, asset_type,
//     rights_basis, tier, dedup_key }
//
// Sources unified:
//   A. public/flyers-found/property-flyers.json  (Tier-1 broker OMs/brochures)
//   B. reflyers PG external_marketing_asset       (classified deal assets)
//   C. out/spotlight-*.html                       (Tier-2 self-generated recaps)

import { readFileSync, writeFileSync, readdirSync, statSync, existsSync } from 'node:fs';
import { execFileSync } from 'node:child_process';
import { fileURLToPath } from 'node:url';
import { dirname, join } from 'node:path';
import { createHash } from 'node:crypto';

// This script lives in re-flyer-aggregator/scripts/, so ROOT = the aggregator project root.
const ROOT = dirname(dirname(fileURLToPath(import.meta.url)));
const AGG_DIR = ROOT; // sources are THIS project's own outputs (post-consolidation)
const OUT = join(ROOT, 'data', 'flyers-normalized.json');

const rows = [];
const dedup = new Set();
function push(r) {
  const key = (r.flyer_url || r.title || '').trim().toLowerCase();
  r.dedup_key = createHash('sha1').update(key).digest('hex').slice(0, 12);
  if (dedup.has(r.dedup_key)) return false;
  dedup.add(r.dedup_key);
  rows.push(r);
  return true;
}

// ---- Source A: broker-owned property flyers (Tier-1, real OMs/brochures) ----
try {
  const p = join(AGG_DIR, 'public', 'flyers-found', 'property-flyers.json');
  if (existsSync(p)) {
    const arr = JSON.parse(readFileSync(p, 'utf8'));
    for (const f of arr) {
      const title = (f.property || '').trim();
      const isOM = /\bOM\b|offering.?memo/i.test(title) || /_OM_/i.test(f.pdf || '');
      push({
        source_build: 'broker-site',
        listing_id: null,
        title: title || 'Untitled brochure',
        address: null, // broker flyers carry a property name, not a normalized address
        flyer_url: f.pdf || f.from || null,
        generated_at: f.found_at || null,
        asset_type: isOM ? 'offering_memorandum' : 'marketing_flyer',
        rights_basis: 'first_party_broker',
        tier: 1,
        firm: f.firm || null,
      });
    }
  }
} catch (e) { console.error('source A skipped:', e.message); }

// ---- Source B: classified deal assets from the reflyers staging DB ----
try {
  const dbList = execFileSync('psql', ['-lqt'], { encoding: 'utf8' });
  if (/\breflyers\b/.test(dbList)) {
    const json = execFileSync('psql', ['reflyers', '-tAc',
      `SELECT row_to_json(t) FROM (
         SELECT asset_type, tier, rights_basis, title, source_landing_url,
                document_url, last_verified_at, created_at
         FROM external_marketing_asset) t`], { encoding: 'utf8' }).trim();
    for (const line of json.split('\n').filter(Boolean)) {
      const a = JSON.parse(line);
      push({
        source_build: 'usre-deal',
        listing_id: null,
        title: a.title || 'Deal asset',
        address: null,
        flyer_url: a.document_url || a.source_landing_url || null,
        generated_at: a.last_verified_at || a.created_at || null,
        asset_type: a.asset_type,
        rights_basis: a.rights_basis || null,
        tier: a.tier,
      });
    }
  }
} catch (e) { console.error('source B skipped:', e.message); }

// ---- Source C: Tier-2 self-generated spotlight recaps (our own flyers) ----
try {
  const outDir = join(AGG_DIR, 'out');
  if (existsSync(outDir)) {
    for (const fn of readdirSync(outDir)) {
      if (!/^spotlight-.*\.html$/.test(fn)) continue;
      const full = join(outDir, fn);
      let title = fn.replace(/\.html$/, '');
      try {
        const html = readFileSync(full, 'utf8');
        const m = html.match(/<title>([^<]+)<\/title>/i);
        if (m) title = m[1].trim();
      } catch {}
      const st = statSync(full);
      push({
        source_build: 'rentv-spotlight',
        listing_id: fn.replace(/^spotlight-|\.html$/g, ''),
        title,
        address: null,
        flyer_url: 'file://' + full, // local Tier-2 artifact (print → PDF)
        generated_at: st.mtime.toISOString(),
        asset_type: 'property_spotlight',
        rights_basis: 'self_generated',
        tier: 2,
      });
    }
  }
} catch (e) { console.error('source C skipped:', e.message); }

// sort newest-first (nulls last)
rows.sort((a, b) => (b.generated_at || '').localeCompare(a.generated_at || ''));

const payload = {
  generated_at: new Date().toISOString(),
  count: rows.length,
  by_source: rows.reduce((m, r) => ((m[r.source_build] = (m[r.source_build] || 0) + 1), m), {}),
  note: 'Normalized cross-source flyer index (TK-10708). Read-only aggregate of existing re-flyer-aggregator outputs; no re-scrape. Tier-3/GATED assets carried with rights_basis for viewer filtering, never auto-downloaded.',
  flyers: rows,
};
writeFileSync(OUT, JSON.stringify(payload, null, 2));
console.log(`wrote ${OUT}: ${rows.length} flyers  by_source=${JSON.stringify(payload.by_source)}`);