← back to Re Flyer Aggregator
TK-10708 (decision B): consolidate re-flyers normalizer into aggregator as scripts/build-normalized-flyers.mjs (224 rows byte-identical: broker 202/spotlight 7/usre 15); gitignore generated feed; retire standalone
ff6543bf030986073dfe9fde0e4401362c1c45c5 · 2026-08-24 20:46:20 -0700 · Steve Abrams
Files touched
M .gitignoreA scripts/build-normalized-flyers.mjs
Diff
commit ff6543bf030986073dfe9fde0e4401362c1c45c5
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Mon Aug 24 20:46:20 2026 -0700
TK-10708 (decision B): consolidate re-flyers normalizer into aggregator as scripts/build-normalized-flyers.mjs (224 rows byte-identical: broker 202/spotlight 7/usre 15); gitignore generated feed; retire standalone
---
.gitignore | 1 +
scripts/build-normalized-flyers.mjs | 132 ++++++++++++++++++++++++++++++++++++
2 files changed, 133 insertions(+)
diff --git a/.gitignore b/.gitignore
index 90f3a06..a47d07a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,3 +25,4 @@ public/flyers/index.html
public/sources.json
data/refine-loop.log
data/*-cron.log
+data/flyers-normalized.json
diff --git a/scripts/build-normalized-flyers.mjs b/scripts/build-normalized-flyers.mjs
new file mode 100644
index 0000000..76bd3db
--- /dev/null
+++ b/scripts/build-normalized-flyers.mjs
@@ -0,0 +1,132 @@
+#!/usr/bin/env node
+// TK-10708 Build data/flyers-normalized.json — the NORMALIZED cross-source flyer index.
+// CONSOLIDATED into re-flyer-aggregator (2026-08-25, decision B): this was the standalone
+// ~/Projects/re-flyers scaffold; it only ever read THIS project's own outputs + local PG,
+// so it now lives here as a script. The standalone project is retired.
+//
+// Reads the EXISTING, already-produced flyer/marketing-asset outputs from this project
+// (does NOT re-scrape anything) and normalizes them into one common schema, deduped.
+// $0 local, read-only against this project's files + local PG.
+//
+// Common schema (per TK-10708):
+// { source_build, listing_id, title, address, flyer_url, generated_at, asset_type,
+// rights_basis, tier, dedup_key }
+//
+// Sources unified:
+// A. public/flyers-found/property-flyers.json (Tier-1 broker OMs/brochures)
+// B. reflyers PG external_marketing_asset (classified deal assets)
+// C. out/spotlight-*.html (Tier-2 self-generated recaps)
+
+import { readFileSync, writeFileSync, readdirSync, statSync, existsSync } from 'node:fs';
+import { execFileSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import { dirname, join } from 'node:path';
+import { createHash } from 'node:crypto';
+
+// This script lives in re-flyer-aggregator/scripts/, so ROOT = the aggregator project root.
+const ROOT = dirname(dirname(fileURLToPath(import.meta.url)));
+const AGG_DIR = ROOT; // sources are THIS project's own outputs (post-consolidation)
+const OUT = join(ROOT, 'data', 'flyers-normalized.json');
+
+const rows = [];
+const dedup = new Set();
+function push(r) {
+ const key = (r.flyer_url || r.title || '').trim().toLowerCase();
+ r.dedup_key = createHash('sha1').update(key).digest('hex').slice(0, 12);
+ if (dedup.has(r.dedup_key)) return false;
+ dedup.add(r.dedup_key);
+ rows.push(r);
+ return true;
+}
+
+// ---- Source A: broker-owned property flyers (Tier-1, real OMs/brochures) ----
+try {
+ const p = join(AGG_DIR, 'public', 'flyers-found', 'property-flyers.json');
+ if (existsSync(p)) {
+ const arr = JSON.parse(readFileSync(p, 'utf8'));
+ for (const f of arr) {
+ const title = (f.property || '').trim();
+ const isOM = /\bOM\b|offering.?memo/i.test(title) || /_OM_/i.test(f.pdf || '');
+ push({
+ source_build: 'broker-site',
+ listing_id: null,
+ title: title || 'Untitled brochure',
+ address: null, // broker flyers carry a property name, not a normalized address
+ flyer_url: f.pdf || f.from || null,
+ generated_at: f.found_at || null,
+ asset_type: isOM ? 'offering_memorandum' : 'marketing_flyer',
+ rights_basis: 'first_party_broker',
+ tier: 1,
+ firm: f.firm || null,
+ });
+ }
+ }
+} catch (e) { console.error('source A skipped:', e.message); }
+
+// ---- Source B: classified deal assets from the reflyers staging DB ----
+try {
+ const dbList = execFileSync('psql', ['-lqt'], { encoding: 'utf8' });
+ if (/\breflyers\b/.test(dbList)) {
+ const json = execFileSync('psql', ['reflyers', '-tAc',
+ `SELECT row_to_json(t) FROM (
+ SELECT asset_type, tier, rights_basis, title, source_landing_url,
+ document_url, last_verified_at, created_at
+ FROM external_marketing_asset) t`], { encoding: 'utf8' }).trim();
+ for (const line of json.split('\n').filter(Boolean)) {
+ const a = JSON.parse(line);
+ push({
+ source_build: 'usre-deal',
+ listing_id: null,
+ title: a.title || 'Deal asset',
+ address: null,
+ flyer_url: a.document_url || a.source_landing_url || null,
+ generated_at: a.last_verified_at || a.created_at || null,
+ asset_type: a.asset_type,
+ rights_basis: a.rights_basis || null,
+ tier: a.tier,
+ });
+ }
+ }
+} catch (e) { console.error('source B skipped:', e.message); }
+
+// ---- Source C: Tier-2 self-generated spotlight recaps (our own flyers) ----
+try {
+ const outDir = join(AGG_DIR, 'out');
+ if (existsSync(outDir)) {
+ for (const fn of readdirSync(outDir)) {
+ if (!/^spotlight-.*\.html$/.test(fn)) continue;
+ const full = join(outDir, fn);
+ let title = fn.replace(/\.html$/, '');
+ try {
+ const html = readFileSync(full, 'utf8');
+ const m = html.match(/<title>([^<]+)<\/title>/i);
+ if (m) title = m[1].trim();
+ } catch {}
+ const st = statSync(full);
+ push({
+ source_build: 'rentv-spotlight',
+ listing_id: fn.replace(/^spotlight-|\.html$/g, ''),
+ title,
+ address: null,
+ flyer_url: 'file://' + full, // local Tier-2 artifact (print → PDF)
+ generated_at: st.mtime.toISOString(),
+ asset_type: 'property_spotlight',
+ rights_basis: 'self_generated',
+ tier: 2,
+ });
+ }
+ }
+} catch (e) { console.error('source C skipped:', e.message); }
+
+// sort newest-first (nulls last)
+rows.sort((a, b) => (b.generated_at || '').localeCompare(a.generated_at || ''));
+
+const payload = {
+ generated_at: new Date().toISOString(),
+ count: rows.length,
+ by_source: rows.reduce((m, r) => ((m[r.source_build] = (m[r.source_build] || 0) + 1), m), {}),
+ note: 'Normalized cross-source flyer index (TK-10708). Read-only aggregate of existing re-flyer-aggregator outputs; no re-scrape. Tier-3/GATED assets carried with rights_basis for viewer filtering, never auto-downloaded.',
+ flyers: rows,
+};
+writeFileSync(OUT, JSON.stringify(payload, null, 2));
+console.log(`wrote ${OUT}: ${rows.length} flyers by_source=${JSON.stringify(payload.by_source)}`);
← 4c87c25 auto-data-snapshot: 2026-08-24T16:31:44 (6 data files) — dat
·
back to Re Flyer Aggregator
·
auto-data-snapshot: 2026-08-25T00:22:08 (6 data files) — dat f109f23 →