← back to Rentv Adintel
db/seed/sources.js
472 lines
'use strict';
/**
* Seed source_policies — RENTV-owned/authorized sources (enabled=true) plus
* CA/AZ publication/association sources requiring manual review (enabled=false).
* Idempotent via ON CONFLICT (source_key) DO UPDATE.
*/
const REVIEWED_AT = '2026-08-07T00:00:00Z';
// ---- RENTV-owned / authorized sources (enabled, automated or upload OK) ----
const RENTV_SOURCES = [
{
source_key: 'rentv_public_web',
display_name: 'RENTV.com public website',
owner: 'RENTV',
base_url: 'https://www.rentv.com',
access_method: 'first_party_public_web',
allows_automated_access: true,
allows_screenshot_capture: true,
allows_internal_storage: true,
allows_export: true,
enabled: true,
review_notes: 'RENTV-owned property. Full automated access permitted.',
},
{
source_key: 'rentvreview_public_web',
display_name: 'RENTVReview.com public website',
owner: 'RENTV',
base_url: 'https://www.rentvreview.com',
access_method: 'first_party_public_web',
allows_automated_access: true,
allows_screenshot_capture: true,
allows_internal_storage: true,
allows_export: true,
enabled: true,
review_notes: 'RENTV-owned property. Full automated access permitted.',
},
{
source_key: 'rentv_conference_pages',
display_name: 'RENTV conference and event pages',
owner: 'RENTV',
base_url: 'https://www.rentv.com',
access_method: 'first_party_public_web',
allows_automated_access: true,
allows_screenshot_capture: true,
allows_internal_storage: true,
allows_export: true,
enabled: true,
review_notes: 'RENTV-owned conference pages. Sponsor data may be exported as evidence.',
},
{
source_key: 'rentv_cre_talk_pages',
display_name: 'RENTV CRE Talk pages',
owner: 'RENTV',
base_url: 'https://www.rentv.com',
access_method: 'first_party_public_web',
allows_automated_access: true,
allows_screenshot_capture: true,
allows_internal_storage: true,
allows_export: true,
enabled: true,
review_notes: 'RENTV CRE Talk sponsor listings are first-party evidence.',
},
{
source_key: 'rentv_newsletters',
display_name: 'RENTV newsletters and browser-view archives',
owner: 'RENTV',
base_url: 'https://www.rentv.com',
access_method: 'first_party_public_web',
allows_automated_access: true,
allows_screenshot_capture: true,
allows_internal_storage: true,
allows_export: true,
enabled: true,
review_notes: 'RENTV-authored newsletters. Sponsor and ad evidence may be retained.',
},
{
source_key: 'rentv_constant_contact_export',
display_name: 'RENTV Constant Contact export (CSV upload)',
owner: 'RENTV',
base_url: 'https://app.constantcontact.com',
access_method: 'manual_upload',
allows_automated_access: false,
allows_screenshot_capture: false,
allows_internal_storage: true,
allows_export: true,
enabled: true,
review_notes: 'Manual CSV export from authorized RENTV Constant Contact account. Upload only.',
},
{
source_key: 'rentv_gmail_authorized',
display_name: 'RENTV authorized Gmail mailbox (Gmail API)',
owner: 'RENTV',
base_url: 'https://mail.google.com',
access_method: 'authorized_mailbox',
allows_automated_access: false,
allows_screenshot_capture: false,
allows_internal_storage: true,
allows_export: false,
enabled: false,
review_notes: 'Disabled by default. Enable only after OAuth setup and admin review. Store selected messages only.',
},
{
source_key: 'rentv_manual_upload',
display_name: 'RENTV manual file upload (.eml, HTML, PDF, JPG, PNG, CSV, XLSX)',
owner: 'RENTV',
base_url: null,
access_method: 'manual_upload',
allows_automated_access: false,
allows_screenshot_capture: false,
allows_internal_storage: true,
allows_export: true,
enabled: true,
review_notes: 'Admin-uploaded files. Rights status per file.',
},
{
source_key: 'rentv_media_kit',
display_name: 'RENTV media kit and rate card',
owner: 'RENTV',
base_url: 'https://www.rentv.com',
access_method: 'first_party_public_web',
allows_automated_access: true,
allows_screenshot_capture: true,
allows_internal_storage: true,
allows_export: true,
enabled: true,
review_notes: 'First-party RENTV rate/audience data. Export allowed.',
},
];
// ---- External CA/AZ sources — manual_review_only, automation disabled -------
const EXTERNAL_SOURCES = [
// CA Business Journals
{
source_key: 'la_business_journal',
display_name: 'Los Angeles Business Journal',
owner: 'American City Business Journals',
base_url: 'https://labusinessjournal.com',
access_method: 'manual_review_only',
review_notes: 'Paywall/subscr. No automated scraping. Manual review of public event/sponsor pages only.',
},
{
source_key: 'oc_business_journal',
display_name: 'Orange County Business Journal',
owner: 'Orange County Business Journal',
base_url: 'https://www.ocbj.com',
access_method: 'manual_review_only',
review_notes: 'Paywall. Manual review only.',
},
{
source_key: 'san_diego_business_journal',
display_name: 'San Diego Business Journal',
owner: 'American City Business Journals',
base_url: 'https://www.sdbj.com',
access_method: 'manual_review_only',
review_notes: 'Paywall. Manual review only.',
},
{
source_key: 'sf_business_times',
display_name: 'San Francisco Business Times',
owner: 'American City Business Journals',
base_url: 'https://www.bizjournals.com/sanfrancisco',
access_method: 'manual_review_only',
review_notes: 'Paywall. Manual review only.',
},
{
source_key: 'sv_business_journal',
display_name: 'Silicon Valley Business Journal',
owner: 'American City Business Journals',
base_url: 'https://www.bizjournals.com/sanjose',
access_method: 'manual_review_only',
review_notes: 'Paywall. Manual review only.',
},
{
source_key: 'sacramento_business_journal',
display_name: 'Sacramento Business Journal',
owner: 'American City Business Journals',
base_url: 'https://www.bizjournals.com/sacramento',
access_method: 'manual_review_only',
review_notes: 'Paywall. Manual review only.',
},
// CRE publications
{
source_key: 'bisnow',
display_name: 'Bisnow market and event pages',
owner: 'Bisnow',
base_url: 'https://www.bisnow.com',
access_method: 'manual_review_only',
review_notes: 'Terms restrict scraping. Manual review of public event/sponsor pages only.',
},
{
source_key: 'connect_cre',
display_name: 'Connect CRE',
owner: 'Connect Media',
base_url: 'https://www.connectcre.com',
access_method: 'manual_review_only',
review_notes: 'Manual review only. Robots policy not verified as permitting automation.',
},
{
source_key: 'globest',
display_name: 'GlobeSt',
owner: 'ALM Media',
base_url: 'https://www.globest.com',
access_method: 'manual_review_only',
review_notes: 'ALM paywall. Manual review only.',
},
{
source_key: 'the_registry',
display_name: 'The Registry',
owner: 'The Registry Media',
base_url: 'https://theregistrysf.com',
access_method: 'manual_review_only',
review_notes: 'Manual review only.',
},
{
source_key: 'commercial_observer',
display_name: 'Commercial Observer',
owner: 'Observer Media',
base_url: 'https://commercialobserver.com',
access_method: 'manual_review_only',
review_notes: 'Manual review only.',
},
{
source_key: 'the_real_deal_ca',
display_name: 'The Real Deal (California)',
owner: 'The Real Deal',
base_url: 'https://therealdeal.com',
access_method: 'manual_review_only',
review_notes: 'Manual review only.',
},
// Associations — CA
{
source_key: 'naiop_so_cal',
display_name: 'NAIOP Southern California',
owner: 'NAIOP SoCal Chapter',
base_url: 'https://www.naiopsc.com',
access_method: 'manual_review_only',
review_notes: 'Public event/sponsor pages only. Manual review.',
},
{
source_key: 'naiop_inland_empire',
display_name: 'NAIOP Inland Empire',
owner: 'NAIOP IE Chapter',
base_url: 'https://www.naiopie.com',
access_method: 'manual_review_only',
review_notes: 'Manual review only.',
},
{
source_key: 'naiop_san_diego',
display_name: 'NAIOP San Diego',
owner: 'NAIOP SD Chapter',
base_url: 'https://www.naiopsd.com',
access_method: 'manual_review_only',
review_notes: 'Manual review only.',
},
{
source_key: 'naiop_nor_cal',
display_name: 'NAIOP Northern California',
owner: 'NAIOP NorCal Chapter',
base_url: 'https://www.naiopnc.com',
access_method: 'manual_review_only',
review_notes: 'Manual review only.',
},
{
source_key: 'uli_los_angeles',
display_name: 'ULI Los Angeles',
owner: 'Urban Land Institute — LA District',
base_url: 'https://la.uli.org',
access_method: 'manual_review_only',
review_notes: 'Manual review only.',
},
{
source_key: 'uli_oc_ie',
display_name: 'ULI Orange County / Inland Empire',
owner: 'Urban Land Institute — OC/IE District',
base_url: 'https://ocie.uli.org',
access_method: 'manual_review_only',
review_notes: 'Manual review only.',
},
{
source_key: 'uli_san_diego',
display_name: 'ULI San Diego / Tijuana',
owner: 'Urban Land Institute — SD District',
base_url: 'https://sandiego.uli.org',
access_method: 'manual_review_only',
review_notes: 'Manual review only.',
},
{
source_key: 'uli_san_francisco',
display_name: 'ULI San Francisco',
owner: 'Urban Land Institute — SF District',
base_url: 'https://sf.uli.org',
access_method: 'manual_review_only',
review_notes: 'Manual review only.',
},
{
source_key: 'boma_greater_la',
display_name: 'BOMA Greater Los Angeles',
owner: 'BOMA Greater LA',
base_url: 'https://www.bomagla.org',
access_method: 'manual_review_only',
review_notes: 'Manual review only.',
},
{
source_key: 'boma_oc',
display_name: 'BOMA Orange County',
owner: 'BOMA OC',
base_url: 'https://bomaoc.org',
access_method: 'manual_review_only',
review_notes: 'Manual review only.',
},
{
source_key: 'boma_san_diego',
display_name: 'BOMA San Diego',
owner: 'BOMA San Diego',
base_url: 'https://bomasd.org',
access_method: 'manual_review_only',
review_notes: 'Manual review only.',
},
{
source_key: 'crew_los_angeles',
display_name: 'CREW Los Angeles',
owner: 'CREW Network — LA Chapter',
base_url: 'https://www.crewla.org',
access_method: 'manual_review_only',
review_notes: 'Manual review only.',
},
{
source_key: 'crew_orange_county',
display_name: 'CREW Orange County',
owner: 'CREW Network — OC Chapter',
base_url: 'https://www.crewoc.com',
access_method: 'manual_review_only',
review_notes: 'Manual review only.',
},
{
source_key: 'crew_san_diego',
display_name: 'CREW San Diego',
owner: 'CREW Network — SD Chapter',
base_url: 'https://www.crewsd.org',
access_method: 'manual_review_only',
review_notes: 'Manual review only.',
},
{
source_key: 'crew_san_francisco',
display_name: 'CREW San Francisco',
owner: 'CREW Network — SF Chapter',
base_url: 'https://www.crewsf.org',
access_method: 'manual_review_only',
review_notes: 'Manual review only.',
},
// AZ sources
{
source_key: 'phoenix_business_journal',
display_name: 'Phoenix Business Journal',
owner: 'American City Business Journals',
base_url: 'https://www.bizjournals.com/phoenix',
access_method: 'manual_review_only',
review_notes: 'Paywall. Manual review only.',
},
{
source_key: 'az_big_media',
display_name: 'AZ Big Media / AZRE',
owner: 'AZ Big Media',
base_url: 'https://azbigmedia.com',
access_method: 'manual_review_only',
review_notes: 'Manual review only.',
},
{
source_key: 'naiop_arizona',
display_name: 'NAIOP Arizona',
owner: 'NAIOP AZ Chapter',
base_url: 'https://www.naiop-az.org',
access_method: 'manual_review_only',
review_notes: 'Manual review only.',
},
{
source_key: 'uli_arizona',
display_name: 'ULI Arizona',
owner: 'Urban Land Institute — AZ District',
base_url: 'https://arizona.uli.org',
access_method: 'manual_review_only',
review_notes: 'Manual review only.',
},
{
source_key: 'boma_greater_phoenix',
display_name: 'BOMA Greater Phoenix',
owner: 'BOMA Phoenix',
base_url: 'https://www.bomaphoenix.org',
access_method: 'manual_review_only',
review_notes: 'Manual review only.',
},
{
source_key: 'valley_partnership',
display_name: 'Valley Partnership',
owner: 'Valley Partnership',
base_url: 'https://www.valleypartnership.org',
access_method: 'manual_review_only',
review_notes: 'Manual review only.',
},
{
source_key: 'azcrew',
display_name: 'AZCREW',
owner: 'CREW Network — AZ Chapter',
base_url: 'https://www.azcrew.org',
access_method: 'manual_review_only',
review_notes: 'Manual review only.',
},
];
async function seedSources(client) {
let inserted = 0;
let updated = 0;
const allSources = [...RENTV_SOURCES, ...EXTERNAL_SOURCES];
for (const s of allSources) {
// external sources default to manual_review_only restrictions
const isExternal = !RENTV_SOURCES.includes(s);
const allows_automated = isExternal ? false : (s.allows_automated_access ?? false);
const allows_screenshot = isExternal ? false : (s.allows_screenshot_capture ?? false);
const allows_internal = s.allows_internal_storage ?? true;
const allows_export = isExternal ? false : (s.allows_export ?? false);
const enabled = isExternal ? false : (s.enabled ?? false);
const res = await client.query(
`INSERT INTO source_policies (
source_key, display_name, owner, base_url, access_method,
allows_automated_access, allows_screenshot_capture,
allows_internal_storage, allows_export,
prohibited_hosts, permitted_paths, prohibited_paths,
reviewed_at, review_notes, enabled
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15)
ON CONFLICT (source_key) DO UPDATE SET
display_name=EXCLUDED.display_name,
owner=EXCLUDED.owner,
base_url=EXCLUDED.base_url,
access_method=EXCLUDED.access_method,
allows_automated_access=EXCLUDED.allows_automated_access,
allows_screenshot_capture=EXCLUDED.allows_screenshot_capture,
allows_internal_storage=EXCLUDED.allows_internal_storage,
allows_export=EXCLUDED.allows_export,
reviewed_at=EXCLUDED.reviewed_at,
review_notes=EXCLUDED.review_notes,
enabled=EXCLUDED.enabled
RETURNING (xmax = 0) AS is_insert`,
[
s.source_key,
s.display_name,
s.owner || null,
s.base_url || null,
s.access_method,
allows_automated,
allows_screenshot,
allows_internal,
allows_export,
JSON.stringify([]),
JSON.stringify([]),
JSON.stringify([]),
REVIEWED_AT,
s.review_notes || null,
enabled,
]
);
if (res.rows[0].is_insert) inserted++; else updated++;
}
return { inserted, updated };
}
module.exports = { seedSources };