← back to Rentv
PR intel: OpenClaw masthead extractor + per-person phone/email capture
25e5df285c910c3e3dbe2de39fbbd80504594aa1 · 2026-08-05 14:02:11 -0700 · Steve
- src/pr/tools/openclaw-media-extract.js: drives real Chrome (openclaw) to
render bot-walled publication mastheads (CoStar, The Real Deal, bizjournals,
etc.) that block plain-fetch AND headless; extracts staff, POSTs to the PR
CRM via the audited /api/pr/people (name-gate + dedupe apply). Backend,
data-only, $0 (real Chrome).
- extractPeople now captures each person's nearby email + phone (blockContact)
and threads public_work_email / public_business_phone through both extractors
into the CRM (people.create already stores them).
- extractPeople gains an opt-in allTeam mode (kept off by default — the
title-gated pass already captures the full masthead without nav-label noise).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Files touched
M src/pr/adapters/website.jsM src/pr/tools/headless-team-extract.jsA src/pr/tools/openclaw-media-extract.js
Diff
commit 25e5df285c910c3e3dbe2de39fbbd80504594aa1
Author: Steve <steve@designerwallcoverings.com>
Date: Wed Aug 5 14:02:11 2026 -0700
PR intel: OpenClaw masthead extractor + per-person phone/email capture
- src/pr/tools/openclaw-media-extract.js: drives real Chrome (openclaw) to
render bot-walled publication mastheads (CoStar, The Real Deal, bizjournals,
etc.) that block plain-fetch AND headless; extracts staff, POSTs to the PR
CRM via the audited /api/pr/people (name-gate + dedupe apply). Backend,
data-only, $0 (real Chrome).
- extractPeople now captures each person's nearby email + phone (blockContact)
and threads public_work_email / public_business_phone through both extractors
into the CRM (people.create already stores them).
- extractPeople gains an opt-in allTeam mode (kept off by default — the
title-gated pass already captures the full masthead without nav-label noise).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
src/pr/adapters/website.js | 43 +++++++++++----
src/pr/tools/headless-team-extract.js | 6 ++-
src/pr/tools/openclaw-media-extract.js | 99 ++++++++++++++++++++++++++++++++++
3 files changed, 137 insertions(+), 11 deletions(-)
diff --git a/src/pr/adapters/website.js b/src/pr/adapters/website.js
index 080cf0dd..ae6ac755 100644
--- a/src/pr/adapters/website.js
+++ b/src/pr/adapters/website.js
@@ -38,9 +38,14 @@ function extractLinks(html, baseUrl) {
/** Titles worth extracting from team pages (kept intentionally broad; classify later). */
const TITLE_HINT = /officer|director|president|principal|partner|manager|vp|vice president|head of|chief|founder|lead|communications|marketing|media|escrow|lending|relations|editor|editorial|reporter|writer|journalist|correspondent|columnist|publisher|contributor|newsroom/i;
-/** Extract (name, title) candidate pairs from a team/leadership page. Conservative:
- * requires a plausible Name-Case name adjacent to a title-looking string. */
-function extractPeople(html) {
+/** Extract (name, title, email, phone) candidate records from a team/leadership page.
+ * Default is conservative (name adjacent to a title-looking string). Pass
+ * { allTeam:true } for masthead/staff pages where the goal is EVERY person in the
+ * CRM — then any real Name-Case name is captured even without a recognized title,
+ * and each person's nearby email/phone is attached. The server name-gate still
+ * rejects non-person names. */
+function extractPeople(html, opts = {}) {
+ const allTeam = !!opts.allTeam;
const text = String(html)
.replace(/<script[\s\S]*?<\/script>/gi, ' ').replace(/<style[\s\S]*?<\/style>/gi, ' ')
.replace(/<(h[1-6]|p|div|li|td|span|strong|em|a|br)[^>]*>/gi, '\n').replace(/<[^>]+>/g, ' ')
@@ -63,22 +68,42 @@ function extractPeople(html) {
// "Jane Smith | Chief Credit Officer" — common on small-bank officer pages (cycle 4:
// 36 team pages yielded 0 because name+title share one line).
const INLINE_RE = /^([A-Z][a-z]+(?:\s+[A-Z]\.?)?(?:\s+[A-Z][a-z'’-]+){1,2})\s*[,|–—-]\s+(.{3,90})$/;
+ // nav/boilerplate lines that are not a job title (only used to gate the allTeam pass)
+ const NOT_TITLE = /^(read more|view|see all|subscribe|sign ?in|sign ?up|log ?in|newsletter|menu|search|share|follow|contact|home|about|privacy|terms|cookie|advertise|copyright|©|all rights|back to|next|previous|load more)/i;
+ const EMAIL1 = /[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}/i;
+ const PHONE1 = /(?:\+?1[\s.-]?)?\(?\d{3}\)?[\s.-]\d{3}[\s.-]\d{4}/;
+ // pull an email + phone from the person's block (their line + next 3)
+ const blockContact = (idx) => {
+ const blk = lines.slice(idx, idx + 4).join(' ');
+ const out = {};
+ const em = (blk.match(EMAIL1) || [])[0];
+ if (em && !/example\.|sentry|wixpress|schema\.org|\.png|\.jpg|@2x/i.test(em)) out.public_work_email = em.toLowerCase();
+ const ph = (blk.match(PHONE1) || [])[0];
+ if (ph) out.public_business_phone = ph.trim();
+ return out;
+ };
+ // is `s` a plausible title line (allTeam pass) — real text, not a name, not nav, not a headline
+ const titleOk = (s) => !!s && s.length <= 90 && /[a-z]/i.test(s) && !NAME_RE.test(s) && !NOT_TITLE.test(s) && !isHeadline('', s);
+ const CAP = allTeam ? 120 : 60;
for (let i = 0; i < lines.length; i++) {
const inline = lines[i].match(INLINE_RE);
- if (inline && !NOT_NAME.test(inline[1]) && TITLE_HINT.test(inline[2]) && !isHeadline(inline[1], inline[2])) {
- people.push({ full_name: inline[1], exact_title: inline[2].trim() });
- if (people.length >= 60) break;
+ if (inline && !NOT_NAME.test(inline[1]) && !isHeadline(inline[1], inline[2]) && (allTeam ? titleOk(inline[2]) : TITLE_HINT.test(inline[2]))) {
+ people.push({ full_name: inline[1], exact_title: inline[2].trim(), ...blockContact(i) });
+ if (people.length >= CAP) break;
continue;
}
const nm = lines[i].match(NAME_RE);
if (!nm || NOT_NAME.test(nm[1])) continue;
// look ahead up to 2 lines for a title
+ let title = null;
for (let j = 1; j <= 2 && i + j < lines.length; j++) {
const cand = lines[i + j];
- if (cand.length > 90 || NAME_RE.test(cand)) break;
- if (TITLE_HINT.test(cand) && !isHeadline(nm[1], cand)) { people.push({ full_name: nm[1], exact_title: cand }); break; }
+ if (NAME_RE.test(cand)) break;
+ if (allTeam ? titleOk(cand) : (cand.length <= 90 && TITLE_HINT.test(cand) && !isHeadline(nm[1], cand))) { title = cand; break; }
}
- if (people.length >= 60) break; // sane cap per page
+ // allTeam: keep the person even with no recognized title (a name IS the CRM row)
+ if (title || allTeam) { people.push({ full_name: nm[1], exact_title: title, ...blockContact(i) }); }
+ if (people.length >= CAP) break; // sane cap per page
}
// de-dupe by name
const seen = new Set();
diff --git a/src/pr/tools/headless-team-extract.js b/src/pr/tools/headless-team-extract.js
index ac6f3ce9..c52142a3 100644
--- a/src/pr/tools/headless-team-extract.js
+++ b/src/pr/tools/headless-team-extract.js
@@ -101,7 +101,7 @@ async function main() {
const resp = await page.goto(cand, { waitUntil: 'domcontentloaded' });
if (resp && resp.status() >= 400) continue; // probed path doesn't exist
await page.waitForTimeout(2000); // rendered people grids are late paints
- people = extractPeople(await page.content());
+ people = extractPeople(await page.content()); // title-gated: the full masthead, minus nav junk
if (people.length) { teamUrl = cand; break; }
} catch { /* dead candidate — try next */ }
}
@@ -109,7 +109,9 @@ async function main() {
for (const cand of people.slice(0, 40)) {
try {
const r = await api('/people', { method: 'POST', body: {
- full_name: cand.full_name, exact_title: cand.exact_title,
+ full_name: cand.full_name, exact_title: cand.exact_title || null,
+ public_work_email: cand.public_work_email || null,
+ public_business_phone: cand.public_business_phone || null,
organization_id: org.id, state: (org.state_presence || [])[0] || STATE, metro: (org.metros || [])[0] || null,
lifecycle_status: 'discovered',
evidence: {
diff --git a/src/pr/tools/openclaw-media-extract.js b/src/pr/tools/openclaw-media-extract.js
new file mode 100644
index 00000000..b36e68e3
--- /dev/null
+++ b/src/pr/tools/openclaw-media-extract.js
@@ -0,0 +1,99 @@
+'use strict';
+// OpenClaw real-Chrome masthead extractor — the anti-automation FALLBACK for the
+// media publications that block plain-fetch AND headless Playwright (CoStar, The Real
+// Deal, bizjournals, etc.). Drives Steve's REAL Google Chrome through the `openclaw`
+// CLI (human fingerprint, real session) to render each pub's About/Staff/Masthead
+// page, runs the SAME extractPeople logic, and POSTs the named editors/reporters to
+// prod through the audited /api/pr/people endpoint (server-side name-gate + dedupe +
+// evidence all apply). BACKEND, DATA-ONLY: public staff names used for outreach
+// targeting, never republished. $0 (local browser).
+//
+// Usage: PR_ORG_IDS="176,602,..." node src/pr/tools/openclaw-media-extract.js
+// Env: PR_API (default prod), PR_AUTH (user:pass), OC_THROTTLE_MS (default 1500)
+const { execFileSync } = require('child_process');
+const { extractPeople } = require('../adapters/website');
+
+const API = process.env.PR_API || 'https://rentv.agentabrams.com';
+const AUTH = 'Basic ' + Buffer.from(process.env.PR_AUTH || 'admin:DW2024!').toString('base64');
+const THROTTLE = Number(process.env.OC_THROTTLE_MS || 1500);
+const IDS = (process.env.PR_ORG_IDS || '').split(',').map((s) => s.trim()).filter(Boolean);
+
+const oc = (args) => execFileSync('openclaw', args, { encoding: 'utf8', maxBuffer: 32 * 1024 * 1024, timeout: 60000 });
+const sleep = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+const cleanTitle = (t) => String(t || '').replace(/\s+/g, ' ').replace(/[.\s]+$/, '').trim().slice(0, 120);
+
+async function api(path, opts = {}) {
+ const r = await fetch(API + '/api/pr' + path, {
+ ...opts,
+ headers: { Authorization: AUTH, 'Content-Type': 'application/json', ...(opts.headers || {}) },
+ body: opts.body ? JSON.stringify(opts.body) : undefined,
+ });
+ if (!r.ok) throw new Error(path + ' → ' + r.status + ': ' + (await r.text()).slice(0, 120));
+ return r.json();
+}
+
+// Read a value the openclaw `evaluate` printed to stdout. It echoes the raw JS return;
+// for HTML we slice from the first '<', for JSON we slice from the first '['/'{'.
+function evalRaw(fn) { try { return oc(['browser', 'evaluate', '--fn', fn]); } catch { return ''; } }
+function evalJSON(fn) { const s = evalRaw(fn); const i = s.search(/[[{]/); if (i < 0) return null; try { return JSON.parse(s.slice(i)); } catch { return null; } }
+
+const LINK_FN = "() => Array.from(document.querySelectorAll('a[href]'))"
+ + ".map(a=>({t:(a.textContent||'').trim().slice(0,50),h:a.href}))"
+ + ".filter(x=>/about|team|staff|masthead|contributor|our-people|people|leadership|editor|contact/i.test(x.t+' '+x.h))"
+ + ".slice(0,40)";
+
+function rankLink(u) {
+ return /masthead|editorial|staff|our-people|meet-the-team/i.test(u) ? 0
+ : /about[-/](us|team|people|staff)|\/(team|people|leadership)\b/i.test(u) ? 1
+ : /about/i.test(u) ? 2 : /contact/i.test(u) ? 3 : 4;
+}
+
+async function main() {
+ if (!IDS.length) { console.error('set PR_ORG_IDS="id,id,..."'); process.exit(1); }
+ // confirm openclaw is up
+ const status = (() => { try { return oc(['browser', 'status']); } catch (e) { return 'ERR ' + e.message; } })();
+ if (!/enabled:\s*true/.test(status)) { console.error('openclaw not enabled:\n' + status); process.exit(1); }
+
+ let totalCreated = 0, orgsWithPeople = 0;
+ for (const id of IDS) {
+ let org;
+ try { org = await api('/organizations/' + id); } catch { console.log(` #${id}: fetch failed — skip`); continue; }
+ if (!org.website_url) { console.log(` ${org.display_name}: no website — skip`); continue; }
+ let created = 0, teamUrl = null;
+ try {
+ oc(['browser', 'navigate', org.website_url]); sleep(THROTTLE);
+ const links = (evalJSON(LINK_FN) || []).map((x) => x.h).filter(Boolean);
+ const cands = [...new Set(links)].sort((a, b) => rankLink(a) - rankLink(b)).slice(0, 3);
+ let people = [];
+ for (const cand of cands) {
+ oc(['browser', 'navigate', cand]); sleep(THROTTLE + 1000);
+ let html = evalRaw("() => document.body.innerHTML");
+ const i = html.indexOf('<'); if (i > 0) html = html.slice(i);
+ const found = extractPeople(html); // title-gated: the full masthead, minus nav junk
+ if (found.length) { people = found; teamUrl = cand; break; }
+ }
+ for (const cand of people.slice(0, 60)) {
+ const title = cleanTitle(cand.exact_title); // may be '' — a name alone is still a CRM row
+ try {
+ const r = await api('/people', { method: 'POST', body: {
+ full_name: cand.full_name, exact_title: title || null,
+ public_work_email: cand.public_work_email || null,
+ public_business_phone: cand.public_business_phone || null,
+ organization_id: org.id, state: (org.state_presence || [])[0] || null, metro: (org.metros || [])[0] || null,
+ lifecycle_status: 'discovered',
+ evidence: { source: { source_type: 'org_website', source_name: 'Masthead (openclaw real Chrome): ' + org.display_name, url: teamUrl, is_primary_source: true, adapter: 'openclaw', license_or_usage_note: 'Public masthead/staff page rendered in real Chrome; names used for outreach targeting only.' }, fields: { organization: 80, exact_title: 80 } },
+ } });
+ if (r.created) created++;
+ } catch { /* name-gate / dedupe rejection — expected */ }
+ }
+ } catch (e) { console.log(` ${org.display_name}: ${e.message.slice(0, 80)}`); }
+ if (created) orgsWithPeople++;
+ totalCreated += created;
+ console.log(` ${org.display_name.padEnd(34)} team:${teamUrl ? 'y' : '-'} created:${created}`);
+ sleep(THROTTLE);
+ }
+ console.log(`[openclaw-media] DONE — people created: ${totalCreated} across ${orgsWithPeople} orgs · $0 (real Chrome)`);
+ process.exit(0);
+}
+
+main().catch((e) => { console.error('fatal:', e.message); process.exit(1); });
← 5c464c00 pr-intelligence dashboard: every data point hrefs deeper — b
·
back to Rentv
·
pr-intelligence dashboard: KPI cards now drill to filtered l b34da7ac →