← back to Ventura Corridor
iter 63: 🏇 crawl derby — racetrack-style live progress viewer for the corridor enrichment pipeline; /api/crawl/status returns 5 lanes (geocode/website/phone/audit/enrich) with done/total/pct + per-lane runtime status (idle/running/done/error); /api/crawl/start spawns the lane's npm script as a child process (safety gate: unsafe lanes like Brave-website require {force:true}); /api/crawl/stop kills it; /crawl-derby.html shows horses on a turf oval (start gate, finish line w/ checkered pattern, repeating-stripe rake marks), each lane has emoji label + pct pill + horse positioned by % across the track + galloping animation when running; roster table below with 🏇 start / ⏹ stop buttons + last-log tail; marquee announcer text changes from 'and they're at the post' to 'down the stretch they come' when any lane runs; auto-refresh 5s; derby link wired into all 8 admin nav bars
1c3e46f42c5b52939af4c2b572454fd090432e63 · 2026-05-06 14:35:21 -0700 · SteveStudio2
Files touched
M public/buildings.htmlA public/crawl-derby.htmlM public/linkedin.htmlM public/pitches.htmlM public/postcards.htmlM public/responses.htmlM public/sales-nav-signup.htmlM public/today.htmlM public/walk-route.htmlM src/server/index.ts
Diff
commit 1c3e46f42c5b52939af4c2b572454fd090432e63
Author: SteveStudio2 <steve@designerwallcoverings.com>
Date: Wed May 6 14:35:21 2026 -0700
iter 63: 🏇 crawl derby — racetrack-style live progress viewer for the corridor enrichment pipeline; /api/crawl/status returns 5 lanes (geocode/website/phone/audit/enrich) with done/total/pct + per-lane runtime status (idle/running/done/error); /api/crawl/start spawns the lane's npm script as a child process (safety gate: unsafe lanes like Brave-website require {force:true}); /api/crawl/stop kills it; /crawl-derby.html shows horses on a turf oval (start gate, finish line w/ checkered pattern, repeating-stripe rake marks), each lane has emoji label + pct pill + horse positioned by % across the track + galloping animation when running; roster table below with 🏇 start / ⏹ stop buttons + last-log tail; marquee announcer text changes from 'and they're at the post' to 'down the stretch they come' when any lane runs; auto-refresh 5s; derby link wired into all 8 admin nav bars
---
public/buildings.html | 1 +
public/crawl-derby.html | 337 +++++++++++++++++++++++++++++++++++++++++++
public/linkedin.html | 1 +
public/pitches.html | 1 +
public/postcards.html | 1 +
public/responses.html | 1 +
public/sales-nav-signup.html | 1 +
public/today.html | 1 +
public/walk-route.html | 1 +
src/server/index.ts | 115 +++++++++++++++
10 files changed, 460 insertions(+)
diff --git a/public/buildings.html b/public/buildings.html
index 6bb54be..08877c2 100644
--- a/public/buildings.html
+++ b/public/buildings.html
@@ -131,6 +131,7 @@
<a href="/responses.html">responses</a>
<a href="/postcards.html">postcards</a>
<a href="/buildings.html" class="active">buildings</a>
+ <a href="/crawl-derby.html">🏇 derby</a>
<span class="theme-toggle-host" style="display:inline-flex;align-items:center;margin-left:6px"></span>
</nav>
</header>
diff --git a/public/crawl-derby.html b/public/crawl-derby.html
new file mode 100644
index 0000000..ce52a68
--- /dev/null
+++ b/public/crawl-derby.html
@@ -0,0 +1,337 @@
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<title>Crawl Derby · DW · Ventura Corridor</title>
+<meta name="viewport" content="width=device-width,initial-scale=1">
+<script>(function(){try{var t=localStorage.getItem('vc-theme');document.documentElement.dataset.theme=(t==='light'||t==='dark')?t:'dark';}catch(e){document.documentElement.dataset.theme='dark';}})();</script>
+<link rel="stylesheet" href="/theme.css">
+<script src="/theme.js" defer></script>
+<style>
+ :root {
+ --noir: #0a0a0c; --noir-rise: #131316; --turf: #1a3318; --turf-rise: #244a21;
+ --ink: #f0ece2; --ink-mute: #888475;
+ --metal: #b89968; --metal-glow: #d4b683;
+ --rule: #2a2622; --green: #6a9b73; --red: #b66565;
+ --serif: 'Cormorant Garamond', 'Times New Roman', serif;
+ --sans: 'Inter', -apple-system, system-ui, sans-serif;
+ --mono: 'JetBrains Mono', ui-monospace, monospace;
+ }
+ @import url('https://fonts.googleapis.com/css2?family=Cormorant+Garamond:ital,wght@0,300;0,400;0,600;1,400;1,600&family=Inter:wght@300;400;600&family=JetBrains+Mono:wght@200;300;400&display=swap');
+ html, body { margin: 0; background: var(--noir); color: var(--ink); font-family: var(--sans); font-weight: 300; min-height: 100vh; }
+ * { box-sizing: border-box; }
+ header { padding: 22px 32px 16px; border-bottom: 1px solid var(--rule);
+ display: flex; justify-content: space-between; align-items: baseline; gap: 24px; flex-wrap: wrap; }
+ header h1 { font-family: var(--serif); font-weight: 400; font-size: 28px; margin: 0; letter-spacing: -0.01em; }
+ header h1 em { font-style: italic; color: var(--metal); }
+ .sub { font-size: 11px; letter-spacing: .26em; text-transform: uppercase; color: var(--ink-mute); margin-top: 2px; }
+ nav.tabs { display: flex; gap: 4px; flex-wrap: wrap; }
+ nav.tabs a { font-size: 9px; letter-spacing: .26em; text-transform: uppercase;
+ color: var(--ink-mute); text-decoration: none; padding: 5px 10px; border: 1px solid var(--rule); }
+ nav.tabs a.active, nav.tabs a:hover { color: var(--metal); border-color: var(--metal); }
+
+ /* The track */
+ .stadium {
+ padding: 32px 24px;
+ background:
+ radial-gradient(ellipse at center, rgba(20,40,18,0.6) 0%, transparent 70%),
+ var(--noir);
+ }
+ .marquee {
+ text-align: center; margin-bottom: 28px;
+ font-family: var(--serif); font-style: italic;
+ color: var(--metal-glow); letter-spacing: 0.05em;
+ }
+ .marquee .big { font-size: 32px; line-height: 1.05; }
+ .marquee .sm { font-size: 12px; letter-spacing: .26em; text-transform: uppercase;
+ color: var(--ink-mute); font-style: normal; font-family: var(--sans); margin-top: 6px; }
+
+ .track {
+ position: relative;
+ max-width: 1100px; margin: 0 auto;
+ padding: 22px 0;
+ background: linear-gradient(180deg, var(--turf) 0%, var(--turf-rise) 100%);
+ border: 2px solid #553e23;
+ border-radius: 200px / 80px;
+ overflow: hidden;
+ box-shadow: inset 0 8px 28px rgba(0,0,0,0.6), 0 0 0 4px #2e2517;
+ }
+ /* Track stripes for that "well-raked turf" look */
+ .track::before {
+ content: '';
+ position: absolute; inset: 0;
+ background:
+ repeating-linear-gradient(90deg, transparent 0, transparent 39px, rgba(0,0,0,0.07) 40px, rgba(0,0,0,0.07) 41px);
+ pointer-events: none;
+ }
+ /* Finish line */
+ .finish {
+ position: absolute; right: 22px; top: 0; bottom: 0;
+ width: 12px; background:
+ repeating-linear-gradient(90deg, white 0 6px, black 6px 12px,
+ black 0 6px, white 6px 12px);
+ background-size: 12px 12px;
+ box-shadow: -2px 0 6px rgba(0,0,0,0.4);
+ z-index: 5;
+ }
+ .finish::after {
+ content: '🏁';
+ position: absolute; top: -6px; right: -6px;
+ font-size: 18px;
+ }
+ /* Start line */
+ .start-gate {
+ position: absolute; left: 22px; top: 0; bottom: 0;
+ width: 6px; background: white; opacity: 0.85; z-index: 5;
+ }
+
+ /* Lanes */
+ .lane {
+ position: relative;
+ height: 56px; margin: 6px 22px;
+ background: rgba(0,0,0,0.18);
+ border-radius: 28px;
+ display: flex; align-items: center;
+ }
+ .lane .label-strip {
+ position: absolute; left: 14px; top: 50%; transform: translateY(-50%);
+ z-index: 2; font-family: var(--mono); font-size: 10px; letter-spacing: .15em;
+ text-transform: uppercase; color: rgba(255,255,255,0.55); pointer-events: none;
+ text-shadow: 0 1px 2px rgba(0,0,0,0.7);
+ }
+ .lane .horse {
+ position: absolute;
+ top: 50%; transform: translate(-50%, -50%);
+ font-size: 38px;
+ filter: drop-shadow(0 2px 4px rgba(0,0,0,0.6));
+ transition: left 0.8s cubic-bezier(0.22, 0.61, 0.36, 1);
+ z-index: 4;
+ cursor: pointer;
+ user-select: none;
+ }
+ .lane .horse.galloping { animation: bob 0.45s ease-in-out infinite; }
+ @keyframes bob {
+ 0% { transform: translate(-50%, -50%) translateY(0px); }
+ 50% { transform: translate(-50%, -50%) translateY(-3px); }
+ 100% { transform: translate(-50%, -50%) translateY(0px); }
+ }
+ .lane .horse .hat {
+ position: absolute; top: -6px; left: -2px;
+ font-size: 16px; filter: none;
+ }
+ .lane .pct-pill {
+ position: absolute; right: 60px; top: 50%; transform: translateY(-50%);
+ background: rgba(0,0,0,0.6); color: var(--metal-glow);
+ padding: 3px 10px; font-family: var(--mono); font-size: 11px;
+ border: 1px solid rgba(212,182,131,0.4);
+ z-index: 3;
+ }
+
+ /* Lane info table below the track */
+ .roster {
+ max-width: 1100px; margin: 32px auto 0; padding: 0 24px;
+ }
+ .roster .row {
+ display: grid; grid-template-columns: auto 1fr auto auto auto auto;
+ gap: 18px; align-items: center;
+ padding: 12px 16px; margin-bottom: 8px;
+ background: var(--noir-rise); border: 1px solid var(--rule);
+ font-size: 12px;
+ }
+ .roster .row .emoji { font-size: 22px; }
+ .roster .row .name { font-family: var(--serif); font-size: 17px; color: var(--ink); }
+ .roster .row .name small { display: block; color: var(--ink-mute); font-size: 10px; letter-spacing: .12em; margin-top: 2px; text-transform: uppercase; }
+ .roster .row .stat { font-family: var(--mono); font-size: 11px; color: var(--metal); min-width: 90px; text-align: right; }
+ .roster .row .stat .v { font-size: 14px; color: var(--metal-glow); display: block; }
+ .roster .row .runtime {
+ font-family: var(--mono); font-size: 9px; letter-spacing: .15em; text-transform: uppercase;
+ padding: 4px 9px; border: 1px solid var(--rule);
+ }
+ .roster .row .runtime.running { color: var(--green); border-color: var(--green); animation: pulse 1.4s ease-in-out infinite; }
+ .roster .row .runtime.idle { color: var(--ink-mute); }
+ .roster .row .runtime.done { color: var(--metal-glow); border-color: var(--metal-glow); }
+ .roster .row .runtime.error { color: var(--red); border-color: var(--red); }
+ @keyframes pulse { 0%,100% { opacity: 1; } 50% { opacity: 0.5; } }
+ .roster .row .actions button {
+ background: transparent; border: 1px solid var(--metal); color: var(--metal);
+ padding: 5px 11px; font-size: 9px; letter-spacing: .18em; text-transform: uppercase;
+ cursor: pointer; font-family: var(--sans);
+ }
+ .roster .row .actions button:hover { background: rgba(184,153,104,0.06); }
+ .roster .row .actions button.danger { border-color: var(--red); color: var(--red); }
+
+ .last-log {
+ grid-column: 1 / -1; margin-top: 6px; padding: 6px 12px;
+ background: rgba(0,0,0,0.3); font-family: var(--mono); font-size: 10px;
+ color: var(--ink-mute); white-space: pre-wrap; word-break: break-all;
+ }
+
+ .legend {
+ text-align: center; font-size: 10px; letter-spacing: .26em; text-transform: uppercase;
+ color: var(--ink-mute); margin-top: 14px;
+ }
+</style>
+</head>
+<body>
+
+<header>
+ <div>
+ <h1>🏇 Crawl <em>derby</em></h1>
+ <div class="sub">17,462 corridor businesses · 5 lanes · live progress · auto-refresh 5s</div>
+ </div>
+ <nav class="tabs">
+ <a href="/">map</a>
+ <a href="/today.html">today</a>
+ <a href="/pitches.html">pitches</a>
+ <a href="/walk-route.html">walk route</a>
+ <a href="/responses.html">responses</a>
+ <a href="/postcards.html">postcards</a>
+ <a href="/buildings.html">buildings</a>
+ <a href="/crawl-derby.html" class="active">crawl derby</a>
+ <span class="theme-toggle-host" style="display:inline-flex;align-items:center;margin-left:6px"></span>
+ </nav>
+</header>
+
+<section class="stadium">
+ <div class="marquee">
+ <div class="big" id="marquee-title">And they're <em>off!</em></div>
+ <div class="sm" id="marquee-stat">live data race · 17,462 corridor businesses to enrich</div>
+ </div>
+ <div class="track" id="track">
+ <div class="start-gate"></div>
+ <div class="finish"></div>
+ <!-- lanes injected -->
+ </div>
+ <div class="legend">click any horse to see what fuel it needs · 🏁 = 100% complete</div>
+</section>
+
+<section class="roster" id="roster">
+ <div style="text-align:center;color:var(--ink-mute);padding:40px;font-style:italic">Loading the field…</div>
+</section>
+
+<script>
+function escHtml(s) { return String(s ?? '').replace(/[&<>"']/g, c => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c])); }
+
+const HORSE_HATS = ['🏇', '🐎', '🐴', '🦓', '🦄'];
+
+async function loadStatus() {
+ try {
+ const data = await fetch('/api/crawl/status').then(r => r.json());
+ renderTrack(data.lanes);
+ renderRoster(data.lanes);
+ renderMarquee(data.lanes, data.corridor_total);
+ } catch (e) {
+ document.getElementById('roster').innerHTML = `<div style="color:var(--red);padding:20px">load failed: ${e.message}</div>`;
+ }
+}
+
+function renderTrack(lanes) {
+ const track = document.getElementById('track');
+ // Clear any existing lanes (keep start/finish)
+ track.querySelectorAll('.lane').forEach(el => el.remove());
+ for (const l of lanes) {
+ const galloping = l.runtime?.status === 'running';
+ const pct = Math.min(100, l.pct);
+ // The horse position: start at left ~22px, end at right ~22px (just before finish line)
+ // Use pct% across the lane width minus padding
+ const lane = document.createElement('div');
+ lane.className = 'lane';
+ lane.dataset.id = l.id;
+ lane.innerHTML = `
+ <span class="label-strip">${l.emoji} ${escHtml(l.name)}</span>
+ <span class="pct-pill">${l.pct}% · ${l.done.toLocaleString()}/${l.total.toLocaleString()}</span>
+ <span class="horse ${galloping ? 'galloping' : ''}" style="left:calc(${pct}% - ${pct * 0.18}px + 22px)" title="${escHtml(l.bottleneck || '')}">
+ ${HORSE_HATS[lanes.indexOf(l) % HORSE_HATS.length]}
+ </span>
+ `;
+ track.appendChild(lane);
+ }
+}
+
+function renderRoster(lanes) {
+ const root = document.getElementById('roster');
+ root.innerHTML = lanes.map((l, i) => {
+ const hat = HORSE_HATS[i % HORSE_HATS.length];
+ const status = l.runtime?.status || 'idle';
+ const log = l.runtime?.last_log || '';
+ const runtimeClass = status;
+ const runtimeLabel = ({ running: '⚡ running', idle: '◯ idle', done: '✓ finished', error: '✕ stopped' })[status] || status;
+ return `
+ <div class="row" data-id="${l.id}">
+ <div class="emoji">${hat}</div>
+ <div class="name">
+ ${escHtml(l.name)}
+ <small>${escHtml(l.bottleneck || '')}</small>
+ </div>
+ <div class="stat"><span class="v">${l.pct}%</span>complete</div>
+ <div class="stat"><span class="v">${l.done.toLocaleString()}</span>${l.total.toLocaleString()} total</div>
+ <div class="runtime ${runtimeClass}">${runtimeLabel}</div>
+ <div class="actions">
+ ${status === 'running'
+ ? `<button class="danger" onclick="stopLane('${l.id}')">⏹ stop</button>`
+ : `<button onclick="startLane('${l.id}')">🏇 start</button>`}
+ </div>
+ ${log ? `<div class="last-log">${escHtml(log)}</div>` : ''}
+ </div>
+ `;
+ }).join('');
+}
+
+function renderMarquee(lanes, total) {
+ const totalDone = lanes.reduce((a, l) => a + l.done, 0);
+ const totalSlots = lanes.reduce((a, l) => a + l.total, 0);
+ const overall = totalSlots ? (100 * totalDone / totalSlots).toFixed(1) : 0;
+ document.getElementById('marquee-stat').innerHTML = `Overall <b style="color:var(--metal-glow);font-family:var(--mono)">${overall}%</b> · <b style="color:var(--metal);font-family:var(--mono)">${total.toLocaleString()}</b> corridor businesses · refresh 5s`;
+ const someRunning = lanes.some(l => l.runtime?.status === 'running');
+ document.getElementById('marquee-title').innerHTML = someRunning
+ ? `<em>Down the stretch they come!</em>`
+ : `<em>And they're at the post.</em>`;
+}
+
+async function startLane(lane) {
+ if (!confirm(`Start the "${lane}" lane?\n\nIt'll spawn the corresponding npm script as a child process. Some lanes are quota-blocked (Brave) or need API keys (HERE). Click cancel if you're not sure.`)) return;
+ try {
+ const r = await fetch('/api/crawl/start', {
+ method: 'POST', headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ lane })
+ });
+ const d = await r.json();
+ if (!r.ok) {
+ if (d.note && confirm(`Lane "${lane}" is flagged unsafe.\n\nNote: ${d.note}\n\nForce-start anyway?`)) {
+ const r2 = await fetch('/api/crawl/start', {
+ method: 'POST', headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ lane, force: true })
+ });
+ const d2 = await r2.json();
+ if (!r2.ok) throw new Error(d2.error || 'force-start failed');
+ alert(`Force-started ${lane} (PID ${d2.pid})`);
+ }
+ } else {
+ alert(d.already_running
+ ? `Lane "${lane}" was already running (PID ${d.pid})`
+ : `Started "${lane}" (PID ${d.pid})${d.note ? '\n\n' + d.note : ''}`);
+ }
+ loadStatus();
+ } catch (e) {
+ alert('start failed: ' + e.message);
+ }
+}
+
+async function stopLane(lane) {
+ if (!confirm(`Stop the "${lane}" lane?`)) return;
+ try {
+ await fetch('/api/crawl/stop', {
+ method: 'POST', headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ lane })
+ });
+ loadStatus();
+ } catch (e) {
+ alert('stop failed: ' + e.message);
+ }
+}
+
+loadStatus();
+setInterval(loadStatus, 5000);
+</script>
+</body>
+</html>
diff --git a/public/linkedin.html b/public/linkedin.html
index d6a040b..ea7b1e4 100644
--- a/public/linkedin.html
+++ b/public/linkedin.html
@@ -78,6 +78,7 @@
<a href="/responses.html">responses</a>
<a href="/postcards.html">postcards</a>
<a href="/buildings.html">buildings</a>
+ <a href="/crawl-derby.html">🏇 derby</a>
<a href="/linkedin.html" class="active">linkedin</a>
<span class="theme-toggle-host" style="display:inline-flex;align-items:center;margin-left:6px"></span>
</nav>
diff --git a/public/pitches.html b/public/pitches.html
index efc8765..d04e543 100644
--- a/public/pitches.html
+++ b/public/pitches.html
@@ -112,6 +112,7 @@
<a href="/responses.html">responses</a>
<a href="/postcards.html">postcards</a>
<a href="/buildings.html">buildings</a>
+ <a href="/crawl-derby.html">🏇 derby</a>
<a href="/linkedin.html">linkedin</a>
<span class="theme-toggle-host" style="display:inline-flex;align-items:center;margin-left:6px"></span>
</nav>
diff --git a/public/postcards.html b/public/postcards.html
index 04ccdde..fe1874b 100644
--- a/public/postcards.html
+++ b/public/postcards.html
@@ -118,6 +118,7 @@
<a href="/responses.html">responses</a>
<a href="/postcards.html" class="active">postcards</a>
<a href="/buildings.html">buildings</a>
+ <a href="/crawl-derby.html">🏇 derby</a>
<span class="theme-toggle-host" style="display:inline-flex;align-items:center;margin-left:6px"></span>
</nav>
</header>
diff --git a/public/responses.html b/public/responses.html
index 7118ba1..bc274f3 100644
--- a/public/responses.html
+++ b/public/responses.html
@@ -186,6 +186,7 @@
<a href="/responses.html" class="active">responses</a>
<a href="/postcards.html">postcards</a>
<a href="/buildings.html">buildings</a>
+ <a href="/crawl-derby.html">🏇 derby</a>
<span class="theme-toggle-host" style="display:inline-flex;align-items:center;margin-left:6px"></span>
</nav>
</header>
diff --git a/public/sales-nav-signup.html b/public/sales-nav-signup.html
index a63f7ff..d0d304f 100644
--- a/public/sales-nav-signup.html
+++ b/public/sales-nav-signup.html
@@ -96,6 +96,7 @@
<a href="/responses.html">responses</a>
<a href="/postcards.html">postcards</a>
<a href="/buildings.html">buildings</a>
+ <a href="/crawl-derby.html">🏇 derby</a>
<a href="/linkedin.html">linkedin</a>
<a href="/sales-nav-signup.html" class="active">signup</a>
<span class="theme-toggle-host" style="display:inline-flex;align-items:center;margin-left:6px"></span>
diff --git a/public/today.html b/public/today.html
index aa31855..1cc496d 100644
--- a/public/today.html
+++ b/public/today.html
@@ -117,6 +117,7 @@
<a href="/responses.html">responses</a>
<a href="/postcards.html">postcards</a>
<a href="/buildings.html">buildings</a>
+ <a href="/crawl-derby.html">🏇 derby</a>
<a href="/today.html" class="active">today</a>
<span class="theme-toggle-host" style="display:inline-flex;align-items:center;margin-left:6px"></span>
</nav>
diff --git a/public/walk-route.html b/public/walk-route.html
index 600d39d..a41e275 100644
--- a/public/walk-route.html
+++ b/public/walk-route.html
@@ -158,6 +158,7 @@
<a href="/responses.html">responses</a>
<a href="/postcards.html">postcards</a>
<a href="/buildings.html">buildings</a>
+ <a href="/crawl-derby.html">🏇 derby</a>
<a href="/linkedin.html">linkedin</a>
<span class="theme-toggle-host" style="display:inline-flex;align-items:center;margin-left:6px"></span>
</nav>
diff --git a/src/server/index.ts b/src/server/index.ts
index 7c8d907..29c6b01 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -68,6 +68,8 @@ const ADMIN_PATHS = [
/^\/api\/followups\/?$/i,
/^\/api\/snapshots\/?$/i,
/^\/api\/activity\/?$/i,
+ /^\/crawl-derby(\.html)?\/?$/i,
+ /^\/api\/crawl(\/.*)?$/i,
/^\/postcards(\.html)?\/?$/i,
/^\/buildings(\.html)?\/?$/i,
/^\/api\/buildings(\/.*)?$/i,
@@ -922,6 +924,119 @@ app.patch('/api/pitches/:id', async (req, res) => {
}
});
+// ─── Crawl derby: per-lane progress for the corridor enrichment pipeline ──
+// Powers /crawl-derby.html, the racetrack-style live status viewer.
+// Each "horse" is one enrichment dimension; lap = % complete.
+import { spawn } from 'node:child_process';
+import { existsSync as fsExistsSync } from 'node:fs';
+
+const _crawlChildren: Record<string, { pid: number; startedAt: number; status: 'running'|'done'|'error'; lastLog: string }> = {};
+
+app.get('/api/crawl/status', async (_req, res) => {
+ try {
+ const r = await query(`
+ SELECT
+ count(*) FILTER (WHERE on_corridor) AS corridor_total,
+ count(*) FILTER (WHERE on_corridor AND lat IS NOT NULL) AS geocoded,
+ count(*) FILTER (WHERE on_corridor AND website IS NOT NULL) AS has_website,
+ count(*) FILTER (WHERE on_corridor AND phone IS NOT NULL) AS has_phone
+ FROM businesses
+ `);
+ const a = await query(`SELECT count(*) AS audited FROM front_page_audits`);
+ const e = await query(`SELECT count(*) FILTER (WHERE headline IS NOT NULL) AS enriched FROM business_enrichment`);
+ const total = Number(r.rows[0].corridor_total);
+ const lanes = [
+ {
+ id: 'geocode', name: 'Geocode', emoji: '📍',
+ done: Number(r.rows[0].geocoded), total,
+ bottleneck: total - Number(r.rows[0].geocoded) < 100 ? 'almost done' : 'OSM/HERE backfill needed'
+ },
+ {
+ id: 'website', name: 'Website discovery', emoji: '🌐',
+ done: Number(r.rows[0].has_website), total,
+ bottleneck: 'needs HERE Maps key (free 30k/mo)'
+ },
+ {
+ id: 'phone', name: 'Phone discovery', emoji: '📞',
+ done: Number(r.rows[0].has_phone), total,
+ bottleneck: 'piggybacks on website crawl'
+ },
+ {
+ id: 'audit', name: 'Front-page audit', emoji: '🔍',
+ done: Number(a.rows[0].audited), total: Math.max(Number(r.rows[0].has_website), 1),
+ bottleneck: 'only audits businesses w/ a known website'
+ },
+ {
+ id: 'enrich', name: 'Headline + meta', emoji: '📰',
+ done: Number(e.rows[0].enriched), total: Math.max(Number(r.rows[0].has_website), 1),
+ bottleneck: 'depends on audit'
+ }
+ ].map(l => ({ ...l, pct: l.total ? Math.round(1000 * l.done / l.total) / 10 : 0 }));
+
+ // Lane runtime status: running/idle
+ for (const l of lanes) {
+ const child = _crawlChildren[l.id];
+ (l as any).runtime = child
+ ? { pid: child.pid, status: child.status, started_at: child.startedAt, last_log: child.lastLog }
+ : { status: 'idle' };
+ }
+ res.json({ lanes, corridor_total: total });
+ } catch (e: any) {
+ res.status(500).json({ error: e.message });
+ }
+});
+
+const LANE_SCRIPTS: Record<string, { script: string; safe: boolean; note?: string }> = {
+ geocode: { script: 'src/enrich/restaurant_owners.ts', safe: false, note: 'no dedicated geocode script wired yet — use HERE batch' },
+ website: { script: 'src/enrich/website_via_brave.ts', safe: false, note: 'BRAVE quota — Steve has flagged Brave-blocked' },
+ phone: { script: 'src/enrich/scrape_contacts.ts', safe: true, note: 'scrapes contact pages from already-discovered sites' },
+ audit: { script: 'src/crawl/front_page.ts', safe: true, note: 'crawls 153 known websites — fast' },
+ enrich: { script: 'src/enrich/ad_signals.ts', safe: true, note: 'pixel/DNS sniff on known sites' },
+};
+
+app.post('/api/crawl/start', express.json(), async (req, res) => {
+ try {
+ const lane = String(req.body?.lane || '');
+ const force = req.body?.force === true;
+ const meta = LANE_SCRIPTS[lane];
+ if (!meta) return res.status(400).json({ error: 'unknown lane' });
+ if (!meta.safe && !force) return res.status(400).json({ error: 'unsafe lane — pass {force:true} to run anyway', note: meta.note });
+ if (_crawlChildren[lane] && _crawlChildren[lane].status === 'running') {
+ return res.json({ ok: true, already_running: true, pid: _crawlChildren[lane].pid });
+ }
+ const scriptPath = path.join(process.cwd(), meta.script);
+ if (!fsExistsSync(scriptPath)) return res.status(404).json({ error: 'script missing', path: scriptPath });
+
+ const child = spawn('npx', ['tsx', meta.script], {
+ cwd: process.cwd(), env: process.env, stdio: ['ignore', 'pipe', 'pipe'], detached: false
+ });
+ _crawlChildren[lane] = { pid: child.pid || -1, startedAt: Date.now(), status: 'running', lastLog: '' };
+ child.stdout.on('data', d => { _crawlChildren[lane].lastLog = String(d).slice(-200); });
+ child.stderr.on('data', d => { _crawlChildren[lane].lastLog = '[err] ' + String(d).slice(-200); });
+ child.on('exit', code => {
+ _crawlChildren[lane].status = code === 0 ? 'done' : 'error';
+ _crawlChildren[lane].lastLog += ` [exit ${code}]`;
+ });
+ res.json({ ok: true, lane, pid: child.pid, note: meta.note });
+ } catch (e: any) {
+ res.status(500).json({ error: e.message });
+ }
+});
+
+app.post('/api/crawl/stop', express.json(), async (req, res) => {
+ try {
+ const lane = String(req.body?.lane || '');
+ const child = _crawlChildren[lane];
+ if (!child || child.status !== 'running') return res.json({ ok: true, was_running: false });
+ try { process.kill(child.pid); } catch {}
+ child.status = 'error';
+ child.lastLog += ' [stopped]';
+ res.json({ ok: true, lane });
+ } catch (e: any) {
+ res.status(500).json({ error: e.message });
+ }
+});
+
// ─── Recent activity feed: stream of pitch_event_log events with business name ──
// Powers the "Recent activity" card on /today.html so Steve sees the corridor breathe.
app.get('/api/activity', async (req, res) => {
← 31c4243 iter 62: walk-route priority-aware ordering — /api/pitches/w
·
back to Ventura Corridor
·
iter 64: building pitch-type heatmap — /api/buildings/:bldg/ e654bdf →