← back to Stayclaim
src/app/admin/sources/page.tsx
248 lines
import Link from 'next/link';
import { pool } from '@/lib/db';
export const dynamic = 'force-dynamic';
type Source = {
key: string;
name: string;
category: 'permits' | 'parcels' | 'newspapers' | 'film' | 'civic' | 'archive';
url: string;
status: 'live' | 'partial' | 'paused' | 'walled' | 'paid' | 'dead';
rows: string;
notes: string;
ingestScript?: string;
};
/** Curated source registry — the canonical list. Keep this in sync with the
* la-research-agent's source map at ~/.claude/agents/la-research-agent.md. */
async function getRegistry(): Promise<Source[]> {
// counts from the DB where applicable. Wrapped in try/catch so a single
// missing column / table-rename doesn't 500 the entire registry view.
// Bounded statement_timeout protects the pool from large unindexed scans.
// Dropped the unused 'ladbs_permit' count (was never consumed below — it
// would otherwise be a wasted full scan of `listing` on every page load).
const noVal = '—';
let c: Record<string, string> = {};
try {
const client = await pool.connect();
try {
await client.query("SET LOCAL statement_timeout = '5s'");
const counts = await client.query<{ k: string; n: string }>(`
SELECT 'pi9x-tg5x' as k, count(*)::text as n FROM permit WHERE source_dataset='pi9x-tg5x'
UNION ALL SELECT 'dyxf-7hc4', count(*)::text FROM permit WHERE source_dataset='dyxf-7hc4'
UNION ALL SELECT 'e67z-kt2n', count(*)::text FROM permit WHERE source_dataset='e67z-kt2n'
UNION ALL SELECT 'weho_film_permit', count(*)::text FROM filming_location WHERE source_label LIKE '%WeHo%'
UNION ALL SELECT 'la_daily_herald', count(*)::text FROM news_issue WHERE paper_short='LADailyHerald'
UNION ALL SELECT 'la_times', count(*)::text FROM news_issue WHERE paper_short='LATimes'
UNION ALL SELECT 'pomona_progress', count(*)::text FROM news_issue WHERE paper_short='PomonaProgress'
UNION ALL SELECT 'santa_ana_register', count(*)::text FROM news_issue WHERE paper_short='SantaAnaRegister'
`);
c = Object.fromEntries(
counts.rows.map(r => {
const n = Number(r.n);
return [r.k, Number.isFinite(n) ? n.toLocaleString() : noVal];
})
);
} finally {
client.release();
}
} catch {
// Schema drift / DB unavailable / timeout: fall through with empty counts;
// the static registry below still renders with `noVal` placeholders.
c = {};
}
return [
// Permits — LA City
{ key: 'pi9x-tg5x', name: 'LADBS Building Permits 2020+', category: 'permits',
url: 'https://data.lacity.org/resource/pi9x-tg5x.json',
status: 'live', rows: c['pi9x-tg5x'] ?? noVal, ingestScript: 'ingest-la-city-permits.ts',
notes: 'Daily refresh. Canonical. Replaces stale xnhu-aczu (dead 2023-05-19).' },
{ key: 'dyxf-7hc4', name: 'LADBS Building Permits 2010-2019', category: 'permits',
url: 'https://data.lacity.org/resource/dyxf-7hc4.json',
status: 'live', rows: c['dyxf-7hc4'] ?? noVal, ingestScript: 'ingest-la-city-permits.ts',
notes: 'Static historical.' },
{ key: 'e67z-kt2n', name: 'LADBS Building Permits <2010', category: 'permits',
url: 'https://data.lacity.org/resource/e67z-kt2n.json',
status: 'live', rows: c['e67z-kt2n'] ?? noVal, ingestScript: 'ingest-la-city-permits.ts',
notes: 'Static historical.' },
{ key: 'u82d-eh7z', name: 'LADBS Code Enforcement Open', category: 'permits',
url: 'https://data.lacity.org/resource/u82d-eh7z.json',
status: 'partial', rows: '0 (pending)', notes: '28,931 expected. Ingester pending.' },
{ key: 'rken-a55j', name: 'LADBS Code Enforcement Closed', category: 'permits',
url: 'https://data.lacity.org/resource/rken-a55j.json',
status: 'partial', rows: '0 (pending)', notes: '810,798 expected.' },
// Permits — adjacent cities
{ key: 'bh-permits', name: 'Beverly Hills Issued Permits', category: 'permits',
url: 'https://opendata-hub.beverlyhills.org/datasets/847ebacb94a44b7b880f1c0a7296decf_0/about',
status: 'partial', rows: '8 (manual seed)', notes: 'ArcGIS Hub item 847ebacb…. Needs full ingester.' },
{ key: 'pasadena-permits', name: 'Pasadena Permit Activity', category: 'permits',
url: 'https://data.cityofpasadena.net/datasets/24525f546e5e4346957bb55695afc448_0',
status: 'partial', rows: '0 (pending)', notes: 'ArcGIS FeatureServer; daily refresh.' },
{ key: 'sm-permits', name: 'Santa Monica Active Permits', category: 'permits',
url: 'https://data.santamonica.gov/dataset/active-building-and-safety-permits',
status: 'partial', rows: '0 (pending)', notes: 'CKAN id d6867c7d…. ODC-BY 1.0.' },
// Parcels
{ key: 'la-county-roll', name: 'LA County Assessor Roll 2021+', category: 'parcels',
url: 'https://data.lacounty.gov/datasets/lacounty::assessor-parcel-data-rolls-2021-present/about',
status: 'partial', rows: '0 (pending)', notes: '~2.7M parcels, 674 MB CSV.' },
{ key: 'navigatela', name: 'NavigateLA ArcGIS REST', category: 'parcels',
url: 'https://maps.lacity.org/arcgis/rest/services/Mapping/NavigateLA/MapServer',
status: 'live', rows: '451 layers', notes: 'Layer 397=parcels, 71=zoning, 72/74=HCM, 75=HPOZ.' },
// Film
{ key: 'weho-film', name: 'WeHo Film Permits xd5y-dzvf', category: 'film',
url: 'https://data.weho.org/Filming/WeHo-Filming-Activity/xd5y-dzvf',
status: 'live', rows: c['weho_film_permit'] ?? noVal, ingestScript: 'ingest-weho-film-permits.ts',
notes: '1,821 expected.' },
{ key: 'filmla', name: 'FilmLA permits (LA City + County)', category: 'film',
url: 'https://www.filmla.com',
status: 'walled', rows: 'CPRA pending', notes: 'CPRA request submitted 2026-04-29; deadline 2026-05-09.' },
// Newspapers
{ key: 'ia_la_daily_herald', name: 'LA Daily Herald (Internet Archive)', category: 'newspapers',
url: 'https://archive.org/details/newspaperarchive-los-angeles-daily-herald',
status: 'live', rows: c['la_daily_herald'] ?? noVal, ingestScript: 'pull-la-newspapers-ocr.ts',
notes: '2,408 issues 1873-1890. OCR pull active.' },
{ key: 'ia_la_times', name: 'LA Times (Internet Archive)', category: 'newspapers',
url: 'https://archive.org/details/pub_los-angeles-times',
status: 'live', rows: c['la_times'] ?? noVal, ingestScript: 'pull-la-newspapers-ocr.ts',
notes: '17,298 issues 1881-1951. OCR pull active.' },
{ key: 'ia_pomona', name: 'Pomona Progress Bulletin (IA)', category: 'newspapers',
url: 'https://archive.org/details/newspaperarchive-pomona-progress-bulletin',
status: 'partial', rows: c['pomona_progress'] ?? noVal,
notes: '3,065 issues; metadata only.' },
{ key: 'ia_santa_ana', name: 'Santa Ana OC Register (IA)', category: 'newspapers',
url: 'https://archive.org/details/newspaperarchive-santa-ana-orange-county-register',
status: 'partial', rows: c['santa_ana_register'] ?? noVal,
notes: '339 issues; metadata only.' },
{ key: 'cdnc', name: 'California Digital Newspaper Collection', category: 'newspapers',
url: 'https://cdnc.ucr.edu/',
status: 'walled', rows: '—', notes: 'Cloudflare anti-bot blocks programmatic ingest. LA Times + Herald-Examiner NOT here.' },
// Civic / records
{ key: 'la_rrcc', name: 'LA County Recorder/Registrar (deeds)', category: 'civic',
url: 'https://www.lavote.gov/home/recorder/real-estate-records/faqs',
status: 'walled', rows: '—', notes: 'No free online deed-index search per Gov Code §6254.21.' },
{ key: 'la_court', name: 'LA Superior Court Civil Index', category: 'civic',
url: 'https://www.lacourt.org/paos/v2public/CivilIndex/',
status: 'paid', rows: '—', notes: '$1-$4.75 per search; not bulk-ingestible.' },
{ key: 'courtlistener', name: 'CourtListener / RECAP', category: 'civic',
url: 'https://www.courtlistener.com/help/api/rest/',
status: 'live', rows: '0 (not ingested)', notes: '~500M PACER objects via free API.' },
// Archives
{ key: 'usc_diglib', name: 'USC Digital Library', category: 'archive',
url: 'https://digitallibrary.usc.edu/',
status: 'live', rows: '0', notes: '~2M images, OAI-PMH likely available.' },
{ key: 'calisphere', name: 'UCLA via Calisphere API', category: 'archive',
url: 'https://calisphere.org/institution/247/',
status: 'live', rows: '0', notes: 'Open API.' },
{ key: 'historicplacesla', name: 'HistoricPlacesLA (SurveyLA)', category: 'archive',
url: 'https://hpla.lacity.org/',
status: 'walled', rows: '—', notes: 'Arches v3, no public API. Use NavigateLA HCM/HPOZ layers instead.' },
// DEAD / DEPRECATED
{ key: 'xnhu-aczu', name: 'LADBS xnhu-aczu (LEGACY)', category: 'permits',
url: 'https://data.lacity.org/resource/xnhu-aczu.json',
status: 'dead', rows: '—', notes: 'DO NOT USE. Last issue 2023-05-19. Replaced by pi9x-tg5x family.' },
{ key: 'long_beach', name: 'Long Beach data portal', category: 'permits',
url: 'https://data.longbeach.gov/',
status: 'dead', rows: '—', notes: 'Only 13 datasets, no permits/parcels.' },
{ key: 'culver_city', name: 'Culver City Socrata', category: 'permits',
url: 'https://data.culvercity.org/',
status: 'dead', rows: '—', notes: 'Redirects to ClearGov financial-only.' },
];
}
const STATUS_STYLE: Record<Source['status'], string> = {
live: 'bg-moss text-sand',
partial: 'bg-coral/20 text-coral',
paused: 'bg-ink/10 text-ink/60',
walled: 'bg-yellow-200 text-ink',
paid: 'bg-yellow-300 text-ink',
dead: 'bg-ink text-sand',
};
export default async function AdminSourcesPage() {
const sources = await getRegistry();
const byCat: Record<string, Source[]> = {};
for (const s of sources) (byCat[s.category] ??= []).push(s);
const categoryOrder: Array<[string, string]> = [
['permits', 'Building permits + code enforcement'],
['parcels', 'Parcels + assessor'],
['film', 'Film permits'],
['newspapers', 'Newspapers (OCR)'],
['civic', 'Civic / court records'],
['archive', 'Archival / historical'],
];
return (
<div className="space-y-10">
<section>
<h1 className="font-display text-4xl text-ink mb-2">Sources</h1>
<p className="text-sm text-ink/60 max-w-2xl">
Every dataset that feeds the archive — verified state, current row count, ingest script if any. Status colors:
<span className="mx-2 px-2 py-0.5 bg-moss text-sand text-[10px] uppercase tracking-wider">live</span>
<span className="mx-2 px-2 py-0.5 bg-coral/20 text-coral text-[10px] uppercase tracking-wider">partial</span>
<span className="mx-2 px-2 py-0.5 bg-yellow-200 text-ink text-[10px] uppercase tracking-wider">walled</span>
<span className="mx-2 px-2 py-0.5 bg-yellow-300 text-ink text-[10px] uppercase tracking-wider">paid</span>
<span className="mx-2 px-2 py-0.5 bg-ink text-sand text-[10px] uppercase tracking-wider">dead</span>
</p>
</section>
{categoryOrder.map(([cat, label]) => byCat[cat] && (
<section key={cat}>
<h2 className="font-display text-2xl text-ink mb-3">{label}</h2>
<table className="w-full text-sm border border-ink/10 bg-white">
<thead className="bg-ink/[0.03] text-[10px] uppercase tracking-[0.15em] text-ink/55">
<tr>
<th className="text-left p-3">source</th>
<th className="text-left p-3">status</th>
<th className="text-right p-3">rows</th>
<th className="text-left p-3">ingest</th>
<th className="text-left p-3">notes</th>
</tr>
</thead>
<tbody>
{byCat[cat]!.map(s => (
<tr key={s.key} className="border-t border-ink/5 align-top">
<td className="p-3">
{/^https?:\/\//i.test(s.url) ? (
<a href={s.url} target="_blank" rel="noreferrer noopener"
className="font-display text-base text-ink hover:text-coral transition">
{s.name} ↗
</a>
) : (
<span className="font-display text-base text-ink">{s.name}</span>
)}
<p className="font-mono text-[10px] text-ink/40 mt-0.5">{s.key}</p>
</td>
<td className="p-3">
<span className={`text-[10px] uppercase tracking-wider px-2 py-0.5 ${STATUS_STYLE[s.status]}`}>
{s.status}
</span>
</td>
<td className="p-3 text-right tabular-nums font-mono text-xs">{s.rows}</td>
<td className="p-3 font-mono text-[11px] text-ink/65">
{s.ingestScript ?? <span className="text-ink/30">—</span>}
</td>
<td className="p-3 text-xs text-ink/65 leading-relaxed">{s.notes}</td>
</tr>
))}
</tbody>
</table>
</section>
))}
<p className="text-xs text-ink/40 max-w-prose leading-relaxed">
Source registry is hand-curated; keep in sync with <code>~/.claude/agents/la-research-agent.md</code>.
Verify any new dataset slug with the LA Research Agent before adding here — Steve has been burned by stale Socrata IDs.
</p>
</div>
);
}