← back to Nationalrealestate
src/ingest/parcels/clark_nv.ts
186 lines
/**
* Clark County NV (Las Vegas, FIPS 32003) parcel ingest — dedicated full-universe
* adapter (applies the Wake/Miami-Dade DTD verdict B precedent).
*
* Source: the Clark County GIS Office keyless hosted ArcGIS FeatureServer ($0,
* supportsPagination, 2k/page). NOTE the parcels layer is at index 1, not 0:
* https://services1.arcgis.com/F1v0ufATbBQScMtY/arcgis/rest/services/CC_PARCELS_SHP/FeatureServer/1
* (The county's gisgate.co.clark.nv.us REST server and the Hub open-data host are
* both blocked/private to headless; this hosted layer is the reachable path.)
*
* A rich PRICED-SALE county: OWNER (~90%), ADDRESS (situs, ~90%), SALEPRICE
* (real dollars, ~81%) + SALEDATE (YYYYMMDD) + DOCNO (recorded-doc number),
* CONSTYR (year built, ~82%), LOTSQFT (lot sqft), LUCODE (land-use code). The one
* gap: assessed/market VALUE is NOT in this GIS layer (the IMPVAL/LANDVAL columns
* exist but are 100% zero) — so land/improvement/total value are left NULL rather
* than fabricated. This is still richer than the parcel-detail-only TX counties
* because it carries a genuine priced last-sale.
*
* source_id = PARCEL (the APN). Rows with no PARCEL are skipped.
*
* Run: NODE_OPTIONS=--max-old-space-size=4096 npm run ingest:parcels clark
*/
import { pool, query } from '../../../db/pool.ts';
import { openRun, closeRun } from '../run.ts';
import { upsertParcels, registerParcelSource, registerSourceFieldMap, normAddress, type ParcelUpsertRow } from './upsert.ts';
import { sanitizeEventDate } from './date_guard.ts';
import { sanitizeSalePrice } from './price_guard.ts';
const FIPS = '32003';
const SOURCE_KEY = 'clark_nv';
const LAYER = 'https://services1.arcgis.com/F1v0ufATbBQScMtY/arcgis/rest/services/CC_PARCELS_SHP/FeatureServer/1';
const PAGE = 2000;
// TK-50 field-level provenance. No value mapping — the assessed-value columns in
// this feed are all zero (see header); we do not map a fabricated value.
const FIELD_MAP: Record<string, string> = {
source_id: 'PARCEL', address: 'ADDRESS', city: 'STRCITY', zip: 'ZIPMAIN',
year_built: 'CONSTYR', use_desc: 'LUCODE', owner_name: 'OWNER',
last_sale_price: 'SALEPRICE', last_sale_date: 'SALEDATE',
};
const s = (v: unknown): string | null => { const t = v == null ? '' : String(v).trim(); return t ? t : null; };
const num = (v: unknown): number | null => { const x = Number(v); return Number.isFinite(x) && x !== 0 ? x : null; };
/** SALEDATE is a "YYYYMMDD" string → ISO 'YYYY-MM-DD', then the shared sanity
* guard (future / pre-1900 / non-calendar reject). Null on malformed. */
function saleDateToISO(v: unknown): string | null {
const t = s(v);
if (!t) return null;
const m = t.match(/^(\d{4})(\d{2})(\d{2})$/);
if (!m) return null;
return sanitizeEventDate(`${m[1]}-${m[2]}-${m[3]}`);
}
/** plain vertex-average of ring[0] — NOT a true polygon centroid (fine for a map
* pin; multipart parcels only get the first ring). */
function centroid(geom: any): [number | null, number | null] {
const ring = geom?.rings?.[0]; if (!Array.isArray(ring) || !ring.length) return [null, null];
let sx = 0, sy = 0; for (const [x, y] of ring) { sx += x; sy += y; }
return [+(sx / ring.length).toFixed(6), +(sy / ring.length).toFixed(6)];
}
// NOTE: use ZIPMAIN (the clean 5-digit zip), NOT ZIPCODE — ZIPCODE is a packed
// 9-digit integer (zip+plus4, e.g. 891490000) that would store as garbage.
const OUT = ['PARCEL', 'OWNER', 'ADDRESS', 'STRCITY', 'ZIPMAIN', 'CONSTYR', 'LOTSQFT',
'LUCODE', 'LANDUSE', 'SALEPRICE', 'SALEDATE', 'SALETYPE', 'DOCNO', 'DOCDATE'].join(',');
async function fetchPage(offset: number): Promise<any[]> {
const u = new URL(LAYER + '/query');
u.searchParams.set('where', '1=1');
u.searchParams.set('outFields', OUT);
// page by the layer's system-maintained unique id (OBJECTID_1, NOT OBJECTID) so
// resultOffset paging is stable + complete across the full ~840k universe. The
// low-id head is blank-owner shells (real but empty parcels); real owner/sale
// rows fill in beyond it — a full run captures all ~754k owned parcels.
u.searchParams.set('orderByFields', 'OBJECTID_1 ASC');
u.searchParams.set('resultOffset', String(offset));
u.searchParams.set('resultRecordCount', String(PAGE));
u.searchParams.set('returnGeometry', 'true');
u.searchParams.set('outSR', '4326');
u.searchParams.set('f', 'json');
let lastErr: any;
for (let a = 0; a < 3; a++) {
try {
const res = await fetch(u, { headers: { 'User-Agent': 'Mozilla/5.0 (usre-parcel-ingest)' }, signal: AbortSignal.timeout(120_000) });
if (!res.ok) throw new Error(`clark ${res.status}: ${(await res.text()).slice(0, 140)}`);
const j: any = await res.json();
if (j.error) throw new Error(`clark error: ${JSON.stringify(j.error).slice(0, 140)}`);
return j.features || [];
} catch (e) { lastErr = e; if (a < 2) await new Promise(r => setTimeout(r, 2000 * (a + 1))); }
}
throw lastErr;
}
export async function ingestClark(opts: { maxPages?: number } = {}): Promise<{ upserted: number }> {
const runId = await openRun('parcel_clark_nv', LAYER);
const fetchedAt = new Date().toISOString();
try {
await registerSourceFieldMap(SOURCE_KEY, FIELD_MAP, 'Clark County NV GIS Office (CC_PARCELS_SHP hosted ArcGIS layer)');
let offset = 0, page = 0, seen = 0, skipped = 0, upserted = 0, sales = 0;
for (;;) {
const feats = await fetchPage(offset);
if (!feats.length) break;
const batch: ParcelUpsertRow[] = [];
const events: { apn: string; price: number; date: string; docno: string | null }[] = [];
for (const f of feats) {
const a = f.attributes || {};
const apn = s(a.PARCEL);
if (!apn) { skipped++; continue; }
seen++;
const [lng, lat] = centroid(f.geometry);
// fixed-width address string ("000713 E LAKE MEAD BLVD") → normAddress
// collapses the internal padding.
const addr = s(a.ADDRESS);
const norm = addr ? normAddress(addr) : null;
const salePrice = sanitizeSalePrice(a.SALEPRICE);
const saleDate = saleDateToISO(a.SALEDATE);
const docno = s(a.DOCNO);
batch.push({
county_fips: FIPS, source_id: apn,
address: norm, norm_address: norm,
// ZIPMAIN is an integer 5-digit zip — pad defensively (NV zips don't lead
// with 0, but keep it safe). STRCITY is a Clark internal abbreviation
// ('LV','NLV','HEND','MES'...), NOT a display city — noted in extra below.
city: s(a.STRCITY), zip: a.ZIPMAIN ? String(a.ZIPMAIN).padStart(5, '0') : null,
lat, lng,
year_built: num(a.CONSTYR), sqft: null, beds: null, baths: null, units: null,
use_desc: s(a.LUCODE),
// assessed value not present in this feed (columns all zero) — leave NULL.
land_value: null, improvement_value: null, total_value: null,
tax_year: null, owner_name: s(a.OWNER), zoning: null,
last_sale_date: saleDate, last_sale_price: salePrice,
extra: JSON.stringify({
land_use: s(a.LANDUSE) || undefined,
lot_sqft: num(a.LOTSQFT) || undefined,
sale_type: s(a.SALETYPE) || undefined,
doc_date: s(a.DOCDATE) || undefined,
city_abbrev: s(a.STRCITY) || undefined, // STRCITY is a Clark abbreviation (LV/NLV/HEND/MES), not a display-ready city name
note: 'Clark County NV GIS — owner + priced last-sale + year built + use; assessed value NOT in this GIS layer (value columns all zero, left NULL); city is a Clark internal abbreviation',
}),
// ── TK-50 field-level provenance (record-level: one feature → this row) ──
sourceKey: SOURCE_KEY,
sourceUrl: `${LAYER}/query?where=${encodeURIComponent(`PARCEL='${apn}'`)}&outFields=*&f=html`,
fetchedAt,
rawSource: JSON.stringify(a),
});
if (salePrice && saleDate) events.push({ apn, price: salePrice, date: saleDate, docno });
}
upserted += await upsertParcels(batch);
// priced last-sale events (amount = SALEPRICE). doc_number = DOCNO (real
// recorded-document number) when present, else the sale date.
for (let i = 0; i < events.length; i += 500) {
const chunk = events.slice(i, i + 500);
await query(
`INSERT INTO parcel_event (county_fips, source_id, event_type, event_date, amount, doc_type, doc_number, source, source_url, detail)
VALUES ${chunk.map((_, j) => { const b = j * 9; return `($${b + 1},$${b + 2},'sale',$${b + 3}::date,$${b + 4},$${b + 5},$${b + 6},$${b + 7},$${b + 8},$${b + 9}::jsonb)`; }).join(',')}
ON CONFLICT (county_fips, source_id, event_type, event_date, doc_number) DO NOTHING`,
chunk.flatMap(e => [FIPS, e.apn, e.date, e.price, 'Deed', e.docno ?? e.date,
'parcel_clark_nv', `https://maps.clarkcountynv.gov/assessor/AssessorParcelDetail/parcel.aspx?instance=pcl1&parcel=${e.apn}`, JSON.stringify({ docno: e.docno })]),
);
sales += chunk.length;
}
offset += feats.length;
page++;
if (seen % 100000 < feats.length) console.log(`[clark] ${upserted} upserted, ${sales} sales, ${skipped} skipped`);
if (opts.maxPages && page >= opts.maxPages) break;
if (feats.length < PAGE) break;
}
if (!opts.maxPages) {
if (upserted < 780000) throw new Error(`only ${upserted} Clark parcels (expected ~840k) — layer/paging drift?`);
const n = await registerParcelSource(FIPS, LAYER,
`Clark County NV (Las Vegas) GIS Office parcels (CC_PARCELS_SHP hosted ArcGIS layer) — ${upserted} parcels: owner (~90%), situs address, priced last-sale (SALEPRICE + SALEDATE + DOCNO, ~81%) → ${sales} parcel_event sale rows, year built (CONSTYR, ~82%), lot sqft, land-use code. Assessed/market value NOT in this GIS layer (value columns all zero, left NULL). ${skipped} PARCEL-less rows skipped.`);
console.log(`[clark] registry ${n}`);
}
await closeRun(runId, 'ok', { upserted, skipped, notes: `Clark NV: ${upserted} parcels, ${sales} sales, ${skipped} skipped${opts.maxPages ? ' (bounded verify)' : ''}` });
console.log(`[clark] ok: ${upserted} parcels, ${sales} sales, ${skipped} skipped${opts.maxPages ? ` (bounded ${opts.maxPages}p)` : ''}`);
return { upserted };
} catch (e: any) {
await closeRun(runId, 'failed', { notes: String(e.message || e).slice(0, 500) });
throw e;
}
}
if (import.meta.url === `file://${process.argv[1]}`) {
ingestClark().then(() => pool.end()).catch(e => { console.error(e); process.exit(1); });
}