← back to Nationalrealestate
Scout round 2: add Crook OR (w/ names), Sonoma CA, Colorado composite (6+ counties), Maricopa AZ, Thurston WA deed adapters + robust price/date parsers; loop jobs
9a14bfd02678bd4c296cda0deca8926c538a1c67 · 2026-07-26 15:10:50 -0700 · Steve Abrams
Files touched
M src/ingest/parcels/arcgis_sales.tsM src/ingest/parcels/engine.tsM src/jobs/hourly_loop.ts
Diff
commit 9a14bfd02678bd4c296cda0deca8926c538a1c67
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Sun Jul 26 15:10:50 2026 -0700
Scout round 2: add Crook OR (w/ names), Sonoma CA, Colorado composite (6+ counties), Maricopa AZ, Thurston WA deed adapters + robust price/date parsers; loop jobs
---
src/ingest/parcels/arcgis_sales.ts | 60 ++++++++++++++++++++++++++++++++++++++
src/ingest/parcels/engine.ts | 5 ++++
src/jobs/hourly_loop.ts | 6 ++++
3 files changed, 71 insertions(+)
diff --git a/src/ingest/parcels/arcgis_sales.ts b/src/ingest/parcels/arcgis_sales.ts
index 3cfff48..595cfe7 100644
--- a/src/ingest/parcels/arcgis_sales.ts
+++ b/src/ingest/parcels/arcgis_sales.ts
@@ -46,6 +46,17 @@ function centroid(geom: any): [number | null, number | null] {
return [+(sx / ring.length).toFixed(6), +(sy / ring.length).toFixed(6)];
}
const clean = (v: unknown): string | null => { const t = s(v); return t ? t.replace(/\s+/g, ' ') : null; };
+// price may be a real number OR a string ('117500', ' 000001060000.'); pull the digits.
+const anyPrice = (v: unknown): number | null => { if (v == null) return null; const x = Number(String(v).replace(/[^\d.]/g, '')); return Number.isFinite(x) && x > 0 ? x : null; };
+// date may be epoch-ms, 'YYYY-MM-DD', or 'M/D/YYYY'.
+const anyDate = (v: unknown): string | null => {
+ if (v == null || v === '') return null;
+ if (typeof v === 'number' || /^\d{11,}$/.test(String(v).trim())) return epochToISO(Number(v));
+ const t = String(v).trim();
+ let m = t.match(/^(\d{4})-(\d{2})-(\d{2})/); if (m) return `${m[1]}-${m[2]}-${m[3]}`;
+ m = t.match(/^(\d{1,2})\/(\d{1,2})\/(\d{4})/); if (m) return `${m[3]}-${m[1].padStart(2, '0')}-${m[2].padStart(2, '0')}`; // M/D/YYYY -> YYYY-MM-DD
+ return null;
+};
// ── source configs ────────────────────────────────────────────────────────
const OR_FIPS: Record<string, string> = { M: '41051', W: '41067', C: '41005' };
@@ -117,6 +128,55 @@ const SOURCES: Record<string, Source> = {
return out;
},
},
+ crook: {
+ key: 'crook', label: 'Crook County OR (Prineville)', cursorKey: 'crook', geometry: false, orderBy: 'OBJECTID', pageSize: 2000,
+ layer: 'https://gis.crookcountyor.gov/server/rest/services/OpenData/TaxlotandTables/MapServer/6',
+ where: 'SALES_PRICE>0',
+ outFields: 'MAPTAXLOT,SALES_PRICE,SALES_DATE,GRANTOR_NAME,GRANTEE_NAME,BOOK,NUMBER,SALE_ID',
+ toRecords: (fs) => fs.map(f => { const a = f.attributes; const apn = s(a.MAPTAXLOT); if (!apn) return null;
+ return { fips: '41013', apn, price: anyPrice(a.SALES_PRICE), saleDate: anyDate(a.SALES_DATE),
+ grantor: clean(a.GRANTOR_NAME), grantee: clean(a.GRANTEE_NAME),
+ docNum: [s(a.BOOK), s(a.NUMBER)].filter(Boolean).join('-') || s(a.SALE_ID) } as Rec; }).filter(Boolean) as Rec[],
+ },
+ sonoma: {
+ key: 'sonoma', label: 'Sonoma County CA', cursorKey: 'sonoma', geometry: false, orderBy: 'OBJECTID', pageSize: 2000,
+ layer: 'https://socogis.sonomacounty.ca.gov/map/rest/services/CRAPublic/ParcelsPublic/FeatureServer/0',
+ where: 'SaleSalesPrice>0',
+ outFields: 'APN,SaleSalesPrice,SaleRecordingDate,SaleDocNum,SalePriorSalesPrice,SalePriorRecordingDate,SalePriorDocNum',
+ toRecords: (fs) => { const out: Rec[] = [];
+ for (const f of fs) { const a = f.attributes; const apn = s(a.APN); if (!apn) continue;
+ const cur = anyPrice(a.SaleSalesPrice), cd = anyDate(a.SaleRecordingDate);
+ if (cur && cd) out.push({ fips: '06097', apn, price: cur, saleDate: cd, docNum: s(a.SaleDocNum) } as Rec);
+ const pp = anyPrice(a.SalePriorSalesPrice), pd = anyDate(a.SalePriorRecordingDate);
+ if (pp && pd) out.push({ fips: '06097', apn, price: pp, saleDate: pd, docNum: s(a.SalePriorDocNum) } as Rec);
+ } return out; },
+ },
+ colorado: {
+ key: 'colorado', label: 'Colorado (statewide composite)', cursorKey: 'colorado', geometry: false, orderBy: 'OBJECTID', pageSize: 2000,
+ layer: 'https://gis.colorado.gov/Public/rest/services/Address_and_Parcel/Colorado_Public_Parcels/MapServer/0',
+ where: "salePrice>'0'",
+ outFields: 'parcel_id,salePrice,saleDate,countyFips,countyName',
+ toRecords: (fs) => fs.map(f => { const a = f.attributes; const apn = s(a.parcel_id); if (!apn) return null;
+ const cf = (s(a.countyFips) || '').replace(/\D/g, '');
+ const fips = cf.length >= 5 ? cf.slice(0, 5) : '08' + cf.padStart(3, '0').slice(-3);
+ return { fips, apn, city: clean(a.countyName), price: anyPrice(a.salePrice), saleDate: anyDate(a.saleDate) } as Rec; }).filter(Boolean) as Rec[],
+ },
+ maricopa: {
+ key: 'maricopa', label: 'Maricopa County AZ (Phoenix)', cursorKey: 'maricopa', geometry: false, orderBy: 'OBJECTID', pageSize: 1000,
+ layer: 'https://gis.mcassessor.maricopa.gov/arcgis/rest/services/MaricopaDynamicQueryService/MapServer/3',
+ where: "SALE_PRICE>'0'",
+ outFields: 'APN,SALE_PRICE,SALE_DATE,DEED_DATE,DEED_NUMBER',
+ toRecords: (fs) => fs.map(f => { const a = f.attributes; const apn = s(a.APN); if (!apn) return null;
+ return { fips: '04013', apn, price: anyPrice(a.SALE_PRICE), saleDate: anyDate(a.SALE_DATE) || anyDate(a.DEED_DATE), docNum: s(a.DEED_NUMBER) } as Rec; }).filter(Boolean) as Rec[],
+ },
+ thurston: {
+ key: 'thurston', label: 'Thurston County WA (Olympia)', cursorKey: 'thurston', geometry: false, orderBy: 'OBJECTID', pageSize: 1000,
+ layer: 'https://map.co.thurston.wa.us/arcgis/rest/services/Thurston/Thurston_Parcels/FeatureServer/0',
+ where: 'SALE_PRICE>0',
+ outFields: 'PARCEL_NO,SALE_PRICE,SALE_DATE',
+ toRecords: (fs) => fs.map(f => { const a = f.attributes; const apn = s(a.PARCEL_NO); if (!apn) return null;
+ return { fips: '53067', apn, price: anyPrice(a.SALE_PRICE), saleDate: anyDate(a.SALE_DATE) } as Rec; }).filter(Boolean) as Rec[],
+ },
};
async function fetchPage(src: Source, offset: number): Promise<any[]> {
diff --git a/src/ingest/parcels/engine.ts b/src/ingest/parcels/engine.ts
index 4d74eb1..2b470f3 100644
--- a/src/ingest/parcels/engine.ts
+++ b/src/ingest/parcels/engine.ts
@@ -20,6 +20,11 @@ const ADAPTERS: Record<string, () => Promise<{ run: () => Promise<{ upserted: nu
spokane: async () => { const m = await import('./arcgis_sales.ts'); return { run: m.ingestArcgisSales('spokane') }; },
alameda: async () => { const m = await import('./arcgis_sales.ts'); return { run: m.ingestArcgisSales('alameda') }; },
deschutes: async () => { const m = await import('./arcgis_sales.ts'); return { run: m.ingestArcgisSales('deschutes') }; },
+ crook: async () => { const m = await import('./arcgis_sales.ts'); return { run: m.ingestArcgisSales('crook') }; },
+ sonoma: async () => { const m = await import('./arcgis_sales.ts'); return { run: m.ingestArcgisSales('sonoma') }; },
+ colorado: async () => { const m = await import('./arcgis_sales.ts'); return { run: m.ingestArcgisSales('colorado') }; },
+ maricopa: async () => { const m = await import('./arcgis_sales.ts'); return { run: m.ingestArcgisSales('maricopa') }; },
+ thurston: async () => { const m = await import('./arcgis_sales.ts'); return { run: m.ingestArcgisSales('thurston') }; },
};
async function main() {
diff --git a/src/jobs/hourly_loop.ts b/src/jobs/hourly_loop.ts
index 1b413ec..a9f3b07 100644
--- a/src/jobs/hourly_loop.ts
+++ b/src/jobs/hourly_loop.ts
@@ -55,6 +55,12 @@ const JOBS: Job[] = [
// King County WA — 2.4M full-history priced deeds (grantor+grantee). Heavy 150MB
// bulk CSV, so weekly cadence (not hourly).
{ name: 'deeds-king', script: 'src/ingest/parcels/engine.ts', args: ['king'], minIntervalHours: 168, runSources: ['parcel_king_wa'] },
+ // Scout round 2 — more free priced-deed feeds (OR/CA/CO/AZ/WA).
+ { name: 'deeds-crook',script: 'src/ingest/parcels/engine.ts', args: ['crook'], minIntervalHours: 4, runSources: ['crook'] },
+ { name: 'deeds-sonoma',script:'src/ingest/parcels/engine.ts', args: ['sonoma'], minIntervalHours: 3, runSources: ['sonoma'] },
+ { name: 'deeds-co', script: 'src/ingest/parcels/engine.ts', args: ['colorado'], minIntervalHours: 2, runSources: ['colorado'] },
+ { name: 'deeds-maricopa',script:'src/ingest/parcels/engine.ts', args: ['maricopa'], minIntervalHours: 2, runSources: ['maricopa'] },
+ { name: 'deeds-thur', script: 'src/ingest/parcels/engine.ts', args: ['thurston'], minIntervalHours: 4, runSources: ['thurston'] },
];
/** hours since this job's freshest OK run across its runSources (∞ if never). */
← 467ca96 Add King County WA (2.4M full-history deeds) to loop on week
·
back to Nationalrealestate
·
wake: optimize bulk-flag UPDATE to pkey join (was 53min JSON d32c870 →