← back to Nationalrealestate
parcels: centralize deed-date sanity guard across all 4 deed ingesters
89344107e3a71c7918ff1097555c0d18cfdb7822 · 2026-07-27 21:04:37 -0700 · Steve Abrams
The future-date guard was a one-off inline check in arcgis_sales.ts; wake_nc
had only a 1971 lower bound and miami_dade/sandiego had no future/absurd-date
guard at all — so a bad-data run against those feeds could re-introduce
impossible parcel_event dates. Add a shared, self-tested sanitizeEventDate()
(rejects future dates, sub-1900, and non-calendar dates like Feb 30) and wire
it into all four deed ingesters at the date-normalization point. Ships a
19-case tsx self-test (no test framework in repo).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Files touched
M src/ingest/parcels/arcgis_sales.tsA src/ingest/parcels/date_guard.tsM src/ingest/parcels/miami_dade.tsM src/ingest/parcels/sandiego_ca.tsM src/ingest/parcels/wake_nc.ts
Diff
commit 89344107e3a71c7918ff1097555c0d18cfdb7822
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Mon Jul 27 21:04:37 2026 -0700
parcels: centralize deed-date sanity guard across all 4 deed ingesters
The future-date guard was a one-off inline check in arcgis_sales.ts; wake_nc
had only a 1971 lower bound and miami_dade/sandiego had no future/absurd-date
guard at all — so a bad-data run against those feeds could re-introduce
impossible parcel_event dates. Add a shared, self-tested sanitizeEventDate()
(rejects future dates, sub-1900, and non-calendar dates like Feb 30) and wire
it into all four deed ingesters at the date-normalization point. Ships a
19-case tsx self-test (no test framework in repo).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
src/ingest/parcels/arcgis_sales.ts | 4 +-
src/ingest/parcels/date_guard.ts | 108 +++++++++++++++++++++++++++++++++++++
src/ingest/parcels/miami_dade.ts | 6 ++-
src/ingest/parcels/sandiego_ca.ts | 4 +-
src/ingest/parcels/wake_nc.ts | 3 +-
5 files changed, 119 insertions(+), 6 deletions(-)
diff --git a/src/ingest/parcels/arcgis_sales.ts b/src/ingest/parcels/arcgis_sales.ts
index e1c4f38..586bab7 100644
--- a/src/ingest/parcels/arcgis_sales.ts
+++ b/src/ingest/parcels/arcgis_sales.ts
@@ -17,10 +17,10 @@
import { query } from '../../../db/pool.ts';
import { openRun, closeRun } from '../run.ts';
import { upsertParcels, normAddress, registerParcelSource, type ParcelUpsertRow } from './upsert.ts';
+import { sanitizeEventDate } from './date_guard.ts';
const PAGE = 2000;
const MAX_PER_RUN = Number(process.env.SALES_MAX_PER_RUN || 6000);
-const TODAY = new Date().toISOString().slice(0, 10); // future-date guard: county feeds carry occasional bad-entry future sales
interface Rec {
fips: string; apn: string;
@@ -265,7 +265,7 @@ export function ingestArcgisSales(key: string) {
if (!feats.length) break;
for (const r of src.toRecords(feats)) {
if (!r.apn) continue;
- if (r.saleDate && r.saleDate > TODAY) r.saleDate = null; // drop impossible future dates (bad source entries)
+ r.saleDate = sanitizeEventDate(r.saleDate); // drop future/absurd bad-entry dates (shared guard)
seenFips.add(r.fips);
const gisUrl = `${src.layer}/query?where=${encodeURIComponent(src.outFields.split(',')[0] + "='" + r.apn + "'")}&outFields=*&f=html`;
rows.push({
diff --git a/src/ingest/parcels/date_guard.ts b/src/ingest/parcels/date_guard.ts
new file mode 100644
index 0000000..980edcd
--- /dev/null
+++ b/src/ingest/parcels/date_guard.ts
@@ -0,0 +1,108 @@
+// Shared deed-date sanity guard for every parcel-deed ingester.
+//
+// Why this exists: each county feed carries occasional bad-entry dates — a
+// future sale ("2027-04-01"), a mainframe near-epoch artifact ("1901-01-01"),
+// or a typo'd century ("2921-06-23"). Before this util the future-date guard
+// lived as a one-off inline check in arcgis_sales.ts (`> TODAY`), while
+// wake_nc.ts had only a 1971 lower bound and miami_dade.ts had NO sanity guard
+// at all. A bad-data run against Miami-Dade or Wake could therefore re-introduce
+// impossible dates into parcel_event, exactly the class the arcgis guard was
+// added to stop. This centralizes ONE tested rule so all four deed ingesters
+// (arcgis_sales, miami_dade, wake_nc, sandiego_ca) reject the same defects.
+//
+// Contract: input is an already-format-normalized 'YYYY-MM-DD' string (or null).
+// Output is that same string iff it is a plausible real recorded-deed date, else
+// null. It never throws and never mutates its input — a rejected date simply
+// drops the event (the parcel's last_sale_date should be nulled alongside it).
+
+// Oldest plausible digitized US deed record. Nothing in these county feeds
+// predates ~1900; anything earlier is an epoch/precision artifact, not a sale.
+const MIN_DEED_YEAR = 1900;
+
+/**
+ * Return `iso` iff it is a well-formed 'YYYY-MM-DD' calendar date that is not
+ * in the future and not older than MIN_DEED_YEAR; otherwise null.
+ *
+ * @param iso an ISO date string 'YYYY-MM-DD', or null/undefined/''
+ * @param today today's date as 'YYYY-MM-DD' (injectable for deterministic tests)
+ */
+export function sanitizeEventDate(
+ iso: string | null | undefined,
+ today: string = new Date().toISOString().slice(0, 10),
+): string | null {
+ if (!iso) return null;
+ const t = String(iso).trim();
+ // strict shape: 4-digit year, 2-digit month, 2-digit day
+ const m = t.match(/^(\d{4})-(\d{2})-(\d{2})$/);
+ if (!m) return null;
+ const year = Number(m[1]);
+ const month = Number(m[2]);
+ const day = Number(m[3]);
+ if (year < MIN_DEED_YEAR) return null;
+ // reject non-calendar dates (month 00/13, day 00/32, Feb 30, etc.) by
+ // round-tripping through Date and checking the parts survived unchanged.
+ const d = new Date(Date.UTC(year, month - 1, day));
+ if (
+ d.getUTCFullYear() !== year ||
+ d.getUTCMonth() !== month - 1 ||
+ d.getUTCDate() !== day
+ ) {
+ return null;
+ }
+ // future-date guard: a recorded sale cannot be dated after today.
+ if (t > today) return null;
+ return t;
+}
+
+// --- self-test (run: `tsx src/ingest/parcels/date_guard.ts`) -----------------
+// No test framework is installed in this repo, so the util ships its own
+// deterministic self-test that exits non-zero on any failure. Runs only when
+// executed directly, never on import.
+const isMain = (() => {
+ try {
+ // import.meta.url is defined in ESM; process.argv[1] is the entry file.
+ return typeof process !== 'undefined'
+ && process.argv[1] != null
+ && import.meta.url === new URL(`file://${process.argv[1]}`).href;
+ } catch { return false; }
+})();
+
+if (isMain) {
+ const TODAY = '2026-07-27';
+ const cases: Array<[string | null | undefined, string | null, string]> = [
+ // [input, expected, label]
+ ['2021-06-23', '2021-06-23', 'ordinary valid deed date'],
+ ['2026-07-27', '2026-07-27', 'today is allowed (boundary)'],
+ ['2026-07-28', null, 'tomorrow rejected (future by 1 day)'],
+ ['2027-04-01', null, 'future year rejected'],
+ ['2921-06-23', null, 'typo century (2921) rejected'],
+ ['1899-12-31', null, 'below 1900 floor rejected'],
+ ['1900-01-01', '1900-01-01', '1900 floor allowed (boundary)'],
+ ['1901-01-01', '1901-01-01', 'near-epoch-looking but valid 1901'],
+ ['2021-13-01', null, 'month 13 rejected'],
+ ['2021-00-10', null, 'month 00 rejected'],
+ ['2021-02-30', null, 'Feb 30 (non-calendar) rejected'],
+ ['2021-06-31', null, 'June 31 (non-calendar) rejected'],
+ ['2021-6-3', null, 'unpadded parts rejected (shape)'],
+ ['20210623', null, 'compact form rejected (shape)'],
+ ['2021-06-23T00:00:00', null, 'datetime rejected (shape)'],
+ ['', null, 'empty string -> null'],
+ [null, null, 'null -> null'],
+ [undefined, null, 'undefined -> null'],
+ [' 2021-06-23 ', '2021-06-23', 'surrounding whitespace trimmed'],
+ ];
+ let failed = 0;
+ for (const [input, expected, label] of cases) {
+ const got = sanitizeEventDate(input, TODAY);
+ const ok = got === expected;
+ if (!ok) {
+ failed++;
+ console.error(`FAIL: ${label}\n input=${JSON.stringify(input)} expected=${JSON.stringify(expected)} got=${JSON.stringify(got)}`);
+ }
+ }
+ if (failed) {
+ console.error(`\n${failed}/${cases.length} sanitizeEventDate cases FAILED`);
+ process.exit(1);
+ }
+ console.log(`sanitizeEventDate: all ${cases.length} cases passed`);
+}
diff --git a/src/ingest/parcels/miami_dade.ts b/src/ingest/parcels/miami_dade.ts
index 9e72e1e..2bc4e80 100644
--- a/src/ingest/parcels/miami_dade.ts
+++ b/src/ingest/parcels/miami_dade.ts
@@ -25,6 +25,7 @@
import { pool, query } from '../../../db/pool.ts';
import { openRun, closeRun } from '../run.ts';
import { upsertParcels, registerParcelSource, normAddress, type ParcelUpsertRow } from './upsert.ts';
+import { sanitizeEventDate } from './date_guard.ts';
const FIPS = '12086';
const LAYER = 'https://gisweb.miamidade.gov/arcgis/rest/services/MD_ComparableSales/MapServer/5';
@@ -32,10 +33,11 @@ const PAGE = 20000;
const s = (v: unknown): string | null => { const t = v == null ? '' : String(v).trim(); return t ? t : null; };
const num = (v: unknown): number | null => { const x = Number(v); return Number.isFinite(x) && x !== 0 ? x : null; };
-/** "20210623" -> "2021-06-23"; anything else -> null */
+/** "20210623" -> "2021-06-23"; future/absurd/non-calendar -> null (shared guard) */
const toDate = (v: unknown): string | null => {
const t = String(v ?? '').trim();
- return /^\d{8}$/.test(t) ? `${t.slice(0, 4)}-${t.slice(4, 6)}-${t.slice(6, 8)}` : null;
+ const iso = /^\d{8}$/.test(t) ? `${t.slice(0, 4)}-${t.slice(4, 6)}-${t.slice(6, 8)}` : null;
+ return sanitizeEventDate(iso);
};
const OUT = [
diff --git a/src/ingest/parcels/sandiego_ca.ts b/src/ingest/parcels/sandiego_ca.ts
index d5a2b00..8e3f366 100644
--- a/src/ingest/parcels/sandiego_ca.ts
+++ b/src/ingest/parcels/sandiego_ca.ts
@@ -20,6 +20,7 @@
import { query } from '../../../db/pool.ts';
import { openRun, closeRun } from '../run.ts';
import { upsertParcels, normAddress, registerParcelSource, type ParcelUpsertRow } from './upsert.ts';
+import { sanitizeEventDate } from './date_guard.ts';
const FIPS = '06073';
const SOURCE = 'sandiego_ca';
@@ -61,7 +62,8 @@ function docDate(v: unknown): string | null {
if (!/^\d{6}$/.test(t)) return null;
const mm = +t.slice(0, 2), dd = +t.slice(2, 4), yy = +t.slice(4, 6);
if (mm < 1 || mm > 12 || dd < 1 || dd > 31) return null;
- return `${fullYear(yy)}-${String(mm).padStart(2, '0')}-${String(dd).padStart(2, '0')}`;
+ // shared guard rejects future/absurd + non-calendar (e.g. Feb 30) dates.
+ return sanitizeEventDate(`${fullYear(yy)}-${String(mm).padStart(2, '0')}-${String(dd).padStart(2, '0')}`);
}
/** year_effective is a 2-digit year. */
function effYear(v: unknown): number | null {
diff --git a/src/ingest/parcels/wake_nc.ts b/src/ingest/parcels/wake_nc.ts
index 18c194c..472dcbc 100644
--- a/src/ingest/parcels/wake_nc.ts
+++ b/src/ingest/parcels/wake_nc.ts
@@ -22,6 +22,7 @@
import { pool, query } from '../../../db/pool.ts';
import { openRun, closeRun } from '../run.ts';
import { upsertParcels, registerParcelSource, normAddress, type ParcelUpsertRow } from './upsert.ts';
+import { sanitizeEventDate } from './date_guard.ts';
const FIPS = '37183';
const LAYER = 'https://maps.wakegov.com/arcgis/rest/services/Property/Parcels/MapServer/0';
@@ -31,7 +32,7 @@ const s = (v: unknown): string | null => { const t = v == null ? '' : String(v).
const num = (v: unknown): number | null => { const x = Number(v); return Number.isFinite(x) && x !== 0 ? x : null; };
// reject <=0 AND sub-1971 (31_536_000_000ms) — Wake's oldest real digital deed is
// 1971; anything earlier is a mainframe near-epoch precision artifact, not a sale.
-const epochToISO = (v: unknown): string | null => { const x = Number(v); if (!Number.isFinite(x) || x < 31_536_000_000) return null; return new Date(x).toISOString().slice(0, 10); };
+const epochToISO = (v: unknown): string | null => { const x = Number(v); if (!Number.isFinite(x) || x < 31_536_000_000) return null; return sanitizeEventDate(new Date(x).toISOString().slice(0, 10)); };
/** plain vertex-average of ring[0] — NOT a true polygon centroid (fine for a map
* pin; multipart parcels only get the first ring). */
function centroid(geom: any): [number | null, number | null] {
← de29c39 parcels: add Marion County OR / Salem (41047) free priced-de
·
back to Nationalrealestate
·
parcels: extend deed-date sanity guard to franklin_oh, king_ 0bae506 →