← back to Unclaimed Property Platform
Cycle 5: NAUPA III XML adapter (completes CSV/NAUPA II/III trio) + XXE guard
ac6d777f7eb92bba9e97763a9f49231d88a83ddd · 2026-08-01 20:07:08 -0700 · Steve Abrams
- services/ingestion/naupa3.py: namespace-tolerant XML parser via the lazy adapter registry
(format_name naupa3_v1); same CanonicalProperty as CSV/NAUPA II.
- SECURITY: rejects any DOCTYPE/ENTITY document (XmlSecurityError) -> closes XXE /
billion-laughs entity-expansion at the door before parsing. build_naupa3_xml helper for
aligned fixtures (optional default xmlns to exercise namespace tolerance).
- tests/test_cycle6_naupa3.py: parity, namespace tolerance, XXE refusal, malformed-record
reject, idempotency.
- Logged backlog: apostrophe/hyphen search RECALL (normalize_text collapses ' to space, so
LIKE-search misses 'Oneil' for O'Neil; matching unaffected via phonetic key). Prod = OpenSearch analyzers.
Tests: 6/6 suites green. All local/synthetic/$0.
TK-10097
Files touched
M services/ingestion/ingest.pyA services/ingestion/naupa3.pyA tests/test_cycle6_naupa3.py
Diff
commit ac6d777f7eb92bba9e97763a9f49231d88a83ddd
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Sat Aug 1 20:07:08 2026 -0700
Cycle 5: NAUPA III XML adapter (completes CSV/NAUPA II/III trio) + XXE guard
- services/ingestion/naupa3.py: namespace-tolerant XML parser via the lazy adapter registry
(format_name naupa3_v1); same CanonicalProperty as CSV/NAUPA II.
- SECURITY: rejects any DOCTYPE/ENTITY document (XmlSecurityError) -> closes XXE /
billion-laughs entity-expansion at the door before parsing. build_naupa3_xml helper for
aligned fixtures (optional default xmlns to exercise namespace tolerance).
- tests/test_cycle6_naupa3.py: parity, namespace tolerance, XXE refusal, malformed-record
reject, idempotency.
- Logged backlog: apostrophe/hyphen search RECALL (normalize_text collapses ' to space, so
LIKE-search misses 'Oneil' for O'Neil; matching unaffected via phonetic key). Prod = OpenSearch analyzers.
Tests: 6/6 suites green. All local/synthetic/$0.
TK-10097
---
services/ingestion/ingest.py | 3 +
services/ingestion/naupa3.py | 127 +++++++++++++++++++++++++++++++++++++++++++
tests/test_cycle6_naupa3.py | 109 +++++++++++++++++++++++++++++++++++++
3 files changed, 239 insertions(+)
diff --git a/services/ingestion/ingest.py b/services/ingestion/ingest.py
index 00e1b70..70103f1 100644
--- a/services/ingestion/ingest.py
+++ b/services/ingestion/ingest.py
@@ -148,6 +148,9 @@ def _get_parser(format_name: str):
if format_name == "naupa2_v1":
from services.ingestion.naupa2 import parse_naupa2_feed
return parse_naupa2_feed
+ if format_name == "naupa3_v1":
+ from services.ingestion.naupa3 import parse_naupa3_feed
+ return parse_naupa3_feed
return None
diff --git a/services/ingestion/naupa3.py b/services/ingestion/naupa3.py
new file mode 100644
index 0000000..8d12df9
--- /dev/null
+++ b/services/ingestion/naupa3.py
@@ -0,0 +1,127 @@
+"""NAUPA III XML parser adapter — the third ingestion format.
+
+NAUPA III replaces NAUPA II fixed-width with XML validated against an XSD. Real element
+names/namespaces vary by jurisdiction version; this is a representative, namespace-TOLERANT
+subset sufficient to prove the adapter and its security posture on synthetic data.
+
+Registered lazily by ingest._get_parser under format_name 'naupa3_v1'. Emits the same
+CanonicalProperty as the CSV/NAUPA II adapters.
+
+SECURITY — XXE / entity-expansion defense:
+ Legitimate NAUPA III feeds are XSD-validated DATA documents with NO DTD. We REJECT any
+ document containing a DOCTYPE/ENTITY declaration before parsing, which closes external-
+ entity (XXE) and billion-laughs expansion attacks at the door. Production should also use
+ defusedxml. Python's stdlib ElementTree does not resolve external entities, but a malicious
+ internal-entity bomb is still worth refusing outright.
+"""
+from __future__ import annotations
+
+import hashlib
+import re
+from typing import Iterable
+from xml.etree import ElementTree as ET
+
+from services.common.normalize import (
+ amount_band, mask_name, normalize_business, normalize_postal, normalize_text,
+ parse_decimal,
+)
+
+# Case-insensitive scan for a DTD/entity declaration anywhere in the document prolog/body.
+_DTD_RE = re.compile(rb"<!\s*(DOCTYPE|ENTITY)", re.IGNORECASE)
+
+
+class XmlSecurityError(ValueError):
+ """The XML declared a DTD/ENTITY — refused to protect against XXE / entity expansion."""
+
+
+def _localname(tag: str) -> str:
+ return tag.split("}", 1)[-1] if "}" in tag else tag
+
+
+def _find_text(elem: ET.Element, localname: str) -> str:
+ """First descendant whose local (namespace-stripped) name matches; '' if absent."""
+ for child in elem.iter():
+ if _localname(child.tag) == localname:
+ return (child.text or "").strip()
+ return ""
+
+
+def _looks_like_business(name: str) -> bool:
+ from services.common.normalize import CORPORATE_SUFFIXES
+ return any(tok in CORPORATE_SUFFIXES for tok in normalize_text(name).split())
+
+
+def build_naupa3_xml(records: list[dict], namespace: str | None = None) -> bytes:
+ """Assemble a valid NAUPA-III-style XML document (used by tests/tools).
+
+ If `namespace` is given, it's applied as the default xmlns so the parser's
+ namespace-tolerance is exercised.
+ """
+ ns_attr = f' xmlns="{namespace}"' if namespace else ""
+ parts = [f"<UnclaimedProperty{ns_attr}>"]
+ for r in records:
+ parts.append(" <Property>")
+ parts.append(f" <PropertyId>{r.get('property_id','')}</PropertyId>")
+ parts.append(f" <HolderName>{r.get('holder_name','')}</HolderName>")
+ parts.append(" <Owner>")
+ parts.append(f" <FirstName>{r.get('first_name','')}</FirstName>")
+ parts.append(f" <LastName>{r.get('last_name','')}</LastName>")
+ parts.append(" </Owner>")
+ parts.append(" <Address>")
+ parts.append(f" <Street>{r.get('street','')}</Street>")
+ parts.append(f" <City>{r.get('city','')}</City>")
+ parts.append(f" <State>{r.get('state','')}</State>")
+ parts.append(f" <Zip>{r.get('zip','')}</Zip>")
+ parts.append(" </Address>")
+ parts.append(f" <PropertyType>{r.get('property_type','')}</PropertyType>")
+ parts.append(f" <Amount>{r.get('amount','')}</Amount>")
+ parts.append(" </Property>")
+ parts.append("</UnclaimedProperty>")
+ return ("\n".join(parts) + "\n").encode("utf-8")
+
+
+def parse_naupa3_feed(data: bytes, jurisdiction: str) -> Iterable["object"]:
+ from services.ingestion.ingest import CanonicalProperty # lazy: avoid import cycle
+
+ if _DTD_RE.search(data):
+ raise XmlSecurityError(
+ "XML declares a DOCTYPE/ENTITY — refused (XXE / entity-expansion protection)"
+ )
+
+ root = ET.fromstring(data) # no external-entity resolution in stdlib ElementTree
+ for prop in root.iter():
+ if _localname(prop.tag) != "Property":
+ continue
+ pid = _find_text(prop, "PropertyId")
+ first = _find_text(prop, "FirstName")
+ last = _find_text(prop, "LastName")
+ owner_name = f"{first} {last}".strip()
+ is_business = _looks_like_business(owner_name)
+ amount_raw = _find_text(prop, "Amount")
+ try:
+ amount = parse_decimal(amount_raw)
+ except ValueError:
+ amount = None
+ owner_norm = (
+ normalize_business(owner_name) if is_business else normalize_text(owner_name)
+ )
+ # canonical raw payload: the serialized element (stable across whitespace/ns)
+ raw = ET.tostring(prop, encoding="unicode")
+ raw_hash = hashlib.sha256(raw.encode("utf-8")).hexdigest()
+ yield CanonicalProperty(
+ jurisdiction=jurisdiction,
+ source_property_id=pid,
+ holder_name_raw=_find_text(prop, "HolderName"),
+ owner_type="business" if is_business else "person",
+ owner_name_raw=owner_name,
+ owner_name_normalized=owner_norm,
+ owner_name_masked=mask_name(owner_name),
+ amount=amount,
+ amount_band=amount_band(amount),
+ raw_payload=raw,
+ raw_record_hash=raw_hash,
+ city_normalized=normalize_text(_find_text(prop, "City")) or None,
+ region=normalize_text(_find_text(prop, "State")) or None,
+ postal_code=normalize_postal(_find_text(prop, "Zip")),
+ property_type=normalize_text(_find_text(prop, "PropertyType")) or None,
+ )
diff --git a/tests/test_cycle6_naupa3.py b/tests/test_cycle6_naupa3.py
new file mode 100644
index 0000000..dc38c66
--- /dev/null
+++ b/tests/test_cycle6_naupa3.py
@@ -0,0 +1,109 @@
+"""Cycle 5 tests — NAUPA III XML adapter: parity, namespace tolerance, XXE guard.
+
+Run: python -m tests.test_cycle6_naupa3
+
+Proves:
+ 1. XML ingests via the same pipeline; canonical shape + cents/dollars + masking match.
+ 2. Namespace-tolerant: an xmlns-defaulted document parses identically.
+ 3. XXE / entity-expansion guard: a DTD/ENTITY document is REFUSED (XmlSecurityError).
+ 4. Malformed record (missing PropertyId) is rejected, not fatal.
+ 5. Same-file idempotency holds for the XML path.
+"""
+from __future__ import annotations
+
+import sys
+import tempfile
+from decimal import Decimal
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from services.common.sqlite_repo import FileObjectStore, SqliteRepository
+from services.ingestion.ingest import FeedDefinition, ingest_authorized_feed
+from services.ingestion.naupa3 import XmlSecurityError, build_naupa3_xml, parse_naupa3_feed
+
+
+def ok(msg: str) -> None:
+ print(f" ✓ {msg}")
+
+
+RECORDS = [
+ {"property_id": "SP-X001", "holder_name": "First Synthetic Bank",
+ "first_name": "Catherine", "last_name": "O'Neil", "street": "100 Test St",
+ "city": "Springfield", "state": "SM", "zip": "00001",
+ "property_type": "UNCASHED_CHECK", "amount": "42.50"},
+ {"property_id": "SP-X002", "holder_name": "Made-Up Securities",
+ "first_name": "", "last_name": "ACME WIDGETS INC", "street": "300 Nowhere Blvd",
+ "city": "Rivertown", "state": "SM", "zip": "00002",
+ "property_type": "SECURITIES", "amount": "5300.00"},
+]
+
+
+def _repo(tmp: Path, data: bytes, uri: str) -> tuple:
+ store = FileObjectStore(tmp / "obj")
+ (tmp / "obj").mkdir(parents=True, exist_ok=True)
+ store.write_bytes(uri, data)
+ repo = SqliteRepository(str(tmp / "p.db"))
+ return store, repo
+
+
+def main() -> int:
+ tmp = Path(tempfile.mkdtemp(prefix="upp-cycle6-"))
+
+ print("1) NAUPA III ingest parity (no namespace)")
+ store, repo = _repo(tmp / "a", build_naupa3_xml(RECORDS), "incoming/n3.xml")
+ feed = FeedDefinition("SAMPLE", "incoming/n3.xml", format_name="naupa3_v1")
+ r = ingest_authorized_feed(feed, store, repo)
+ assert r["status"] == "completed" and r["accepted"] == 2, r
+ amt = repo.conn.execute(
+ "SELECT amount FROM property WHERE source_property_id='SP-X001'").fetchone()["amount"]
+ assert Decimal(str(amt)) == Decimal("42.50"), amt
+ # NB: normalize_text turns "O'Neil" -> "O NEIL" (apostrophe->space), so search the
+ # given name. (Apostrophe/hyphen search-recall is a logged backlog item; matching is
+ # unaffected because the phonetic key still links O'Neil<->ONeill.)
+ hits = repo.masked_search("Catherine")
+ assert hits and "CATHERINE" not in (hits[0]["owner_name_masked"] or "").upper()
+ ok(f"accepted=2; $42.50 parsed; masked {hits[0]['owner_name_masked']}")
+
+ print("2) namespace tolerance (default xmlns)")
+ store2, repo2 = _repo(tmp / "b",
+ build_naupa3_xml(RECORDS, namespace="http://naupa.org/naupa3"),
+ "incoming/n3ns.xml")
+ r2 = ingest_authorized_feed(
+ FeedDefinition("SAMPLE", "incoming/n3ns.xml", format_name="naupa3_v1"), store2, repo2)
+ assert r2["accepted"] == 2, r2
+ ok("xmlns-defaulted document parsed identically (accepted=2)")
+
+ print("3) XXE / entity-expansion guard")
+ bomb = (b'<?xml version="1.0"?>\n'
+ b'<!DOCTYPE lolz [ <!ENTITY lol "lol"> <!ENTITY lol2 "&lol;&lol;"> ]>\n'
+ b'<UnclaimedProperty><Property><PropertyId>X</PropertyId>'
+ b'<Owner><LastName>&lol2;</LastName></Owner></Property></UnclaimedProperty>')
+ raised = False
+ try:
+ list(parse_naupa3_feed(bomb, "SAMPLE"))
+ except XmlSecurityError:
+ raised = True
+ assert raised, "DTD/ENTITY document must be refused"
+ ok("DOCTYPE/ENTITY document refused (XmlSecurityError)")
+
+ print("4) malformed record (missing PropertyId) rejected, not fatal")
+ bad = list(RECORDS) + [{"first_name": "No", "last_name": "Id", "amount": "1.00"}]
+ store3, repo3 = _repo(tmp / "c", build_naupa3_xml(bad), "incoming/n3bad.xml")
+ r3 = ingest_authorized_feed(
+ FeedDefinition("SAMPLE", "incoming/n3bad.xml", format_name="naupa3_v1"), store3, repo3)
+ assert r3["accepted"] == 2 and r3["rejected"] == 1, r3
+ ok(f"accepted=2, rejected=1 (missing PropertyId), status={r3['status']}")
+
+ print("5) idempotency on the XML path")
+ n = repo.count_properties("SAMPLE")
+ again = ingest_authorized_feed(feed, store, repo)
+ assert again["status"] == "duplicate" and repo.count_properties("SAMPLE") == n
+ ok(f"re-ingest -> duplicate; count stable at {n}")
+
+ print("\nALL CYCLE-5 NAUPA-III ASSERTIONS PASSED ✅")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
← 9a96426 docs: ledger Cycle-4 record (search service + real limiter);
·
back to Unclaimed Property Platform
·
docs: ledger Cycle-5 record (NAUPA III + XXE guard); TK-1009 48cadf2 →