← back to Unclaimed Property Platform
Cycle 6: apostrophe/hyphen search-recall fix (zero schema change)
f74dc2b440e438d3cc262adeacc07edb660cf2d3 · 2026-08-01 20:10:17 -0700 · Steve Abrams
- normalize_search(): fully-collapsed alphanumeric key (spaces+punct removed) — CATHERINEONEIL.
- masked_search matches REPLACE(owner_name_normalized,' ','') LIKE collapsed-query, so 'Oneil'
now finds "O'Neil" and 'Smith-Jones' variants match. Does NOT widen normalize_text (masks/
blocking/phonetic unchanged). Empty-query guard (m2) preserved.
- tests/test_cycle7_recall.py proves the fix + no-regression.
Tests: 7/7 suites green. All local/synthetic/$0.
TK-10097
Files touched
M services/common/normalize.pyM services/common/sqlite_repo.pyA tests/test_cycle7_recall.py
Diff
commit f74dc2b440e438d3cc262adeacc07edb660cf2d3
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Sat Aug 1 20:10:17 2026 -0700
Cycle 6: apostrophe/hyphen search-recall fix (zero schema change)
- normalize_search(): fully-collapsed alphanumeric key (spaces+punct removed) — CATHERINEONEIL.
- masked_search matches REPLACE(owner_name_normalized,' ','') LIKE collapsed-query, so 'Oneil'
now finds "O'Neil" and 'Smith-Jones' variants match. Does NOT widen normalize_text (masks/
blocking/phonetic unchanged). Empty-query guard (m2) preserved.
- tests/test_cycle7_recall.py proves the fix + no-regression.
Tests: 7/7 suites green. All local/synthetic/$0.
TK-10097
---
services/common/normalize.py | 11 +++++++
services/common/sqlite_repo.py | 17 ++++++-----
tests/test_cycle7_recall.py | 69 ++++++++++++++++++++++++++++++++++++++++++
3 files changed, 90 insertions(+), 7 deletions(-)
diff --git a/services/common/normalize.py b/services/common/normalize.py
index 03bed45..727e6ee 100644
--- a/services/common/normalize.py
+++ b/services/common/normalize.py
@@ -27,6 +27,17 @@ def normalize_text(value: str | None) -> str:
return re.sub(r"\s+", " ", cleaned).strip()
+def normalize_search(value: str | None) -> str:
+ """Fully-collapsed alphanumeric SEARCH key: normalize_text with spaces removed.
+
+ 'Catherine O\\'Neil' -> 'CATHERINEONEIL'. Substring-matching a same-collapsed query
+ ('Oneil' -> 'ONEIL') then finds it regardless of internal apostrophes/hyphens/spaces —
+ fixing the apostrophe/hyphen recall gap without widening normalize_text (which feeds
+ masks, blocking keys, and phonetic codes). Production = an OpenSearch char-filter analyzer.
+ """
+ return normalize_text(value).replace(" ", "")
+
+
def normalize_business(value: str | None) -> str:
"""Normalize a business name, dropping legal suffixes but keeping brand tokens."""
tokens = normalize_text(value).split()
diff --git a/services/common/sqlite_repo.py b/services/common/sqlite_repo.py
index efb651d..f2f854f 100644
--- a/services/common/sqlite_repo.py
+++ b/services/common/sqlite_repo.py
@@ -199,13 +199,16 @@ class SqliteRepository:
Returns the SAME allowlisted projection as the production API and runs each row
through assert_public_safe — so a raw-PII leak fails a test instead of shipping.
"""
- from services.common.normalize import normalize_text
+ from services.common.normalize import normalize_search
from services.common.public_projection import assert_public_safe
- norm = normalize_text(name_query)
- # Reject empty / all-punctuation queries — an empty normalized query would LIKE '%'
- # and walk the whole table, a mass-enumeration primitive (m2).
- if not norm:
+ # Fully-collapsed alphanumeric key on BOTH sides fixes apostrophe/hyphen recall
+ # ("Oneil" finds "O'Neil"). REPLACE(...,' ','') matches the stored space-collapsed
+ # normalized name. (Prototype: full scan; production = analyzer-backed index.)
+ needle = normalize_search(name_query)
+ # Reject empty / all-punctuation queries — an empty needle would LIKE '%' and walk
+ # the whole table, a mass-enumeration primitive (m2).
+ if not needle:
raise ValueError("search query must contain at least one alphanumeric token")
rows = self.conn.execute(
@@ -215,9 +218,9 @@ class SqliteRepository:
JOIN property p ON p.property_id = o.property_id
JOIN search_document_state s ON s.property_id = p.property_id
WHERE s.is_public=1 AND s.is_suppressed=0
- AND o.owner_name_normalized LIKE ?
+ AND REPLACE(o.owner_name_normalized, ' ', '') LIKE ?
LIMIT ?""",
- (f"%{norm.split()[0]}%", limit),
+ (f"%{needle}%", limit),
).fetchall()
# Build ONLY the public projection — holder_name_raw/property_id/city_normalized
diff --git a/tests/test_cycle7_recall.py b/tests/test_cycle7_recall.py
new file mode 100644
index 0000000..49dedc8
--- /dev/null
+++ b/tests/test_cycle7_recall.py
@@ -0,0 +1,69 @@
+"""Cycle 6 test — apostrophe/hyphen search RECALL fix.
+
+Run: python -m tests.test_cycle7_recall
+
+Before: normalize_text collapsed "O'Neil" -> "O NEIL", so masked_search("Oneil") missed it.
+After: a fully-collapsed alphanumeric key on both sides matches regardless of internal
+apostrophes/hyphens/spaces. Matching (phonetic) was always fine; this restores SEARCH recall.
+"""
+from __future__ import annotations
+
+import sys
+import tempfile
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from services.common.normalize import normalize_search
+from services.common.sqlite_repo import FileObjectStore, SqliteRepository
+from services.ingestion.ingest import FeedDefinition, ingest_authorized_feed
+
+SAMPLE = Path(__file__).resolve().parents[1] / "data" / "sample" / "sample_state_feed.csv"
+
+
+def ok(msg: str) -> None:
+ print(f" ✓ {msg}")
+
+
+def main() -> int:
+ tmp = Path(tempfile.mkdtemp(prefix="upp-cycle7-"))
+ store = FileObjectStore(tmp / "obj")
+ (tmp / "obj").mkdir(parents=True, exist_ok=True)
+ store.write_bytes("incoming/s.csv", SAMPLE.read_bytes())
+ repo = SqliteRepository(str(tmp / "p.db"))
+ ingest_authorized_feed(FeedDefinition("SAMPLE", "incoming/s.csv"), store, repo)
+
+ print("0) normalize_search collapses punctuation + spaces")
+ assert normalize_search("Catherine O'Neil") == "CATHERINEONEIL", normalize_search("Catherine O'Neil")
+ assert normalize_search("Smith-Jones") == "SMITHJONES"
+ ok("O'Neil -> CATHERINEONEIL, Smith-Jones -> SMITHJONES")
+
+ print("1) 'Oneil' now finds \"O'Neil\" (the recall fix)")
+ hits = repo.masked_search("Oneil")
+ assert hits, "expected 'Oneil' to match \"O'Neil\" after the fix"
+ assert all("owner_name_raw" not in h for h in hits)
+ ok(f"'Oneil' -> {len(hits)} hit(s), e.g. {hits[0]['owner_name_masked']}")
+
+ print("2) space/punct-free query still matches")
+ assert repo.masked_search("catherineoneil"), "collapsed query should match"
+ ok("'catherineoneil' matches")
+
+ print("3) ordinary token still works")
+ assert repo.masked_search("Testcase"), "expected Maria Testcase"
+ ok("'Testcase' matches")
+
+ print("4) empty/punctuation still rejected (m2 preserved)")
+ for bad in ("", " ", "'-.'"):
+ try:
+ repo.masked_search(bad)
+ raise AssertionError(f"{bad!r} should raise")
+ except ValueError:
+ pass
+ ok("empty/punctuation-only queries rejected")
+
+ print("\nALL CYCLE-6 RECALL ASSERTIONS PASSED ✅")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
← 48cadf2 docs: ledger Cycle-5 record (NAUPA III + XXE guard); TK-1009
·
back to Unclaimed Property Platform
·
docs: ledger Cycle-6 record (recall fix); TK-10097 02a3ead →