← back to Paul Conrad Archive
Dedupe: generic/[bracketed] title matches need identifier or finer-than-year date agreement; different LOC pks never auto-link; negative test
e9c5e1ca697d2fdebd771ce99897adbd93e37ccb · 2026-09-24 17:03:01 -0700 · Steve Abrams
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Files touched
M src/conrad/dedupe.pyM tests/test_dedupe.py
Diff
commit e9c5e1ca697d2fdebd771ce99897adbd93e37ccb
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Sep 24 17:03:01 2026 -0700
Dedupe: generic/[bracketed] title matches need identifier or finer-than-year date agreement; different LOC pks never auto-link; negative test
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
src/conrad/dedupe.py | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++--
tests/test_dedupe.py | 50 +++++++++++++++++++++++++++++++++++++
2 files changed, 117 insertions(+), 2 deletions(-)
diff --git a/src/conrad/dedupe.py b/src/conrad/dedupe.py
index 5a379e1..791021a 100644
--- a/src/conrad/dedupe.py
+++ b/src/conrad/dedupe.py
@@ -6,6 +6,7 @@ Perceptual-hash (image) dedupe is intentionally SKIPPED: the copyright rule forb
"""
from __future__ import annotations
+import re
from collections import defaultdict
from rapidfuzz import fuzz
@@ -18,6 +19,55 @@ REVIEW = 35 # score in [REVIEW, AUTO_LINK) or weak title -> manual r
TITLE_STRONG = 88 # rapidfuzz token_set_ratio
TITLE_SIMILAR = 80
PRIMARY_RANK = {"loc": 0, "wsu": 1, "si": 2, "dpla": 3, "hunt": 4, "syr": 5, "cat": 9}
+GENERIC_NORM = {"cartoon", "cartoons", "political cartoon", "political cartoons", "editorial cartoon",
+ "editorial cartoons", "drawing", "drawings", "cartoon drawing", "cartoon drawings", "untitled",
+ "untitled drawing", "untitled cartoon", "proof", "proofs", "sketch", "sketches", "no title", "title unknown"}
+LOC_PK = re.compile(r"loc\.gov/(?:pictures/)?item/([0-9a-z]+)/?", re.I)
+
+
+def is_generic_title(title: str | None) -> bool:
+ """Cataloger-devised ([bracketed]) or generic titles name a KIND of thing, not a specific cartoon:
+ a title match on them is not evidence two records are the same work."""
+ raw = (title or "").strip()
+ if not raw:
+ return True
+ if raw.startswith("[") and raw.rstrip(".").endswith("]"):
+ return True
+ t = norm_title(raw)
+ return not t or t in GENERIC_NORM or len(t.split()) == 1
+
+
+def _ids(rows) -> set[str]:
+ """Normalised identifier tokens for a record's source rows (LOC pk from any loc.gov item URL, plus raw URLs)."""
+ out = set()
+ for sid, ident, url in rows:
+ if sid in ("loc", "seed_loc") and ident:
+ out.add(f"loc:{ident}")
+ m = LOC_PK.search(url or "")
+ if m:
+ out.add(f"loc:{m[1]}")
+ elif url:
+ out.add("url:" + re.sub(r"^https?://(www\.)?", "", url.strip().lower()).rstrip("/"))
+ return out
+
+
+def id_relation(a: dict, b: dict) -> str:
+ """'agree' (shared identifier), 'conflict' (both carry a LOC pk and they differ) or 'none'."""
+ ia, ib = a.get("ids") or set(), b.get("ids") or set()
+ if ia & ib:
+ return "agree"
+ la, lb = {x for x in ia if x.startswith("loc:")}, {x for x in ib if x.startswith("loc:")}
+ if la and lb:
+ return "conflict"
+ return "none"
+
+
+def date_agree(a: dict, b: dict) -> bool:
+ """Exact day equal, or identical ranges finer than a whole year (identical bare-year spans do NOT count)."""
+ if a.get("date_exact") and a.get("date_exact") == b.get("date_exact"):
+ return True
+ sa, ea, sb, eb = a.get("date_start"), a.get("date_end"), b.get("date_start"), b.get("date_end")
+ return bool(sa and ea and (sa, ea) == (sb, eb) and sa[:7] == ea[:7])
def _pub(p: str | None) -> str:
@@ -56,10 +106,14 @@ def _load_items(conn) -> list[dict]:
subj[cid].add(name.lower())
for cid, name in conn.execute("SELECT cp.cartoon_id, p.name FROM cartoon_people cp JOIN people p ON p.id=cp.person_id"):
ppl[cid].add(name.lower())
+ ids = defaultdict(list)
+ for cid, sid, ident, url in conn.execute("SELECT cartoon_id, source_id, identifier, record_url FROM cartoon_sources"):
+ ids[cid].append((sid, ident, url))
out = []
for r in rows:
d = dict(r)
d["subjects"], d["people"] = subj[r["id"]], ppl[r["id"]]
+ d["ids"] = _ids(ids[r["id"]])
out.append(d)
return out
@@ -91,8 +145,19 @@ def run(conn) -> dict:
continue # same day, clearly different cartoons (e.g. Wichita daily sequence)
keep, dup = sorted([a, b], key=_rank)
date_conflict = bool(a["date_exact"] and b["date_exact"] and a["date_exact"] != b["date_exact"])
- if (s >= AUTO_LINK and parts["title_sim"] >= TITLE_STRONG) or \
- (parts["title_sim"] >= 95 and not date_conflict):
+ rel = id_relation(a, b)
+ title_match = (s >= AUTO_LINK and parts["title_sim"] >= TITLE_STRONG) or \
+ (parts["title_sim"] >= 95 and not date_conflict)
+ # a title match on a generic / [cataloger-devised] title is not evidence of identity: it needs
+ # identifier or (finer-than-year) date agreement. Two different LOC pks are never auto-linked.
+ generic = is_generic_title(a["title"]) or is_generic_title(b["title"])
+ if title_match and rel == "conflict":
+ parts["hold"] = "different LOC identifiers"
+ title_match = False
+ elif title_match and generic and not (rel == "agree" or date_agree(a, b)):
+ parts["hold"] = "generic title: needs identifier or date agreement"
+ title_match = False
+ if title_match:
auto.append((dup, keep, s, parts))
else:
review.append((a, b, s, parts))
diff --git a/tests/test_dedupe.py b/tests/test_dedupe.py
index a831d8f..8e77c70 100644
--- a/tests/test_dedupe.py
+++ b/tests/test_dedupe.py
@@ -49,3 +49,53 @@ def test_run_links_never_deletes(tmpdb):
# box range never merged, but linked as contained_in candidate for the dated items
assert tmpdb.execute("SELECT merged_into FROM cartoons WHERE canonical_id='hunt:CON-5'").fetchone()[0] is None
assert res["contained_in_candidates"] == 2
+
+
+# ---------------------------------------------------------------- NEGATIVE: generic bracketed titles (TK-12199)
+def _merged(conn, a, b) -> bool:
+ ia = conn.execute("SELECT id, merged_into FROM cartoons WHERE canonical_id=?", (a,)).fetchone()
+ ib = conn.execute("SELECT id, merged_into FROM cartoons WHERE canonical_id=?", (b,)).fetchone()
+ return (ia["merged_into"] or ia["id"]) == (ib["merged_into"] or ib["id"])
+
+
+def test_generic_bracketed_titles_different_items_not_merged(tmpdb):
+ """Two DIFFERENT cartoons that share a cataloger-devised / generic title must never auto-merge on title alone."""
+ _setup(tmpdb)
+ db.upsert_source(tmpdb, "loc", "loc")
+ lp = "https://www.loc.gov/pictures/item/{}/"
+ # same bracketed title, different LOC ids, different MONTHS (no exact dates) -> title_sim 100, no date_exact conflict
+ db.save_record(tmpdb, _item("loc:2001000001", "[Herblock as George Washington]", year=2000, start="2000-03-01",
+ end="2000-03-31", record_url=lp.format("2001000001")), "loc")
+ db.save_record(tmpdb, _item("loc:2001000002", "[Herblock as George Washington]", year=2000, start="2000-11-01",
+ end="2000-11-30", record_url=lp.format("2001000002")), "loc")
+ # same generic title, different ids, different exact dates
+ db.save_record(tmpdb, _item("wsu:9001", "[Untitled]", date="1971-02-03"), "seed_wichita")
+ db.save_record(tmpdb, _item("wsu:9002", "[Untitled]", date="1971-08-19"), "seed_wichita")
+ # generic word title, bare-year ranges only, different ids (identical spans are NOT date agreement)
+ db.save_record(tmpdb, _item("loc:2001000003", "[Cartoons]", year=1985, record_url=lp.format("2001000003")), "loc")
+ db.save_record(tmpdb, _item("cat:cartoons-1985", "[Cartoons]", year=1985,
+ record_url=lp.format("2001000004")), "seed_catalog")
+ # positive control: generic bracketed title + the SAME LOC identifier -> still merges (the real herblock case)
+ db.save_record(tmpdb, _item("loc:00652290", "[Herblock as George Washington]", year=1999,
+ record_url=lp.format("00652290")), "loc")
+ db.save_record(tmpdb, _item("cat:loc-herblock-1999", "[Herblock as George Washington]", year=1999,
+ record_url="https://www.loc.gov/item/00652290/"), "seed_catalog")
+ tmpdb.commit()
+ dedupe.run(tmpdb)
+ assert not _merged(tmpdb, "loc:2001000001", "loc:2001000002")
+ assert not _merged(tmpdb, "wsu:9001", "wsu:9002")
+ assert not _merged(tmpdb, "loc:2001000003", "cat:cartoons-1985")
+ assert _merged(tmpdb, "loc:00652290", "cat:loc-herblock-1999")
+ # the non-merged generic pairs are surfaced for manual review, not silently dropped
+ pd = {tuple(sorted(r)) for r in tmpdb.execute(
+ """SELECT a.canonical_id, b.canonical_id FROM cartoon_links l JOIN cartoons a ON a.id=l.cartoon_id
+ JOIN cartoons b ON b.id=l.related_id WHERE l.relation='possible_duplicate'""")}
+ assert ("loc:2001000001", "loc:2001000002") in pd
+ assert tuple(sorted(("loc:2001000003", "cat:cartoons-1985"))) in pd
+
+
+def test_generic_title_detection():
+ assert dedupe.is_generic_title("[Herblock as George Washington]")
+ assert dedupe.is_generic_title("[Untitled]") and dedupe.is_generic_title("Cartoons")
+ assert dedupe.is_generic_title("Political cartoon")
+ assert not dedupe.is_generic_title("S&L's and HUD") and not dedupe.is_generic_title("Death row")
← 88b34d2 Add acquisition_method provenance field: idempotent migratio
·
back to Paul Conrad Archive
·
LOC deep enumeration: robots audit per path, more format end 84113e0 →