← back to Paul Conrad Archive
tests/test_dedupe.py
102 lines
from conrad import db, dedupe
from conrad.models import CartoonRecord
def _item(cid, title, date=None, year=None, pub="Los Angeles Times", src="seed_loc", gran="item", **kw):
return CartoonRecord(canonical_id=cid, identifier=cid, granularity=gran, title=title, date_exact=date,
date_start=kw.pop("start", date or (f"{year}-01-01" if year else None)),
date_end=kw.pop("end", date or (f"{year}-12-31" if year else None)),
year=year or (int(date[:4]) if date else None), publication=pub, repository="R", **kw)
def test_score_components():
a = {"date_exact": "1989-07-16", "title": "S&L's and HUD", "publication": "Los Angeles Times",
"caption": "a cartoon about savings and loans", "subjects": {"banks"}, "people": {"bush"}}
b = dict(a, publication="The Los Angeles Times")
s, parts = dedupe.score_pair(a, b)
assert parts["date"] == 35 and parts["publication"] == 10 and parts["caption"] == 15
assert parts["subjects"] == 5 and parts["people"] == 5 and parts["title"] == 25
assert s == 95
def test_score_different_cartoons_low():
s, parts = dedupe.score_pair({"title": "Death row", "date_exact": "1991-04-19"},
{"title": "The naked truth", "date_exact": "1987-01-02"})
assert s < dedupe.REVIEW
def _setup(conn):
for sid in ("seed_loc", "seed_catalog", "huntington", "seed_wichita"):
db.upsert_source(conn, sid, sid)
def test_run_links_never_deletes(tmpdb):
_setup(tmpdb)
db.save_record(tmpdb, _item("loc:1", "Death row", year=1991), "seed_loc")
db.save_record(tmpdb, _item("cat:x", "Death row", year=1991), "seed_catalog")
db.save_record(tmpdb, _item("wsu:1", "The Graduate", date="1969-06-01"), "seed_wichita")
db.save_record(tmpdb, _item("wsu:2", "Open Just A Little Wider", date="1969-06-01"), "seed_wichita")
db.save_record(tmpdb, _item("hunt:CON-5", None, gran="box_range", start="1969-05-01", end="1969-07-31", year=1969,
box="7"), "huntington")
before = tmpdb.execute("SELECT COUNT(*) FROM cartoons").fetchone()[0]
res = dedupe.run(tmpdb)
assert tmpdb.execute("SELECT COUNT(*) FROM cartoons").fetchone()[0] == before # nothing deleted
cat = tmpdb.execute("SELECT merged_into FROM cartoons WHERE canonical_id='cat:x'").fetchone()[0]
loc = tmpdb.execute("SELECT id FROM cartoons WHERE canonical_id='loc:1'").fetchone()[0]
assert cat == loc and res["auto_linked"] == 1
# same-day, different titles: not duplicates
assert tmpdb.execute("SELECT COUNT(*) FROM cartoon_links WHERE relation!='contained_in_candidate'").fetchone()[0] == 1
# box range never merged, but linked as contained_in candidate for the dated items
assert tmpdb.execute("SELECT merged_into FROM cartoons WHERE canonical_id='hunt:CON-5'").fetchone()[0] is None
assert res["contained_in_candidates"] == 2
# ---------------------------------------------------------------- NEGATIVE: generic bracketed titles (TK-12199)
def _merged(conn, a, b) -> bool:
ia = conn.execute("SELECT id, merged_into FROM cartoons WHERE canonical_id=?", (a,)).fetchone()
ib = conn.execute("SELECT id, merged_into FROM cartoons WHERE canonical_id=?", (b,)).fetchone()
return (ia["merged_into"] or ia["id"]) == (ib["merged_into"] or ib["id"])
def test_generic_bracketed_titles_different_items_not_merged(tmpdb):
"""Two DIFFERENT cartoons that share a cataloger-devised / generic title must never auto-merge on title alone."""
_setup(tmpdb)
db.upsert_source(tmpdb, "loc", "loc")
lp = "https://www.loc.gov/pictures/item/{}/"
# same bracketed title, different LOC ids, different MONTHS (no exact dates) -> title_sim 100, no date_exact conflict
db.save_record(tmpdb, _item("loc:2001000001", "[Herblock as George Washington]", year=2000, start="2000-03-01",
end="2000-03-31", record_url=lp.format("2001000001")), "loc")
db.save_record(tmpdb, _item("loc:2001000002", "[Herblock as George Washington]", year=2000, start="2000-11-01",
end="2000-11-30", record_url=lp.format("2001000002")), "loc")
# same generic title, different ids, different exact dates
db.save_record(tmpdb, _item("wsu:9001", "[Untitled]", date="1971-02-03"), "seed_wichita")
db.save_record(tmpdb, _item("wsu:9002", "[Untitled]", date="1971-08-19"), "seed_wichita")
# generic word title, bare-year ranges only, different ids (identical spans are NOT date agreement)
db.save_record(tmpdb, _item("loc:2001000003", "[Cartoons]", year=1985, record_url=lp.format("2001000003")), "loc")
db.save_record(tmpdb, _item("cat:cartoons-1985", "[Cartoons]", year=1985,
record_url=lp.format("2001000004")), "seed_catalog")
# positive control: generic bracketed title + the SAME LOC identifier -> still merges (the real herblock case)
db.save_record(tmpdb, _item("loc:00652290", "[Herblock as George Washington]", year=1999,
record_url=lp.format("00652290")), "loc")
db.save_record(tmpdb, _item("cat:loc-herblock-1999", "[Herblock as George Washington]", year=1999,
record_url="https://www.loc.gov/item/00652290/"), "seed_catalog")
tmpdb.commit()
dedupe.run(tmpdb)
assert not _merged(tmpdb, "loc:2001000001", "loc:2001000002")
assert not _merged(tmpdb, "wsu:9001", "wsu:9002")
assert not _merged(tmpdb, "loc:2001000003", "cat:cartoons-1985")
assert _merged(tmpdb, "loc:00652290", "cat:loc-herblock-1999")
# the non-merged generic pairs are surfaced for manual review, not silently dropped
pd = {tuple(sorted(r)) for r in tmpdb.execute(
"""SELECT a.canonical_id, b.canonical_id FROM cartoon_links l JOIN cartoons a ON a.id=l.cartoon_id
JOIN cartoons b ON b.id=l.related_id WHERE l.relation='possible_duplicate'""")}
assert ("loc:2001000001", "loc:2001000002") in pd
assert tuple(sorted(("loc:2001000003", "cat:cartoons-1985"))) in pd
def test_generic_title_detection():
assert dedupe.is_generic_title("[Herblock as George Washington]")
assert dedupe.is_generic_title("[Untitled]") and dedupe.is_generic_title("Cartoons")
assert dedupe.is_generic_title("Political cartoon")
assert not dedupe.is_generic_title("S&L's and HUD") and not dedupe.is_generic_title("Death row")