`tests/test_loc_deep.py`

81 lines
"""LOC deep enumeration (TK-12199): group handling, notes-only Conrad attribution, bounded neighbour probing."""
import json

from conrad.crawlers import loc
from conrad.crawlers.base import Blocked

from conftest import FakeHttp

GROUP = {"item": {"title": "[Proofs of political cartoons for the Los Angeles Times]",
                  "call_number": "Unprocessed in PR 13 CN 1983:004 [P&P]", "creators": [], "contributor_names": [],
                  "created_published": "[between 1971 and 1980]", "related_items": [],
                  "notes": [{"note": "Creators include: Paul Conrad."}, {"note": "Gift; Paul Conrad; (PR 13 CN 1983:004)"}]},
         "resources": [], "related": {"neighbors": "https://www.loc.gov/pictures/related/?&pk=2010632868"}}


def _item(title, pk_creator=True):
    return {"item": {"title": title, "creators": [{"title": "Conrad, Paul, 1924-2010"}] if pk_creator else [],
                     "created_published": "1975", "call_number": "CD 1 - Conrad, no. 1 (A size)", "notes": []},
            "resources": [{"image": "https://tile.loc.gov/x.jpg"}]}


def test_group_record_is_folder_and_components_not_addressable():
    rec = loc.item_record("2010632868", GROUP, "t")
    assert rec is not None and rec.granularity == "folder"
    g = loc.group_components("2010632868", GROUP)
    assert g["components_addressable"] is False and "robots.txt disallows" in g["why"]
    assert g["neighbors_link_disallowed"] is True and g["digitized_resources"] == 0


def test_notes_only_attribution_and_media_count():
    d = json.loads(json.dumps(GROUP))
    d["item"]["title"] = "[Original editorial cartoon drawings]"
    d["item"]["notes"].append({"note": "Media includes: 21 Drawings : Ink and pencil."})
    rec = loc.item_record("2010634752", d, "t")
    assert rec and rec.granularity == "folder" and "21 drawings" in rec.notes
    d["item"]["notes"] = [{"note": "Creators include: Pat Oliphant."}]
    assert loc.item_record("2010634752", d, "t") is None  # other cartoonists' lots are not Conrad


def test_neighbour_probe_bounded_and_extends(tmpdb, monkeypatch):
    monkeypatch.setattr(loc, "PROBE_WINDOW", 1)
    monkeypatch.setattr(loc, "PROBE_MAX", 5)
    routes = {
        "/pictures/item/2016685101/": (200, json.dumps(_item("Known cartoon"))),
        "/pictures/item/2016685102/": (200, json.dumps(_item("Neighbour hit"))),
        "/pictures/item/2016685103/": (200, json.dumps(_item("Other man's photo", pk_creator=False))),
        "/pictures/item/2016685100/": (404, "not found"),
        "/pictures/item/": (404, "not found"),
        "robots": (200, ""),
        "/photos/": (200, json.dumps({"results": [{"id": "http://www.loc.gov/item/2016685101/",
                                                   "contributor": ["conrad, paul"]}], "pagination": {}})),
        "loc.gov/": (200, json.dumps({"results": [], "pagination": {}})),
    }

    class H(FakeHttp):
        def allowed(self, url):
            return "/pictures/search" not in url and "/pictures/related" not in url and "/search/" not in url

    monkeypatch.setattr(loc, "KNOWN_GROUPS", {})
    monkeypatch.setattr(loc, "EXTRA_CANDIDATES", {})
    monkeypatch.setattr(loc.Checkpoint, "__init__", lambda self, n: setattr(self, "data", {}) or setattr(self, "path", None))
    monkeypatch.setattr(loc.Checkpoint, "set", lambda self, k, v: self.data.__setitem__(k, v))
    c = loc.LOCCrawler(conn=tmpdb, http=H(routes))
    res = c.run()
    assert res["status"] == "worked"
    ids = {r[0] for r in tmpdb.execute("SELECT identifier FROM cartoon_sources WHERE source_id='loc'")}
    assert ids == {"2016685101", "2016685102"}  # hit extended the window; non-Conrad neighbour skipped
    assert c.probe_stats["requests"] <= 5
    assert any("/pictures/related/=DISALLOW" in n for n in c.notes)
    assert not any("/pictures/search" in call or "/pictures/related" in call for call in c.http.calls)


def test_unprocessed_single_print_stays_item():
    d = {"item": {"title": "If you liked Vietnam, you'll love this one! Angola", "creators": [{"title": "Conrad, Paul"}],
                  "call_number": "Unprocessed in PR 13 CN 2023:102 (AA size part) [P&P]", "medium_brief": "1 print ;",
                  "notes": [{"note": "Title transcribed from item."}]}, "resources": []}
    assert loc.item_record("2023631806", d, "t").granularity == "item"
    d["item"].update(title="[Cartoons]", medium_brief="7 drawings.")
    rec = loc.item_record("2010634813", d, "t")
    assert rec.granularity == "folder" and "7 drawings" in rec.notes