`tests/test_parsers.py`

127 lines
"""One parser fixture per crawler (offline)."""
import json

from conrad import config, db
from conrad.crawlers import (daily_iowan, denver_post, discovery, huntington, internet_archive, latimes, loc, oac,
                             ohio_state, seed_import, syracuse, wichita)
from conrad.crawlers.base import Blocked, looks_like_challenge
from conftest import FakeHttp


# ---- huntington
def test_huntington_ead_parse_and_typo_repair(fix):
    ead = huntington.parse_ead(fix("ead_small.xml"))
    assert ead["collection"]["unitid"] == "mssCON 1-12360"
    boxes, repairs = huntington.drawing_boxes(ead["components"])
    assert len(boxes) == 4
    assert boxes[1]["date_start"] == "1969-01-01" and boxes[1]["date_end"] == "1969-03-10"
    assert boxes[3]["date_end"] == "1970-02-28"
    assert repairs and boxes[2]["con_end"] == 129  # 'CON 84– 1290' clamped to next box start - 1
    recs = huntington.slot_records(boxes)
    assert len(recs) == 168 and all(r.granularity == "box_range" and r.date_is_estimate for r in recs)
    assert all(r.image_url is None for r in recs)


def test_parse_span_variants():
    assert huntington.parse_span("1995, May 22- July") == ("1995-05-22", "1995-07-31")
    assert huntington.parse_span("1981, July- 1992") == ("1981-07-01", "1992-12-31")


# ---- oac
def test_oac_parse(fix):
    info = oac.parse_oac(fix("oac.html"))
    assert "272.41 Linear Feet" in (info.get("extent") or "")
    assert info["count_phrase"].startswith("9,500")


# ---- seed importer (wichita md, loc json, syracuse, catalog)
def test_seed_wsu_md(fix):
    it = seed_import.parse_wsu_md(fix("wsu_item.md"), "111938")
    assert it["date"] == "1969-03-05" and it["box"] == "1" and it["series"].startswith("Series 2")
    rec = seed_import.wsu_record(it, "test")
    assert rec.date_exact == "1969-03-05" and "Nixon, Richard M." in rec.people and rec.granularity == "item"


def test_seed_loc_record():
    r = {"creator": "Conrad, Paul, 1924-2010, artist", "pk": "2016685164", "title": "\"We'll bury you\"",
         "created_published_date": "1984 Mar. 13 [publication date]",
         "subjects": ["Khrushchev, Nikita Sergeevich,--1894-1971--Quotations.", "Editorial cartoons--1980-1990."],
         "image": {"full": "https://www.loc.gov/pictures/static/images/item/500x500_notdigitized.png"}}
    rec = seed_import.loc_record(r, "test")
    assert rec.date_exact == "1984-03-13" and rec.image_url is None and rec.access_level == "online_metadata"
    assert seed_import.loc_record({"creator": "Jenkins, R. Michael", "pk": "1"}, "t") is None


def test_seed_syracuse_and_catalog():
    syr = {"folders": [{"folder": "B1F3", "label": "Feb 1964", "year": 1964, "month": 2}],
           "headings": [{"heading": "Johnson, Lyndon B.", "folders": ["B1F3"]}, {"heading": "civil rights", "folders": ["B1F3"]}]}
    (rec,) = seed_import.syracuse_folder_records(syr, "t")
    assert rec.granularity == "folder" and rec.date_end == "1964-02-29"
    assert rec.people == ["Johnson, Lyndon B."] and rec.subjects == ["civil rights"]
    assert seed_import.catalog_record({"id": "pulitzer-1964", "record_type": "documented", "title": "Prize"}, "t") is None


# ---- loc live item json
def test_loc_item_record(fix):
    rec = loc.item_record("2016685164", json.loads(fix("loc_item.json")), "test")
    assert rec.copyright_holder == "Los Angeles Times" and rec.publication == "Los Angeles Times"
    assert "Khrushchev, Nikita Sergeevich" in rec.people and "Deficit financing" in rec.subjects
    assert rec.thumbnail_url and rec.access_level == "online_image"  # metadata link only
    assert rec.granularity == "item"


# ---- blocked sources (syracuse, wichita, ohio state) record status and never crash
def test_blocked_sources_record_status(tmpdb):
    blocked = Blocked("WAF/JS challenge (HTTP 202)")
    for mod in (syracuse, wichita, ohio_state):
        c = mod.CRAWLER(conn=tmpdb, http=FakeHttp({"http": blocked}))
        res = c.run()
        assert res["status"] == "blocked", mod.__name__
    assert tmpdb.execute("SELECT COUNT(*) FROM errors").fetchone()[0] >= 3


def test_challenge_detection():
    assert looks_like_challenge(202, "")
    assert looks_like_challenge(200, "<script src=awswafintegration.js>")
    assert not looks_like_challenge(200, "<html>ok</html>")


# ---- paywalled registrations
def test_paywalled_not_attempted(tmpdb):
    assert denver_post.CRAWLER(conn=tmpdb, http=FakeHttp({})).run()["status"] == "not_attempted"
    cls = {r[0]: r[1] for r in tmpdb.execute("SELECT id, classification FROM sources")}
    assert cls["denver_post"] == "PAYWALLED"
    # latimes (TK-12199): crawls only free latimes.com pages; the paywalled archive hosts are never contacted
    http = FakeHttp({})
    latimes.CRAWLER(conn=tmpdb, http=http, years=(2010, 2010)).run()
    assert http.calls and all("latimes.com" in u for u in http.calls)
    assert not any(h in u for u in http.calls for h in ("newspapers.com", "proquest"))


# ---- daily iowan inventory
def test_daily_iowan_counts(tmpdb, tmp_path, monkeypatch):
    monkeypatch.setattr(config, "EXPORT_DIR", tmp_path)  # never overwrite the real issue list from a test
    page = "".join(f'<a href="https://dailyiowan.lib.uiowa.edu/DI/1948/di1948-01-{d:02d}.pdf">x</a>' for d in range(1, 11))
    c = daily_iowan.CRAWLER(conn=tmpdb, http=FakeHttp({"1948.php": (200, page), "dailyiowan": (200, "<a href='/1948.php'>")}))
    res = c.run()
    assert res["status"] == "partial" and "1948: 10" in res["notes"]


# ---- internet archive books + conflation exclusion
def test_internet_archive_books(tmpdb, fix):
    http = FakeHttp({"advancedsearch": (200, fix("ia_search.json")), "metadata/paulconraddrawin": (200, fix("ia_meta.json"))})
    res = internet_archive.CRAWLER(conn=tmpdb, http=http).run()
    b = dict(tmpdb.execute("SELECT * FROM books WHERE title='Drawing the Line'").fetchone())
    assert b["verified"] == 1 and b["ia_access"] == "controlled_digital_lending" and b["oclc"] == "41888497"
    assert "larryboy" in res["notes"] and "gottistmeinheil" in res["notes"]


# ---- discovery (DPLA) keeps images/objects, drops non-cartoon types
def test_discovery_dpla_filter(tmpdb, fix, monkeypatch):
    from conrad import config
    monkeypatch.setattr(config, "secret", lambda name: "test-key" if name == "DPLA_API_KEY" else None)
    http = FakeHttp({"api.dp.la": (200, fix("dpla.json")), "http": Blocked("probe blocked")})
    discovery.CRAWLER(conn=tmpdb, http=http).run()
    rows = tmpdb.execute("SELECT c.title, cs.repository FROM cartoons c JOIN cartoon_sources cs ON cs.cartoon_id=c.id").fetchall()
    assert [tuple(r) for r in rows] == [("The Law West of Dodge", "History Colorado")]