`tests/test_daily_iowan.py`

41 lines
"""Daily Iowan: inventories issue PDFs + probes for a text layer, never fetches a PDF, never invents cartoons."""
import csv

from conftest import FakeHttp

from conrad import config
from conrad.crawlers.base import Blocked
from conrad.crawlers.daily_iowan import DailyIowan, is_google_cse, parse_issues

YEAR_HTML = """<a href="https://dailyiowan.lib.uiowa.edu/DI/{y}/di{y}-01-03.pdf">Jan 3</a>
<a href="DI/{y}/di{y}-01-04.pdf">Jan 4</a><a href="DI/{y}/di{y}-01-04.pdf">dup</a>"""
CSE = "<script>var cx='x'; gcse.src='https://cse.google.com/cse.js?cx='+cx;</script>"


class Fake(FakeHttp):
    def allowed(self, url):
        return False  # cse.google.com Disallow: / ; digital.lib.uiowa.edu 403 robots


def test_parse_issues_dedupes_and_absolutises():
    got = parse_issues(YEAR_HTML.format(y=1948))
    assert got == [("1948-01-03", "https://dailyiowan.lib.uiowa.edu/DI/1948/di1948-01-03.pdf"),
                   ("1948-01-04", "https://dailyiowan.lib.uiowa.edu/DI/1948/di1948-01-04.pdf")]
    assert is_google_cse(CSE)


def test_crawl_no_pdf_no_records_and_issue_list(tmpdb, tmp_path, monkeypatch):
    monkeypatch.setattr(config, "EXPORT_DIR", tmp_path)
    routes = {f"{y}.php": (200, YEAR_HTML.format(y=y)) for y in range(1945, 1951)}
    routes.update({".txt": (404, "nf"), ".xml": (404, "nf"), ".hocr": (404, "nf"), ".html": (404, "nf"),
                   "DI/1948/": Blocked("HTTP 403"), "search/": (200, CSE)})
    http = Fake(routes)
    res = DailyIowan(conn=tmpdb, http=http).run()
    assert res["status"] == "partial" and res["added"] == 0
    assert "NO separate text layer" in res["notes"] and "Google CSE" in res["notes"]
    assert not [c for c in http.calls if c.split("?")[0].endswith(".pdf")], "a PDF was requested"
    assert tmpdb.execute("SELECT COUNT(*) FROM cartoons").fetchone()[0] == 0
    rows = list(csv.DictReader((tmp_path / "daily_iowan_issues_1945_1950.csv").open()))
    assert len(rows) == 12 and rows[0]["issue_date"] == "1945-01-03" and rows[0]["conrad_ui_years_1946_1950"] == "0"
    assert all(r["issue_pdf_url"].endswith(".pdf") for r in rows)