`tests/test_ia_fulltext.py`

136 lines
"""IA full-text appearance parser + the base-client truncation fix (TK-12199)."""
from conftest import FakeHttp
from conrad.crawlers import base, ia_fulltext
from conrad.crawlers.ia_fulltext import classify, issue_date

CREDIT = ["Views / A portfolio from around the nation\n{{{Paul Conrad}}}\nThe Los Angeles Times\nLos Angeles Times Syndicate"]
CAPS = ["OKAY ... LET'S MOVE IT\nPAUL {{{CONRAD}}} Los Angeles Times © Los Angeles Times Syndicate"]
PROSE = ["commissions was for {{{Paul Conrad}}}, editorial cartoonist for the Los Angeles Times Syndicate. He wanted a Richard"]
OTHER = ["Pricing Flukes Create Asia Arbitrage Play\nBy {{{Conrad}}} de Aenlle\nInternational Herald Tribune"]
JOSEPH = ["Lord Jim and other tales. Joseph {{{Conrad}}}\n© Los Angeles Times Syndicate"]


def test_classify_credit_lines():
    assert classify(CREDIT)[0] == "credit"
    kind, ev = classify(CAPS)
    assert kind == "credit" and "Los Angeles Times" in ev


def test_classify_rejects_prose_and_other_conrads():
    # negative cases: an article ABOUT Conrad (long prose line) and other Conrads must never become records
    assert classify(PROSE)[0] == "mention"
    assert classify(OTHER)[0] == "other_conrad"
    assert classify(JOSEPH)[0] == "other_conrad"
    assert classify([])[0] == "mention"


def test_issue_date():
    assert issue_date({"file_basename": "Aug 11 1998, The Jerusalem Post, #20003, Israel (en)"}) == ("1998-08-11", 1998)
    assert issue_date({"identifier": "The_Times_News_Idaho_Newspaper_1978_11_01"}) == ("1978-11-01", 1978)
    assert issue_date({"identifier": "x", "year": 1975}) == (None, 1975)


def _fts_page(hits):
    import json
    return json.dumps({"response": {"body": {"hits": {"total": len(hits), "hits": hits}}}})


def test_crawl_saves_only_credit_pages(tmpdb):
    hits = [
        {"fields": {"identifier": "JP1980", "file_basename": "Mar 3 1980, The Jerusalem Post Magazine", "page_num": 7,
                    "title": "The Jerusalem Post Magazine , 1980, Israel, English", "year": 1980,
                    "collection": ["newspapers"], "result_in_subfile": True}, "highlight": {"text": CREDIT}},
        {"fields": {"identifier": "IHT1998", "file_basename": "May 7 1998, IHT", "page_num": 3, "year": 1998,
                    "title": "International Herald Tribune , 1998", "collection": ["newspapers"]},
         "highlight": {"text": OTHER}},
        {"fields": {"identifier": "bestcartoons1975", "page_num": 40, "year": 1975, "title": "Best Editorial Cartoons",
                    "collection": ["inlibrary", "internetarchivebooks"]}, "highlight": {"text": CAPS}},
    ]
    http = FakeHttp({"page_production": (200, _fts_page(hits)),
                     "iiif.archive.org/iiif/3/JP1980/manifest.json": (200, '{"items": []}')})
    c = ia_fulltext.CRAWLER(conn=tmpdb, http=http, years=[1980], variants=[("t", '"Paul Conrad"', range(1980, 1981))])
    res = c.run()
    rows = tmpdb.execute("SELECT c.title, c.date_exact, c.publication, cs.image_url FROM cartoons c "
                         "JOIN cartoon_sources cs ON cs.cartoon_id=c.id ORDER BY c.title").fetchall()
    assert len(rows) == 2, rows
    news = [r for r in rows if r["date_exact"] == "1980-03-03"][0]
    assert news["publication"] == "The Jerusalem Post Magazine" and news["title"].startswith("[Paul Conrad cartoon")
    book = [r for r in rows if r["date_exact"] is None][0]
    assert book["image_url"] is None  # lending-library book: no image URL recorded
    # the lending-library book never asked IA for a manifest
    assert not any("bestcartoons1975/manifest" in u for u in http.calls)
    assert "other Conrads=1" in res["notes"]


# ---- base client: the whole body must be read through ONE iter_content generator
class _BufferingResp:
    """Mimics a gzip response: a generator drains the raw stream into its own buffer on first use, so a SECOND
    iter_content() call sees nothing (the pre-fix code truncated bodies to the first chunk this way)."""

    status_code = 200
    headers = {"content-type": "text/html; charset=utf-8"}
    encoding = "utf-8"

    def __init__(self, body: bytes):
        self._raw = [body]

    def iter_content(self, n):
        buf = b"".join(self._raw)
        self._raw = []
        for i in range(0, len(buf), n):
            yield buf[i:i + n]

    def close(self):
        pass

    @property
    def content(self):
        return self._content

    @property
    def text(self):
        return self._content.decode("utf-8")


def test_fetch_reads_whole_body_with_one_generator(monkeypatch):
    body = b"<!DOCTYPE html>" + b"x" * 200000
    h = base.Http(use_cache=False)
    monkeypatch.setattr(h.s, "get", lambda *a, **k: _BufferingResp(body))
    monkeypatch.setattr(base.config, "REQUEST_DELAY", 0)
    monkeypatch.setattr(h, "delay_for", lambda url: 0)
    st, text = h._fetch("https://example.org/page", None)
    assert st == 200 and len(text) == len(body)


def test_fetch_negative_two_generators_would_truncate():
    # negative control: the buffering fake really does lose data when iter_content is called twice
    r = _BufferingResp(b"<!DOCTYPE html>" + b"x" * 1000)
    head = next(r.iter_content(16))
    rest = b"".join(r.iter_content(65536))
    assert len(head + rest) == 16


def test_cartoon_title_and_more_negatives():
    from conrad.crawlers.ia_fulltext import cartoon_title
    assert cartoon_title('noted. "AT YOUR SERVICE, MADAM" was the title of this cartoon by Paul Conrad') == \
        "AT YOUR SERVICE, MADAM"
    assert cartoon_title("by Dennis Renault 20 The View from Watts. By Paul Conrad. Los Angeles Times 34") == \
        "The View from Watts"
    assert cartoon_title("Paul Conrad, Los Angeles Times") is None
    # negatives: a music copyright entry, a prize roster and a name list are not cartoon appearances
    assert classify(["Wallis; 5Aug59.\nCHINA CLIPPER; m {{{Paul Conrad}}}. -\n© Lo ridge Music Inc."])[0] == "mention"
    assert classify(["Frank Miller, Des Moines Register.\n1964 — {{{Paul Conrad}}}, Denver Post.\n1966 — Don Wright"])[0] \
        == "mention"
    assert classify(["Warren Christophel\n{{{Paul Conrad}}}\nMrs. Chauncey Crossgrove"])[0] == "mention"


def test_editor_publisher_syndicate_ad():
    from conrad.crawlers.ia_fulltext import classify_hit
    ep = {"identifier": "sim_editor-publisher_1970-08-01_103_31"}
    ad = ["Thought-Provoking Editorial Cartoons\n{{{PAUL CONRAD}}}\nFive cartoons a week\nLos Angeles Times Syndicate"]
    assert classify_hit(ep, ad)[0] == "syndicate_ad"
    # negatives: the same ad text outside E&P, and an E&P prize roster, are not syndicate ads
    assert classify_hit({"identifier": "other_1970"}, ad)[0] == "mention"
    roster = ["Pulitzer Prize winners\n{{{Paul Conrad}}}\nLos Angeles Times Syndicate cartoons"]
    assert classify_hit(ep, roster)[0] == "mention"