`tests/test_secondary_citations.py`

108 lines
"""Cycle 3 secondary citations: a record is saved ONLY when its verbatim quote is on the fetched page; Wayback copies
are used only when the origin's robots.txt allows the path; no image URL is ever requested."""
from urllib.robotparser import RobotFileParser

import pytest
from conftest import FakeHttp

from conrad import config
from conrad.crawlers import secondary_citations as sc
from conrad.crawlers.base import IMAGE_URL, Blocked, Http

PAGE = """<html><body><p>In one image from 1968, Conrad drew a thing.</p>
<img alt="A cartoon of a man. The caption reads: &ldquo;It&rsquo;s all fine.&rdquo;" src="x.jpg">
<p>Time , 89 (15)</p></body></html>"""
WB = "https://web.archive.org/web/20100101000000/http://example.org/gallery.html"


class Fake(FakeHttp):
    def __init__(self, routes, disallow=()):
        super().__init__(routes)
        self.disallow = disallow

    def robots(self, url):
        if any(d in url for d in self.disallow):
            rp = RobotFileParser()
            rp.parse(["User-agent: *", "Disallow: /"])
            return rp
        return None


@pytest.fixture
def crawler(tmpdb, tmp_path, monkeypatch):
    monkeypatch.setattr(config, "CHECKPOINT_DIR", tmp_path)
    monkeypatch.setattr(sc, "PROBES", [])

    def make(cards, routes, disallow=()):
        monkeypatch.setattr(sc, "C", cards)
        http = Fake(routes, disallow)
        http.s = type("S", (), {"headers": {"User-Agent": "t"}})()
        return sc.SecondaryCitations(conn=tmpdb, http=http), http
    return make


def test_quote_must_be_on_page(crawler, tmpdb):
    cards = [
        dict(slug="good-1968", title="A thing", year=1968, pub="Los Angeles Times", desc="d",
             cites=[("https://ex.org/a", "In one image from 1968, Conrad drew a thing.", None)]),
        dict(slug="invented-1970", title="Not on the page", year=1970, pub="Los Angeles Times", desc="d",
             cites=[("https://ex.org/a", "Conrad drew an invented cartoon in 1970", None)]),
        dict(slug="alt-caption", title="It's all fine.", date="1967-04-14", pub="Time", desc="d",
             cites=[("https://ex.org/a", "The caption reads: \"It's all fine.\"", "Time, 89 (15)"),
                    ("https://ex.org/missing", "anything", None)]),
    ]
    c, http = crawler(cards, {"https://ex.org/a": (200, PAGE), "https://ex.org/missing": (404, "nf")})
    res = c.run()
    ids = {r[0] for r in tmpdb.execute("SELECT canonical_id FROM cartoons")}
    assert ids == {"sec:good-1968", "sec:alt-caption"}, "a citation whose quote is not on the page was saved"
    assert "invented-1970" in res["notes"] and "REJECTED" in res["notes"]
    row = tmpdb.execute("SELECT * FROM cartoons WHERE canonical_id='sec:good-1968'").fetchone()
    assert row["year"] == 1968 and row["date_is_estimate"] == 1 and row["date_exact"] is None
    assert row["notes"].startswith("Cited by") and "In one image from 1968" in row["notes"]
    exact = tmpdb.execute("SELECT * FROM cartoons WHERE canonical_id='sec:alt-caption'").fetchone()
    assert exact["date_exact"] == "1967-04-14" and exact["date_is_estimate"] == 0
    meth = {r[0] for r in tmpdb.execute("SELECT acquisition_method FROM cartoon_sources")}
    assert meth == {"secondary_citation"}
    assert tmpdb.execute("SELECT COUNT(*) FROM cartoon_sources").fetchone()[0] == 2  # missing page -> no source row
    assert not [u for u in http.calls if IMAGE_URL.search(u)]


def test_corroborating_citation_does_not_overwrite_fields(crawler, tmpdb):
    page2 = "<p>Somebody else also mentions a thing without any year.</p>"
    cards = [dict(slug="k", title="A thing", year=1968, pub="Los Angeles Times", desc="d",
                  cites=[("https://ex.org/a", "In one image from 1968, Conrad drew a thing.", None),
                         ("https://ex.org/b", "also mentions a thing without any year", None)])]
    c, _ = crawler(cards, {"https://ex.org/a": (200, PAGE), "https://ex.org/b": (200, page2)})
    c.run()
    row = tmpdb.execute("SELECT * FROM cartoons WHERE canonical_id='sec:k'").fetchone()
    assert row["year"] == 1968 and row["title"] == "A thing"
    urls = {r[0] for r in tmpdb.execute("SELECT record_url FROM cartoon_sources")}
    assert urls == {"https://ex.org/a", "https://ex.org/b"}


def test_wayback_not_used_when_origin_robots_disallow(crawler, tmpdb):
    cards = [dict(slug="wb", title="A thing", year=1968, pub="x", desc="d",
                  cites=[(WB, "In one image from 1968, Conrad drew a thing.", None)])]
    c, http = crawler(cards, {WB: (200, PAGE)}, disallow=("example.org",))
    c.run()
    assert tmpdb.execute("SELECT COUNT(*) FROM cartoons").fetchone()[0] == 0
    assert WB not in http.calls, "Wayback copy fetched although the origin robots.txt disallows the path"


def test_http_refuses_image_urls_before_any_io():
    h = Http(use_cache=False)

    def boom(*a, **k):
        raise AssertionError("network I/O attempted for an image URL")
    h.s.get = boom
    for u in ("https://web.archive.org/web/2010/http://www.pbs.org/independentlens/paulconrad/mag/mag_01.jpg",
              "https://tile.loc.gov/x/cartoon.GIF?w=150", "https://ex.org/a.png#frag"):
        with pytest.raises(Blocked, match="image URL refused"):
            h.get(u)


def test_page_text_reads_alt_and_normalises():
    t = sc.norm(sc.page_text(PAGE))
    assert "the caption reads: \"it's all fine.\"" in t
    assert "time, 89 (15)" in t