← back to Paul Conrad Archive
tests/test_latimes.py
92 lines
"""latimes.com crawler: sitemap discovery, gallery slide parser, quote-verified citation (TK-12199)."""
from pathlib import Path
import pytest
from conftest import FakeHttp
from conrad import config
from conrad.crawlers import base
from conrad.crawlers import latimes
from conrad.crawlers.latimes import caption_year, parse_gallery
IMG = "https://ca-times.brightspotcdn.com/dims4/default/abc/2147483647/strip/true/crop/300x410+0+0/resize/300x410!/quality/75/?url=x"
def _slide(bsp, info, attr, alt, cap):
return (f'<div class="gallery-slide"><div class="gallery-slide-media" data-image-bsp-id="{bsp}" '
f'data-info-title="{info}" data-info-attribution="{attr}" ><picture><img class="image" alt="{alt}" '
f'src="{IMG}"></picture></div><div class="gallery-slide-caption">{cap}</div></div>')
GALLERY = ('<html><meta property="og:title" content="Images: Paul Conrad dies at 86">'
+ _slide("b1", "Abortion", "Paul Conrad / Los Angeles Times", "A 1976 panel",
"A 1976 panel showing Conrad's view on abortion then. (Paul Conrad / Los Angeles Times)")
+ _slide("b2", "Oil spill", "Paul Conrad / Los Angeles Times", "oil",
"Union Oil's involvement in the 1969 oil spill. (Paul Conrad / Los Angeles Times)")
+ _slide("b3", "Paul Conrad", "Los Angeles Times", "Conrad at his desk",
"Paul Conrad at his drawing table in 1984. (Los Angeles Times)") + "</html>")
LETTER = ("<html><p>Re Paul Conrad’s Dec. 25 editorial cartoon (Commentary): the tripe that depicted a "
"flag-draped coffin with the caption, “I’ll be home for Christmas.... “</p></html>")
SITEMAP_IDX = "<sitemapindex><sitemap><loc>https://www.latimes.com/sitemaps/sitemap-201009.xml</loc></sitemap></sitemapindex>"
SITEMAP = ("<urlset><url><loc>https://www.latimes.com/nation/la-me-paul-conrad-pictures-photogallery.html</loc></url>"
"<url><loc>https://www.latimes.com/x/lauren-conrad-mtv</loc></url></urlset>")
def test_parse_gallery_and_caption_year():
s = parse_gallery(GALLERY)
assert [x["bsp"] for x in s] == ["b1", "b2", "b3"] and s[0]["image"] == IMG
assert caption_year(s[0]["caption"]) == 1976
assert caption_year(s[1]["caption"]) is None # an EVENT year is not the cartoon's date (negative case)
def test_crawl_gallery_skips_photos_and_verifies_quotes(tmpdb):
http = FakeHttp({"sitemaps/sitemap.xml": (200, SITEMAP_IDX), "sitemap-201009": (200, SITEMAP),
"photogallery": (200, GALLERY), "le-conrad29.1": (200, LETTER)})
res = latimes.CRAWLER(conn=tmpdb, http=http, years=(2010, 2010)).run()
rows = {r["canonical_id"]: r for r in tmpdb.execute("SELECT * FROM cartoons")}
assert set(rows) == {"latg:b1", "latg:b2", "lat:lat-home-for-christmas-2003"} # b3 = a PHOTO of Conrad: skipped
assert rows["latg:b1"]["year"] == 1976 and rows["latg:b2"]["year"] is None
assert rows["lat:lat-home-for-christmas-2003"]["date_exact"] == "2003-12-25"
assert res["status"] == "partial" and "slug=1" in res["notes"]
# no image was ever requested
assert not any("brightspotcdn" in u for u in http.calls)
def test_citation_rejected_when_quote_missing(tmpdb):
http = FakeHttp({"sitemaps/sitemap.xml": (200, SITEMAP_IDX), "sitemap-201009": (200, "<urlset/>"),
"photogallery": (200, "<html></html>"), "le-conrad29.1": (200, "<html>unrelated letters</html>")})
res = latimes.CRAWLER(conn=tmpdb, http=http, years=(2010, 2010)).run()
assert tmpdb.execute("SELECT COUNT(*) FROM cartoons").fetchone()[0] == 0
assert res["status"] == "failed" and res["errors"] >= 1
# ---------------------------------------------------------------- robots (TK-12199 reconcile, 2026-09-25)
ROBOTS = Path(__file__).parent / "fixtures" / "latimes_robots_20260925.txt"
def _rules():
rr = base.RobotRules()
rr.parse(ROBOTS.read_text().splitlines())
return rr
def test_latimes_robots_gallery_allowed_for_our_ua_only_googlebot_news_disallows():
"""Live latimes.com robots.txt (snapshot 2026-09-25): 'Disallow: /*photogallery' sits ONLY in the Googlebot-News
group. Our UA falls to 'User-agent: *', which does not list it, so the Conrad gallery is robots-permitted for us."""
rr = _rules()
assert rr.can_fetch(config.USER_AGENT, latimes.GALLERY)
assert not rr.can_fetch("Googlebot-News", latimes.GALLERY)
assert not rr.can_fetch("ClaudeBot/1.0", latimes.GALLERY) # AI-crawler groups are Disallow: /
assert not rr.can_fetch(config.USER_AGENT, "https://www.latimes.com/search?q=conrad")
assert rr.can_fetch(config.USER_AGENT, latimes.SITEMAP_INDEX)
def test_gallery_refused_if_robots_ever_disallows_it(monkeypatch):
"""If latimes.com ever adds '/*photogallery' to the '*' group, the client refuses the gallery before any I/O."""
rr = base.RobotRules()
rr.parse(["User-agent: *", "Disallow: /*photogallery"])
monkeypatch.setitem(base._robots, "https://www.latimes.com", rr)
h = base.Http(use_cache=False)
monkeypatch.setattr(h, "_fetch", lambda *a, **k: pytest.fail("network touched for a robots-disallowed URL"))
with pytest.raises(base.Blocked, match="robots.txt disallows"):
h.get(latimes.GALLERY)