← back to Paul Conrad Archive
src/conrad/crawlers/latimes.py
203 lines
"""Los Angeles Times (Conrad's paper 1964-1993; syndicated via LA Times Syndicate / Tribune Media 1993-2010).
The historical page archive is PAYWALLED (ProQuest Historical Newspapers; latimes.newspapers.com) — never touched.
What IS free and robots-permitted on latimes.com (robots.txt `User-agent: *` disallows only /search, /config, ...):
(Robots re-audit 2026-09-25: `Disallow: /*photogallery` exists ONLY in the `Googlebot-News` group; our UA falls to
`*`, so the gallery below is permitted. Every fetch still goes through Http.get's RobotRules check, so if the `*`
group ever gains that rule the gallery is refused before any I/O — tests/test_latimes.py pins both facts.)
1. DISCOVERY via the site's own monthly sitemaps (robots.txt `Sitemap:` lines; 1985-today). latimes.com's /search
is robots-disallowed and web search engines do not index it for this client, so the sitemap slugs are the only
permitted discovery path. Slugs are descriptive only from ~2000 on (older la-xpm slugs are opaque numbers), so
pre-2000 articles about individual cartoons cannot be discovered this way — recorded as a gap, not guessed.
2. PHOTO GALLERIES (e.g. the 2010 obituary gallery "Images: Times political cartoonist Paul Conrad ... dies at 86")
are parsed slide by slide: LA Times caption (used as a descriptive title), attribution, the image's own URL on
the LA Times image CDN (stored as image_url METADATA — never fetched), and a year ONLY when the caption states
the cartoon's year ("A 1976 panel ..."). Undated slides stay undated.
3. CURATED quote-verified citations from free latimes.com article pages (letters, columns) that name a specific
dated Conrad cartoon; saved only if the quote is on the fetched page (same rule as secondary_citations).
"""
from __future__ import annotations
import html as _html
import re
from .. import db, rights
from ..models import CartoonRecord
from ..normalize import presidents_in_text
from .base import Blocked, Checkpoint, Crawler, Transient
from .secondary_citations import norm, page_text, quote_on_page
SITEMAP_INDEX = "https://www.latimes.com/sitemaps/sitemap.xml"
SITEMAP_YEARS = (1985, 2026)
CONRAD_SLUG = re.compile(r"paul-conrads?\b|paul-conrad-", re.I)
GALLERY = "https://www.latimes.com/nation/la-me-paul-conrad-pictures-photogallery.html"
LAT = "Los Angeles Times"
# Curated LA Times pages that name a specific, dated Conrad cartoon. (url, quote, date_quote)
CITES = [
dict(slug="lat-home-for-christmas-2003", title="I'll be home for Christmas....",
caption="I'll be home for Christmas....", date="2003-12-25",
desc="A flag-draped coffin (Iraq war), Christmas Day 2003 Commentary page.",
url="https://www.latimes.com/archives/la-xpm-2003-dec-29-le-conrad29.1-story.html",
quote="depicted a flag-draped coffin with the caption, \"I'll be home for Christmas",
date_quote="Re Paul Conrad's Dec. 25 editorial cartoon (Commentary)",
note="letters to the editor, Los Angeles Times, 29 Dec 2003 (year from the letters page date)"),
]
def parse_gallery(body: str) -> list[dict]:
"""Slides of a latimes.com Brightspot photo gallery -> [{bsp, info_title, attribution, alt, caption, image}]."""
out = []
for chunk in body.split('<div class="gallery-slide">')[1:]:
media = re.search(r'class="gallery-slide-media"([^>]*)>', chunk)
attrs = media[1] if media else ""
def attr(name):
m = re.search(name + r'="([^"]*)"', attrs)
return _html.unescape(m[1]).strip() if m else None
img = re.search(r"<img\b[^>]*>", chunk)
alt = src = None
if img:
a = re.search(r'\balt="([^"]*)"', img[0])
alt = _html.unescape(a[1]).strip() if a else None
s = re.search(r'\bsrc="(https://[^"]+)"', img[0]) or re.search(r'\bsrcset="(https://[^" ]+)', img[0])
src = _html.unescape(s[1]) if s else None
cap = re.search(r'class="[^"]*gallery-slide-caption[^"]*"[^>]*>(.*?)</div>', chunk, re.S) or \
re.search(r"<figcaption[^>]*>(.*?)</figcaption>", chunk, re.S)
caption = re.sub(r"\s+", " ", _html.unescape(re.sub(r"<[^>]+>", " ", cap[1]))).strip() if cap else None
if caption:
caption = re.sub(r"\s+([.,;:)])", r"\1", caption)
out.append(dict(bsp=attr("data-image-bsp-id"), info_title=attr("data-info-title"),
attribution=attr("data-info-attribution"), alt=alt, caption=caption or alt, image=src))
return out
def caption_year(text: str | None) -> int | None:
"""Year of the CARTOON only when the caption says so ('A 1976 panel', 'this 1984 cartoon'); event years are
ignored ('bus boycott of 1955')."""
m = re.search(r"\b(?:a|an|this|the|his)\s+(19[4-9]\d|200\d|2010)\s+(?:panel|cartoon|drawing)", text or "", re.I)
return int(m[1]) if m else None
class LATimes(Crawler):
source_id = "latimes"
name = "Los Angeles Times — free latimes.com galleries/articles (historical archive paywalled)"
repository = "Los Angeles Times"
url = "https://www.latimes.com/"
classification = "PUBLIC_HTML"
access_notes = ("Historical page archive (ProQuest / latimes.newspapers.com) is paywalled and NOT crawled. Free, "
"robots-permitted latimes.com pages found via the site's own sitemaps are parsed; images are "
"recorded as URLs only. Robots re-audit 2026-09-25: the photogallery Disallow applies only to "
"Googlebot-News, not to '*' (our UA) — gallery permitted.")
def __init__(self, *a, years=SITEMAP_YEARS, **kw):
super().__init__(*a, **kw)
self.years = years
self.ck = Checkpoint("latimes")
self.found: list[str] = []
# ---------------------------------------------------------------- discovery
def discover(self) -> list[str]:
try:
_, idx = self.http.get(SITEMAP_INDEX)
except (Blocked, Transient) as e:
self.error(SITEMAP_INDEX, e)
return []
maps = [u for u in re.findall(r"<loc>([^<]+)</loc>", idx)
if (m := re.search(r"sitemap-(\d{4})\d\d\.xml$", u)) and self.years[0] <= int(m[1]) <= self.years[1]]
hits: set[str] = set()
for u in maps:
try:
st, b = self.http.get(u)
self.stats["pages"] += 1
except (Blocked, Transient) as e:
self.error(u, e)
continue
hits.update(loc for loc in re.findall(r"<loc>([^<]+)</loc>", b) if CONRAD_SLUG.search(loc))
self.notes.append(f"sitemaps scanned={len(maps)} ({self.years[0]}-{self.years[1]}); pages with a "
f"'paul-conrad' slug={len(hits)}")
self.ck.set("discovered", sorted(hits))
return sorted(hits)
# ---------------------------------------------------------------- galleries
def gallery(self, url: str) -> int:
try:
st, body = self.http.get(url)
self.stats["pages"] += 1
except (Blocked, Transient) as e:
self.error(url, e)
return 0
if st != 200:
self.error(url, f"HTTP {st}")
return 0
title = re.search(r'<meta property="og:title" content="([^"]*)"', body)
gtitle = _html.unescape(title[1]) if title else url
n = 0
for i, s in enumerate(parse_gallery(body), 1):
text = s["caption"] or ""
credited = ("conrad" in (s["attribution"] or "").lower() or re.search(r"\(Paul Conrad / Los Angeles Times\)\s*$", text)
or re.match(r"Conrad.s editorial cartoon", text))
if not text or not credited:
continue # only slides credited to Paul Conrad (the gallery also holds photos of him)
desc = re.sub(r"\s*\(Paul Conrad / Los Angeles Times\)\s*$", "", text).strip()
y = caption_year(desc)
rec = CartoonRecord(
canonical_id=f"latg:{s['bsp'] or re.sub(r'[^a-z0-9]+', '-', desc.lower())[:60]}",
identifier=f"{url}#slide-{i}", granularity="item",
title=desc if len(desc) <= 160 else desc[:157] + "...", description=desc, caption=None,
year=y, date_start=f"{y}-01-01" if y else None, date_end=f"{y}-12-31" if y else None,
date_is_estimate=bool(y), publication=LAT,
notes=(f"LA Times photo gallery '{gtitle}' slide {i}; title = the Times' descriptive caption (the "
f"cartoon's own caption is not given); {'year stated in caption' if y else 'undated in source'}; "
f"slide label '{s['info_title'] or ''}', credit '{s['attribution'] or 'Paul Conrad / Los Angeles Times (caption)'}'")[:1000],
rights_text=rights.COPYRIGHT_NOTE, repository=LAT, collection_name=gtitle[:200],
record_url=url, image_url=s["image"], access_level=rights.ONLINE_IMAGE if s["image"] else rights.ONLINE_METADATA,
provenance=f"live:latimes.com gallery {db.now()[:10]} slide {i}", acquisition_method="direct_html",
people=presidents_in_text(desc + " " + (s["info_title"] or "")), subjects=["Editorial cartoons"])
self.save(rec)
n += 1
return n
# ---------------------------------------------------------------- curated citations
def cites(self) -> int:
n = 0
for c in CITES:
try:
st, body = self.http.get(c["url"])
self.stats["pages"] += 1
except (Blocked, Transient) as e:
self.error(c["url"], e)
continue
t = norm(page_text(body))
if not (quote_on_page(c["quote"], t) and quote_on_page(c["date_quote"], t)):
self.error(c["url"], f"citation rejected for {c['slug']}: quote not found on page")
continue
d = c["date"]
rec = CartoonRecord(
canonical_id=f"lat:{c['slug']}", identifier=c["url"], granularity="item", title=c["title"],
caption=c["caption"], description=c["desc"], date_exact=d, date_start=d, date_end=d, year=int(d[:4]),
publication=LAT, rights_text=rights.COPYRIGHT_NOTE, repository=LAT, collection_name="latimes.com archive page",
record_url=c["url"], access_level=rights.ONLINE_METADATA,
notes=f"Quoted from {c['note']}: \"{c['quote']}\" | date evidence: \"{c['date_quote']}\"",
provenance=f"live:latimes.com quote-verified {db.now()[:10]}", acquisition_method="direct_html",
people=presidents_in_text(c["desc"]), subjects=["Editorial cartoons"])
self.save(rec)
n += 1
return n
def crawl(self) -> None:
found = self.discover()
galleries = sorted({u for u in found if "photogallery" in u} | {GALLERY})
g = sum(self.gallery(u) for u in galleries)
c = self.cites()
n_att = self.conn.execute("SELECT COUNT(*) FROM cartoons WHERE publication LIKE 'Los Angeles Times%' "
"AND granularity='item'").fetchone()[0]
self.notes.append(f"gallery slides saved={g} (from {len(galleries)} gallery page(s)); quote-verified article "
f"citations={c}; historical archive paywalled — not crawled; {n_att} item records overall are "
f"attributed to the LA Times; pre-2000 la-xpm slugs are opaque, so older articles about "
f"single cartoons are undiscoverable without the robots-disallowed /search")
self.status = "partial" if (g or c) else "failed"
CRAWLER = LATimes