← back to Paul Conrad Archive
LA Times: sitemap discovery + 2010 obituary gallery parser (26 cartoons w/ image URLs) + quote-verified letters citation; DPLA NPG Time covers; base.py gzip truncation fix; IA full-text + newspaper-probe crawlers; tests
25daa261a36802c31e26c00b113dce22e068355e · 2026-09-25 10:52:07 -0700 · Steve Abrams
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SW5KHgwrh2Rr9TKHM6ndqM
Files touched
M src/conrad/crawlers/__init__.pyM src/conrad/crawlers/base.pyM src/conrad/crawlers/discovery.pyA src/conrad/crawlers/ia_fulltext.pyM src/conrad/crawlers/latimes.pyA src/conrad/crawlers/newspaper_probes.pyM src/conrad/crawlers/secondary_citations.pyA tests/test_dpla_authority.pyA tests/test_ia_fulltext.pyA tests/test_latimes.pyM tests/test_parsers.py
Diff
commit 25daa261a36802c31e26c00b113dce22e068355e
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Fri Sep 25 10:52:07 2026 -0700
LA Times: sitemap discovery + 2010 obituary gallery parser (26 cartoons w/ image URLs) + quote-verified letters citation; DPLA NPG Time covers; base.py gzip truncation fix; IA full-text + newspaper-probe crawlers; tests
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SW5KHgwrh2Rr9TKHM6ndqM
---
src/conrad/crawlers/__init__.py | 3 +-
src/conrad/crawlers/base.py | 7 +-
src/conrad/crawlers/discovery.py | 16 +-
src/conrad/crawlers/ia_fulltext.py | 280 +++++++++++++++++++++++++++++
src/conrad/crawlers/latimes.py | 198 ++++++++++++++++++--
src/conrad/crawlers/newspaper_probes.py | 76 ++++++++
src/conrad/crawlers/secondary_citations.py | 23 ++-
tests/test_dpla_authority.py | 31 ++++
tests/test_ia_fulltext.py | 110 ++++++++++++
tests/test_latimes.py | 54 ++++++
tests/test_parsers.py | 10 +-
11 files changed, 785 insertions(+), 23 deletions(-)
diff --git a/src/conrad/crawlers/__init__.py b/src/conrad/crawlers/__init__.py
index 772e670..f38af05 100644
--- a/src/conrad/crawlers/__init__.py
+++ b/src/conrad/crawlers/__init__.py
@@ -1,3 +1,4 @@
"""Source crawlers. Each module exposes a Crawler subclass named CRAWLER."""
ORDER = ["seed_import", "huntington", "oac", "syracuse", "loc", "ohio_state", "wichita",
- "latimes", "denver_post", "daily_iowan", "internet_archive", "discovery", "secondary_citations"]
+ "latimes", "denver_post", "daily_iowan", "internet_archive", "discovery", "secondary_citations",
+ "newspaper_probes", "ia_fulltext"]
diff --git a/src/conrad/crawlers/base.py b/src/conrad/crawlers/base.py
index 03e1c06..9b153da 100644
--- a/src/conrad/crawlers/base.py
+++ b/src/conrad/crawlers/base.py
@@ -164,11 +164,14 @@ class Http:
if ctype.startswith(BINARY_CTYPES):
r.close()
raise Blocked(f"copyright rail: {ctype or 'binary'} response refused, body never read: {url}")
- head = next(r.iter_content(16), b"")
+ # ONE generator for the whole body: a second iter_content() call loses whatever the first
+ # generator's gzip decoder had already buffered (bodies were silently truncated to ~10 bytes).
+ chunks = r.iter_content(65536)
+ head = next(chunks, b"")
if head.startswith(IMAGE_MAGIC):
r.close()
raise Blocked(f"copyright rail: image bytes refused, body never read: {url}")
- r._content = head + b"".join(r.iter_content(65536))
+ r._content = head + b"".join(chunks)
r._content_consumed = True
log.info("GET %s -> %s (%.1fs, %d bytes)", url, r.status_code, time.time() - t0, len(r.content))
except requests.RequestException as e:
diff --git a/src/conrad/crawlers/discovery.py b/src/conrad/crawlers/discovery.py
index bd4820a..f54a3b7 100644
--- a/src/conrad/crawlers/discovery.py
+++ b/src/conrad/crawlers/discovery.py
@@ -16,6 +16,10 @@ from .base import Blocked, Crawler, Transient
CONRAD_RE = re.compile(r"conrad,\s*paul(?:,?\s*1924)?|paul conrad", re.I)
CARTOON_RE = re.compile(r"cartoon|caricature|drawing|editorial|los angeles times|denver post", re.I)
+# Name-authority forms that pin THIS Paul Conrad (b. 27 Jun 1924): the NPG/Smithsonian form carries the birth date,
+# LC form the life dates. A record whose creator matches one of these is kept without the cartoon-word test
+# (NPG's Time-cover originals describe the Time donation, not the medium).
+AUTHORITY_RE = re.compile(r"paul conrad, born 27 jun 1924|conrad,\s*paul,\s*1924", re.I)
PROBES = [ # (source_id, name, url, classification-if-blocked, note)
("calisphere", "Calisphere (UC Libraries)", "https://calisphere.org/search/?q=%22paul+conrad%22", "REQUIRES_PERMISSION",
@@ -79,10 +83,13 @@ class Discovery(Crawler):
classification="PUBLIC_API", status="running")
providers: dict[str, int] = {}
kept = 0
- for q in ('"paul conrad" cartoon', '"conrad, paul"', '"paul conrad" los angeles times'):
+ queries = [{"q": '"paul conrad" cartoon'}, {"q": '"conrad, paul"'}, {"q": '"paul conrad" los angeles times'},
+ {"sourceResource.creator": '"Paul Conrad, born 27 Jun 1924"'}]
+ for qp in queries:
+ q = " ".join(f"{k}={v}" for k, v in qp.items())
for page in range(1, 6):
try:
- d = self.http.get_json("https://api.dp.la/v2/items", params={"q": q, "page_size": 100, "page": page},
+ d = self.http.get_json("https://api.dp.la/v2/items", params={**qp, "page_size": 100, "page": page},
secret_params={"api_key": key})
except (Blocked, Transient) as e:
self.error("https://api.dp.la/v2/items", e)
@@ -95,7 +102,7 @@ class Discovery(Crawler):
title = " ".join(sr.get("title") or []) if isinstance(sr.get("title"), list) else str(sr.get("title") or "")
subj = " ".join(s.get("name", "") for s in sr.get("subject") or [] if isinstance(s, dict))
blob = " ".join([creators, title, subj, " ".join(sr.get("description") or [] if isinstance(sr.get("description"), list) else [str(sr.get("description") or "")])])
- if not (CONRAD_RE.search(creators) and CARTOON_RE.search(blob)):
+ if not ((CONRAD_RE.search(creators) and CARTOON_RE.search(blob)) or AUTHORITY_RE.search(creators)):
continue
types = sr.get("type") or []
types = types if isinstance(types, list) else [types]
@@ -106,8 +113,11 @@ class Discovery(Crawler):
prov = prov.get("name") if isinstance(prov, dict) else str(prov)
providers[prov] = providers.get(prov, 0) + 1
dt = parse_date((sr.get("date") or [{}])[0].get("displayDate") if isinstance(sr.get("date"), list) else None)
+ time_cover = "time magazine donated" in blob.lower() and "cover art" in blob.lower()
rec = CartoonRecord(
canonical_id=f"dpla:{doc['id']}", identifier=doc["id"], granularity="item", title=title or None,
+ publication="Time" if time_cover else None,
+ notes="original Time cover art (NPG Time Collection, donated by Time 1978)" if time_cover else None,
description=blob[:1000], date_exact=dt["date_exact"], date_start=dt["date_start"],
date_end=dt["date_end"], year=dt["year"], repository=prov, collection_name="via DPLA",
record_url=doc.get("isShownAt"), thumbnail_url=doc.get("object"),
diff --git a/src/conrad/crawlers/ia_fulltext.py b/src/conrad/crawlers/ia_fulltext.py
new file mode 100644
index 0000000..d9cf0ab
--- /dev/null
+++ b/src/conrad/crawlers/ia_fulltext.py
@@ -0,0 +1,280 @@
+"""Internet Archive FULL-TEXT (OCR) search for printed APPEARANCES of Paul Conrad cartoons — newspapers,
+magazines and cartoon anthologies whose scanned pages carry a Conrad credit line
+("PAUL CONRAD / Los Angeles Times / © Los Angeles Times Syndicate", "CONRAD © 1971 Los Angeles Times", ...).
+
+How it works (robots-permitted, metadata only):
+ * archive.org/robots.txt disallows only /control/ and /report/; the public full-text search service
+ (`/services/search/beta/page_production/?service_backend=fts`) is queried year by year, 1950-2010, for several
+ query variants, and EVERY result page is walked (no sampling). Each hit is one scanned PAGE
+ (identifier + sub-file + page_num) with OCR highlight snippets.
+ * a hit becomes a record ONLY if an OCR snippet carries a Conrad CREDIT-LINE signature (see `classify`) — a short
+ line naming Conrad next to "Los Angeles Times" / "Times Syndicate" / "Denver Post" / "©". Articles merely
+ *about* Conrad, other Conrads (Joseph, Conrad Hilton, Conrad de Aenlle, ...) and bare byline noise are rejected
+ and counted, never saved.
+ * granularity 'item', but the record is an APPEARANCE of an (unidentified) cartoon on a dated page, not a titled
+ cartoon: title is a supplied, bracketed description "[Paul Conrad cartoon — <publication>, <date>, p. N]";
+ the OCR snippet is kept verbatim in notes (flagged "OCR, unverified"). Dedupe folds reprints of the same page.
+ * IMAGES are never downloaded. For items IA serves openly, the page image URL is taken from IA's own IIIF
+ presentation manifest (JSON) and stored as image_url on host iiif.archive.org for hotlinking; lending-library
+ (access-restricted) books get no image URL — their page is only viewable through IA's lending UI.
+"""
+from __future__ import annotations
+
+import json
+import re
+from urllib.parse import quote, urlencode
+
+from .. import config, db, rights
+from ..models import CartoonRecord
+from ..normalize import presidents_in_text
+from .base import Blocked, Checkpoint, Crawler, Transient
+
+FTS = "https://archive.org/services/search/beta/page_production/"
+YEARS = range(1950, 2011)
+# query variants (coordinator/Steve, TK-12199). "Los Angeles Times Syndicate" alone is covered by V2: the FTS index
+# is PAGE-level, so '"Conrad" "Los Angeles Times Syndicate"' is exactly "LATS pages whose text also carries Conrad".
+VARIANTS = [
+ ("paul_conrad", '"Paul Conrad"', YEARS),
+ ("conrad_lats", '"Conrad" "Los Angeles Times Syndicate"', YEARS),
+ ("conrad_lat", '"Conrad" "Los Angeles Times"', YEARS),
+ ("conrad_denver", '"Conrad" "Denver Post"', range(1950, 1965)),
+ ("conrad_cartoon", '"Conrad cartoon"', YEARS),
+]
+PAGE_SIZE = 100
+MAX_PAGES = 40 # per (variant, year); hitting the cap is logged, never silent
+
+HL = re.compile(r"\{\{\{(.*?)\}\}\}")
+OTHER_CONRADS = re.compile(
+ r"joseph\s+conrad|conrad\s+(?:de\s*a|hilton|burns|black|lister|veidt|n\.? ?hilton|jr|adenauer|bain|dobler|"
+ r"birdie|richter|aiken|kent|jones)|(?:robert|kent|frank|charles|pete|william|andree|michael|chris|dr\.?|rev\.?|"
+ r"deacon|the rev)\s+conrad|conrad\s+international|conrad\s+hotel", re.I)
+CREDIT_CONTEXT = re.compile(r"los\s*angeles\s*times|l\.\s?a\.\s?times|times\s*syndicate|denver\s*post|syndicate|©|\(c\)", re.I)
+PAUL = re.compile(r"paul\s+conrad", re.I)
+
+
+def _norm_line(s: str) -> str:
+ return re.sub(r"\s+", " ", HL.sub(r"\1", s)).strip()
+
+
+def classify(snippets: list[str]) -> tuple[str, str | None]:
+ """-> (kind, evidence line). kind: 'credit' (a printed Conrad cartoon credit), 'other_conrad', 'mention'.
+
+ A CREDIT needs a line that (a) contains the highlighted Conrad match, (b) is short (<= 45 chars, i.e. a credit or
+ signature line, not running prose — or <= 90 chars when the line STARTS with the Conrad
+ credit, e.g. 'PAUL CONRAD Los Angeles Times © Los Angeles Times Syndicate'), (c) is not another Conrad, and (d) has credit context (LA Times / Syndicate /
+ Denver Post / ©) on the same line or within the next two lines, or reads exactly 'PAUL CONRAD'."""
+ other = False
+ for sn in snippets:
+ lines = [ln for ln in re.split(r"\n", sn)]
+ for i, raw in enumerate(lines):
+ if not re.search(r"\{\{\{[^}]*conrad[^}]*\}\}\}", raw, re.I):
+ continue
+ line = _norm_line(raw)
+ window = " ".join(_norm_line(x) for x in lines[max(0, i - 1): i + 3])
+ if OTHER_CONRADS.search(window):
+ other = True
+ continue
+ lead = re.match(r"(?:[©@]\s*)?(?:paul\s+)?conrad\b\W*(?:\(?c\)?\s*)?(?:the\s+)?(?:los\s*angeles\s*times|l\.\s?a\."
+ r"|denver\s*post|©|copyright|courtesy)", line, re.I)
+ if len(line) > 45 and not (lead and len(line) <= 90):
+ continue
+ ctx = CREDIT_CONTEXT.search(line) or CREDIT_CONTEXT.search(" ".join(_norm_line(x) for x in lines[i + 1: i + 3]))
+ if ctx or re.fullmatch(r"(?:[©@]\s*)?paul\s+conrad[.,]?", line, re.I):
+ return "credit", _norm_line(" / ".join(lines[max(0, i - 1): i + 3]))[:300]
+ return ("other_conrad" if other else "mention"), None
+
+
+MONTHS = {m: i for i, m in enumerate(["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov",
+ "dec"], 1)}
+
+
+def issue_date(fields: dict) -> tuple[str | None, int | None]:
+ """Best issue date for the scanned page: from the sub-file name ('Aug 11 1998, The Jerusalem Post ...'), the
+ identifier ('..._1978_11_01'), else the item date. Returns (ISO date or None, year)."""
+ for s in (fields.get("file_basename") or "", fields.get("identifier") or ""):
+ m = re.search(r"\b([A-Z][a-z]{2})[a-z]*\.? (\d{1,2}),? (19[4-9]\d|20[01]\d)\b", s)
+ if m and m[1].lower() in MONTHS:
+ return f"{m[3]}-{MONTHS[m[1].lower()]:02d}-{int(m[2]):02d}", int(m[3])
+ m = re.search(r"(19[4-9]\d|20[01]\d)[_-](\d{2})[_-](\d{2})", s)
+ if m and 1 <= int(m[2]) <= 12 and 1 <= int(m[3]) <= 31:
+ return f"{m[1]}-{m[2]}-{m[3]}", int(m[1])
+ d = str(fields.get("date") or "")[:10]
+ y = fields.get("year")
+ return None, int(y) if y else (int(d[:4]) if d[:4].isdigit() else None)
+
+
+def publication(fields: dict) -> str:
+ t = re.sub(r"\s*,\s*(19|20)\d\d.*$", "", str(fields.get("title") or fields.get("identifier") or "")).strip()
+ t = re.sub(r"\s+(19|20)\d\d[ _-]\d\d[ _-]\d\d$", "", t)
+ return t[:120] or fields.get("identifier")
+
+
+def is_book(fields: dict) -> bool:
+ cols = set(fields.get("collection") or [])
+ return bool(cols & {"inlibrary", "printdisabled", "internetarchivebooks", "americana"}) and "newspapers" not in cols
+
+
+def restricted(fields: dict) -> bool:
+ return bool(set(fields.get("collection") or []) & {"inlibrary", "printdisabled", "lendinglibrary"})
+
+
+class IAFullText(Crawler):
+ source_id = "ia_fulltext"
+ name = "Internet Archive full-text search — printed Conrad cartoon appearances (newspapers, anthologies)"
+ repository = "Internet Archive"
+ url = "https://archive.org/search?query=%22Paul+Conrad%22&sin=TXT"
+ classification = "PUBLIC_API"
+ access_notes = ("OCR full-text search (page-level). Records = pages carrying a Conrad credit line. Page images are "
+ "hotlinked from iiif.archive.org only for openly served items; lending-library books: no image.")
+
+ def __init__(self, *a, max_pages: int = MAX_PAGES, years=None, variants=None, **kw):
+ super().__init__(*a, **kw)
+ self.max_pages = max_pages
+ self.years = years
+ self.variants = variants or VARIANTS
+ self.ck = Checkpoint("ia_fulltext")
+ self.counts = {"hits": 0, "pages": 0, "credit": 0, "mention": 0, "other_conrad": 0, "images": 0,
+ "restricted_no_image": 0, "capped": 0}
+ self._manifests: dict[str, list | None] = {}
+
+ # ------------------------------------------------------------------ search
+ def fts(self, query: str, page: int) -> dict:
+ params = {"user_query": query, "hits_per_page": PAGE_SIZE, "page": page, "service_backend": "fts"}
+ return self.http.get_json(FTS, params=params)
+
+ def harvest(self) -> dict:
+ pages: dict[tuple, dict] = {}
+ for vid, q, yrs in self.variants:
+ for y in (self.years or yrs):
+ if y not in yrs:
+ continue
+ qy = f"{q} AND year:{y}"
+ p, seen = 1, 0
+ while True:
+ try:
+ d = self.fts(qy, p)
+ except (Blocked, Transient, ValueError) as e:
+ self.error(FTS + "?" + urlencode({"user_query": qy, "page": p}), f"{type(e).__name__}: {e}")
+ break
+ self.stats["pages"] += 1
+ body = (d.get("response") or {}).get("body") or {}
+ hits = (body.get("hits") or {}).get("hits") or []
+ total = (body.get("hits") or {}).get("total") or 0
+ for h in hits:
+ f = h.get("fields") or {}
+ key = (f.get("identifier"), f.get("file_basename") or "", f.get("page_num"))
+ ent = pages.setdefault(key, {"fields": f, "snippets": [], "variants": set()})
+ ent["snippets"] += (h.get("highlight") or {}).get("text") or []
+ ent["variants"].add(vid)
+ seen += len(hits)
+ self.counts["hits"] += len(hits)
+ if not hits or seen >= total:
+ break
+ if p >= self.max_pages:
+ self.counts["capped"] += 1
+ self.notes.append(f"CAPPED {vid} {y}: {seen}/{total} hits walked")
+ break
+ p += 1
+ return pages
+
+ # ------------------------------------------------------------------ images
+ def manifest_images(self, identifier: str) -> list | None:
+ """Canvas image URLs from IA's own IIIF presentation manifest (JSON; no image is requested)."""
+ if identifier in self._manifests:
+ return self._manifests[identifier]
+ out = None
+ try:
+ m = self.http.get_json(f"https://iiif.archive.org/iiif/3/{quote(identifier, safe='')}/manifest.json")
+ out = []
+ for c in m.get("items") or []:
+ url, label = None, json.dumps(c.get("label") or "")
+ for ap in c.get("items") or []:
+ for an in ap.get("items") or []:
+ b = an.get("body") or {}
+ url = url or b.get("id")
+ out.append((url, label))
+ except (Blocked, Transient, ValueError) as e:
+ self.error(f"iiif manifest {identifier}", f"{type(e).__name__}: {e}")
+ self._manifests[identifier] = out
+ return out
+
+ def page_image(self, f: dict) -> str | None:
+ if restricted(f):
+ return None
+ canv = self.manifest_images(f["identifier"])
+ if not canv:
+ return None
+ n, base = f.get("page_num"), f.get("file_basename") or ""
+ if n is None:
+ return None
+ # multi-file items: the manifest concatenates every sub-file; pick the canvas whose image path names the
+ # sub-file and page. Single-file items: canvas index == page_num.
+ stem = f"{base}_{int(n):04d}.jp2"
+ for url, _ in canv:
+ if url and stem and (quote(stem, safe="") in url or stem in url or quote(stem) in url):
+ return url
+ if not f.get("result_in_subfile") and int(n) < len(canv):
+ return canv[int(n)][0]
+ return None
+
+ # ------------------------------------------------------------------ records
+ def crawl(self) -> None:
+ pages = self.harvest()
+ self.counts["pages"] = len(pages)
+ seen_ids: set[str] = set()
+ for (ident, base, pn), ent in pages.items():
+ f = ent["fields"]
+ kind, evidence = classify(ent["snippets"])
+ self.counts[kind] += 1
+ if kind != "credit" or not ident:
+ continue
+ iso, year = issue_date(f)
+ pub = publication(f)
+ book = is_book(f)
+ page_url = f"https://archive.org/details/{quote(ident)}" + (f"/{quote(base)}" if f.get("result_in_subfile")
+ and base else "") + f"/page/n{pn}"
+ img = self.page_image(f)
+ if img:
+ self.counts["images"] += 1
+ elif restricted(f):
+ self.counts["restricted_no_image"] += 1
+ ymatch = re.search(r"(?:©|\(c\)|copyright)\s*(19[5-9]\d|20[01]\d)|times,?\s*(19[5-9]\d|20[01]\d)", evidence or "", re.I)
+ cy = int(ymatch[1] or ymatch[2]) if ymatch else None
+ if book:
+ # an anthology reprint: the cartoon's own year only if the credit line states it
+ d = dict(date_exact=None, year=cy, date_start=f"{cy}-01-01" if cy else None,
+ date_end=f"{cy}-12-31" if cy else (f"{year}-12-31" if year else None), date_is_estimate=True)
+ where = f"reprinted in {pub}" + (f" ({year})" if year else "")
+ else:
+ d = dict(date_exact=iso, year=year, date_start=iso or (f"{year}-01-01" if year else None),
+ date_end=iso or (f"{year}-12-31" if year else None), date_is_estimate=not iso)
+ where = f"{pub}, {iso or year or 'n.d.'}"
+ cid = f"iafts:{ident}:{base}:{pn}"
+ if cid in seen_ids:
+ continue
+ seen_ids.add(cid)
+ note = (f"PRINTED APPEARANCE (newspaper/periodical page)" if not book else "PRINTED APPEARANCE (book reprint)")
+ note += f" found by IA full-text OCR search {sorted(ent['variants'])}; credit line (OCR, unverified): \"{evidence}\""
+ rec = CartoonRecord(
+ canonical_id=cid, identifier=f"{ident}/{base}#n{pn}", granularity="item",
+ title=f"[Paul Conrad cartoon — {where}, p. {int(pn) + 1}]", publication=pub if not book else None,
+ syndicate="Los Angeles Times Syndicate" if re.search(r"syndicate", evidence or "", re.I) else None,
+ notes=note[:1000], rights_text=rights.COPYRIGHT_NOTE, repository="Internet Archive",
+ collection_name=pub, page=str(int(pn) + 1), record_url=page_url, image_url=img,
+ access_level=rights.ONLINE_IMAGE if img else rights.ONLINE_METADATA,
+ provenance=f"live:archive.org FTS {db.now()[:10]}; {ident}/{base} n{pn}",
+ acquisition_method="direct_api", subjects=["Editorial cartoons"],
+ people=presidents_in_text(evidence or ""), **d)
+ self.save(rec)
+ self.ck.set("last_run", db.now())
+ self.ck.set("counts", self.counts)
+ c = self.counts
+ self.notes.insert(0, f"FTS hits walked={c['hits']} distinct pages={c['pages']}; credit-line pages saved={c['credit']} "
+ f"(page images hotlinkable={c['images']}, lending-library/no image={c['restricted_no_image']}); "
+ f"rejected: about-Conrad mentions={c['mention']}, other Conrads={c['other_conrad']}; "
+ f"capped query-years={c['capped']}")
+ if c["credit"] == 0 and c["hits"] == 0:
+ self.status = "failed"
+
+
+CRAWLER = IAFullText
diff --git a/src/conrad/crawlers/latimes.py b/src/conrad/crawlers/latimes.py
index f312f81..98161fb 100644
--- a/src/conrad/crawlers/latimes.py
+++ b/src/conrad/crawlers/latimes.py
@@ -1,26 +1,198 @@
"""Los Angeles Times (Conrad's paper 1964-1993; syndicated via LA Times Syndicate / Tribune Media 1993-2010).
-The historical archive is PAYWALLED (ProQuest Historical Newspapers; latimes.newspapers.com). No paywall is
-bypassed and no article/page is fetched. This module registers the source and computes coverage from
-records already attributed to the LA Times by other repositories (LOC rights statements, Huntington spans)."""
+
+The historical page archive is PAYWALLED (ProQuest Historical Newspapers; latimes.newspapers.com) — never touched.
+What IS free and robots-permitted on latimes.com (robots.txt `User-agent: *` disallows only /search, /config, ...):
+
+ 1. DISCOVERY via the site's own monthly sitemaps (robots.txt `Sitemap:` lines; 1985-today). latimes.com's /search
+ is robots-disallowed and web search engines do not index it for this client, so the sitemap slugs are the only
+ permitted discovery path. Slugs are descriptive only from ~2000 on (older la-xpm slugs are opaque numbers), so
+ pre-2000 articles about individual cartoons cannot be discovered this way — recorded as a gap, not guessed.
+ 2. PHOTO GALLERIES (e.g. the 2010 obituary gallery "Images: Times political cartoonist Paul Conrad ... dies at 86")
+ are parsed slide by slide: LA Times caption (used as a descriptive title), attribution, the image's own URL on
+ the LA Times image CDN (stored as image_url METADATA — never fetched), and a year ONLY when the caption states
+ the cartoon's year ("A 1976 panel ..."). Undated slides stay undated.
+ 3. CURATED quote-verified citations from free latimes.com article pages (letters, columns) that name a specific
+ dated Conrad cartoon; saved only if the quote is on the fetched page (same rule as secondary_citations).
+"""
from __future__ import annotations
-from .base import Crawler
+import html as _html
+import re
+
+from .. import db, rights
+from ..models import CartoonRecord
+from ..normalize import presidents_in_text
+from .base import Blocked, Checkpoint, Crawler, Transient
+from .secondary_citations import norm, page_text, quote_on_page
+
+SITEMAP_INDEX = "https://www.latimes.com/sitemaps/sitemap.xml"
+SITEMAP_YEARS = (1985, 2026)
+CONRAD_SLUG = re.compile(r"paul-conrads?\b|paul-conrad-", re.I)
+GALLERY = "https://www.latimes.com/nation/la-me-paul-conrad-pictures-photogallery.html"
+LAT = "Los Angeles Times"
+
+# Curated LA Times pages that name a specific, dated Conrad cartoon. (url, quote, date_quote)
+CITES = [
+ dict(slug="lat-home-for-christmas-2003", title="I'll be home for Christmas....",
+ caption="I'll be home for Christmas....", date="2003-12-25",
+ desc="A flag-draped coffin (Iraq war), Christmas Day 2003 Commentary page.",
+ url="https://www.latimes.com/archives/la-xpm-2003-dec-29-le-conrad29.1-story.html",
+ quote="depicted a flag-draped coffin with the caption, \"I'll be home for Christmas",
+ date_quote="Re Paul Conrad's Dec. 25 editorial cartoon (Commentary)",
+ note="letters to the editor, Los Angeles Times, 29 Dec 2003 (year from the letters page date)"),
+]
+
+
+def parse_gallery(body: str) -> list[dict]:
+ """Slides of a latimes.com Brightspot photo gallery -> [{bsp, info_title, attribution, alt, caption, image}]."""
+ out = []
+ for chunk in body.split('<div class="gallery-slide">')[1:]:
+ media = re.search(r'class="gallery-slide-media"([^>]*)>', chunk)
+ attrs = media[1] if media else ""
+ def attr(name):
+ m = re.search(name + r'="([^"]*)"', attrs)
+ return _html.unescape(m[1]).strip() if m else None
+ img = re.search(r"<img\b[^>]*>", chunk)
+ alt = src = None
+ if img:
+ a = re.search(r'\balt="([^"]*)"', img[0])
+ alt = _html.unescape(a[1]).strip() if a else None
+ s = re.search(r'\bsrc="(https://[^"]+)"', img[0]) or re.search(r'\bsrcset="(https://[^" ]+)', img[0])
+ src = _html.unescape(s[1]) if s else None
+ cap = re.search(r'class="[^"]*gallery-slide-caption[^"]*"[^>]*>(.*?)</div>', chunk, re.S) or \
+ re.search(r"<figcaption[^>]*>(.*?)</figcaption>", chunk, re.S)
+ caption = re.sub(r"\s+", " ", _html.unescape(re.sub(r"<[^>]+>", " ", cap[1]))).strip() if cap else None
+ if caption:
+ caption = re.sub(r"\s+([.,;:)])", r"\1", caption)
+ out.append(dict(bsp=attr("data-image-bsp-id"), info_title=attr("data-info-title"),
+ attribution=attr("data-info-attribution"), alt=alt, caption=caption or alt, image=src))
+ return out
+
+
+def caption_year(text: str | None) -> int | None:
+ """Year of the CARTOON only when the caption says so ('A 1976 panel', 'this 1984 cartoon'); event years are
+ ignored ('bus boycott of 1955')."""
+ m = re.search(r"\b(?:a|an|this|the|his)\s+(19[4-9]\d|200\d|2010)\s+(?:panel|cartoon|drawing)", text or "", re.I)
+ return int(m[1]) if m else None
class LATimes(Crawler):
source_id = "latimes"
- name = "Los Angeles Times archive (ProQuest / newspapers.com)"
+ name = "Los Angeles Times — free latimes.com galleries/articles (historical archive paywalled)"
repository = "Los Angeles Times"
- url = "https://latimes.newspapers.com/"
- classification = "PAYWALLED"
- access_notes = ("Full-page archive via ProQuest Historical Newspapers (library subscription) or "
- "latimes.newspapers.com. Discovery/metadata only; not crawled.")
+ url = "https://www.latimes.com/"
+ classification = "PUBLIC_HTML"
+ access_notes = ("Historical page archive (ProQuest / latimes.newspapers.com) is paywalled and NOT crawled. Free, "
+ "robots-permitted latimes.com pages found via the site's own sitemaps are parsed; images are "
+ "recorded as URLs only.")
+
+ def __init__(self, *a, years=SITEMAP_YEARS, **kw):
+ super().__init__(*a, **kw)
+ self.years = years
+ self.ck = Checkpoint("latimes")
+ self.found: list[str] = []
+
+ # ---------------------------------------------------------------- discovery
+ def discover(self) -> list[str]:
+ try:
+ _, idx = self.http.get(SITEMAP_INDEX)
+ except (Blocked, Transient) as e:
+ self.error(SITEMAP_INDEX, e)
+ return []
+ maps = [u for u in re.findall(r"<loc>([^<]+)</loc>", idx)
+ if (m := re.search(r"sitemap-(\d{4})\d\d\.xml$", u)) and self.years[0] <= int(m[1]) <= self.years[1]]
+ hits: set[str] = set()
+ for u in maps:
+ try:
+ st, b = self.http.get(u)
+ self.stats["pages"] += 1
+ except (Blocked, Transient) as e:
+ self.error(u, e)
+ continue
+ hits.update(loc for loc in re.findall(r"<loc>([^<]+)</loc>", b) if CONRAD_SLUG.search(loc))
+ self.notes.append(f"sitemaps scanned={len(maps)} ({self.years[0]}-{self.years[1]}); pages with a "
+ f"'paul-conrad' slug={len(hits)}")
+ self.ck.set("discovered", sorted(hits))
+ return sorted(hits)
+
+ # ---------------------------------------------------------------- galleries
+ def gallery(self, url: str) -> int:
+ try:
+ st, body = self.http.get(url)
+ self.stats["pages"] += 1
+ except (Blocked, Transient) as e:
+ self.error(url, e)
+ return 0
+ if st != 200:
+ self.error(url, f"HTTP {st}")
+ return 0
+ title = re.search(r'<meta property="og:title" content="([^"]*)"', body)
+ gtitle = _html.unescape(title[1]) if title else url
+ n = 0
+ for i, s in enumerate(parse_gallery(body), 1):
+ text = s["caption"] or ""
+ credited = ("conrad" in (s["attribution"] or "").lower() or re.search(r"\(Paul Conrad / Los Angeles Times\)\s*$", text)
+ or re.match(r"Conrad.s editorial cartoon", text))
+ if not text or not credited:
+ continue # only slides credited to Paul Conrad (the gallery also holds photos of him)
+ desc = re.sub(r"\s*\(Paul Conrad / Los Angeles Times\)\s*$", "", text).strip()
+ y = caption_year(desc)
+ rec = CartoonRecord(
+ canonical_id=f"latg:{s['bsp'] or re.sub(r'[^a-z0-9]+', '-', desc.lower())[:60]}",
+ identifier=f"{url}#slide-{i}", granularity="item",
+ title=desc if len(desc) <= 160 else desc[:157] + "...", description=desc, caption=None,
+ year=y, date_start=f"{y}-01-01" if y else None, date_end=f"{y}-12-31" if y else None,
+ date_is_estimate=bool(y), publication=LAT,
+ notes=(f"LA Times photo gallery '{gtitle}' slide {i}; title = the Times' descriptive caption (the "
+ f"cartoon's own caption is not given); {'year stated in caption' if y else 'undated in source'}; "
+ f"slide label '{s['info_title'] or ''}', credit '{s['attribution'] or 'Paul Conrad / Los Angeles Times (caption)'}'")[:1000],
+ rights_text=rights.COPYRIGHT_NOTE, repository=LAT, collection_name=gtitle[:200],
+ record_url=url, image_url=s["image"], access_level=rights.ONLINE_IMAGE if s["image"] else rights.ONLINE_METADATA,
+ provenance=f"live:latimes.com gallery {db.now()[:10]} slide {i}", acquisition_method="direct_html",
+ people=presidents_in_text(desc + " " + (s["info_title"] or "")), subjects=["Editorial cartoons"])
+ self.save(rec)
+ n += 1
+ return n
+
+ # ---------------------------------------------------------------- curated citations
+ def cites(self) -> int:
+ n = 0
+ for c in CITES:
+ try:
+ st, body = self.http.get(c["url"])
+ self.stats["pages"] += 1
+ except (Blocked, Transient) as e:
+ self.error(c["url"], e)
+ continue
+ t = norm(page_text(body))
+ if not (quote_on_page(c["quote"], t) and quote_on_page(c["date_quote"], t)):
+ self.error(c["url"], f"citation rejected for {c['slug']}: quote not found on page")
+ continue
+ d = c["date"]
+ rec = CartoonRecord(
+ canonical_id=f"lat:{c['slug']}", identifier=c["url"], granularity="item", title=c["title"],
+ caption=c["caption"], description=c["desc"], date_exact=d, date_start=d, date_end=d, year=int(d[:4]),
+ publication=LAT, rights_text=rights.COPYRIGHT_NOTE, repository=LAT, collection_name="latimes.com archive page",
+ record_url=c["url"], access_level=rights.ONLINE_METADATA,
+ notes=f"Quoted from {c['note']}: \"{c['quote']}\" | date evidence: \"{c['date_quote']}\"",
+ provenance=f"live:latimes.com quote-verified {db.now()[:10]}", acquisition_method="direct_html",
+ people=presidents_in_text(c["desc"]), subjects=["Editorial cartoons"])
+ self.save(rec)
+ n += 1
+ return n
def crawl(self) -> None:
- n = self.conn.execute("SELECT COUNT(*) FROM cartoons WHERE publication LIKE 'Los Angeles Times%' "
- "AND granularity='item'").fetchone()[0]
- self.status = "not_attempted"
- self.notes.append(f"paywalled — not crawled; {n} item records elsewhere are attributed to the LA Times")
+ found = self.discover()
+ galleries = sorted({u for u in found if "photogallery" in u} | {GALLERY})
+ g = sum(self.gallery(u) for u in galleries)
+ c = self.cites()
+ n_att = self.conn.execute("SELECT COUNT(*) FROM cartoons WHERE publication LIKE 'Los Angeles Times%' "
+ "AND granularity='item'").fetchone()[0]
+ self.notes.append(f"gallery slides saved={g} (from {len(galleries)} gallery page(s)); quote-verified article "
+ f"citations={c}; historical archive paywalled — not crawled; {n_att} item records overall are "
+ f"attributed to the LA Times; pre-2000 la-xpm slugs are opaque, so older articles about "
+ f"single cartoons are undiscoverable without the robots-disallowed /search")
+ self.status = "partial" if (g or c) else "failed"
CRAWLER = LATimes
diff --git a/src/conrad/crawlers/newspaper_probes.py b/src/conrad/crawlers/newspaper_probes.py
new file mode 100644
index 0000000..f754ecb
--- /dev/null
+++ b/src/conrad/crawlers/newspaper_probes.py
@@ -0,0 +1,76 @@
+"""Probes of free digitized-newspaper / museum archives that could show Conrad cartoons as printed (TK-12199 image
+priority). Each is tested through the project's polite client (robots.txt per path, descriptive UA) and registered
+with an honest status. Nothing is bypassed: a robots 'Disallow: /' or a 403 to our UA is recorded as blocked.
+
+Sources that actually yield records live in their own crawlers (ia_fulltext, discovery/DPLA, latimes)."""
+from __future__ import annotations
+
+import json
+
+from .. import db
+from .base import Blocked, Crawler, Transient
+
+# (source_id, name, probe url, params or None, how)
+PROBES = [
+ ("cdnc", "California Digital Newspaper Collection (cdnc.ucr.edu, Veridian)", "https://cdnc.ucr.edu/?a=q&txq=%22Paul+Conrad%22",
+ None, "plain"),
+ ("colorado_newspapers", "Colorado Historic Newspapers (Veridian)",
+ "https://www.coloradohistoricnewspapers.org/?a=q&txq=%22Paul+Conrad%22", None, "plain"),
+ ("chronicling_america", "Chronicling America (loc.gov, digitized newspapers to 1963)",
+ "https://www.loc.gov/collections/chronicling-america/",
+ {"q": '"paul conrad"', "fo": "json", "c": 100, "dates": "1946/1963"}, "loc_json"),
+ ("nara_artifacts", "NARA presidential-library museum collections (jfk/lbj/reagan/carter .artifacts.archives.gov)",
+ "https://jfk.artifacts.archives.gov/people/6554/paul-conrad-denver-post", None, "plain"),
+ ("google_news_archive", "Google News Archive (news.google.com/newspapers)",
+ "https://news.google.com/newspapers?nid=conrad", None, "plain"),
+]
+NOTES = {
+ "cdnc": "robots.txt 'User-agent: * Disallow: /' (only archive.org_bot/bingbot allowed, and even they may not query)",
+ "colorado_newspapers": "robots.txt 'User-agent: * Disallow: /' (Veridian default); Denver Post 1950-64 not reachable",
+ "nara_artifacts": "eMuseum sites list Conrad originals given to presidents (e.g. LBJ 'In the front door') but answer "
+ "HTTP 403 to this crawler's UA",
+ "google_news_archive": "robots.txt disallows /newspapers for generic agents",
+}
+
+
+class NewspaperProbes(Crawler):
+ source_id = "newspaper_probes"
+ name = "Digitized-newspaper / museum archive probes"
+ repository = None
+ url = "https://www.loc.gov/collections/chronicling-america/"
+ classification = "PUBLIC_HTML"
+ access_notes = "Reachability probes only; each probed archive is registered as its own source."
+
+ def crawl(self) -> None:
+ summary = []
+ for sid, name, url, params, how in PROBES:
+ status, cls, note = "worked", "PUBLIC_HTML", ""
+ try:
+ st, body = self.http.get(url, params=params)
+ self.stats["pages"] += 1
+ if st != 200:
+ status, note = "failed", f"HTTP {st}"
+ elif how == "loc_json":
+ d = json.loads(body)
+ n = (d.get("pagination") or {}).get("of") or 0
+ cls = "PUBLIC_API"
+ note = (f"q=\"paul conrad\" 1946-1963: {n} page hit(s); Conrad drew for the Denver Post (not in "
+ f"Chronicling America) and was syndicated only from 1964 -> 0 records")
+ if n:
+ note += "; hits: " + "; ".join(f"{r.get('date')} {str(r.get('title'))[:50]}"
+ for r in (d.get("results") or [])[:10])
+ else:
+ note = "reachable (HTTP 200); no parser for item-level results"
+ except Blocked as e:
+ status, cls = "blocked", "REQUIRES_PERMISSION"
+ note = f"{NOTES.get(sid, '')} ({e})".strip()
+ except Transient as e: # timeouts / 5xx after retries: unmeasured, NOT blocked
+ status, cls = "failed", ("PUBLIC_API" if how == "loc_json" else "PUBLIC_HTML")
+ note = f"not measured — endpoint timed out / errored after retries ({e})"
+ db.upsert_source(self.conn, sid, name, url=url, classification=cls, status=status, notes=note[:1500])
+ summary.append(f"{sid}={status}")
+ self.conn.commit()
+ self.notes.append("probes: " + ", ".join(summary))
+
+
+CRAWLER = NewspaperProbes
diff --git a/src/conrad/crawlers/secondary_citations.py b/src/conrad/crawlers/secondary_citations.py
index 092b5d2..94aa1c7 100644
--- a/src/conrad/crawlers/secondary_citations.py
+++ b/src/conrad/crawlers/secondary_citations.py
@@ -34,6 +34,8 @@ LAT = "https://www.latimes.com/archives/la-xpm-2010-sep-05-la-me-paul-conrad-201
CANYON = ("https://web.archive.org/web/20140819082434/http://www.canyon-news.com/artman2/publish/LifeStyleMillerTimes/"
"Paul_Conrad_A_Bitter_Appreciation_printer.php")
NCR = "http://natcath.org/NCR_Online/archives2/2001d/102601/102601j.htm"
+WRMEA = "https://www.wrmea.org/1988-july/pacific-perspective-conrad-cartoons.html"
+AP_NBC = "https://www.nbcnews.com/news/amp/wbna39006857"
PBS_STATE = "https://web.archive.org/web/20171101005025/http://www.pbs.org/independentlens/paulconrad/state.html"
_PBS_TS = {1: "20061201210139", 2: "20100907193806", 3: "20100908073239", 4: "20100909220206", 5: "20100908073123",
6: "20100908073128", 7: "20100908072321", 8: "20100909221123", 9: "20100910032059", 10: "20100908072326",
@@ -51,7 +53,9 @@ def PBS(n: int) -> str:
SITE = {WIKI: "Wikipedia, 'Paul Conrad'", LCIB: "Library of Congress Information Bulletin, Oct 1999",
LAT: "Los Angeles Times obituary, 5 Sep 2010", CANYON: "Canyon News, 3 Oct 2010 (Wayback copy)",
NCR: "National Catholic Reporter, 26 Oct 2001 (Wayback copy)",
- PBS_STATE: "PBS Independent Lens, 'The State of Political Cartooning' (Wayback copy)"}
+ PBS_STATE: "PBS Independent Lens, 'The State of Political Cartooning' (Wayback copy)",
+ WRMEA: "Washington Report on Middle East Affairs, July 1988, p. 12",
+ AP_NBC: "Associated Press obituary, 4 Sep 2010 (NBC News copy)"}
SITE.update({PBS(n): f"PBS Independent Lens 'Conrad Gallery' {n} of 27 (Wayback copy)" for n in _PBS_TS})
LAT_ = "Los Angeles Times"
@@ -220,6 +224,23 @@ C = [
"jockeys weighing in.",
cites=[(WIKI, "In April 1967, Conrad drew the cover for Time magazine in an issue about the potential "
"candidates for the 1968 United States presidential election", "(1967, April 14). Time, 89 (15)")]),
+ # ---------------- Washington Report on Middle East Affairs, July 1988 (TK-12199 cycle 4)
+ dict(slug="wrmea-talk-peace-1988",
+ title="I'm willing to talk peace, but there aren't any Palestinians to talk peace with.",
+ caption="I'm willing to talk peace, but there aren't any Palestinians to talk peace with.",
+ date="1988-04-27", pub=LAT_,
+ desc="A gun-wielding Israeli soldier standing atop Palestinian corpses (first intifada).",
+ note="year from the article's own date (WRMEA, July 1988): 'April 27' = 27 Apr 1988",
+ cites=[(WRMEA, "Palestinian corpses captioned: \"I'm willing to talk peace, but there aren't any Palestinians "
+ "to talk peace with.\"", "The struggle reached a climax April 27 with publication of a Conrad "
+ "cartoon")]),
+ # ---------------- Associated Press obituary (NBC News copy), 4 Sep 2010
+ dict(slug="ap-cuckoos-nest-1974", title="One flew over the cuckoo's nest", caption="One flew over the cuckoo's nest.",
+ year=1974, range=("1974-08-08", "1974-12-31"), pub=LAT_,
+ desc="Nixon's helicopter leaving the White House at his resignation.", seed="cat:obit-cuckoos-nest-1974",
+ note="date range = Nixon's resignation (8-9 Aug 1974) .. end of 1974; AP gives no exact date",
+ cites=[(AP_NBC, "Conrad drew Nixon's helicopter leaving the White House with the caption: \"One flew over the "
+ "cuckoo's nest.\"", "At the time of the president's resignation")]),
# ---------------- National Catholic Reporter, 26 Oct 2001
dict(slug="ncr-band-of-brothers-2001", title="Band of Brothers", year=2001, range=("2001-09-11", "2001-10-26"),
pub=LAT_, desc="Firemen at the World Trade Center, drawn in the wake of the Sept. 11 attacks.",
diff --git a/tests/test_dpla_authority.py b/tests/test_dpla_authority.py
new file mode 100644
index 0000000..b2e732a
--- /dev/null
+++ b/tests/test_dpla_authority.py
@@ -0,0 +1,31 @@
+"""DPLA: records pinned to Paul Conrad (b. 27 Jun 1924) by name authority are kept even without a cartoon keyword
+(NPG Time-cover originals); a same-authority SOUND record and a different Paul Conrad are still dropped."""
+import json
+
+from conftest import FakeHttp
+from conrad.crawlers import discovery
+from conrad.crawlers.base import Blocked
+
+NPG_DESC = ("In 1978, Time magazine donated approximately eight hundred works of original cover art to the National "
+ "Portrait Gallery.")
+
+
+def _doc(i, title, creators, types, prov="National Portrait Gallery", desc=NPG_DESC, obj=None):
+ return {"id": f"id{i}", "isShownAt": f"http://n2t.net/ark:/{i}", "object": obj, "dataProvider": {"name": prov},
+ "sourceResource": {"title": [title], "creator": creators, "type": types, "description": [desc],
+ "date": [{"displayDate": "1967"}]}}
+
+
+def test_dpla_authority_keeps_npg_time_covers(tmpdb, monkeypatch):
+ from conrad import config
+ monkeypatch.setattr(config, "secret", lambda name: "k" if name == "DPLA_API_KEY" else None)
+ docs = [_doc(1, "Weighing in for '68", ["Paul Conrad, born 27 Jun 1924", "Lyndon Baines Johnson"], ["image"],
+ obj="https://ids.si.edu/ids/deliveryService?id=NPG-NPG_78_TC179-000005"),
+ _doc(2, "Paul Conrad cartoons and drawings", ["Conrad, Paul, 1924-2010"], ["sound"], prov="LACMA"),
+ _doc(3, "Die Reformation", ["Conrad, Paul, 1865-1927"], ["text"], prov="SRLF", desc="Festschrift")]
+ http = FakeHttp({"api.dp.la": (200, json.dumps({"count": 3, "docs": docs})), "http": Blocked("probe blocked")})
+ discovery.CRAWLER(conn=tmpdb, http=http).run()
+ rows = tmpdb.execute("SELECT c.title, c.publication, cs.thumbnail_url FROM cartoons c "
+ "JOIN cartoon_sources cs ON cs.cartoon_id=c.id").fetchall()
+ assert [tuple(r) for r in rows] == [("Weighing in for '68", "Time",
+ "https://ids.si.edu/ids/deliveryService?id=NPG-NPG_78_TC179-000005")]
diff --git a/tests/test_ia_fulltext.py b/tests/test_ia_fulltext.py
new file mode 100644
index 0000000..9950c49
--- /dev/null
+++ b/tests/test_ia_fulltext.py
@@ -0,0 +1,110 @@
+"""IA full-text appearance parser + the base-client truncation fix (TK-12199)."""
+from conftest import FakeHttp
+from conrad.crawlers import base, ia_fulltext
+from conrad.crawlers.ia_fulltext import classify, issue_date
+
+CREDIT = ["Views / A portfolio from around the nation\n{{{Paul Conrad}}}\nThe Los Angeles Times\nLos Angeles Times Syndicate"]
+CAPS = ["OKAY ... LET'S MOVE IT\nPAUL {{{CONRAD}}} Los Angeles Times © Los Angeles Times Syndicate"]
+PROSE = ["commissions was for {{{Paul Conrad}}}, editorial cartoonist for the Los Angeles Times Syndicate. He wanted a Richard"]
+OTHER = ["Pricing Flukes Create Asia Arbitrage Play\nBy {{{Conrad}}} de Aenlle\nInternational Herald Tribune"]
+JOSEPH = ["Lord Jim and other tales. Joseph {{{Conrad}}}\n© Los Angeles Times Syndicate"]
+
+
+def test_classify_credit_lines():
+ assert classify(CREDIT)[0] == "credit"
+ kind, ev = classify(CAPS)
+ assert kind == "credit" and "Los Angeles Times" in ev
+
+
+def test_classify_rejects_prose_and_other_conrads():
+ # negative cases: an article ABOUT Conrad (long prose line) and other Conrads must never become records
+ assert classify(PROSE)[0] == "mention"
+ assert classify(OTHER)[0] == "other_conrad"
+ assert classify(JOSEPH)[0] == "other_conrad"
+ assert classify([])[0] == "mention"
+
+
+def test_issue_date():
+ assert issue_date({"file_basename": "Aug 11 1998, The Jerusalem Post, #20003, Israel (en)"}) == ("1998-08-11", 1998)
+ assert issue_date({"identifier": "The_Times_News_Idaho_Newspaper_1978_11_01"}) == ("1978-11-01", 1978)
+ assert issue_date({"identifier": "x", "year": 1975}) == (None, 1975)
+
+
+def _fts_page(hits):
+ import json
+ return json.dumps({"response": {"body": {"hits": {"total": len(hits), "hits": hits}}}})
+
+
+def test_crawl_saves_only_credit_pages(tmpdb):
+ hits = [
+ {"fields": {"identifier": "JP1980", "file_basename": "Mar 3 1980, The Jerusalem Post Magazine", "page_num": 7,
+ "title": "The Jerusalem Post Magazine , 1980, Israel, English", "year": 1980,
+ "collection": ["newspapers"], "result_in_subfile": True}, "highlight": {"text": CREDIT}},
+ {"fields": {"identifier": "IHT1998", "file_basename": "May 7 1998, IHT", "page_num": 3, "year": 1998,
+ "title": "International Herald Tribune , 1998", "collection": ["newspapers"]},
+ "highlight": {"text": OTHER}},
+ {"fields": {"identifier": "bestcartoons1975", "page_num": 40, "year": 1975, "title": "Best Editorial Cartoons",
+ "collection": ["inlibrary", "internetarchivebooks"]}, "highlight": {"text": CAPS}},
+ ]
+ http = FakeHttp({"page_production": (200, _fts_page(hits)),
+ "iiif.archive.org/iiif/3/JP1980/manifest.json": (200, '{"items": []}')})
+ c = ia_fulltext.CRAWLER(conn=tmpdb, http=http, years=[1980], variants=[("t", '"Paul Conrad"', range(1980, 1981))])
+ res = c.run()
+ rows = tmpdb.execute("SELECT c.title, c.date_exact, c.publication, cs.image_url FROM cartoons c "
+ "JOIN cartoon_sources cs ON cs.cartoon_id=c.id ORDER BY c.title").fetchall()
+ assert len(rows) == 2, rows
+ news = [r for r in rows if r["date_exact"] == "1980-03-03"][0]
+ assert news["publication"] == "The Jerusalem Post Magazine" and news["title"].startswith("[Paul Conrad cartoon")
+ book = [r for r in rows if r["date_exact"] is None][0]
+ assert book["image_url"] is None # lending-library book: no image URL recorded
+ # the lending-library book never asked IA for a manifest
+ assert not any("bestcartoons1975/manifest" in u for u in http.calls)
+ assert "other Conrads=1" in res["notes"]
+
+
+# ---- base client: the whole body must be read through ONE iter_content generator
+class _BufferingResp:
+ """Mimics a gzip response: a generator drains the raw stream into its own buffer on first use, so a SECOND
+ iter_content() call sees nothing (the pre-fix code truncated bodies to the first chunk this way)."""
+
+ status_code = 200
+ headers = {"content-type": "text/html; charset=utf-8"}
+ encoding = "utf-8"
+
+ def __init__(self, body: bytes):
+ self._raw = [body]
+
+ def iter_content(self, n):
+ buf = b"".join(self._raw)
+ self._raw = []
+ for i in range(0, len(buf), n):
+ yield buf[i:i + n]
+
+ def close(self):
+ pass
+
+ @property
+ def content(self):
+ return self._content
+
+ @property
+ def text(self):
+ return self._content.decode("utf-8")
+
+
+def test_fetch_reads_whole_body_with_one_generator(monkeypatch):
+ body = b"<!DOCTYPE html>" + b"x" * 200000
+ h = base.Http(use_cache=False)
+ monkeypatch.setattr(h.s, "get", lambda *a, **k: _BufferingResp(body))
+ monkeypatch.setattr(base.config, "REQUEST_DELAY", 0)
+ monkeypatch.setattr(h, "delay_for", lambda url: 0)
+ st, text = h._fetch("https://example.org/page", None)
+ assert st == 200 and len(text) == len(body)
+
+
+def test_fetch_negative_two_generators_would_truncate():
+ # negative control: the buffering fake really does lose data when iter_content is called twice
+ r = _BufferingResp(b"<!DOCTYPE html>" + b"x" * 1000)
+ head = next(r.iter_content(16))
+ rest = b"".join(r.iter_content(65536))
+ assert len(head + rest) == 16
diff --git a/tests/test_latimes.py b/tests/test_latimes.py
new file mode 100644
index 0000000..1d539c4
--- /dev/null
+++ b/tests/test_latimes.py
@@ -0,0 +1,54 @@
+"""latimes.com crawler: sitemap discovery, gallery slide parser, quote-verified citation (TK-12199)."""
+from conftest import FakeHttp
+from conrad.crawlers import latimes
+from conrad.crawlers.latimes import caption_year, parse_gallery
+
+IMG = "https://ca-times.brightspotcdn.com/dims4/default/abc/2147483647/strip/true/crop/300x410+0+0/resize/300x410!/quality/75/?url=x"
+
+
+def _slide(bsp, info, attr, alt, cap):
+ return (f'<div class="gallery-slide"><div class="gallery-slide-media" data-image-bsp-id="{bsp}" '
+ f'data-info-title="{info}" data-info-attribution="{attr}" ><picture><img class="image" alt="{alt}" '
+ f'src="{IMG}"></picture></div><div class="gallery-slide-caption">{cap}</div></div>')
+
+
+GALLERY = ('<html><meta property="og:title" content="Images: Paul Conrad dies at 86">'
+ + _slide("b1", "Abortion", "Paul Conrad / Los Angeles Times", "A 1976 panel",
+ "A 1976 panel showing Conrad's view on abortion then. (Paul Conrad / Los Angeles Times)")
+ + _slide("b2", "Oil spill", "Paul Conrad / Los Angeles Times", "oil",
+ "Union Oil's involvement in the 1969 oil spill. (Paul Conrad / Los Angeles Times)")
+ + _slide("b3", "Paul Conrad", "Los Angeles Times", "Conrad at his desk",
+ "Paul Conrad at his drawing table in 1984. (Los Angeles Times)") + "</html>")
+LETTER = ("<html><p>Re Paul Conrad’s Dec. 25 editorial cartoon (Commentary): the tripe that depicted a "
+ "flag-draped coffin with the caption, “I’ll be home for Christmas.... “</p></html>")
+SITEMAP_IDX = "<sitemapindex><sitemap><loc>https://www.latimes.com/sitemaps/sitemap-201009.xml</loc></sitemap></sitemapindex>"
+SITEMAP = ("<urlset><url><loc>https://www.latimes.com/nation/la-me-paul-conrad-pictures-photogallery.html</loc></url>"
+ "<url><loc>https://www.latimes.com/x/lauren-conrad-mtv</loc></url></urlset>")
+
+
+def test_parse_gallery_and_caption_year():
+ s = parse_gallery(GALLERY)
+ assert [x["bsp"] for x in s] == ["b1", "b2", "b3"] and s[0]["image"] == IMG
+ assert caption_year(s[0]["caption"]) == 1976
+ assert caption_year(s[1]["caption"]) is None # an EVENT year is not the cartoon's date (negative case)
+
+
+def test_crawl_gallery_skips_photos_and_verifies_quotes(tmpdb):
+ http = FakeHttp({"sitemaps/sitemap.xml": (200, SITEMAP_IDX), "sitemap-201009": (200, SITEMAP),
+ "photogallery": (200, GALLERY), "le-conrad29.1": (200, LETTER)})
+ res = latimes.CRAWLER(conn=tmpdb, http=http, years=(2010, 2010)).run()
+ rows = {r["canonical_id"]: r for r in tmpdb.execute("SELECT * FROM cartoons")}
+ assert set(rows) == {"latg:b1", "latg:b2", "lat:lat-home-for-christmas-2003"} # b3 = a PHOTO of Conrad: skipped
+ assert rows["latg:b1"]["year"] == 1976 and rows["latg:b2"]["year"] is None
+ assert rows["lat:lat-home-for-christmas-2003"]["date_exact"] == "2003-12-25"
+ assert res["status"] == "partial" and "slug=1" in res["notes"]
+ # no image was ever requested
+ assert not any("brightspotcdn" in u for u in http.calls)
+
+
+def test_citation_rejected_when_quote_missing(tmpdb):
+ http = FakeHttp({"sitemaps/sitemap.xml": (200, SITEMAP_IDX), "sitemap-201009": (200, "<urlset/>"),
+ "photogallery": (200, "<html></html>"), "le-conrad29.1": (200, "<html>unrelated letters</html>")})
+ res = latimes.CRAWLER(conn=tmpdb, http=http, years=(2010, 2010)).run()
+ assert tmpdb.execute("SELECT COUNT(*) FROM cartoons").fetchone()[0] == 0
+ assert res["status"] == "failed" and res["errors"] >= 1
diff --git a/tests/test_parsers.py b/tests/test_parsers.py
index affa470..2709684 100644
--- a/tests/test_parsers.py
+++ b/tests/test_parsers.py
@@ -88,10 +88,14 @@ def test_challenge_detection():
# ---- paywalled registrations
def test_paywalled_not_attempted(tmpdb):
- for mod in (latimes, denver_post):
- assert mod.CRAWLER(conn=tmpdb, http=FakeHttp({})).run()["status"] == "not_attempted"
+ assert denver_post.CRAWLER(conn=tmpdb, http=FakeHttp({})).run()["status"] == "not_attempted"
cls = {r[0]: r[1] for r in tmpdb.execute("SELECT id, classification FROM sources")}
- assert cls["latimes"] == "PAYWALLED" and cls["denver_post"] == "PAYWALLED"
+ assert cls["denver_post"] == "PAYWALLED"
+ # latimes (TK-12199): crawls only free latimes.com pages; the paywalled archive hosts are never contacted
+ http = FakeHttp({})
+ latimes.CRAWLER(conn=tmpdb, http=http, years=(2010, 2010)).run()
+ assert http.calls and all("latimes.com" in u for u in http.calls)
+ assert not any(h in u for u in http.calls for h in ("newspapers.com", "proquest"))
# ---- daily iowan inventory
← 020434a auto-data-snapshot: 2026-09-25T00:34:03 (1 data files) — dat
·
back to Paul Conrad Archive
·
auto-data-snapshot: 2026-09-25T10:52:57 (1 data files) — dat 851fc79 →