← back to Paul Conrad Archive
src/conrad/crawlers/daily_iowan.py
137 lines
"""The Daily Iowan (University of Iowa student paper; Conrad drew for it c.1946-1950).
The UI Libraries host issue-level PDFs at dailyiowan.lib.uiowa.edu/DI/<yyyy>/di<yyyy-mm-dd>.pdf.
This crawler:
1. inventories every issue PDF URL for 1945-1950 from the public year pages (it NEVER downloads a PDF);
2. probes, robots-checked and non-PDF only, whether a separate TEXT/OCR layer is exposed anywhere a polite crawler
may read it (per-issue .txt / ALTO .xml / .hocr / .html sidecars, the site search endpoint, the Iowa Digital
Library host) — the text layer, if any, would otherwise live only inside the PDFs, which we may not fetch;
3. writes the full issue list to data/exports/daily_iowan_issues_1945_1950.csv for later human page review.
It never OCRs scans and never invents cartoon records: with no permitted text layer there is nothing item-level to add.
"""
from __future__ import annotations
import csv
import re
from datetime import date
from .. import config
from .base import Blocked, Crawler, Transient
BASE = "https://dailyiowan.lib.uiowa.edu/"
YEARS = range(1945, 1951)
ISSUE_RE = re.compile(r"(?:https://dailyiowan\.lib\.uiowa\.edu/)?DI/(19[45]\d)/di(\d{4}-\d{2}-\d{2})\.pdf")
SIDE_EXTS = (".txt", ".xml", ".hocr", ".html") # text-layer sidecars a digitisation pipeline might expose (never .pdf)
EXPORT = "daily_iowan_issues_1945_1950.csv"
# Checked by hand 2026-09-24 (TK-12199): loc.gov Chronicling America Iowa titles 1946-1950 do not include The Daily
# Iowan (only 9 small Iowa titles); archive.org has no Daily Iowan run (2 unrelated items). Recorded, not re-queried.
OFFSITE_NOTE = ("not in Chronicling America (loc.gov Iowa titles 1946-1950 exclude it) and no Internet Archive run "
"(checked 2026-09-24)")
def parse_issues(html: str) -> list[tuple[str, str]]:
"""-> sorted unique (iso_date, pdf_url) pairs listed on a year page."""
out = {d: f"{BASE}DI/{y}/di{d}.pdf" for y, d in ISSUE_RE.findall(html)}
return sorted(out.items())
def is_google_cse(html: str) -> bool:
return "cse.google.com/cse.js" in html or "gcse" in html
class DailyIowan(Crawler):
source_id = "daily_iowan"
name = "The Daily Iowan digital archive (University of Iowa Libraries)"
repository = "University of Iowa Libraries"
url = BASE
classification = "PUBLIC_HTML"
access_notes = ("Issue-level PDFs only, public (robots allows /). No separate text/OCR layer is exposed: no "
".txt/ALTO/hOCR sidecars, search is a Google CSE embed (cse.google.com robots Disallow: /), "
"Iowa Digital Library host 403s even robots.txt. Any OCR text lives only inside the PDFs, which "
"this project may not download. Conrad's student cartoons need manual page review — issue list in "
f"data/exports/{EXPORT}.")
def crawl(self) -> None:
issues: dict[int, list[tuple[str, str]]] = {}
for y in YEARS:
url = f"{BASE}{y}.php"
try:
st, html = self.http.get(url)
except (Blocked, Transient) as e:
self.error(url, e)
continue
self.stats["pages"] += 1
if st == 200:
issues[y] = [(d, u) for d, u in parse_issues(html) if d.startswith(str(y))]
if not self.stats["pages"]:
self.status = "blocked"
return
self._write_issue_list(issues)
counts = {y: len(v) for y, v in issues.items()}
self.notes.append(f"issue PDFs listed 1945-1950: {counts} (total {sum(counts.values())}); "
f"full list -> data/exports/{EXPORT}; no PDF downloaded")
self.notes.append("text-layer probe: " + self._probe_text_layer(issues))
self.notes.append(OFFSITE_NOTE)
self.notes.append("0 cartoon records added: no permitted text layer; OCR of scans is out of bounds")
self.status = "partial"
# ------------------------------------------------------------------ helpers
def _write_issue_list(self, issues: dict[int, list[tuple[str, str]]]) -> None:
path = config.EXPORT_DIR / EXPORT
with path.open("w", newline="") as f:
w = csv.writer(f)
w.writerow(["issue_date", "weekday", "year", "conrad_ui_years_1946_1950", "year_page", "issue_pdf_url"])
for y in sorted(issues):
for d, u in issues[y]:
w.writerow([d, date.fromisoformat(d).strftime("%a"), y, int(1946 <= y <= 1950),
f"{BASE}{y}.php", u])
def _probe(self, url: str) -> str:
try:
st, body = self.http.get(url)
except Blocked as e:
return f"blocked ({str(e)[:80]})"
except Transient as e:
self.error(url, e)
return f"error ({str(e)[:60]})"
self.stats["pages"] += 1
return f"HTTP {st}"
def _allowed(self, url: str) -> bool:
fn = getattr(self.http, "allowed", None)
return bool(fn(url)) if fn else False # unknown robots state -> treat as disallowed
def _probe_text_layer(self, issues: dict[int, list[tuple[str, str]]]) -> str:
found, parts = [], []
sample = [v[len(v) // 2][1] for v in (issues.get(1947), issues.get(1949)) if v]
for pdf in sample:
stem = pdf[:-4]
for ext in SIDE_EXTS:
r = self._probe(stem + ext)
parts.append(f"{stem.rsplit('/', 1)[1]}{ext}={r}")
if r == "HTTP 200":
found.append(stem + ext)
# issue directory listing
parts.append(f"DI/1948/ listing={self._probe(BASE + 'DI/1948/')}")
# site search endpoint: is it a real full-text API or a third-party embed?
try:
st, html = self.http.get(BASE + "search/", params={"q": "Paul Conrad"})
self.stats["pages"] += 1
if is_google_cse(html):
cse_ok = self._allowed("https://cse.google.com/cse/element/v1?q=conrad")
parts.append("search/=Google CSE embed (cse.google.com robots " + ("allows" if cse_ok else "Disallow: /")
+ " -> not used)")
else:
parts.append(f"search/=HTTP {st} non-CSE (manual follow-up)")
except (Blocked, Transient) as e:
parts.append(f"search/=unavailable ({str(e)[:60]})")
# Iowa Digital Library host
idl = "https://digital.lib.uiowa.edu/"
parts.append("digital.lib.uiowa.edu=" + ("robots readable" if self._allowed(idl) else
"robots unreadable/disallowed (403) -> not crawled"))
verdict = ("TEXT LAYER FOUND: " + ", ".join(found)) if found else "NO separate text layer exposed"
return verdict + " [" + "; ".join(parts) + "]"
CRAWLER = DailyIowan