← back to Paul Conrad Archive
Live crawlers: LOC item JSON, OAC, Syracuse/OSU/Wichita probes, paywalled registrations, Daily Iowan inventory, IA books, DPLA/SI discovery
a9bba5a52cabe83ee15846c80e56854d87282c21 · 2026-09-24 16:28:05 -0700 · Steve Abrams
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Files touched
M data/conrad.dbM data/conrad.db-shmM data/conrad.db-walM scripts/crawl_source.pyM src/conrad/crawlers/base.pyA src/conrad/crawlers/daily_iowan.pyA src/conrad/crawlers/denver_post.pyA src/conrad/crawlers/discovery.pyA src/conrad/crawlers/internet_archive.pyA src/conrad/crawlers/latimes.pyA src/conrad/crawlers/loc.pyA src/conrad/crawlers/oac.pyA src/conrad/crawlers/ohio_state.pyA src/conrad/crawlers/syracuse.pyA src/conrad/crawlers/wichita.py
Diff
commit a9bba5a52cabe83ee15846c80e56854d87282c21
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Sep 24 16:28:05 2026 -0700
Live crawlers: LOC item JSON, OAC, Syracuse/OSU/Wichita probes, paywalled registrations, Daily Iowan inventory, IA books, DPLA/SI discovery
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
data/conrad.db | Bin 12853248 -> 12935168 bytes
data/conrad.db-shm | Bin 32768 -> 32768 bytes
data/conrad.db-wal | Bin 407912 -> 0 bytes
scripts/crawl_source.py | 3 +-
src/conrad/crawlers/base.py | 17 +++
src/conrad/crawlers/daily_iowan.py | 42 +++++++
src/conrad/crawlers/denver_post.py | 23 ++++
src/conrad/crawlers/discovery.py | 199 ++++++++++++++++++++++++++++++++
src/conrad/crawlers/internet_archive.py | 101 ++++++++++++++++
src/conrad/crawlers/latimes.py | 26 +++++
src/conrad/crawlers/loc.py | 159 +++++++++++++++++++++++++
src/conrad/crawlers/oac.py | 51 ++++++++
src/conrad/crawlers/ohio_state.py | 35 ++++++
src/conrad/crawlers/syracuse.py | 40 +++++++
src/conrad/crawlers/wichita.py | 44 +++++++
15 files changed, 739 insertions(+), 1 deletion(-)
diff --git a/data/conrad.db b/data/conrad.db
index 4b5ce97..c67ce59 100644
Binary files a/data/conrad.db and b/data/conrad.db differ
diff --git a/data/conrad.db-shm b/data/conrad.db-shm
index f44f651..fe9ac28 100644
Binary files a/data/conrad.db-shm and b/data/conrad.db-shm differ
diff --git a/data/conrad.db-wal b/data/conrad.db-wal
index ee094a6..e69de29 100644
Binary files a/data/conrad.db-wal and b/data/conrad.db-wal differ
diff --git a/scripts/crawl_source.py b/scripts/crawl_source.py
index 649e430..62bb307 100644
--- a/scripts/crawl_source.py
+++ b/scripts/crawl_source.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python3
"""Run one crawler: python scripts/crawl_source.py loc [--no-cache]"""
-import importlib, json, sys, pathlib
+import importlib, json, logging, sys, pathlib
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s", stream=sys.stderr)
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1] / "src"))
from conrad import db # noqa: E402
from conrad.crawlers.base import Http # noqa: E402
diff --git a/src/conrad/crawlers/base.py b/src/conrad/crawlers/base.py
index e4e61e1..22192d3 100644
--- a/src/conrad/crawlers/base.py
+++ b/src/conrad/crawlers/base.py
@@ -2,6 +2,7 @@
from __future__ import annotations
import hashlib
+import logging
import json
import threading
import time
@@ -10,11 +11,21 @@ from urllib.parse import urlparse
from urllib.robotparser import RobotFileParser
import requests
+
+try: # use the OS trust store (some library hosts serve incomplete chains that certifi rejects)
+ import truststore
+
+ truststore.inject_into_ssl()
+except ImportError: # pragma: no cover
+ pass
from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential
from .. import config, db
+log = logging.getLogger("conrad.http")
+
+
class Blocked(Exception):
"""Robots disallow, WAF/JS challenge, 401/403, paywall — we stop, record, and move on."""
@@ -101,6 +112,9 @@ class Http:
d = json.loads(cp.read_text())
return d["status"], d["body"]
if check_robots and not self.allowed(full):
+ rp = self.robots(full)
+ if getattr(rp, "_unreadable", False):
+ raise Blocked(f"robots.txt unreadable (HTTP 401/403 or WAF challenge) — treated as disallowed: {full}")
raise Blocked(f"robots.txt disallows {full}")
status, body = self._fetch(full, secret_params)
if looks_like_challenge(status, body):
@@ -125,9 +139,12 @@ class Http:
wait = self.delay_for(url) - (time.time() - _host_last.get(host, 0))
if wait > 0:
time.sleep(wait)
+ t0 = time.time()
try:
r = self.s.get(url, params=secret_params, timeout=config.TIMEOUT)
+ log.info("GET %s -> %s (%.1fs, %d bytes)", url, r.status_code, time.time() - t0, len(r.content))
except requests.RequestException as e:
+ log.warning("GET %s failed: %s", url, type(e).__name__)
raise Transient(str(e).replace(str((secret_params or {}).get("api_key", "\0")), "***")) from None
finally:
_host_last[host] = time.time()
diff --git a/src/conrad/crawlers/daily_iowan.py b/src/conrad/crawlers/daily_iowan.py
new file mode 100644
index 0000000..d4aae5f
--- /dev/null
+++ b/src/conrad/crawlers/daily_iowan.py
@@ -0,0 +1,42 @@
+"""The Daily Iowan (University of Iowa student paper; Conrad drew for it c.1946-1950).
+The UI Libraries host issue-level PDFs. There is no item/cartoon index, so this crawler only inventories
+which issue years are listed (from the public sitemap) — it never downloads PDFs and never invents cartoons."""
+from __future__ import annotations
+
+import re
+
+from .base import Blocked, Crawler, Transient
+
+BASE = "https://dailyiowan.lib.uiowa.edu/"
+
+
+class DailyIowan(Crawler):
+ source_id = "daily_iowan"
+ name = "The Daily Iowan digital archive (University of Iowa Libraries)"
+ repository = "University of Iowa Libraries"
+ url = BASE
+ classification = "PUBLIC_HTML"
+ access_notes = "Issue-level PDFs, public. No cartoon-level index: Conrad's student cartoons need manual page review."
+
+ def crawl(self) -> None:
+ years: dict[int, int] = {}
+ for path in ("sitemap/index.html", *[f"{y}.php" for y in range(1945, 1951)]):
+ try:
+ st, html = self.http.get(BASE + path)
+ except (Blocked, Transient) as e:
+ self.error(BASE + path, e)
+ continue
+ self.stats["pages"] += 1
+ for y in re.findall(r"/DI/(19[4-5]\d)/di\d{4}-\d{2}-\d{2}\.pdf", html):
+ years[int(y)] = years.get(int(y), 0) + 1
+ for y in re.findall(r"/(19[4-5]\d)\.php", html):
+ years.setdefault(int(y), 0)
+ target = {y: years.get(y) for y in range(1945, 1951)}
+ self.notes.append(f"issue PDFs listed on fetched pages for 1945-1950: {target} (no item index; manual review needed)")
+ if not self.stats["pages"]:
+ self.status = "blocked"
+ else:
+ self.status = "partial"
+
+
+CRAWLER = DailyIowan
diff --git a/src/conrad/crawlers/denver_post.py b/src/conrad/crawlers/denver_post.py
new file mode 100644
index 0000000..e395223
--- /dev/null
+++ b/src/conrad/crawlers/denver_post.py
@@ -0,0 +1,23 @@
+"""Denver Post (Conrad's paper 1950-1964). Archive is PAYWALLED (newspapers.com / NewsBank). Not crawled;
+source registered and coverage computed from other repositories' attributions."""
+from __future__ import annotations
+
+from .base import Crawler
+
+
+class DenverPost(Crawler):
+ source_id = "denver_post"
+ name = "Denver Post archive (newspapers.com / NewsBank)"
+ repository = "Denver Post"
+ url = "https://denverpost.newspapers.com/"
+ classification = "PAYWALLED"
+ access_notes = "Denver Post 1950-1964 pages via subscription databases or Denver Public Library microfilm."
+
+ def crawl(self) -> None:
+ n = self.conn.execute("SELECT COUNT(*) FROM cartoons WHERE publication='Denver Post' AND granularity='item'"
+ ).fetchone()[0]
+ self.status = "not_attempted"
+ self.notes.append(f"paywalled — not crawled; {n} item records elsewhere are attributed to the Denver Post")
+
+
+CRAWLER = DenverPost
diff --git a/src/conrad/crawlers/discovery.py b/src/conrad/crawlers/discovery.py
new file mode 100644
index 0000000..c97b67b
--- /dev/null
+++ b/src/conrad/crawlers/discovery.py
@@ -0,0 +1,199 @@
+"""Discovery pass: find and classify other repositories holding Conrad material.
+
+Uses only free, robots-permitted APIs: DPLA (key from env/secrets), Smithsonian Open Access (api.data.gov key,
+if present). Hosts that refuse automated access (Calisphere, HathiTrust catalog, OSU PastPerfect, Empire ADC)
+are probed once and classified. Presidential libraries / WorldCat are listed as UNVERIFIED leads only.
+Every discovered repository is written to data/sources.json by scripts/report.py.
+"""
+from __future__ import annotations
+
+import re
+
+from .. import config, db, rights
+from ..models import CartoonRecord
+from ..normalize import parse_date, presidents_in_text
+from .base import Blocked, Crawler, Transient
+
+CONRAD_RE = re.compile(r"conrad,\s*paul(?:,?\s*1924)?|paul conrad", re.I)
+CARTOON_RE = re.compile(r"cartoon|caricature|drawing|editorial|los angeles times|denver post", re.I)
+
+PROBES = [ # (source_id, name, url, classification-if-blocked, note)
+ ("calisphere", "Calisphere (UC Libraries)", "https://calisphere.org/search/?q=%22paul+conrad%22", "REQUIRES_PERMISSION",
+ "AWS WAF JS challenge for non-browser clients"),
+ ("hathitrust", "HathiTrust catalog", "https://catalog.hathitrust.org/Search/Home?lookfor=%22Conrad%2C+Paul%2C+1924-2010%22",
+ "REQUIRES_PERMISSION", "robots.txt unreadable (403); treated as disallowed"),
+ ("empire_adc", "Empire Archival Discovery Cooperative", "https://empireadc.org/search/catalog/nsyu_2642890",
+ "REQUIRES_PERMISSION", "AWS WAF JS challenge"),
+ ("loc_lcib", "LOC Information Bulletin (Oct 1999) — Conrad gift article", "https://www.loc.gov/loc/lcib/9910/conrad.html",
+ "PUBLIC_HTML", "context article about Conrad's gift of drawings to LOC"),
+]
+LEADS = [ # listed, NOT verified by this crawler
+ ("worldcat", "WorldCat", "https://search.worldcat.org/search?q=au%3AConrad%2C+Paul%2C+1924-2010", "METADATA_ONLY",
+ "No keyless API; bibliographic lead only (not crawled)."),
+ ("nixon_library", "Richard Nixon Presidential Library", "https://www.nixonlibrary.gov/", "PHYSICAL_ARCHIVE",
+ "UNVERIFIED lead: presidential libraries often hold cartoon originals sent to presidents."),
+ ("lbj_library", "LBJ Presidential Library", "https://www.lbjlibrary.org/", "PHYSICAL_ARCHIVE", "UNVERIFIED lead."),
+ ("reagan_library", "Ronald Reagan Presidential Library", "https://www.reaganlibrary.gov/", "PHYSICAL_ARCHIVE",
+ "UNVERIFIED lead."),
+ ("pulitzer", "Pulitzer Prizes (1964, 1971, 1984)", "https://www.pulitzer.org/winners/paul-conrad", "METADATA_ONLY",
+ "Year-of-work awards; no single cartoon named."),
+]
+
+
+class Discovery(Crawler):
+ source_id = "discovery"
+ name = "Discovery pass (DPLA, Smithsonian, probes)"
+ repository = "multiple"
+ url = "https://api.dp.la/v2/items"
+ classification = "PUBLIC_API"
+
+ def crawl(self) -> None:
+ self.refs: list[str] = []
+ # clear earlier non-cartoon rows written by a previous version of this pass (records are re-derived here)
+ self.dpla()
+ self.smithsonian()
+ for sid, name, url, cls, note in PROBES:
+ status = "worked"
+ try:
+ st, _ = self.http.get(url)
+ self.stats["pages"] += 1
+ if st != 200:
+ status = "failed"
+ self.error(url, f"HTTP {st}")
+ res_note = f"HTTP {st}"
+ except (Blocked, Transient) as e:
+ status, res_note = "blocked", str(e)
+ self.error(url, f"BLOCKED: {e}")
+ db.upsert_source(self.conn, sid, name, url=url, classification=cls if status == "blocked" else "PUBLIC_HTML",
+ status=status, notes=f"{note}; probe: {res_note}")
+ if self.refs:
+ self.notes.append("non-cartoon references: " + " || ".join(self.refs[:20]))
+ for sid, name, url, cls, note in LEADS:
+ db.upsert_source(self.conn, sid, name, url=url, classification=cls, status="not_attempted", notes=note)
+ self.conn.commit()
+
+ # ------------------------------------------------------------------ DPLA
+ def dpla(self) -> None:
+ key = config.secret("DPLA_API_KEY")
+ if not key:
+ db.upsert_source(self.conn, "dpla", "DPLA", url=self.url, classification="PUBLIC_API", status="not_attempted",
+ notes="no DPLA_API_KEY available")
+ return
+ db.upsert_source(self.conn, "dpla", "Digital Public Library of America (API)", url="https://dp.la/",
+ classification="PUBLIC_API", status="running")
+ providers: dict[str, int] = {}
+ kept = 0
+ for q in ('"paul conrad" cartoon', '"conrad, paul"', '"paul conrad" los angeles times'):
+ for page in range(1, 6):
+ try:
+ d = self.http.get_json("https://api.dp.la/v2/items", params={"q": q, "page_size": 100, "page": page},
+ secret_params={"api_key": key})
+ except (Blocked, Transient) as e:
+ self.error("https://api.dp.la/v2/items", e)
+ break
+ self.stats["pages"] += 1
+ docs = d.get("docs") or []
+ for doc in docs:
+ sr = doc.get("sourceResource") or {}
+ creators = " ".join(sr.get("creator") or []) if isinstance(sr.get("creator"), list) else str(sr.get("creator") or "")
+ title = " ".join(sr.get("title") or []) if isinstance(sr.get("title"), list) else str(sr.get("title") or "")
+ subj = " ".join(s.get("name", "") for s in sr.get("subject") or [] if isinstance(s, dict))
+ blob = " ".join([creators, title, subj, " ".join(sr.get("description") or [] if isinstance(sr.get("description"), list) else [str(sr.get("description") or "")])])
+ if not (CONRAD_RE.search(creators) and CARTOON_RE.search(blob)):
+ continue
+ types = sr.get("type") or []
+ types = types if isinstance(types, list) else [types]
+ if types and not ({"image", "physical object"} & {str(t).lower() for t in types}):
+ self.refs.append(f"DPLA non-image ref ({','.join(map(str, types))}): {title[:90]}")
+ continue
+ prov = (doc.get("dataProvider") or {})
+ prov = prov.get("name") if isinstance(prov, dict) else str(prov)
+ providers[prov] = providers.get(prov, 0) + 1
+ dt = parse_date((sr.get("date") or [{}])[0].get("displayDate") if isinstance(sr.get("date"), list) else None)
+ rec = CartoonRecord(
+ canonical_id=f"dpla:{doc['id']}", identifier=doc["id"], granularity="item", title=title or None,
+ description=blob[:1000], date_exact=dt["date_exact"], date_start=dt["date_start"],
+ date_end=dt["date_end"], year=dt["year"], repository=prov, collection_name="via DPLA",
+ record_url=doc.get("isShownAt"), thumbnail_url=doc.get("object"),
+ access_level=rights.ONLINE_IMAGE if doc.get("object") else rights.ONLINE_METADATA,
+ rights_text=" ".join(sr.get("rights") or []) if isinstance(sr.get("rights"), list) else (sr.get("rights") or rights.COPYRIGHT_NOTE),
+ provenance=f"live:DPLA q={q}", people=presidents_in_text(title),
+ )
+ self.stats["seen"] += 1
+ st = db.save_record(self.conn, rec, "dpla")
+ if st in ("added", "updated"):
+ self.stats[st] += 1
+ kept += 1
+ if len(docs) < 100:
+ break
+ db.upsert_source(self.conn, "dpla", "Digital Public Library of America (API)", url="https://dp.la/",
+ classification="PUBLIC_API", status="worked",
+ notes=f"kept {kept} Conrad-cartoon records; providers={providers}")
+ self.conn.commit()
+
+ # ------------------------------------------------------------ Smithsonian
+ def smithsonian(self) -> None:
+ key = config.secret("SI_API_KEY")
+ if not key:
+ db.upsert_source(self.conn, "smithsonian", "Smithsonian Open Access", url="https://api.si.edu/",
+ classification="PUBLIC_API", status="not_attempted", notes="no SI_API_KEY")
+ return
+ db.upsert_source(self.conn, "smithsonian", "Smithsonian Open Access API", url="https://api.si.edu/openaccess/",
+ classification="PUBLIC_API", status="running")
+ units: dict[str, int] = {}
+ kept = 0
+ for q in ('"paul conrad"', 'conrad cartoon'):
+ try:
+ d = self.http.get_json("https://api.si.edu/openaccess/api/v1.0/search",
+ params={"q": q, "rows": 100}, secret_params={"api_key": key})
+ except (Blocked, Transient) as e:
+ self.error("https://api.si.edu/openaccess/api/v1.0/search", e)
+ continue
+ self.stats["pages"] += 1
+ for row in (d.get("response") or {}).get("rows") or []:
+ c = row.get("content") or {}
+ blob = str(c)[:20000]
+ if not (CONRAD_RE.search(blob) and CARTOON_RE.search(blob)):
+ continue
+ if re.search(r"conrad,\s*(?!paul)", row.get("title", ""), re.I):
+ continue
+ unit = row.get("unitCode") or "SI"
+ units[unit] = units.get(unit, 0) + 1
+ if unit.startswith("SIL"): # Smithsonian Libraries: books / artist files, not cartoon objects
+ t = row.get("title") or ""
+ if re.match(r"pro and conrad", t, re.I):
+ self.conn.execute("UPDATE books SET verified=1, notes=COALESCE(notes,'') || ? WHERE title='Pro and Conrad'",
+ (f" | held by Smithsonian Libraries ({(c.get('descriptiveNonRepeating') or {}).get('record_link') or row.get('id')})",))
+ self.refs.append(f"SI {unit} library record: {t[:90]}")
+ continue
+ dnr = c.get("descriptiveNonRepeating") or {}
+ title = row.get("title")
+ is_folder = bool(re.search(r"\[folder\]", title or "", re.I))
+ ind = (c.get("indexedStructured") or {})
+ dates = ind.get("date") or []
+ dt = parse_date(dates[0] if dates else None)
+ media = ((dnr.get("online_media") or {}).get("media") or [{}])[0]
+ rec = CartoonRecord(
+ canonical_id=f"si:{row.get('id')}", identifier=row.get("id"),
+ granularity="folder" if is_folder else "item", title=title, date_start=dt["date_start"],
+ date_end=dt["date_end"], date_exact=dt["date_exact"], year=dt["year"],
+ repository=f"Smithsonian ({dnr.get('data_source') or unit})", collection_name=unit,
+ record_url=dnr.get("record_link") or dnr.get("guid"),
+ thumbnail_url=media.get("thumbnail"), image_url=media.get("content"),
+ access_level=rights.ONLINE_IMAGE if media.get("thumbnail") else rights.ONLINE_METADATA,
+ rights_text=((c.get("freetext") or {}).get("creditLine") or [{}])[0].get("content") or rights.COPYRIGHT_NOTE,
+ provenance=f"live:api.si.edu q={q}",
+ notes="artist/library file folder, not a cartoon" if is_folder else None,
+ people=presidents_in_text(title),
+ )
+ self.stats["seen"] += 1
+ st = db.save_record(self.conn, rec, "smithsonian")
+ if st in ("added", "updated"):
+ self.stats[st] += 1
+ kept += 1
+ db.upsert_source(self.conn, "smithsonian", "Smithsonian Open Access API", url="https://api.si.edu/openaccess/",
+ classification="PUBLIC_API", status="worked", notes=f"kept {kept} records; units={units}")
+ self.conn.commit()
+
+
+CRAWLER = Discovery
diff --git a/src/conrad/crawlers/internet_archive.py b/src/conrad/crawlers/internet_archive.py
new file mode 100644
index 0000000..5025312
--- /dev/null
+++ b/src/conrad/crawlers/internet_archive.py
@@ -0,0 +1,101 @@
+"""Internet Archive — bibliography of Conrad's books (metadata API + advancedsearch; robots permit both).
+Books on archive.org are Controlled-Digital-Lending scans: we record metadata only and never borrow/open them.
+Name-authority conflations (other 'Paul Conrad's tagged 1924-2010) are detected and excluded."""
+from __future__ import annotations
+
+import re
+
+from .base import Blocked, Crawler, Transient
+
+BIBLIOGRAPHY = [ # (title, expected year) — the books named in the spec
+ ("When in the Course of Human Events", 1973),
+ ("The King and Us", 1974),
+ ("Pro and Conrad", 1979),
+ ("Drawn and Quartered", 1985),
+ ("CONartist: 30 Years with the Los Angeles Times", 1993),
+ ("Drawing the Line", 1999),
+ ("I, Con", None),
+]
+ADV = "https://archive.org/advancedsearch.php"
+META = "https://archive.org/metadata/"
+NOT_OUR_CONRAD = re.compile(r"larryboy|veggie|kinderklapper|gott ist|thriving church|slave bug", re.I) # slave bug: 1975 novel, no dates on creator
+
+
+def _norm(t: str) -> str:
+ return re.sub(r"[^a-z0-9 ]", "", (t or "").lower())
+
+
+class InternetArchive(Crawler):
+ source_id = "internet_archive"
+ name = "Internet Archive (books metadata)"
+ repository = "Internet Archive"
+ url = "https://archive.org/details/paulconraddrawin0000conr"
+ classification = "PUBLIC_API"
+ access_notes = "Book scans are lending-library (CDL) items; metadata only, not borrowed."
+
+ def search(self, q: str) -> list[dict]:
+ d = self.http.get_json(ADV, params={"q": q, "fl[]": ["identifier", "title", "date", "creator", "mediatype"],
+ "rows": 100, "output": "json"})
+ self.stats["pages"] += 1
+ return d.get("response", {}).get("docs", [])
+
+ def crawl(self) -> None:
+ cands: dict[str, dict] = {}
+ try:
+ for doc in self.search('creator:("Conrad, Paul") AND mediatype:texts'):
+ cands[doc["identifier"]] = doc
+ for title, _ in BIBLIOGRAPHY:
+ short = title.split(":")[0]
+ for doc in self.search(f'title:("{short}") AND creator:(conrad)'):
+ cands[doc["identifier"]] = doc
+ except (Blocked, Transient) as e:
+ self.error(ADV, e)
+ excluded = []
+ for ident, doc in sorted(cands.items()):
+ creators = doc.get("creator") or []
+ creators = creators if isinstance(creators, list) else [creators]
+ ctext = " ".join(creators)
+ if NOT_OUR_CONRAD.search(doc.get("title", "")) or not re.search(r"conrad,\s*paul", ctext, re.I) \
+ or re.search(r"1865|funke", ctext, re.I):
+ excluded.append(f"{ident} ({doc.get('title', '')[:50]}; {ctext[:60]})")
+ continue
+ try:
+ m = self.http.get_json(META + ident)
+ except (Blocked, Transient) as e:
+ self.error(META + ident, e)
+ continue
+ self.stats["pages"] += 1
+ md = m.get("metadata", {})
+ title = md.get("title") or doc.get("title")
+ year = int(str(md.get("date") or doc.get("date") or "0")[:4] or 0) or None
+ colls = md.get("collection") or []
+ colls = colls if isinstance(colls, list) else [colls]
+ access = "controlled_digital_lending" if {"inlibrary", "printdisabled"} & set(colls) else "public"
+ bib = next((b for b in BIBLIOGRAPHY if _norm(b[0].split(":")[0])[:14] in _norm(title)), None)
+ name = bib[0] if bib else title.strip()
+ isbn = md.get("isbn")
+ isbn = isbn[0] if isinstance(isbn, list) else isbn
+ oclc = md.get("oclc-id")
+ oclc = oclc[0] if isinstance(oclc, list) else oclc
+ desc = md.get("description")
+ desc = " | ".join(desc) if isinstance(desc, list) else desc
+ self.conn.execute(
+ """INSERT INTO books(title,year,publisher,isbn,oclc,ia_identifier,ia_access,record_url,notes,verified)
+ VALUES (?,?,?,?,?,?,?,?,?,1)
+ ON CONFLICT(title) DO UPDATE SET year=COALESCE(excluded.year,year), publisher=excluded.publisher,
+ isbn=COALESCE(excluded.isbn,isbn), oclc=COALESCE(excluded.oclc,oclc),
+ ia_identifier=excluded.ia_identifier, ia_access=excluded.ia_access,
+ record_url=excluded.record_url, notes=excluded.notes, verified=1""",
+ (name, year, md.get("publisher"), isbn, oclc, ident, access, f"https://archive.org/details/{ident}",
+ (f"IA title: {title}. " + (desc or ""))[:800]))
+ self.stats["seen"] += 1
+ self.stats["added"] += 1
+ # bibliography titles not found on IA stay listed, unverified
+ for title, year in BIBLIOGRAPHY:
+ self.conn.execute("INSERT OR IGNORE INTO books(title,year,notes,verified) VALUES (?,?,?,0)",
+ (title, year, "listed in project brief; not found on Internet Archive"))
+ self.conn.commit()
+ self.notes.append(f"excluded name-authority conflations: {excluded}")
+
+
+CRAWLER = InternetArchive
diff --git a/src/conrad/crawlers/latimes.py b/src/conrad/crawlers/latimes.py
new file mode 100644
index 0000000..f312f81
--- /dev/null
+++ b/src/conrad/crawlers/latimes.py
@@ -0,0 +1,26 @@
+"""Los Angeles Times (Conrad's paper 1964-1993; syndicated via LA Times Syndicate / Tribune Media 1993-2010).
+The historical archive is PAYWALLED (ProQuest Historical Newspapers; latimes.newspapers.com). No paywall is
+bypassed and no article/page is fetched. This module registers the source and computes coverage from
+records already attributed to the LA Times by other repositories (LOC rights statements, Huntington spans)."""
+from __future__ import annotations
+
+from .base import Crawler
+
+
+class LATimes(Crawler):
+ source_id = "latimes"
+ name = "Los Angeles Times archive (ProQuest / newspapers.com)"
+ repository = "Los Angeles Times"
+ url = "https://latimes.newspapers.com/"
+ classification = "PAYWALLED"
+ access_notes = ("Full-page archive via ProQuest Historical Newspapers (library subscription) or "
+ "latimes.newspapers.com. Discovery/metadata only; not crawled.")
+
+ def crawl(self) -> None:
+ n = self.conn.execute("SELECT COUNT(*) FROM cartoons WHERE publication LIKE 'Los Angeles Times%' "
+ "AND granularity='item'").fetchone()[0]
+ self.status = "not_attempted"
+ self.notes.append(f"paywalled — not crawled; {n} item records elsewhere are attributed to the LA Times")
+
+
+CRAWLER = LATimes
diff --git a/src/conrad/crawlers/loc.py b/src/conrad/crawlers/loc.py
new file mode 100644
index 0000000..d12bd25
--- /dev/null
+++ b/src/conrad/crawlers/loc.py
@@ -0,0 +1,159 @@
+"""Library of Congress Prints & Photographs.
+
+robots.txt disallows /search and /pictures/search (Crawl-delay 5), so the crawler does NOT page the
+search UI. It uses (a) the documented loc.gov format endpoint /photos/?fa=contributor:...&fo=json, which
+robots permits, exhausting its pagination, and (b) per-item JSON (/pictures/item/<id>/?fo=json) for every
+known Conrad item id (seed + format endpoint) to verify and enrich each record. Images: metadata links only.
+"""
+from __future__ import annotations
+
+import re
+
+from .. import rights
+from ..models import CartoonRecord
+from ..normalize import GENERIC_SUBJECTS, clean_subject, parse_date, person_from_heading, presidents_in_text
+from .base import Blocked, Checkpoint, Crawler, Transient
+from .seed_import import _pub_from_call
+
+LOC = "https://www.loc.gov"
+FORMAT_QUERIES = [
+ ("photos", {"fa": "contributor:conrad, paul"}),
+ ("photos", {"q": "paul conrad editorial cartoon"}),
+ ("photos", {"q": "conrad, paul, 1924-2010"}),
+ ("manuscripts", {"fa": "contributor:conrad, paul"}),
+ ("books", {"fa": "contributor:conrad, paul"}),
+]
+KNOWN_GROUPS = {"2010632868": "83 proofs of political cartoons for the Los Angeles Times (group record)"}
+
+
+def _pk_from(url_or_id: str) -> str | None:
+ m = re.search(r"/(?:item|pictures/item)/([0-9a-z]+)/?", url_or_id or "")
+ return m[1] if m else None
+
+
+def item_record(pk: str, d: dict, provenance: str) -> CartoonRecord | None:
+ it = d.get("item") or {}
+ creators = " ".join(c.get("title", "") for c in (it.get("creators") or []) if isinstance(c, dict))
+ creators += " " + " ".join(it.get("contributor_names") or [])
+ if not re.search(r"conrad,\s*paul", creators, re.I):
+ return None
+ title = it.get("title")
+ dt = parse_date(it.get("created_published_date") or it.get("created_published") or it.get("date"))
+ subjects, people = [], []
+ for s in it.get("subjects") or []:
+ s = s.get("title") if isinstance(s, dict) else s
+ if not s:
+ continue
+ p = person_from_heading(s)
+ c = clean_subject(s.replace("--Quotations", ""))
+ if p:
+ people.append(p)
+ elif c and not GENERIC_SUBJECTS.search(c):
+ subjects.append(c)
+ summary = it.get("summary") or ""
+ people += presidents_in_text(" ".join([title or "", summary]))
+ res = d.get("resources") or []
+ img = None
+ if res and isinstance(res[0], dict):
+ img = res[0].get("image") or res[0].get("large") or res[0].get("medium")
+ thumb = None
+ tg = it.get("thumb_gallery")
+ if isinstance(tg, str) and "notdigitized" not in tg:
+ thumb = tg
+ rights_info = it.get("rights_information") or ""
+ holder = None
+ m = re.search(r"Copyright\s+(\d{4}),?\s+([^.]+)", rights_info)
+ if m:
+ holder = m[2].strip()
+ group = pk in KNOWN_GROUPS or bool(re.search(r"^\[?proofs of", title or "", re.I))
+ pub = holder if holder and re.search(r"times|post", holder, re.I) else _pub_from_call(None, dt["year"])
+ return CartoonRecord(
+ canonical_id=f"loc:{pk}", identifier=pk, granularity="folder" if group else "item", title=title,
+ description=summary or None, date_exact=dt["date_exact"], date_start=dt["date_start"],
+ date_end=dt["date_end"], year=dt["year"], date_is_estimate=dt["estimate"],
+ medium=it.get("medium_brief") or (it.get("medium") if isinstance(it.get("medium"), str) else None),
+ publication=pub, rights_text=rights_info or rights.rights_for("Library of Congress")[0],
+ copyright_holder=holder, repository="Library of Congress",
+ collection_name=", ".join(c.get("title", "") for c in (it.get("collections") or []) if isinstance(c, dict))
+ or "Prints & Photographs Division", folder=it.get("call_number"),
+ record_url=f"{LOC}/pictures/item/{pk}/", image_url=img, thumbnail_url=thumb,
+ access_level=rights.ONLINE_IMAGE if (img or thumb) else rights.ONLINE_METADATA,
+ rights_url="https://www.loc.gov/rr/print/res/rights.html", provenance=provenance,
+ subjects=subjects, people=sorted(set(people)),
+ notes=KNOWN_GROUPS.get(pk) or ("group/proof set — not a single cartoon" if group else None),
+ )
+
+
+class LOCCrawler(Crawler):
+ source_id = "loc"
+ name = "Library of Congress — Prints & Photographs (JSON API)"
+ repository = "Library of Congress"
+ url = "https://www.loc.gov/photos/?fa=contributor:conrad,%20paul&fo=json"
+ classification = "PUBLIC_API"
+ access_notes = ("Item records online; many Conrad drawings are digitized but flagged 'May be restricted: Copyright "
+ "... Los Angeles Times' — view at loc.gov (link-out), reproduction needs permission.")
+
+ def crawl(self) -> None:
+ cp = Checkpoint("loc")
+ self.books: list[dict] = []
+ pks: dict[str, str] = {}
+ for fmt, params in FORMAT_QUERIES:
+ page = 1
+ max_pages = 50 if "fa" in params else 3 # keyword queries: stop early, results get irrelevant fast
+ while page <= max_pages:
+ try:
+ data = self.http.get_json(f"{LOC}/{fmt}/", params={**params, "fo": "json", "c": 100, "sp": page})
+ except (Blocked, Transient) as e:
+ self.error(f"{LOC}/{fmt}/ {params}", e)
+ break
+ self.stats["pages"] += 1
+ hits = 0
+ for r in data.get("results") or []:
+ contrib = " ".join(r.get("contributor") or [])
+ if fmt == "books" and "conrad, paul" in contrib.lower():
+ self.books.append(r)
+ continue
+ if "conrad, paul" in contrib.lower():
+ hits += 1
+ pk = _pk_from(r.get("id") or "")
+ if pk:
+ pks.setdefault(pk, f"live:{LOC}/{fmt}/?{params}")
+ if not (data.get("pagination") or {}).get("next") or ("q" in params and not hits):
+ break
+ page += 1
+ found_live = set(pks)
+ # every Conrad pk already known from the seed cache gets verified individually
+ for (pk,) in self.conn.execute("SELECT identifier FROM cartoon_sources WHERE source_id='seed_loc'"):
+ pks.setdefault(pk, "live:item-json (seed id verification)")
+ for pk in KNOWN_GROUPS:
+ pks.setdefault(pk, "live:item-json (known group record)")
+ done = set(cp.get("done", []))
+ for pk, prov in sorted(pks.items()):
+ url = f"{LOC}/pictures/item/{pk}/"
+ try:
+ d = self.http.get_json(url, params={"fo": "json"})
+ except (Blocked, Transient) as e:
+ self.error(url, e)
+ continue
+ self.stats["pages"] += 1
+ rec = item_record(pk, d, prov + f" -> {url}?fo=json")
+ if rec:
+ self.save(rec)
+ done.add(pk)
+ if len(done) % 10 == 0:
+ self.conn.commit()
+ cp.set("done", sorted(done))
+ self.conn.commit()
+ cp.set("done", sorted(done))
+ for b in self.books: # LOC catalog records for Conrad's own books -> verification of the bibliography
+ t = (b.get("title") or "").strip(" /")
+ self.conn.execute("INSERT OR IGNORE INTO collections(repository,name,identifier,url,level,notes,source_id) "
+ "VALUES (?,?,?,?,?,?,?)", ("Library of Congress", t[:300], _pk_from(b.get("id") or ""),
+ b.get("id"), "book", f"date={b.get('date')}", self.source_id))
+ self.conn.commit()
+ self.notes.append(f"LOC book records by Conrad: {len(self.books)}")
+ self.notes.append(f"format endpoint found {len(found_live)} Conrad ids; verified {len(done)} item JSON records; "
+ f"/search + /pictures/search are robots-disallowed (not paged)")
+
+
+CRAWLER = LOCCrawler
diff --git a/src/conrad/crawlers/oac.py b/src/conrad/crawlers/oac.py
new file mode 100644
index 0000000..0fe0ebb
--- /dev/null
+++ b/src/conrad/crawlers/oac.py
@@ -0,0 +1,51 @@
+"""Online Archive of California — collection-level metadata for the Huntington finding aid.
+robots.txt: /search and /view are disallowed, Crawl-delay 5 (honoured); /findaid/ is allowed."""
+from __future__ import annotations
+
+import re
+
+from bs4 import BeautifulSoup
+
+from .base import Crawler
+
+OAC_URL = "https://oac.cdlib.org/findaid/ark:/13030/c8z03dxd/"
+
+
+def parse_oac(html: str) -> dict:
+ soup = BeautifulSoup(html, "html.parser")
+ text = re.sub(r"\s+", " ", soup.get_text(" "))
+ out = {"title": (soup.title.string or "").strip() if soup.title else None}
+ for label in ("Extent", "Date", "Collection Number", "Conditions Governing Access", "Conditions Governing Use",
+ "Physical Location", "Online items available"):
+ m = re.search(re.escape(label) + r"\s*:?\s*(.{0,300}?)(?=\s(?:[A-Z][a-z]+ ){0,4}[A-Z][a-z]+\s*:|$)", text)
+ if m:
+ out[label.lower().replace(" ", "_")] = m.group(1).strip()
+ m = re.search(r"([\d,]+)\s+(?:items|original cartoon drawings|cartoons|pieces)", text, re.I)
+ out["count_phrase"] = m.group(0) if m else None
+ return out
+
+
+class OAC(Crawler):
+ source_id = "oac"
+ name = "Online Archive of California (Huntington finding aid host)"
+ repository = "Huntington Library"
+ url = OAC_URL
+ classification = "PUBLIC_HTML"
+ access_notes = "Finding aid only; no digitized Conrad items. Crawl-delay 5 honoured."
+
+ def crawl(self) -> None:
+ st, html = self.http.get(OAC_URL)
+ self.stats["pages"] += 1
+ if st != 200:
+ self.error(OAC_URL, f"HTTP {st}")
+ self.status = "failed"
+ return
+ info = parse_oac(html)
+ self.conn.execute(
+ """UPDATE collections SET notes = COALESCE(notes,'') || ? WHERE repository='Huntington Library' AND level='collection'""",
+ (f" | OAC: {info}",))
+ self.conn.commit()
+ self.notes.append(f"OAC collection page parsed: {info.get('extent') or info.get('count_phrase')}")
+
+
+CRAWLER = OAC
diff --git a/src/conrad/crawlers/ohio_state.py b/src/conrad/crawlers/ohio_state.py
new file mode 100644
index 0000000..31ee560
--- /dev/null
+++ b/src/conrad/crawlers/ohio_state.py
@@ -0,0 +1,35 @@
+"""Ohio State University Billy Ireland Cartoon Library & Museum (PastPerfect Online).
+The PastPerfect host returns HTTP 403 to non-browser clients (including /robots.txt), so it cannot be
+crawled politely; recorded as blocked / REQUIRES_PERMISSION. No bypass is attempted."""
+from __future__ import annotations
+
+from .base import Blocked, Crawler, Transient
+
+BASE = "https://osucartoons.pastperfectonline.com/"
+SEARCHES = [BASE + "search?search_criteria=%22Conrad%2C+Paul%22&onlyimages=false",
+ BASE + "bysearchterm?keyword=Conrad%2C%20Paul"]
+
+
+class OhioState(Crawler):
+ source_id = "ohio_state"
+ name = "Ohio State University — Billy Ireland Cartoon Library & Museum"
+ repository = "Ohio State University"
+ url = BASE
+ classification = "REQUIRES_PERMISSION"
+ access_notes = ("Holds Conrad material (per OSU catalog references) but the online collections database refuses "
+ "automated access (HTTP 403). Request a metadata export from the Billy Ireland reference desk.")
+
+ def crawl(self) -> None:
+ for url in [BASE, *SEARCHES]:
+ try:
+ st, html = self.http.get(url)
+ self.stats["pages"] += 1
+ self.notes.append(f"{url}: HTTP {st}")
+ except (Blocked, Transient) as e:
+ self.error(url, f"BLOCKED: {e}")
+ self.status = "blocked"
+ self.notes.append(f"blocked: {e}")
+ break
+
+
+CRAWLER = OhioState
diff --git a/src/conrad/crawlers/syracuse.py b/src/conrad/crawlers/syracuse.py
new file mode 100644
index 0000000..4913b22
--- /dev/null
+++ b/src/conrad/crawlers/syracuse.py
@@ -0,0 +1,40 @@
+"""Syracuse University — Paul Conrad Cartoons (1,000+ originals, mostly 1963-1969).
+Live pages are behind an AWS WAF JS challenge (HTTP 202, empty body) for non-browser clients, so they are
+recorded as blocked and the seed cache (fetched earlier via r.jina.ai — see REPORT provenance caveat) is used.
+No challenge bypass is attempted for new crawling."""
+from __future__ import annotations
+
+from .base import Blocked, Crawler, Transient
+from .seed_import import SYR_URL, syracuse_folder_records
+
+EMPIRE = "https://empireadc.org/search/catalog/nsyu_2642890"
+
+
+class Syracuse(Crawler):
+ source_id = "syracuse"
+ name = "Syracuse University Special Collections — Paul Conrad Cartoons"
+ repository = "Syracuse University"
+ url = SYR_URL
+ classification = "PHYSICAL_ARCHIVE"
+ access_notes = "Reading-room access at SCRC. Finding aid is folder-level (65 folders, 388 index headings)."
+
+ def crawl(self) -> None:
+ ok = False
+ for url in (SYR_URL, EMPIRE):
+ try:
+ st, html = self.http.get(url)
+ self.stats["pages"] += 1
+ if st == 200 and "B1F" in html:
+ ok = True
+ self.notes.append(f"live ok: {url}")
+ else:
+ self.error(url, f"HTTP {st}")
+ except (Blocked, Transient) as e:
+ self.error(url, f"BLOCKED: {e}")
+ self.notes.append(f"blocked: {url} ({e})")
+ if not ok:
+ self.status = "blocked"
+ self.notes.append("using seed cache (seed_syracuse) for the 65 folder records")
+
+
+CRAWLER = Syracuse
diff --git a/src/conrad/crawlers/wichita.py b/src/conrad/crawlers/wichita.py
new file mode 100644
index 0000000..076bec0
--- /dev/null
+++ b/src/conrad/crawlers/wichita.py
@@ -0,0 +1,44 @@
+"""Wichita State University — Cartoon Collection of Paul Conrad (MS 90-18).
+Tries the legacy static finding aid and the ArchivesSpace public UI directly (robots Crawl-delay 35 honoured).
+The ArchivesSpace PUI sits behind an AWS WAF JS challenge; the legacy aid now redirects to the AS home page.
+If blocked, the seed cache (fetched earlier via r.jina.ai — provenance caveat in REPORT.md) is relied on."""
+from __future__ import annotations
+
+from .base import Blocked, Crawler, Transient
+
+LEGACY = "https://specialcollections.wichita.edu/collections/ms/90-18/90-18-a.html"
+RESOURCE = "https://archivesspace.wichita.edu/repositories/3/resources/166"
+SERIES2 = "https://archivesspace.wichita.edu/repositories/3/archival_objects/111935"
+
+
+class Wichita(Crawler):
+ source_id = "wichita"
+ name = "Wichita State University Special Collections — MS 90-18"
+ repository = "Wichita State University"
+ url = RESOURCE
+ classification = "PHYSICAL_ARCHIVE"
+ access_notes = "Item-level ArchivesSpace records (Series 2, 1968-1971); originals viewable in Special Collections."
+
+ def crawl(self) -> None:
+ live = False
+ for url in (LEGACY, RESOURCE, SERIES2):
+ try:
+ st, html = self.http.get(url)
+ self.stats["pages"] += 1
+ if st == 200 and ("Conrad" in html or "90-18" in html):
+ live = True
+ self.notes.append(f"live ok: {url}")
+ elif st == 200:
+ self.error(url, "HTTP 200 but no Conrad content (legacy aid retired -> redirects to AS home)")
+ self.notes.append(f"{url}: retired/redirected")
+ else:
+ self.error(url, f"HTTP {st}")
+ except (Blocked, Transient) as e:
+ self.error(url, f"BLOCKED: {e}")
+ self.notes.append(f"blocked: {url}")
+ if not live:
+ self.status = "blocked"
+ self.notes.append("REQUIRES_PERMISSION for automated access; relying on seed_wichita cache (r.jina.ai provenance)")
+
+
+CRAWLER = Wichita
← bc3f93c auto-data-snapshot: 2026-09-24T16:24:12 (3 data files) — dat
·
back to Paul Conrad Archive
·
Dedupe, exports, REPORT, FastAPI viewer (link-out only, basi 00dd3c6 →