← back to Paul Conrad Archive
Core schema, models, normalizers, polite HTTP base and seed importer
380110b140a0399856c4f795a4e9707153929189 · 2026-09-24 16:13:53 -0700 · Steve Abrams
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Files touched
A .env.exampleA data/conrad.dbA data/conrad.db-shmA data/conrad.db-walA pyproject.tomlA scripts/crawl_source.pyA scripts/init_db.pyA src/conrad/__init__.pyA src/conrad/config.pyA src/conrad/crawlers/__init__.pyA src/conrad/crawlers/base.pyA src/conrad/crawlers/seed_import.pyA src/conrad/db.pyA src/conrad/models.pyA src/conrad/normalize.pyA src/conrad/rights.py
Diff
commit 380110b140a0399856c4f795a4e9707153929189
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Sep 24 16:13:53 2026 -0700
Core schema, models, normalizers, polite HTTP base and seed importer
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
.env.example | 9 ++
data/conrad.db | Bin 0 -> 9650176 bytes
data/conrad.db-shm | Bin 0 -> 32768 bytes
data/conrad.db-wal | 0
pyproject.toml | 27 +++++
scripts/crawl_source.py | 19 +++
scripts/init_db.py | 9 ++
src/conrad/__init__.py | 7 ++
src/conrad/config.py | 49 ++++++++
src/conrad/crawlers/__init__.py | 3 +
src/conrad/crawlers/base.py | 208 +++++++++++++++++++++++++++++++++
src/conrad/crawlers/seed_import.py | 234 +++++++++++++++++++++++++++++++++++++
src/conrad/db.py | 204 ++++++++++++++++++++++++++++++++
src/conrad/models.py | 41 +++++++
src/conrad/normalize.py | 145 +++++++++++++++++++++++
src/conrad/rights.py | 36 ++++++
16 files changed, 991 insertions(+)
diff --git a/.env.example b/.env.example
new file mode 100644
index 0000000..9ab86ac
--- /dev/null
+++ b/.env.example
@@ -0,0 +1,9 @@
+# All optional. Defaults shown.
+MAX_CONCURRENCY=3
+REQUEST_DELAY=2.0
+TIMEOUT=30
+CONRAD_DB=data/conrad.db
+CONRAD_SEED_ROOT=~/Projects/paul-conrad-cartoons
+# Free keys (read from env or ~/Projects/secrets-manager/.env; never commit values)
+DPLA_API_KEY=
+SI_API_KEY=
diff --git a/data/conrad.db b/data/conrad.db
new file mode 100644
index 0000000..729fbf9
Binary files /dev/null and b/data/conrad.db differ
diff --git a/data/conrad.db-shm b/data/conrad.db-shm
new file mode 100644
index 0000000..fe9ac28
Binary files /dev/null and b/data/conrad.db-shm differ
diff --git a/data/conrad.db-wal b/data/conrad.db-wal
new file mode 100644
index 0000000..e69de29
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..bedf517
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,27 @@
+[project]
+name = "paul-conrad-archive"
+version = "0.1.0"
+description = "Metadata-only master index of Paul Conrad (1924-2010) editorial cartoons and where each can be legally viewed."
+requires-python = ">=3.11"
+dependencies = [
+ "requests==2.32.3",
+ "tenacity==9.0.0",
+ "rapidfuzz==3.10.1",
+ "beautifulsoup4==4.12.3",
+ "fastapi==0.115.6",
+ "uvicorn==0.34.0",
+]
+
+[project.optional-dependencies]
+dev = ["pytest==8.3.4", "httpx==0.28.1"]
+
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+pythonpath = ["src", "."]
+testpaths = ["tests"]
diff --git a/scripts/crawl_source.py b/scripts/crawl_source.py
new file mode 100644
index 0000000..649e430
--- /dev/null
+++ b/scripts/crawl_source.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+"""Run one crawler: python scripts/crawl_source.py loc [--no-cache]"""
+import importlib, json, sys, pathlib
+sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1] / "src"))
+from conrad import db # noqa: E402
+from conrad.crawlers.base import Http # noqa: E402
+
+
+def run(name: str, use_cache: bool = True) -> dict:
+ mod = importlib.import_module(f"conrad.crawlers.{name}")
+ conn = db.connect()
+ db.init_db(conn)
+ return mod.CRAWLER(conn=conn, http=Http(use_cache=use_cache)).run()
+
+
+if __name__ == "__main__":
+ args = [a for a in sys.argv[1:] if not a.startswith("--")]
+ for n in args:
+ print(json.dumps(run(n, use_cache="--no-cache" not in sys.argv)))
diff --git a/scripts/init_db.py b/scripts/init_db.py
new file mode 100644
index 0000000..45aedbf
--- /dev/null
+++ b/scripts/init_db.py
@@ -0,0 +1,9 @@
+#!/usr/bin/env python3
+"""Create data/conrad.db (idempotent)."""
+import sys, pathlib
+sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1] / "src"))
+from conrad import config, db # noqa: E402
+
+conn = db.connect()
+db.init_db(conn)
+print(f"initialized {config.DB_PATH}")
diff --git a/src/conrad/__init__.py b/src/conrad/__init__.py
new file mode 100644
index 0000000..58c16a1
--- /dev/null
+++ b/src/conrad/__init__.py
@@ -0,0 +1,7 @@
+"""Paul Conrad Master Archive — a METADATA-ONLY index of Paul Conrad (1924-2010) cartoons.
+
+Copyright rule (Steve, TK-12179): Conrad cartoon images are research-only and are never
+downloaded, stored, or displayed. image_url / thumbnail_url are kept as metadata links only;
+cartoon_sources.local_image is always NULL (enforced by a CHECK constraint).
+"""
+__version__ = "0.1.0"
diff --git a/src/conrad/config.py b/src/conrad/config.py
new file mode 100644
index 0000000..ee87c31
--- /dev/null
+++ b/src/conrad/config.py
@@ -0,0 +1,49 @@
+"""Central configuration. Values can be overridden by environment variables (see .env.example)."""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+DATA = Path(os.environ.get("CONRAD_DATA_DIR", ROOT / "data"))
+DB_PATH = Path(os.environ.get("CONRAD_DB", DATA / "conrad.db"))
+CACHE_DIR = DATA / "cache"
+CHECKPOINT_DIR = CACHE_DIR / "checkpoints"
+EXPORT_DIR = DATA / "exports"
+SOURCES_JSON = DATA / "sources.json"
+
+# READ-ONLY seed corpus (owned by TK-12179 shadow-man-build; never write there).
+SEED_ROOT = Path(os.environ.get("CONRAD_SEED_ROOT", Path.home() / "Projects/paul-conrad-cartoons"))
+SEED_RESEARCH = SEED_ROOT / "research"
+
+MAX_CONCURRENCY = int(os.environ.get("MAX_CONCURRENCY", 3))
+REQUEST_DELAY = float(os.environ.get("REQUEST_DELAY", 2.0)) # seconds per host (robots Crawl-delay wins if larger)
+TIMEOUT = int(os.environ.get("TIMEOUT", 30))
+USER_AGENT = os.environ.get(
+ "CONRAD_USER_AGENT",
+ "paul-conrad-archive/0.1 research metadata-only; contact steve@designerwallcoverings.com",
+)
+
+CREATOR = "Conrad, Paul, 1924-2010"
+YEAR_MIN, YEAR_MAX = 1945, 2010
+
+# Optional free API keys, read from the environment or ~/Projects/secrets-manager/.env
+# (never printed, never committed).
+SECRETS_ENV = Path.home() / "Projects/secrets-manager/.env"
+
+
+def secret(name: str) -> str | None:
+ if os.environ.get(name):
+ return os.environ[name]
+ try:
+ for line in SECRETS_ENV.read_text().splitlines():
+ if line.startswith(name + "="):
+ v = line.split("=", 1)[1].strip().strip('"').strip("'")
+ return v or None
+ except OSError:
+ return None
+ return None
+
+
+for _d in (DATA, CACHE_DIR, CHECKPOINT_DIR, EXPORT_DIR):
+ _d.mkdir(parents=True, exist_ok=True)
diff --git a/src/conrad/crawlers/__init__.py b/src/conrad/crawlers/__init__.py
new file mode 100644
index 0000000..1e03de8
--- /dev/null
+++ b/src/conrad/crawlers/__init__.py
@@ -0,0 +1,3 @@
+"""Source crawlers. Each module exposes a Crawler subclass named CRAWLER."""
+ORDER = ["seed_import", "huntington", "oac", "syracuse", "loc", "ohio_state", "wichita",
+ "latimes", "denver_post", "daily_iowan", "internet_archive", "discovery"]
diff --git a/src/conrad/crawlers/base.py b/src/conrad/crawlers/base.py
new file mode 100644
index 0000000..fd37491
--- /dev/null
+++ b/src/conrad/crawlers/base.py
@@ -0,0 +1,208 @@
+"""Polite HTTP client + crawler base: robots.txt, per-host delay, cache, retries, checkpoints."""
+from __future__ import annotations
+
+import hashlib
+import json
+import threading
+import time
+import traceback
+from urllib.parse import urlparse
+from urllib.robotparser import RobotFileParser
+
+import requests
+from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential
+
+from .. import config, db
+
+
+class Blocked(Exception):
+ """Robots disallow, WAF/JS challenge, 401/403, paywall — we stop, record, and move on."""
+
+
+class Transient(Exception):
+ pass
+
+
+_host_locks: dict[str, threading.Lock] = {}
+_host_last: dict[str, float] = {}
+_robots: dict[str, RobotFileParser | None] = {}
+_glock = threading.Lock()
+
+
+def _lock_for(host: str) -> threading.Lock:
+ with _glock:
+ return _host_locks.setdefault(host, threading.Lock())
+
+
+def looks_like_challenge(status: int, body: str) -> bool:
+ b = body[:5000].lower()
+ return (status == 202 and len(body.strip()) == 0) or "awswafintegration" in b or "challenge-platform" in b \
+ or ("just a moment" in b and "cloudflare" in b) or "captcha" in b and status != 200
+
+
+class Http:
+ def __init__(self, use_cache: bool = True):
+ self.s = requests.Session()
+ self.s.headers.update({"User-Agent": config.USER_AGENT, "Accept": "*/*"})
+ self.use_cache = use_cache
+ self.cache_dir = config.CACHE_DIR / "http"
+ self.cache_dir.mkdir(parents=True, exist_ok=True)
+ self.requests_made = 0
+
+ # ---------------- robots
+ def robots(self, url: str) -> RobotFileParser | None:
+ p = urlparse(url)
+ base = f"{p.scheme}://{p.netloc}"
+ with _glock:
+ if base in _robots:
+ return _robots[base]
+ rp = RobotFileParser()
+ try:
+ r = self.s.get(base + "/robots.txt", timeout=config.TIMEOUT)
+ if r.status_code in (401, 403) or looks_like_challenge(r.status_code, r.text):
+ rp.parse(["User-agent: *", "Disallow: /"]) # conservative: cannot read robots -> treat as disallowed
+ rp._unreadable = True # type: ignore[attr-defined]
+ elif r.status_code >= 400:
+ rp.parse([]) # no robots.txt -> allowed
+ else:
+ rp.parse(r.text.splitlines())
+ except requests.RequestException:
+ rp = None
+ with _glock:
+ _robots[base] = rp
+ return rp
+
+ def allowed(self, url: str) -> bool:
+ rp = self.robots(url)
+ return True if rp is None else rp.can_fetch(config.USER_AGENT, url)
+
+ def delay_for(self, url: str) -> float:
+ rp = self.robots(url)
+ cd = None
+ if rp is not None:
+ try:
+ cd = rp.crawl_delay(config.USER_AGENT) or rp.crawl_delay("*")
+ except Exception: # noqa: BLE001
+ cd = None
+ return max(config.REQUEST_DELAY, float(cd or 0))
+
+ # ---------------- fetch
+ def _cache_path(self, url: str):
+ return self.cache_dir / (hashlib.sha1(url.encode()).hexdigest() + ".json")
+
+ def get(self, url: str, *, params: dict | None = None, check_robots: bool = True, refresh: bool = False,
+ secret_params: dict | None = None) -> tuple[int, str]:
+ """GET text. secret_params (API keys) are sent but never cached or logged."""
+ full = requests.Request("GET", url, params=params).prepare().url
+ cp = self._cache_path(full)
+ if self.use_cache and not refresh and cp.exists():
+ d = json.loads(cp.read_text())
+ return d["status"], d["body"]
+ if check_robots and not self.allowed(full):
+ raise Blocked(f"robots.txt disallows {full}")
+ status, body = self._fetch(full, secret_params)
+ if looks_like_challenge(status, body):
+ raise Blocked(f"WAF/JS challenge (HTTP {status}) at {full}")
+ if status in (401, 403):
+ raise Blocked(f"HTTP {status} (access denied) at {full}")
+ if status == 200 and self.use_cache:
+ cp.write_text(json.dumps({"url": full, "status": status, "body": body, "at": db.now()}))
+ return status, body
+
+ def get_json(self, url: str, **kw):
+ status, body = self.get(url, **kw)
+ if status != 200:
+ raise Transient(f"HTTP {status} at {url}")
+ return json.loads(body)
+
+ @retry(retry=retry_if_exception_type(Transient), stop=stop_after_attempt(4),
+ wait=wait_exponential(multiplier=2, min=4, max=60), reraise=True)
+ def _fetch(self, url: str, secret_params: dict | None) -> tuple[int, str]:
+ host = urlparse(url).netloc
+ with _lock_for(host):
+ wait = self.delay_for(url) - (time.time() - _host_last.get(host, 0))
+ if wait > 0:
+ time.sleep(wait)
+ try:
+ r = self.s.get(url, params=secret_params, timeout=config.TIMEOUT)
+ except requests.RequestException as e:
+ raise Transient(str(e).replace(str((secret_params or {}).get("api_key", "\0")), "***")) from None
+ finally:
+ _host_last[host] = time.time()
+ self.requests_made += 1
+ if r.status_code == 429 or r.status_code >= 500:
+ raise Transient(f"HTTP {r.status_code} at {url}")
+ return r.status_code, r.text
+
+
+class Checkpoint:
+ def __init__(self, name: str):
+ self.path = config.CHECKPOINT_DIR / f"{name}.json"
+ self.data = json.loads(self.path.read_text()) if self.path.exists() else {}
+
+ def get(self, k, default=None):
+ return self.data.get(k, default)
+
+ def set(self, k, v):
+ self.data[k] = v
+ self.path.write_text(json.dumps(self.data, indent=1))
+
+
+class Crawler:
+ source_id = "base"
+ name = "Base"
+ repository: str | None = None
+ url: str | None = None
+ classification: str | None = None
+ access_notes: str | None = None
+
+ def __init__(self, conn=None, http: Http | None = None):
+ self.conn = conn or db.connect()
+ self.http = http or Http()
+ self.stats = {"pages": 0, "seen": 0, "added": 0, "updated": 0, "errors": 0}
+ self.status = "worked"
+ self.notes: list[str] = []
+ self.run_id: int | None = None
+
+ def register(self, status=None, notes=None):
+ db.upsert_source(self.conn, self.source_id, self.name, repository=self.repository, url=self.url,
+ classification=self.classification, status=status, access_notes=self.access_notes,
+ notes=notes)
+ self.conn.commit()
+
+ def save(self, rec) -> None:
+ self.stats["seen"] += 1
+ st = db.save_record(self.conn, rec, self.source_id)
+ if st in ("added", "updated"):
+ self.stats[st] += 1
+
+ def error(self, url, err) -> None:
+ self.stats["errors"] += 1
+ db.log_error(self.conn, self.run_id, self.source_id, url, str(err))
+
+ def blocked(self, url, err) -> None:
+ self.error(url, f"BLOCKED: {err}")
+ self.status = "blocked"
+ self.notes.append(f"blocked: {err}")
+
+ def crawl(self) -> None: # pragma: no cover - implemented by subclasses
+ raise NotImplementedError
+
+ def run(self) -> dict:
+ self.register()
+ self.run_id = db.start_run(self.conn, self.source_id)
+ try:
+ self.crawl()
+ self.conn.commit()
+ except Blocked as e:
+ self.blocked(self.url, e)
+ except Exception as e: # noqa: BLE001 — never stall the whole crawl
+ self.conn.rollback()
+ self.error(self.url, f"{type(e).__name__}: {e}\n{traceback.format_exc()[-1500:]}")
+ self.status = "failed"
+ if self.status == "worked" and self.stats["errors"] and not self.stats["seen"]:
+ self.status = "failed"
+ note = "; ".join(self.notes) or None
+ db.finish_run(self.conn, self.run_id, self.stats, self.status, note)
+ self.register(status=self.status, notes=note)
+ return {"source": self.source_id, "status": self.status, **self.stats, "notes": note}
diff --git a/src/conrad/crawlers/seed_import.py b/src/conrad/crawlers/seed_import.py
new file mode 100644
index 0000000..668b556
--- /dev/null
+++ b/src/conrad/crawlers/seed_import.py
@@ -0,0 +1,234 @@
+"""Seed importer: ingests the READ-ONLY research corpus built under TK-12179
+(~/Projects/paul-conrad-cartoons/research). Every record carries provenance 'seed:<file>'.
+Nothing is ever written to the seed directory."""
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+
+from .. import config, rights
+from ..models import CartoonRecord
+from ..normalize import (GENERIC_SUBJECTS, clean_subject, parse_date, person_from_heading, presidents_in_text)
+from .base import Crawler
+
+OAC_URL = "https://oac.cdlib.org/findaid/ark:/13030/c8z03dxd/"
+WSU_OBJ = "https://archivesspace.wichita.edu/repositories/3/archival_objects/"
+SYR_URL = "https://library.syracuse.edu/digital/guides/c/conrad_p.htm"
+
+
+def read_json(p: Path):
+ return json.loads(p.read_text())
+
+
+def huntington_slot_records(slots: list[dict], provenance: str) -> list[CartoonRecord]:
+ """One box_range record per CON folder number. Dates are the BOX span (range-level, not item-level)."""
+ out = []
+ txt, url = rights.rights_for("Huntington Library")
+ for s in slots:
+ con = s["con_number"]
+ est = s.get("date_est")
+ out.append(CartoonRecord(
+ canonical_id=f"hunt:CON-{con}", identifier=f"CON {con}", granularity="box_range",
+ title=None, date_start=s["date_start"], date_end=s["date_end"],
+ year=int(est[:4]) if est else int(s["date_start"][:4]), date_is_estimate=True,
+ description=f"Original drawing slot CON {con} in Box {s['box']} ({s['series']}); box span "
+ f"{s['date_start']} to {s['date_end']}. Title not cataloged in the public finding aid.",
+ notes=f"range-level slot; interpolated date_est={est} (linear within box span, NOT a cataloged date)",
+ publication="Los Angeles Times" if s["date_start"] >= "1964" and s["date_end"] <= "1993-12-31" else None,
+ medium="Original drawing", rights_text=txt, rights_url=url,
+ repository="Huntington Library", collection_name="Paul Conrad Papers (mssCON 1-12360)",
+ box=str(s["box"]), folder=f"CON {con}", record_url=s.get("source_url") or OAC_URL,
+ access_level=rights.ARCHIVE_VISIT, provenance=provenance,
+ ))
+ return out
+
+
+def loc_record(r: dict, provenance: str) -> CartoonRecord | None:
+ creator = (r.get("creator") or "")
+ if not re.search(r"conrad,\s*paul", creator, re.I):
+ return None
+ pk = str(r.get("pk") or r.get("id"))
+ d = parse_date(r.get("created_published_date"))
+ subj_raw = r.get("subjects") or []
+ subjects, people = [], []
+ for s in subj_raw:
+ c = clean_subject(s.replace("--Quotations", ""))
+ if not c or GENERIC_SUBJECTS.search(c):
+ continue
+ p = person_from_heading(s)
+ (people if p else subjects).append(p or c)
+ people += presidents_in_text(r.get("title"))
+ img = r.get("image") or {}
+ digitized = img.get("full") and "notdigitized" not in (img.get("full") or "")
+ txt, rurl = rights.rights_for("Library of Congress")
+ return CartoonRecord(
+ canonical_id=f"loc:{pk}", identifier=pk, granularity="item", title=r.get("title"),
+ date_exact=d["date_exact"], date_start=d["date_start"], date_end=d["date_end"], year=d["year"],
+ date_is_estimate=d["estimate"], medium=r.get("medium_brief") or r.get("medium"),
+ publication=_pub_from_call(r.get("call_number"), d["year"]),
+ rights_text=txt, rights_url=rurl, repository="Library of Congress",
+ collection_name="Prints & Photographs Division", folder=r.get("call_number"),
+ record_url=(r.get("links") or {}).get("item") or f"https://www.loc.gov/pictures/item/{pk}/",
+ image_url=img.get("full") if digitized else None, thumbnail_url=img.get("thumb") if digitized else None,
+ access_level=rights.ONLINE_IMAGE if digitized else rights.ONLINE_METADATA,
+ provenance=provenance, subjects=subjects, people=sorted(set(people)),
+ )
+
+
+def _pub_from_call(call: str | None, year: int | None) -> str | None:
+ if year and 1950 <= year <= 1963:
+ return "Denver Post"
+ if year and 1964 <= year <= 1993:
+ return "Los Angeles Times"
+ if year and 1994 <= year <= 2010:
+ return "Los Angeles Times Syndicate / Tribune Media Services"
+ return None
+
+
+def parse_wsu_md(md: str, obj_id: str) -> dict | None:
+ h = (re.search(r"^Title:\s*(.+?)\s*\|\s*Wichita State University", md, re.M) or [None, ""])[1]
+ m = re.match(r"^(.*),\s*(\d{4}(?:-\d{2}(?:-\d{2})?)?|undated)$", h)
+ title = (m[1] if m else h).strip()
+ date = m[2] if m and m[2] != "undated" else None
+ scope = re.search(r"## Scope and Contents\s+([\s\S]*?)\n##", md)
+ box = re.search(r"File\s*[—-]\s*Box:\s*(\w+)", md)
+ series = re.search(r"\[(Series \d+ -- [^\]]+?)\]\(https://archivesspace", md)
+ return {"id": obj_id, "title": title, "date": date, "scope": re.sub(r"\s+", " ", scope[1]).strip() if scope else None,
+ "box": box[1] if box else None, "series": series[1] if series else None}
+
+
+def wsu_record(it: dict, provenance: str) -> CartoonRecord:
+ d = parse_date(it["date"])
+ untitled = re.fullmatch(r"untitled", it["title"] or "", re.I) is not None
+ title = None if untitled else it["title"].strip('"').strip()
+ subjects = [s.strip() for s in re.split(r",\s*", (it.get("scope") or "").rstrip(".")) if s.strip()]
+ people = presidents_in_text(" ".join([it["title"] or "", it.get("scope") or ""]))
+ txt, rurl = rights.rights_for("Wichita State University")
+ return CartoonRecord(
+ canonical_id=f"wsu:{it['id']}", identifier=it["id"], granularity="item", title=title,
+ caption=it["title"] if (it["title"] or "").startswith('"') else None,
+ description=it.get("scope"), date_exact=d["date_exact"], date_start=d["date_start"], date_end=d["date_end"],
+ year=d["year"], publication=_pub_from_call(None, d["year"]), rights_text=txt, rights_url=rurl,
+ repository="Wichita State University", collection_name="Cartoon Collection of Paul Conrad (MS 90-18)",
+ box=it.get("box"), record_url=WSU_OBJ + it["id"], access_level=rights.ARCHIVE_VISIT,
+ provenance=provenance, subjects=subjects, people=people,
+ notes=None if it.get("series") else "title/date from a sibling page's collection-organization listing",
+ )
+
+
+def syracuse_folder_records(syr: dict, provenance: str) -> list[CartoonRecord]:
+ heads_by_folder: dict[str, list[str]] = {}
+ for h in syr["headings"]:
+ for f in h["folders"]:
+ heads_by_folder.setdefault(f, []).append(h["heading"])
+ txt, rurl = rights.rights_for("Syracuse University")
+ out = []
+ for f in syr["folders"]:
+ label = f["label"]
+ d = parse_date(label)
+ heads = heads_by_folder.get(f["folder"], [])
+ people = sorted({p for p in (person_from_heading(h) for h in heads) if p})
+ subjects = sorted({h for h in heads if not person_from_heading(h)})
+ out.append(CartoonRecord(
+ canonical_id=f"syr:{f['folder']}", identifier=f["folder"], granularity="folder",
+ title=f"Cartoons, {label} (folder {f['folder']})", date_start=d["date_start"], date_end=d["date_end"],
+ year=d["year"] if d["year"] else None,
+ description=f"Syracuse folder {f['folder']}: original cartoons dated {label}. Folder-level record — "
+ f"the finding aid indexes {len(heads)} subject/name headings to this folder but does not "
+ f"list individual cartoons.",
+ publication="Los Angeles Times" if (d["year"] or 0) >= 1964 else None,
+ rights_text=txt, rights_url=rurl, repository="Syracuse University",
+ collection_name="Paul Conrad Cartoons, [1944?]-1986", box=f["folder"].split("F")[0],
+ folder=f["folder"], record_url=f"{SYR_URL}#{f['folder']}", access_level=rights.ARCHIVE_VISIT,
+ provenance=provenance, subjects=subjects, people=people,
+ ))
+ return out
+
+
+def catalog_record(r: dict, provenance: str) -> CartoonRecord | None:
+ if r.get("record_type") != "documented" or r["id"].startswith("pulitzer-"):
+ return None # year-of-work awards / collection summaries are not individual cartoons
+ d = parse_date(r.get("date"))
+ return CartoonRecord(
+ canonical_id=f"cat:{r['id']}", identifier=r["id"], granularity="item", title=r["title"],
+ description=r.get("subject"), date_exact=d["date_exact"], date_start=d["date_start"], date_end=d["date_end"],
+ year=d["year"], date_is_estimate=d["estimate"], publication=r.get("publication"),
+ rights_text=rights.COPYRIGHT_NOTE, repository="Secondary citation",
+ collection_name="Documented in published secondary sources", record_url=r.get("citation_url"),
+ access_level=rights.ONLINE_METADATA, provenance=provenance,
+ people=presidents_in_text(" ".join([r.get("title") or "", r.get("subject") or ""])),
+ subjects=list(r.get("topics") or []),
+ notes="secondary-source citation (obituary/encyclopedia/LOC page); image intentionally not carried over",
+ )
+
+
+class SeedImport(Crawler):
+ source_id = "seed"
+ name = "Seed corpus (TK-12179, read-only)"
+ repository = "multiple"
+ url = str(config.SEED_RESEARCH)
+ classification = "METADATA_ONLY"
+ access_notes = "Local cached metadata built by paul-conrad-cartoons/scripts/build-corpus.mjs; read-only."
+
+ SUBSOURCES = {
+ "seed_huntington": ("Seed: Huntington EAD slots (corpus.json)", "Huntington Library", OAC_URL, "PHYSICAL_ARCHIVE"),
+ "seed_loc": ("Seed: LOC P&P search cache (raw/loc-page-1.json)", "Library of Congress",
+ "https://www.loc.gov/pictures/search/?q=Conrad%2C%20Paul", "PUBLIC_API"),
+ "seed_wichita": ("Seed: Wichita State ArchivesSpace cache (raw/wsu/*.md via r.jina.ai)",
+ "Wichita State University", "https://archivesspace.wichita.edu/repositories/3/resources/166",
+ "PHYSICAL_ARCHIVE"),
+ "seed_syracuse": ("Seed: Syracuse finding-aid cache (raw/syr.html, syr.md)", "Syracuse University", SYR_URL,
+ "PHYSICAL_ARCHIVE"),
+ "seed_catalog": ("Seed: hand-built secondary-source catalog (data/cartoons.json)", "Secondary citation",
+ str(config.SEED_ROOT / "data/cartoons.json"), "METADATA_ONLY"),
+ }
+
+ def crawl(self) -> None:
+ R = config.SEED_RESEARCH
+ from .. import db
+ for sid, (name, repo, url, cls) in self.SUBSOURCES.items():
+ db.upsert_source(self.conn, sid, name, repository=repo, url=url, classification=cls, status="seed_only",
+ access_notes="seed cache; provenance recorded per row")
+ corpus = read_json(R / "corpus.json")
+ self.stats["pages"] += 1
+ self._save_all(huntington_slot_records(corpus["huntington"], "seed:research/corpus.json#huntington"),
+ "seed_huntington")
+ raw = read_json(R / "raw/loc-page-1.json")
+ self.stats["pages"] += 1
+ self._save_all([x for x in (loc_record(r, "seed:research/raw/loc-page-1.json") for r in raw["results"]) if x],
+ "seed_loc")
+ items: dict[str, dict] = {}
+ for p in sorted((R / "raw/wsu").glob("*.md")):
+ md = p.read_text()
+ self.stats["pages"] += 1
+ it = parse_wsu_md(md, p.stem)
+ if it:
+ items[p.stem] = it
+ # sibling listing in the "Collection organization" sidebar
+ for m in re.finditer(r"\[([^\]\[]*), (\d{4}-\d{2}-\d{2}|undated)\]\(" + re.escape(WSU_OBJ) + r"(\d+)\)", md):
+ oid = m[3]
+ if oid not in items and m[2] != "undated":
+ items[oid] = {"id": oid, "title": m[1], "date": m[2], "scope": None, "box": None, "series": None}
+ self._save_all([wsu_record(it, "seed:research/raw/wsu/" + (k + ".md" if it.get("series") else "*.md sidebar"))
+ for k, it in sorted(items.items())], "seed_wichita")
+ self._save_all(syracuse_folder_records(corpus["syracuse"], "seed:research/corpus.json#syracuse"),
+ "seed_syracuse")
+ catp = config.SEED_ROOT / "data/cartoons.json"
+ if catp.exists():
+ cat = read_json(catp)
+ self._save_all([x for x in (catalog_record(r, "seed:data/cartoons.json") for r in cat["records"]) if x],
+ "seed_catalog")
+ self.notes.append(f"huntington_repairs={corpus.get('huntington_repairs')}")
+
+ def _save_all(self, recs, source_id):
+ from .. import db
+ for rec in recs:
+ self.stats["seen"] += 1
+ st = db.save_record(self.conn, rec, source_id)
+ if st in ("added", "updated"):
+ self.stats[st] += 1
+ self.conn.commit()
+
+
+CRAWLER = SeedImport
diff --git a/src/conrad/db.py b/src/conrad/db.py
new file mode 100644
index 0000000..3b0deed
--- /dev/null
+++ b/src/conrad/db.py
@@ -0,0 +1,204 @@
+"""SQLite schema + write helpers. Records are never deleted; duplicates are linked via merged_into."""
+from __future__ import annotations
+
+import json
+import sqlite3
+from datetime import datetime, timezone
+from pathlib import Path
+
+from . import config
+
+SCHEMA = """
+PRAGMA foreign_keys = ON;
+CREATE TABLE IF NOT EXISTS sources (
+ id TEXT PRIMARY KEY,
+ name TEXT NOT NULL,
+ repository TEXT,
+ url TEXT,
+ classification TEXT CHECK (classification IN ('PUBLIC_API','PUBLIC_HTML','PUBLIC_IMAGE','METADATA_ONLY',
+ 'PAYWALLED','PHYSICAL_ARCHIVE','REQUIRES_PERMISSION')),
+ status TEXT, -- worked | blocked | seed_only | not_attempted | partial
+ access_notes TEXT,
+ notes TEXT,
+ last_checked TEXT
+);
+CREATE TABLE IF NOT EXISTS cartoons (
+ id INTEGER PRIMARY KEY,
+ canonical_id TEXT NOT NULL UNIQUE,
+ creator TEXT NOT NULL DEFAULT 'Conrad, Paul, 1924-2010',
+ granularity TEXT NOT NULL CHECK (granularity IN ('item','folder','box_range')),
+ date_exact TEXT, date_start TEXT, date_end TEXT, year INTEGER,
+ date_is_estimate INTEGER NOT NULL DEFAULT 0,
+ title TEXT, caption TEXT, description TEXT, publication TEXT, syndicate TEXT,
+ medium TEXT, dimensions TEXT, signed_name TEXT, rights_text TEXT, copyright_holder TEXT,
+ notes TEXT,
+ merged_into INTEGER REFERENCES cartoons(id), -- NULL = canonical; else points at the canonical cartoon
+ created_at TEXT NOT NULL, updated_at TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS ix_cartoons_year ON cartoons(year);
+CREATE INDEX IF NOT EXISTS ix_cartoons_gran ON cartoons(granularity);
+CREATE TABLE IF NOT EXISTS cartoon_sources (
+ id INTEGER PRIMARY KEY,
+ cartoon_id INTEGER NOT NULL REFERENCES cartoons(id),
+ source_id TEXT NOT NULL REFERENCES sources(id),
+ repository TEXT, collection_name TEXT, identifier TEXT NOT NULL,
+ box TEXT, folder TEXT, page TEXT,
+ record_url TEXT, image_url TEXT, thumbnail_url TEXT,
+ local_image TEXT CHECK (local_image IS NULL), -- copyright rule: images are NEVER stored locally
+ access_level TEXT, rights_url TEXT, provenance TEXT, retrieved_at TEXT,
+ UNIQUE (source_id, identifier)
+);
+CREATE INDEX IF NOT EXISTS ix_cs_cartoon ON cartoon_sources(cartoon_id);
+CREATE TABLE IF NOT EXISTS people (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, is_president INTEGER NOT NULL DEFAULT 0);
+CREATE TABLE IF NOT EXISTS cartoon_people (cartoon_id INTEGER REFERENCES cartoons(id), person_id INTEGER REFERENCES people(id),
+ PRIMARY KEY (cartoon_id, person_id));
+CREATE TABLE IF NOT EXISTS subjects (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE);
+CREATE TABLE IF NOT EXISTS cartoon_subjects (cartoon_id INTEGER REFERENCES cartoons(id), subject_id INTEGER REFERENCES subjects(id),
+ PRIMARY KEY (cartoon_id, subject_id));
+CREATE TABLE IF NOT EXISTS publications (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, date_start TEXT, date_end TEXT,
+ classification TEXT, notes TEXT);
+CREATE TABLE IF NOT EXISTS collections (
+ id INTEGER PRIMARY KEY, repository TEXT NOT NULL, name TEXT NOT NULL, identifier TEXT, url TEXT,
+ extent TEXT, date_range TEXT, level TEXT, notes TEXT, source_id TEXT,
+ UNIQUE (repository, name, identifier));
+CREATE TABLE IF NOT EXISTS books (
+ id INTEGER PRIMARY KEY, title TEXT NOT NULL UNIQUE, year INTEGER, publisher TEXT, isbn TEXT, oclc TEXT,
+ ia_identifier TEXT, ia_access TEXT, hathi_url TEXT, record_url TEXT, notes TEXT, verified INTEGER NOT NULL DEFAULT 0);
+CREATE TABLE IF NOT EXISTS book_cartoon_refs (book_id INTEGER REFERENCES books(id), cartoon_id INTEGER REFERENCES cartoons(id),
+ page TEXT, note TEXT, PRIMARY KEY (book_id, cartoon_id));
+CREATE TABLE IF NOT EXISTS cartoon_links (
+ cartoon_id INTEGER NOT NULL REFERENCES cartoons(id), related_id INTEGER NOT NULL REFERENCES cartoons(id),
+ relation TEXT NOT NULL CHECK (relation IN ('duplicate_of','possible_duplicate','contained_in_candidate')),
+ score REAL, detail TEXT, PRIMARY KEY (cartoon_id, related_id, relation));
+CREATE TABLE IF NOT EXISTS crawl_runs (
+ id INTEGER PRIMARY KEY, source TEXT NOT NULL, started TEXT NOT NULL, completed TEXT,
+ pages_scanned INTEGER DEFAULT 0, records_seen INTEGER DEFAULT 0, records_added INTEGER DEFAULT 0,
+ records_updated INTEGER DEFAULT 0, errors INTEGER DEFAULT 0, status TEXT, notes TEXT);
+CREATE TABLE IF NOT EXISTS errors (
+ id INTEGER PRIMARY KEY, run_id INTEGER REFERENCES crawl_runs(id), source TEXT, url TEXT, error TEXT, at TEXT);
+"""
+
+
+def now() -> str:
+ return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
+
+
+def connect(path: Path | str | None = None) -> sqlite3.Connection:
+ conn = sqlite3.connect(str(path or config.DB_PATH), timeout=60, check_same_thread=False)
+ conn.row_factory = sqlite3.Row
+ conn.execute("PRAGMA journal_mode=WAL")
+ conn.execute("PRAGMA foreign_keys=ON")
+ return conn
+
+
+def init_db(conn: sqlite3.Connection) -> None:
+ conn.executescript(SCHEMA)
+ conn.commit()
+
+
+# ---------------------------------------------------------------- sources
+def upsert_source(conn, id: str, name: str, *, repository=None, url=None, classification=None,
+ status=None, access_notes=None, notes=None) -> None:
+ conn.execute(
+ """INSERT INTO sources (id,name,repository,url,classification,status,access_notes,notes,last_checked)
+ VALUES (?,?,?,?,?,?,?,?,?)
+ ON CONFLICT(id) DO UPDATE SET name=excluded.name,
+ repository=COALESCE(excluded.repository,repository), url=COALESCE(excluded.url,url),
+ classification=COALESCE(excluded.classification,classification),
+ status=COALESCE(excluded.status,status), access_notes=COALESCE(excluded.access_notes,access_notes),
+ notes=COALESCE(excluded.notes,notes), last_checked=excluded.last_checked""",
+ (id, name, repository, url, classification, status, access_notes, notes, now()),
+ )
+
+
+# ---------------------------------------------------------------- cartoons
+CARTOON_FIELDS = ["granularity", "date_exact", "date_start", "date_end", "year", "date_is_estimate", "title",
+ "caption", "description", "publication", "syndicate", "medium", "dimensions", "signed_name",
+ "rights_text", "copyright_holder", "notes"]
+SOURCE_FIELDS = ["repository", "collection_name", "box", "folder", "page", "record_url", "image_url",
+ "thumbnail_url", "access_level", "rights_url", "provenance"]
+
+
+def _id_for(conn, table: str, name: str) -> int:
+ row = conn.execute(f"SELECT id FROM {table} WHERE name=?", (name,)).fetchone()
+ if row:
+ return row[0]
+ return conn.execute(f"INSERT INTO {table}(name) VALUES (?)", (name,)).lastrowid
+
+
+def save_record(conn, rec, source_id: str) -> str:
+ """Insert or update one cartoon + its cartoon_sources row. Returns 'added' | 'updated' | 'unchanged'."""
+ from .normalize import is_president # local import avoids a cycle
+
+ ts = now()
+ row = conn.execute("SELECT * FROM cartoons WHERE canonical_id=?", (rec.canonical_id,)).fetchone()
+ vals = {f: getattr(rec, f) for f in CARTOON_FIELDS}
+ vals["date_is_estimate"] = int(bool(vals["date_is_estimate"]))
+ if row is None:
+ cols = ["canonical_id", *CARTOON_FIELDS, "created_at", "updated_at"]
+ cid = conn.execute(
+ f"INSERT INTO cartoons ({','.join(cols)}) VALUES ({','.join('?' * len(cols))})",
+ [rec.canonical_id, *vals.values(), ts, ts],
+ ).lastrowid
+ status = "added"
+ else:
+ cid = row["id"]
+ # only fill/overwrite with non-empty new values (never blank out richer data)
+ changes = {k: v for k, v in vals.items() if v not in (None, "") and row[k] != v}
+ if changes:
+ sets = ",".join(f"{k}=?" for k in changes)
+ conn.execute(f"UPDATE cartoons SET {sets}, updated_at=? WHERE id=?", [*changes.values(), ts, cid])
+ status = "updated"
+ else:
+ status = "unchanged"
+ # source row
+ svals = {f: getattr(rec, f) for f in SOURCE_FIELDS}
+ ex = conn.execute("SELECT id FROM cartoon_sources WHERE source_id=? AND identifier=?",
+ (source_id, rec.identifier)).fetchone()
+ if ex:
+ conn.execute(f"UPDATE cartoon_sources SET cartoon_id=?, {','.join(f'{k}=?' for k in svals)}, retrieved_at=? WHERE id=?",
+ [cid, *svals.values(), ts, ex[0]])
+ else:
+ cols = ["cartoon_id", "source_id", "identifier", *SOURCE_FIELDS, "retrieved_at"]
+ conn.execute(f"INSERT INTO cartoon_sources ({','.join(cols)}) VALUES ({','.join('?' * len(cols))})",
+ [cid, source_id, rec.identifier, *svals.values(), ts])
+ for s in rec.subjects or []:
+ s = s.strip()
+ if s:
+ conn.execute("INSERT OR IGNORE INTO cartoon_subjects VALUES (?,?)", (cid, _id_for(conn, "subjects", s)))
+ for p in rec.people or []:
+ p = p.strip()
+ if p:
+ pid = _id_for(conn, "people", p)
+ if is_president(p):
+ conn.execute("UPDATE people SET is_president=1 WHERE id=?", (pid,))
+ conn.execute("INSERT OR IGNORE INTO cartoon_people VALUES (?,?)", (cid, pid))
+ if rec.publication:
+ conn.execute("INSERT OR IGNORE INTO publications(name) VALUES (?)", (rec.publication,))
+ return status
+
+
+# ---------------------------------------------------------------- crawl runs / errors
+def start_run(conn, source: str) -> int:
+ rid = conn.execute("INSERT INTO crawl_runs(source,started,status) VALUES (?,?,?)", (source, now(), "running")).lastrowid
+ conn.commit()
+ return rid
+
+
+def finish_run(conn, run_id: int, stats: dict, status: str, notes: str | None = None) -> None:
+ conn.execute(
+ """UPDATE crawl_runs SET completed=?, pages_scanned=?, records_seen=?, records_added=?, records_updated=?,
+ errors=?, status=?, notes=? WHERE id=?""",
+ (now(), stats.get("pages", 0), stats.get("seen", 0), stats.get("added", 0), stats.get("updated", 0),
+ stats.get("errors", 0), status, notes, run_id),
+ )
+ conn.commit()
+
+
+def log_error(conn, run_id: int | None, source: str, url: str | None, error: str) -> None:
+ conn.execute("INSERT INTO errors(run_id,source,url,error,at) VALUES (?,?,?,?,?)", (run_id, source, url, error[:2000], now()))
+ conn.commit()
+
+
+def dump_json(obj) -> str:
+ return json.dumps(obj, ensure_ascii=False, indent=2, default=str)
diff --git a/src/conrad/models.py b/src/conrad/models.py
new file mode 100644
index 0000000..2493424
--- /dev/null
+++ b/src/conrad/models.py
@@ -0,0 +1,41 @@
+"""Record model shared by all crawlers."""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class CartoonRecord:
+ canonical_id: str # stable, source-derived key (e.g. 'loc:2016685149', 'hunt:CON-123')
+ identifier: str # the repository's own identifier for this record
+ granularity: str = "item" # item | folder | box_range
+ title: str | None = None
+ caption: str | None = None
+ description: str | None = None
+ date_exact: str | None = None
+ date_start: str | None = None
+ date_end: str | None = None
+ year: int | None = None
+ date_is_estimate: bool = False
+ publication: str | None = None
+ syndicate: str | None = None
+ medium: str | None = None
+ dimensions: str | None = None
+ signed_name: str | None = None
+ rights_text: str | None = None
+ copyright_holder: str | None = None
+ notes: str | None = None
+ # cartoon_sources fields
+ repository: str | None = None
+ collection_name: str | None = None
+ box: str | None = None
+ folder: str | None = None
+ page: str | None = None
+ record_url: str | None = None
+ image_url: str | None = None # METADATA ONLY — never fetched, never rendered
+ thumbnail_url: str | None = None # METADATA ONLY — never fetched, never rendered
+ access_level: str | None = None
+ rights_url: str | None = None
+ provenance: str | None = None
+ subjects: list[str] = field(default_factory=list)
+ people: list[str] = field(default_factory=list)
diff --git a/src/conrad/normalize.py b/src/conrad/normalize.py
new file mode 100644
index 0000000..23ae1d6
--- /dev/null
+++ b/src/conrad/normalize.py
@@ -0,0 +1,145 @@
+"""Date, title, subject and name normalisation."""
+from __future__ import annotations
+
+import calendar
+import re
+import unicodedata
+
+MONTHS = {m.lower(): i for i, m in enumerate(calendar.month_abbr) if m}
+MONTHS.update({m.lower(): i for i, m in enumerate(calendar.month_name) if m})
+MONTHS["sept"] = 9
+
+
+def _mon(word: str | None) -> int | None:
+ if not word:
+ return None
+ return MONTHS.get(word.lower().rstrip(".")) or MONTHS.get(word.lower()[:3])
+
+
+def _last(y: int, m: int) -> int:
+ return calendar.monthrange(y, m)[1]
+
+
+def _iso(y, m=None, d=None) -> str:
+ return f"{y:04d}" + (f"-{m:02d}" if m else "") + (f"-{d:02d}" if d else "")
+
+
+def parse_date(text: str | None) -> dict:
+ """Parse free-text archival dates into {date_exact, date_start, date_end, year, estimate}.
+
+ date_exact is only set when a full day is known. Ranges set date_start/date_end.
+ """
+ out = {"date_exact": None, "date_start": None, "date_end": None, "year": None, "estimate": False}
+ if not text:
+ return out
+ t = str(text).strip()
+ t = re.sub(r"\[(publication date|date created|printed later)\]", "", t, flags=re.I).strip()
+ if re.search(r"\?|circa|ca\.|approximately", t, re.I):
+ out["estimate"] = True
+ # ISO: 1989-07-16 / 1969-03 / 1969
+ m = re.fullmatch(r"\[?(\d{4})(?:-(\d{2})(?:-(\d{2}))?)?\]?", t)
+ if m:
+ y, mo, d = int(m[1]), int(m[2]) if m[2] else None, int(m[3]) if m[3] else None
+ return _fill(out, y, mo, d)
+ # "1984 Mar. 13" / "1984 March" / "Mar 13, 1984" / "March 1964"
+ m = re.search(r"(\d{4})\s+([A-Za-z]{3,9})\.?(?:\s+(\d{1,2}))?", t)
+ if m and _mon(m[2]):
+ return _fill(out, int(m[1]), _mon(m[2]), int(m[3]) if m[3] else None)
+ m = re.search(r"([A-Za-z]{3,9})\.?\s+(?:(\d{1,2}),?\s+)?(\d{4})", t)
+ if m and _mon(m[1]):
+ return _fill(out, int(m[3]), _mon(m[1]), int(m[2]) if m[2] else None)
+ # "between 1971 and 1980", "1950s-1963", "1967-1968", "1950s"
+ yrs = [int(y) for y in re.findall(r"(1[89]\d\d|20[0-2]\d)", t)]
+ decade = re.search(r"(1[89]\d0|20[0-2]0)s", t)
+ if yrs:
+ lo, hi = min(yrs), max(yrs)
+ if decade and int(decade[1]) == lo and len(yrs) == 1:
+ hi = lo + 9
+ if lo == hi:
+ return _fill(out, lo, None, None)
+ out.update(date_start=f"{lo}-01-01", date_end=f"{hi}-12-31", year=None)
+ return out
+ return out
+
+
+def _fill(out: dict, y: int, mo: int | None, d: int | None) -> dict:
+ out["year"] = y
+ if mo and d:
+ out["date_exact"] = _iso(y, mo, d)
+ out["date_start"] = out["date_end"] = out["date_exact"]
+ elif mo:
+ out["date_start"], out["date_end"] = _iso(y, mo, 1), _iso(y, mo, _last(y, mo))
+ else:
+ out["date_start"], out["date_end"] = f"{y}-01-01", f"{y}-12-31"
+ return out
+
+
+def norm_title(t: str | None) -> str:
+ if not t:
+ return ""
+ t = unicodedata.normalize("NFKD", t)
+ t = re.sub(r"--\s*news item\.?", "", t, flags=re.I)
+ t = t.lower().replace("…", " ").replace("&", " and ")
+ t = re.sub(r"[^a-z0-9 ]+", " ", t)
+ t = re.sub(r"\s+", " ", t).strip()
+ return "" if t in ("untitled", "untitled drawing", "") else t
+
+
+def clean_subject(s: str) -> str:
+ s = re.sub(r"--\s*\d{4}-\d{4}\.?$", "", s.strip())
+ s = re.sub(r",?\s*\d{4}-(\d{4})?\.?$", "", s)
+ return s.rstrip(" .").strip()
+
+
+GENERIC_SUBJECTS = re.compile(r"^(editorial cartoons|political cartoons|drawings|caricatures|cartoons|"
+ r"american|pen and ink drawings|proofs|prints|graphic arts)\b", re.I)
+
+PERSON_RE = re.compile(r"^([A-Z][A-Za-z'.\-]+(?: [A-Z][A-Za-z'.\-]+)?),\s+([A-Z][A-Za-z.\- ]+?)(?:\s*\(|,\s*\d{4}|--|$)")
+
+PRESIDENTS = {
+ "truman": "Truman, Harry S.", "eisenhower": "Eisenhower, Dwight D.", "kennedy, john": "Kennedy, John F.",
+ "johnson, lyndon": "Johnson, Lyndon B.", "nixon, richard": "Nixon, Richard M.", "ford, gerald": "Ford, Gerald R.",
+ "carter, jimmy": "Carter, Jimmy", "reagan, ronald": "Reagan, Ronald", "bush, george h": "Bush, George H. W.",
+ "clinton, bill": "Clinton, Bill", "bush, george w": "Bush, George W.", "obama": "Obama, Barack",
+}
+PRESIDENT_WORDS = { # free-text mentions in titles/scope notes
+ r"\btruman\b": "Truman, Harry S.", r"\beisenhower|\bike\b": "Eisenhower, Dwight D.",
+ r"\bjfk\b|john f\.? kennedy": "Kennedy, John F.", r"\blbj\b|lyndon": "Johnson, Lyndon B.",
+ r"\bnixon": "Nixon, Richard M.", r"gerald ford|\bford administration": "Ford, Gerald R.",
+ r"jimmy carter|\bcarter administration": "Carter, Jimmy", r"\breagan": "Reagan, Ronald",
+ r"\bclinton\b": "Clinton, Bill", r"\bobama": "Obama, Barack",
+}
+
+
+def is_president(name: str) -> bool:
+ n = name.lower()
+ return any(n.startswith(k) for k in PRESIDENTS) or name in PRESIDENTS.values()
+
+
+def person_from_heading(h: str) -> str | None:
+ """'Nixon, Richard M. (Richard Milhous), 1913-1994' -> 'Nixon, Richard M.'; non-person headings -> None."""
+ h = h.strip()
+ for k, v in PRESIDENTS.items():
+ if h.lower().startswith(k):
+ return v
+ m = PERSON_RE.match(h)
+ if not m:
+ return None
+ first = m[2].strip()
+ # reject corporate/geographic headings that happen to contain a comma
+ if re.search(r"\b(Inc|Company|Corporation|University|Department|Association|Party|Committee|Council|"
+ r"County|State|City|Republic|Church|Industry|Office|Court|Commission|Bureau|Army|Navy)\b", h):
+ return None
+ if len(first) < 2 or first.lower() in {"the"}:
+ return None
+ return f"{m[1]}, {first}".strip()
+
+
+def presidents_in_text(text: str | None) -> list[str]:
+ if not text:
+ return []
+ return sorted({v for k, v in PRESIDENT_WORDS.items() if re.search(k, text, re.I)})
+
+
+def decade(y: int | None) -> str:
+ return f"{y // 10 * 10}s" if y else "undated"
diff --git a/src/conrad/rights.py b/src/conrad/rights.py
new file mode 100644
index 0000000..af8d675
--- /dev/null
+++ b/src/conrad/rights.py
@@ -0,0 +1,36 @@
+"""Rights + access-level vocabulary. Conrad's cartoons are in copyright (LA Times / Denver Post /
+Conrad estate); nothing here is public domain. We store WHERE a work can be legally viewed."""
+from __future__ import annotations
+
+# access_level vocabulary for cartoon_sources
+ONLINE_IMAGE = "online_image" # repository itself shows a digitized image to the public (link out)
+ONLINE_METADATA = "online_metadata" # catalog record online, no public image
+ARCHIVE_VISIT = "archive_visit" # physical item; view in reading room / by appointment
+PAYWALLED = "paywalled" # newspaper database etc.
+CDL = "controlled_digital_lending" # borrowable scan (archive.org); not accessed by this project
+
+PUBLIC_IMAGE_LEVELS = {ONLINE_IMAGE}
+REQUIRES_ARCHIVE = {ARCHIVE_VISIT}
+
+DEFAULT_RIGHTS = {
+ "Huntington Library": ("Physical originals at The Huntington Library, San Marino, CA. Copyright retained by "
+ "the Conrad estate / original publisher; reproduction requires permission.",
+ "https://www.huntington.org/reproduction-requests"),
+ "Library of Congress": ("Library of Congress Prints & Photographs. Rights status per item record; Conrad works are "
+ "in copyright — publication may require permission from the copyright holder.",
+ "https://www.loc.gov/rr/print/res/rights.html"),
+ "Syracuse University": ("Syracuse University Special Collections Research Center; reading-room access. "
+ "Copyright held by the creator's estate / publisher.",
+ "https://library.syracuse.edu/special-collections-research-center/"),
+ "Wichita State University": ("Wichita State University Libraries Special Collections (MS 90-18). Copyright held by "
+ "the creator's estate / publisher.", "https://libraries.wichita.edu/specialcollections"),
+ "Ohio State University": ("Billy Ireland Cartoon Library & Museum. Copyright held by the creator's estate / publisher.",
+ "https://cartoons.osu.edu/"),
+}
+
+COPYRIGHT_NOTE = ("In copyright (Los Angeles Times, Denver Post, or the Paul Conrad estate). This archive stores "
+ "metadata and link-outs only; no image is reproduced.")
+
+
+def rights_for(repository: str | None) -> tuple[str, str | None]:
+ return DEFAULT_RIGHTS.get(repository or "", (COPYRIGHT_NOTE, None))
← e3065ee initial scaffold
·
back to Paul Conrad Archive
·
Live Huntington EAD crawler: 8628 CON slots verified, box in 8ff2c42 →