← back to Paul Conrad Archive
Live Huntington EAD crawler: 8628 CON slots verified, box inventory, charset fix
8ff2c42d729799ac5bfee9e87b601a61b242a4d1 · 2026-09-24 16:16:35 -0700 · Steve Abrams
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Files touched
M data/conrad.dbM src/conrad/crawlers/base.pyA src/conrad/crawlers/huntington.py
Diff
commit 8ff2c42d729799ac5bfee9e87b601a61b242a4d1
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Sep 24 16:16:35 2026 -0700
Live Huntington EAD crawler: 8628 CON slots verified, box inventory, charset fix
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
data/conrad.db | Bin 9650176 -> 12853248 bytes
src/conrad/crawlers/base.py | 9 +-
src/conrad/crawlers/huntington.py | 201 ++++++++++++++++++++++++++++++++++++++
3 files changed, 209 insertions(+), 1 deletion(-)
diff --git a/data/conrad.db b/data/conrad.db
index 729fbf9..4b5ce97 100644
Binary files a/data/conrad.db and b/data/conrad.db differ
diff --git a/src/conrad/crawlers/base.py b/src/conrad/crawlers/base.py
index fd37491..e4e61e1 100644
--- a/src/conrad/crawlers/base.py
+++ b/src/conrad/crawlers/base.py
@@ -59,7 +59,9 @@ class Http:
rp = RobotFileParser()
try:
r = self.s.get(base + "/robots.txt", timeout=config.TIMEOUT)
- if r.status_code in (401, 403) or looks_like_challenge(r.status_code, r.text):
+ if r.status_code == 403 and "amazonaws.com" in p.netloc and "<Code>AccessDenied</Code>" in r.text:
+ rp.parse([]) # S3 answers 403 for any MISSING key in a non-listable bucket: no robots.txt exists
+ elif r.status_code in (401, 403) or looks_like_challenge(r.status_code, r.text):
rp.parse(["User-agent: *", "Disallow: /"]) # conservative: cannot read robots -> treat as disallowed
rp._unreadable = True # type: ignore[attr-defined]
elif r.status_code >= 400:
@@ -132,6 +134,11 @@ class Http:
self.requests_made += 1
if r.status_code == 429 or r.status_code >= 500:
raise Transient(f"HTTP {r.status_code} at {url}")
+ if "charset" not in r.headers.get("content-type", "").lower():
+ try:
+ return r.status_code, r.content.decode("utf-8")
+ except UnicodeDecodeError:
+ pass
return r.status_code, r.text
diff --git a/src/conrad/crawlers/huntington.py b/src/conrad/crawlers/huntington.py
new file mode 100644
index 0000000..f7bfd8b
--- /dev/null
+++ b/src/conrad/crawlers/huntington.py
@@ -0,0 +1,201 @@
+"""Huntington Library — Paul Conrad Papers (mssCON 1-12360).
+
+Live-parses the public EAD finding aid (the same XML OAC renders) for EVERY series / subseries / box /
+item component. Original-drawing boxes carry CON number ranges; each CON number becomes a box_range
+slot (date = box span; NOT item-level). Non-drawing series (correspondence, book files, tear sheets…)
+are inventoried into `collections` for context, never counted as cartoons.
+"""
+from __future__ import annotations
+
+import re
+import xml.etree.ElementTree as ET
+from datetime import date
+
+from bs4 import BeautifulSoup
+
+from .. import rights
+from ..models import CartoonRecord
+from .base import Blocked, Crawler, Transient
+
+EAD_URL = "https://cinco-prd.s3.amazonaws.com/media/ead/conrad.xml"
+OAC_URL = "https://oac.cdlib.org/findaid/ark:/13030/c8z03dxd/"
+CATALOG_URL = "https://catalog.huntington.org/record=b1768021"
+VERSO_URL = "https://www.huntington.org/verso/volunteering-decipher-paul-conrad"
+COLL = "Paul Conrad Papers (mssCON 1-12360)"
+
+MONTHS = {m: i for i, m in enumerate(["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"], 1)}
+
+
+def _mon(w):
+ return MONTHS.get((w or "").lower()[:3])
+
+
+def _last(y, m):
+ import calendar
+ return calendar.monthrange(y, m)[1]
+
+
+def parse_span(span: str):
+ """'1970, Mar- Aug' | '1969, Sept- 1970, Feb' | '1967- 1968' | '1995, May 22- July' -> (start, end)."""
+ parts = [p.strip() for p in re.split(r"\s*[-–—]\s*", span) if p.strip()]
+ if not parts:
+ return None
+ L, R = parts[0], (parts[1] if len(parts) > 1 else "")
+ lm = re.match(r"^(\d{4})(?:,\s*([A-Za-z]+)\.?(?:\s+(\d{1,2}))?)?", L)
+ if not lm:
+ return None
+ y1, m1, d1 = int(lm[1]), _mon(lm[2]) or 1, int(lm[3]) if lm[3] else 1
+ y2 = y1
+ ry = re.match(r"^(\d{4})(?:,\s*([A-Za-z]+)\.?(?:\s+(\d{1,2}))?)?", R)
+ rm = re.match(r"^([A-Za-z]+)\.?(?:\s+(\d{1,2}))?", R)
+ if ry:
+ y2 = int(ry[1]); m2 = _mon(ry[2]) or 12; d2 = int(ry[3]) if ry[3] else _last(y2, m2)
+ elif rm and _mon(rm[1]):
+ m2 = _mon(rm[1]); d2 = int(rm[2]) if rm[2] else _last(y2, m2)
+ elif lm[2]:
+ m2 = m1; d2 = d1 if lm[3] else _last(y1, m1)
+ else:
+ m2, d2 = 12, 31
+ return date(y1, m1, d1).isoformat(), date(y2, m2, d2).isoformat()
+
+
+def _txt(el) -> str:
+ return re.sub(r"\s+", " ", "".join(el.itertext())).strip() if el is not None else ""
+
+
+def parse_ead(xml: str) -> dict:
+ root = ET.fromstring(xml.encode("utf-8"))
+ arch = root.find("archdesc")
+ did = arch.find("did")
+ coll = {
+ "title": _txt(did.find("unittitle")), "unitid": _txt(did.find("unitid")),
+ "dates": _txt(did.find("unitdate")),
+ "extent": "; ".join(_txt(e) for e in did.iter("extent")) or _txt(did.find("physdesc")),
+ "access": _txt(arch.find("accessrestrict")), "use": _txt(arch.find("userestrict")),
+ "abstract": _txt(did.find("abstract")),
+ }
+ comps = []
+
+ def walk(el, path):
+ for c in el:
+ if not re.fullmatch(r"c0\d|c", c.tag):
+ continue
+ cd = c.find("did")
+ title = _txt(cd.find("unittitle")) if cd is not None else ""
+ unitid = _txt(cd.find("unitid")) if cd is not None else ""
+ unitdate = _txt(cd.find("unitdate")) if cd is not None else ""
+ conts = {(k.get("type") or "").lower(): _txt(k) for k in (cd.findall("container") if cd is not None else [])}
+ comps.append({"level": c.get("level"), "tag": c.tag, "title": title, "unitid": unitid, "unitdate": unitdate,
+ "box": conts.get("box"), "folder": conts.get("folder"), "path": path})
+ walk(c, path + [title.rstrip(".")])
+ walk(arch.find("dsc"), [])
+ return {"collection": coll, "components": comps}
+
+
+def drawing_boxes(comps: list[dict]) -> list[dict]:
+ boxes = []
+ for c in comps:
+ sub = c["path"][1] if len(c["path"]) > 1 else ""
+ if not sub.startswith("Original Drawings") or not c["title"].startswith("Drawings,"):
+ continue
+ src = c["title"] if re.search(r"CON\s*\d", c["title"]) else c["unitid"]
+ cm = re.search(r"CON\s*(\d+)\s*[-–]\s*(\d+)", src)
+ span_text = re.sub(r"\.?\s*CON[\s\S]*$", "", c["title"].replace("Drawings,", "", 1)).strip().rstrip(".")
+ sp = parse_span(span_text)
+ boxes.append({"series": " > ".join(c["path"][:2]), "subseries": sub, "box": c["box"] or "", "title": c["title"],
+ "span": span_text, "con_start": int(cm[1]) if cm else None, "con_end": int(cm[2]) if cm else None,
+ "date_start": sp[0] if sp else None, "date_end": sp[1] if sp else None})
+ # repair transcription typos where an end number overruns the next box start (e.g. 'CON 4890- 49152')
+ repairs = []
+ for i, b in enumerate(boxes):
+ nxt = next((x for x in boxes[i + 1:] if x["subseries"] == b["subseries"] and x["con_start"]), None)
+ if b["con_start"] and b["con_end"] and nxt and b["con_end"] >= nxt["con_start"]:
+ repairs.append(f"box {b['box']}: {b['con_start']}-{b['con_end']} -> {b['con_start']}-{nxt['con_start'] - 1}")
+ b["con_end"] = nxt["con_start"] - 1
+ return boxes, repairs
+
+
+def slot_records(boxes: list[dict]) -> list[CartoonRecord]:
+ from datetime import date as D
+ txt, rurl = rights.rights_for("Huntington Library")
+ out = []
+ for b in boxes:
+ if not (b["con_start"] and b["con_end"] and b["date_start"]):
+ continue
+ n = b["con_end"] - b["con_start"] + 1
+ t0, t1 = D.fromisoformat(b["date_start"]).toordinal(), D.fromisoformat(b["date_end"]).toordinal()
+ for k in range(n):
+ con = b["con_start"] + k
+ est = D.fromordinal(round(t0 + (t1 - t0) * (0 if n == 1 else k / (n - 1)))).isoformat()
+ out.append(CartoonRecord(
+ canonical_id=f"hunt:CON-{con}", identifier=f"CON {con}", granularity="box_range",
+ date_start=b["date_start"], date_end=b["date_end"], year=int(est[:4]), date_is_estimate=True,
+ description=f"Original drawing slot CON {con} in Box {b['box']} ({b['series']}); box span "
+ f"'{b['span']}'. Title not cataloged in the public finding aid.",
+ notes=f"range-level slot; interpolated date_est={est} (linear within box span, NOT a cataloged date)",
+ publication="Los Angeles Times" if b["date_start"] >= "1964" and b["date_end"] <= "1993-12-31" else None,
+ medium="Original drawing", rights_text=txt, rights_url=rurl, repository="Huntington Library",
+ collection_name=COLL, box=b["box"], folder=f"CON {con}", record_url=OAC_URL,
+ access_level=rights.ARCHIVE_VISIT, provenance=f"live:{EAD_URL}",
+ ))
+ return out
+
+
+class Huntington(Crawler):
+ source_id = "huntington"
+ name = "Huntington Library — Paul Conrad Papers (EAD finding aid)"
+ repository = "Huntington Library"
+ url = EAD_URL
+ classification = "PHYSICAL_ARCHIVE"
+ access_notes = ("Originals viewable in the Huntington reading room (reader credentials required). Public EAD "
+ "lists box-level CON ranges only; the Huntington's internal item database is not public.")
+
+ def crawl(self) -> None:
+ status, xml = self.http.get(EAD_URL)
+ self.stats["pages"] += 1
+ if status != 200:
+ raise Transient(f"EAD HTTP {status}")
+ ead = parse_ead(xml)
+ c = ead["collection"]
+ self.conn.execute(
+ """INSERT OR REPLACE INTO collections(repository,name,identifier,url,extent,date_range,level,notes,source_id)
+ VALUES (?,?,?,?,?,?,?,?,?)""",
+ ("Huntington Library", c["title"] or COLL, c["unitid"], OAC_URL, c["extent"], c["dates"], "collection",
+ f"ACCESS: {c['access'][:600]} | USE: {c['use'][:600]}", self.source_id))
+ n_boxes = 0
+ for comp in ead["components"]:
+ if comp["level"] in ("box", "Box") or (comp["box"] and comp["level"] not in ("item",)):
+ n_boxes += 1
+ self.conn.execute(
+ """INSERT OR REPLACE INTO collections(repository,name,identifier,url,extent,date_range,level,notes,source_id)
+ VALUES (?,?,?,?,?,?,?,?,?)""",
+ ("Huntington Library", " > ".join(comp["path"] + [comp["title"]])[:400], f"Box {comp['box']}",
+ OAC_URL, None, comp["unitdate"], "box", comp["unitid"], self.source_id))
+ if len(comp["path"]) > 1 and comp["path"][1].startswith("Book Files") and comp["level"] == "item":
+ self.conn.execute("INSERT OR IGNORE INTO collections(repository,name,identifier,url,level,notes,source_id)"
+ " VALUES (?,?,?,?,?,?,?)", ("Huntington Library", comp["title"][:400], comp["unitid"],
+ OAC_URL, "book_file", "Huntington Book Files series", self.source_id))
+ boxes, repairs = drawing_boxes(ead["components"])
+ for rec in slot_records(boxes):
+ self.save(rec)
+ self.conn.commit()
+ self.notes.append(f"EAD: {len(ead['components'])} components, {n_boxes} boxes, {len(boxes)} original-drawing "
+ f"boxes; typo repairs: {repairs}")
+ # collection-level catalog record + volunteer article (context only)
+ for url in (CATALOG_URL, VERSO_URL):
+ try:
+ st, html = self.http.get(url)
+ self.stats["pages"] += 1
+ if st == 200:
+ soup = BeautifulSoup(html, "html.parser")
+ text = re.sub(r"\s+", " ", soup.get_text(" "))
+ m = re.search(r"(\d[\d,]*)\s+(?:original\s+)?(?:cartoons|drawings)", text, re.I)
+ self.notes.append(f"{url}: ok" + (f" (mentions '{m.group(0)}')" if m else ""))
+ else:
+ self.error(url, f"HTTP {st}")
+ except (Blocked, Transient) as e:
+ self.error(url, e)
+ self.notes.append(f"{url}: {e}")
+
+
+CRAWLER = Huntington
← 380110b Core schema, models, normalizers, polite HTTP base and seed
·
back to Paul Conrad Archive
·
auto-data-snapshot: 2026-09-24T16:24:12 (3 data files) — dat bc3f93c →