← back to Paul Conrad Archive
IA full-text classifier: strict credit-line + prose-reprint rules, stated-title extraction, award/roster/music negatives
f2625ece88ad1e9a806acc6cce8397d9f3f54619 · 2026-09-25 10:55:30 -0700 · Steve Abrams
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SW5KHgwrh2Rr9TKHM6ndqM
Files touched
M src/conrad/crawlers/ia_fulltext.pyM tests/test_ia_fulltext.py
Diff
commit f2625ece88ad1e9a806acc6cce8397d9f3f54619
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Fri Sep 25 10:55:30 2026 -0700
IA full-text classifier: strict credit-line + prose-reprint rules, stated-title extraction, award/roster/music negatives
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SW5KHgwrh2Rr9TKHM6ndqM
---
src/conrad/crawlers/ia_fulltext.py | 66 ++++++++++++++++++++++++++++++++------
tests/test_ia_fulltext.py | 14 ++++++++
2 files changed, 70 insertions(+), 10 deletions(-)
diff --git a/src/conrad/crawlers/ia_fulltext.py b/src/conrad/crawlers/ia_fulltext.py
index d9cf0ab..1c928cb 100644
--- a/src/conrad/crawlers/ia_fulltext.py
+++ b/src/conrad/crawlers/ia_fulltext.py
@@ -56,16 +56,39 @@ def _norm_line(s: str) -> str:
return re.sub(r"\s+", " ", HL.sub(r"\1", s)).strip()
+AWARD = re.compile(r"prize|award|pulitzer|winner|best\s+cartoon|citation|honorable|finalist|nominat|\b(?:19|20)\d\d\s*[—–-]\s*paul"
+ r"|cartoons?\s*[—–-]|five cartoons a week|music|copyright entries|;\s*m\s+paul|joined the|work(?:ed)? at|\bmr\.", re.I)
+STRONG = re.compile(r"syndicate|©|\(c\)|copyright|reprinted|courtesy|permission", re.I)
+DASH_CREDIT = re.compile(r"^[—–-]+\s*(?:paul\s+)?conrad\s*[,.]?\s*(?:the\s+)?(?:los\s*angeles\s*times|denver\s*post)\b", re.I)
+# prose credits that mark a REPRINTED cartoon (textbooks / anthologies / magazines)
+PROSE_CREDITS = [
+ re.compile(r"source:\s*paul\s+conrad\s+in\s+the\s+(?:los\s+angeles\s+times|denver\s+post)", re.I),
+ re.compile(r"(?P<title>[A-Z][^./|]{3,60})\.\s*by\s+paul\s+conrad\.?,?\s*(?:the\s+)?(?:los\s+angeles\s+times|denver\s+post)",
+ re.I),
+ re.compile(r"(?:this|the above|the following|original artwork for this)\s+cartoon\s+by\s+paul\s+conrad", re.I),
+ re.compile(r"paul\s+conrad\s+cartoon\s+reprinted", re.I),
+]
+QUOTED_CAPTION = re.compile(r"[’”\"!?]\s*$")
+
+
def classify(snippets: list[str]) -> tuple[str, str | None]:
- """-> (kind, evidence line). kind: 'credit' (a printed Conrad cartoon credit), 'other_conrad', 'mention'.
+ """-> (kind, evidence). kind: 'credit' (a printed Conrad cartoon credit), 'other_conrad', 'mention'.
- A CREDIT needs a line that (a) contains the highlighted Conrad match, (b) is short (<= 45 chars, i.e. a credit or
- signature line, not running prose — or <= 90 chars when the line STARTS with the Conrad
- credit, e.g. 'PAUL CONRAD Los Angeles Times © Los Angeles Times Syndicate'), (c) is not another Conrad, and (d) has credit context (LA Times / Syndicate /
- Denver Post / ©) on the same line or within the next two lines, or reads exactly 'PAUL CONRAD'."""
+ A CREDIT line must (a) carry the highlighted Conrad match, (b) be a short credit/signature line (<= 45 chars, or
+ <= 90 when the line STARTS with the Conrad credit, e.g. 'PAUL CONRAD Los Angeles Times © LA Times Syndicate'),
+ (c) not be another Conrad, (d) not sit in an award / biography / roster context (prize lists, 'Five cartoons a
+ week', 'Mr. Conrad joined ...'), and (e) have STRONG credit evidence within itself and the next two lines
+ (Syndicate / © / copyright / reprinted / courtesy), or be a dash attribution under a quoted caption
+ ('...HEALTH HAZARD!’ / —PAUL CONRAD, THE DENVER POST'). A bare 'Paul Conrad' in a list of names never counts."""
other = False
for sn in snippets:
- lines = [ln for ln in re.split(r"\n", sn)]
+ flat = _norm_line(sn.replace("\n", " "))
+ if not OTHER_CONRADS.search(flat):
+ for rx in PROSE_CREDITS:
+ m = rx.search(flat)
+ if m:
+ return "credit", flat[max(0, m.start() - 60): m.end() + 60][:300]
+ lines = sn.split("\n")
for i, raw in enumerate(lines):
if not re.search(r"\{\{\{[^}]*conrad[^}]*\}\}\}", raw, re.I):
continue
@@ -74,16 +97,37 @@ def classify(snippets: list[str]) -> tuple[str, str | None]:
if OTHER_CONRADS.search(window):
other = True
continue
- lead = re.match(r"(?:[©@]\s*)?(?:paul\s+)?conrad\b\W*(?:\(?c\)?\s*)?(?:the\s+)?(?:los\s*angeles\s*times|l\.\s?a\."
+ if AWARD.search(window):
+ continue
+ lead = re.match(r"(?:[©@—–-]\s*)?(?:paul\s+)?conrad\b\W*(?:\(?c\)?\s*)?(?:the\s+)?(?:los\s*angeles\s*times|l\.\s?a\."
r"|denver\s*post|©|copyright|courtesy)", line, re.I)
if len(line) > 45 and not (lead and len(line) <= 90):
continue
- ctx = CREDIT_CONTEXT.search(line) or CREDIT_CONTEXT.search(" ".join(_norm_line(x) for x in lines[i + 1: i + 3]))
- if ctx or re.fullmatch(r"(?:[©@]\s*)?paul\s+conrad[.,]?", line, re.I):
+ nxt = " ".join(_norm_line(x) for x in lines[i: i + 3])
+ prev = _norm_line(lines[i - 1]) if i else ""
+ if (STRONG.search(nxt) and re.search(r"conrad", line, re.I) and len(re.sub(r"(?i)paul|conrad|\W", "", line)) < 40) \
+ or (DASH_CREDIT.match(line) and QUOTED_CAPTION.search(prev)):
return "credit", _norm_line(" / ".join(lines[max(0, i - 1): i + 3]))[:300]
return ("other_conrad" if other else "mention"), None
+TITLE_RX = [re.compile(r"[“\"]([^”\"]{3,80})[”\"]\s*(?:was|is)\s+the\s+(?:\w+\s+)?title\s+of\s+this\s+cartoon", re.I),
+ re.compile(r"(?:\d+\s+|^)([A-Z][A-Za-z'’ ,-]{3,60}?)\.\s*By\s+Paul\s+Conrad\.?,?\s*(?:the\s+)?(?:Los\s+Angeles\s+Times|Denver\s+Post)")]
+
+
+def cartoon_title(evidence: str | None) -> str | None:
+ """A cartoon title only when the OCR text itself states one ('"X" was the title of this cartoon', a contents line
+ 'X. By Paul Conrad. Los Angeles Times')."""
+ for rx in TITLE_RX:
+ m = rx.search(evidence or "")
+ if m:
+ t = m[1].strip(" ,;:")
+ words = t.split()
+ if len(words) > 1 and not re.search(r"\d{2,}", t):
+ return t
+ return None
+
+
MONTHS = {m: i for i, m in enumerate(["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov",
"dec"], 1)}
@@ -249,6 +293,7 @@ class IAFullText(Crawler):
d = dict(date_exact=iso, year=year, date_start=iso or (f"{year}-01-01" if year else None),
date_end=iso or (f"{year}-12-31" if year else None), date_is_estimate=not iso)
where = f"{pub}, {iso or year or 'n.d.'}"
+ ctitle = cartoon_title(evidence)
cid = f"iafts:{ident}:{base}:{pn}"
if cid in seen_ids:
continue
@@ -257,7 +302,8 @@ class IAFullText(Crawler):
note += f" found by IA full-text OCR search {sorted(ent['variants'])}; credit line (OCR, unverified): \"{evidence}\""
rec = CartoonRecord(
canonical_id=cid, identifier=f"{ident}/{base}#n{pn}", granularity="item",
- title=f"[Paul Conrad cartoon — {where}, p. {int(pn) + 1}]", publication=pub if not book else None,
+ title=ctitle or f"[Paul Conrad cartoon — {where}, p. {int(pn) + 1}]",
+ caption=ctitle, publication=pub if not book else None,
syndicate="Los Angeles Times Syndicate" if re.search(r"syndicate", evidence or "", re.I) else None,
notes=note[:1000], rights_text=rights.COPYRIGHT_NOTE, repository="Internet Archive",
collection_name=pub, page=str(int(pn) + 1), record_url=page_url, image_url=img,
diff --git a/tests/test_ia_fulltext.py b/tests/test_ia_fulltext.py
index 9950c49..26d2151 100644
--- a/tests/test_ia_fulltext.py
+++ b/tests/test_ia_fulltext.py
@@ -108,3 +108,17 @@ def test_fetch_negative_two_generators_would_truncate():
head = next(r.iter_content(16))
rest = b"".join(r.iter_content(65536))
assert len(head + rest) == 16
+
+
+def test_cartoon_title_and_more_negatives():
+ from conrad.crawlers.ia_fulltext import cartoon_title
+ assert cartoon_title('noted. "AT YOUR SERVICE, MADAM" was the title of this cartoon by Paul Conrad') == \
+ "AT YOUR SERVICE, MADAM"
+ assert cartoon_title("by Dennis Renault 20 The View from Watts. By Paul Conrad. Los Angeles Times 34") == \
+ "The View from Watts"
+ assert cartoon_title("Paul Conrad, Los Angeles Times") is None
+ # negatives: a music copyright entry, a prize roster and a name list are not cartoon appearances
+ assert classify(["Wallis; 5Aug59.\nCHINA CLIPPER; m {{{Paul Conrad}}}. -\n© Lo ridge Music Inc."])[0] == "mention"
+ assert classify(["Frank Miller, Des Moines Register.\n1964 — {{{Paul Conrad}}}, Denver Post.\n1966 — Don Wright"])[0] \
+ == "mention"
+ assert classify(["Warren Christophel\n{{{Paul Conrad}}}\nMrs. Chauncey Crossgrove"])[0] == "mention"
← 851fc79 auto-data-snapshot: 2026-09-25T10:52:57 (1 data files) — dat
·
back to Paul Conrad Archive
·
docs: 5 unsent draft research-access letters (CDNC, Colorado 92c289d →