← back to Paul Conrad Archive
IA FTS: Editor & Publisher syndicate-ad pages, IA access-restricted check before recording page images, FTS 400 retry; probes: LBJ/JFK eMuseum 403 (no UA swap), Billy Ireland DC 504, Chronicling America 23 hits/0 cartoons; DPLA creator query
7735e27c7f6f555d4073e5d4b30d7c26f575756f · 2026-09-25 11:08:54 -0700 · Steve Abrams
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SW5KHgwrh2Rr9TKHM6ndqM
Files touched
M src/conrad/crawlers/discovery.pyM src/conrad/crawlers/ia_fulltext.pyM src/conrad/crawlers/newspaper_probes.pyM tests/test_ia_fulltext.py
Diff
commit 7735e27c7f6f555d4073e5d4b30d7c26f575756f
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Fri Sep 25 11:08:54 2026 -0700
IA FTS: Editor & Publisher syndicate-ad pages, IA access-restricted check before recording page images, FTS 400 retry; probes: LBJ/JFK eMuseum 403 (no UA swap), Billy Ireland DC 504, Chronicling America 23 hits/0 cartoons; DPLA creator query
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SW5KHgwrh2Rr9TKHM6ndqM
---
src/conrad/crawlers/discovery.py | 2 +-
src/conrad/crawlers/ia_fulltext.py | 59 ++++++++++++++++++++++++++++-----
src/conrad/crawlers/newspaper_probes.py | 14 +++++---
tests/test_ia_fulltext.py | 11 ++++++
4 files changed, 72 insertions(+), 14 deletions(-)
diff --git a/src/conrad/crawlers/discovery.py b/src/conrad/crawlers/discovery.py
index f54a3b7..960a374 100644
--- a/src/conrad/crawlers/discovery.py
+++ b/src/conrad/crawlers/discovery.py
@@ -84,7 +84,7 @@ class Discovery(Crawler):
providers: dict[str, int] = {}
kept = 0
queries = [{"q": '"paul conrad" cartoon'}, {"q": '"conrad, paul"'}, {"q": '"paul conrad" los angeles times'},
- {"sourceResource.creator": '"Paul Conrad, born 27 Jun 1924"'}]
+ {"sourceResource.creator": '"Paul Conrad, born 27 Jun 1924"'}, {"sourceResource.creator": '"Conrad, Paul"'}]
for qp in queries:
q = " ".join(f"{k}={v}" for k, v in qp.items())
for page in range(1, 6):
diff --git a/src/conrad/crawlers/ia_fulltext.py b/src/conrad/crawlers/ia_fulltext.py
index 1c928cb..21e54f3 100644
--- a/src/conrad/crawlers/ia_fulltext.py
+++ b/src/conrad/crawlers/ia_fulltext.py
@@ -21,6 +21,7 @@ How it works (robots-permitted, metadata only):
from __future__ import annotations
import json
+import time
import re
from urllib.parse import quote, urlencode
@@ -71,6 +72,27 @@ PROSE_CREDITS = [
QUOTED_CAPTION = re.compile(r"[’”\"!?]\s*$")
+SYNDICATE_AD = re.compile(r"syndicate|cartoons a week|editorial cartoons|features", re.I)
+
+
+def classify_hit(fields: dict, snippets: list[str]) -> tuple[str, str | None]:
+ """classify() plus the Editor & Publisher rule: E&P carried syndicate sales ADS for Conrad ('PAUL CONRAD / Five
+ cartoons a week', LA Times Syndicate) that usually reproduce a sample cartoon — recorded as kind 'syndicate_ad'."""
+ kind, ev = classify(snippets)
+ if kind == "credit" or not str(fields.get("identifier") or "").startswith("sim_editor-publisher"):
+ return kind, ev
+ for sn in snippets:
+ lines = sn.split("\n")
+ for i, raw in enumerate(lines):
+ line = _norm_line(raw)
+ if re.fullmatch(r"(?:paul\s+)?conrad[.,]?", line, re.I) and "{{{" in raw:
+ window = " ".join(_norm_line(x) for x in lines[max(0, i - 2): i + 3])
+ if SYNDICATE_AD.search(window) and not OTHER_CONRADS.search(window) \
+ and not re.search(r"prize|award|pulitzer|winner|joined|obituar|died", window, re.I):
+ return "syndicate_ad", window[:300]
+ return kind, ev
+
+
def classify(snippets: list[str]) -> tuple[str, str | None]:
"""-> (kind, evidence). kind: 'credit' (a printed Conrad cartoon credit), 'other_conrad', 'mention'.
@@ -184,7 +206,13 @@ class IAFullText(Crawler):
# ------------------------------------------------------------------ search
def fts(self, query: str, page: int) -> dict:
params = {"user_query": query, "hits_per_page": PAGE_SIZE, "page": page, "service_backend": "fts"}
- return self.http.get_json(FTS, params=params)
+ for attempt in range(3): # the FTS service answers a sporadic HTTP 400 that succeeds on retry
+ try:
+ return self.http.get_json(FTS, params=params)
+ except Transient:
+ if attempt == 2:
+ raise
+ time.sleep(5 * (attempt + 1))
def harvest(self) -> dict:
pages: dict[tuple, dict] = {}
@@ -242,8 +270,18 @@ class IAFullText(Crawler):
self._manifests[identifier] = out
return out
+ def access_restricted(self, identifier: str) -> bool:
+ """IA's own flag (metadata JSON): lending-library / restricted items expose no public page image."""
+ try:
+ m = self.http.get_json(f"https://archive.org/metadata/{quote(identifier, safe='')}/metadata")
+ except (Blocked, Transient, ValueError) as e:
+ self.error(f"ia metadata {identifier}", f"{type(e).__name__}: {e}")
+ return True # unknown -> no image claimed
+ r = (m or {}).get("result") or {}
+ return str(r.get("access-restricted-item", "")).lower() == "true"
+
def page_image(self, f: dict) -> str | None:
- if restricted(f):
+ if restricted(f) or self.access_restricted(f["identifier"]):
return None
canv = self.manifest_images(f["identifier"])
if not canv:
@@ -268,9 +306,9 @@ class IAFullText(Crawler):
seen_ids: set[str] = set()
for (ident, base, pn), ent in pages.items():
f = ent["fields"]
- kind, evidence = classify(ent["snippets"])
- self.counts[kind] += 1
- if kind != "credit" or not ident:
+ kind, evidence = classify_hit(f, ent["snippets"])
+ self.counts[kind] = self.counts.get(kind, 0) + 1
+ if kind not in ("credit", "syndicate_ad") or not ident:
continue
iso, year = issue_date(f)
pub = publication(f)
@@ -280,7 +318,7 @@ class IAFullText(Crawler):
img = self.page_image(f)
if img:
self.counts["images"] += 1
- elif restricted(f):
+ else:
self.counts["restricted_no_image"] += 1
ymatch = re.search(r"(?:©|\(c\)|copyright)\s*(19[5-9]\d|20[01]\d)|times,?\s*(19[5-9]\d|20[01]\d)", evidence or "", re.I)
cy = int(ymatch[1] or ymatch[2]) if ymatch else None
@@ -298,7 +336,9 @@ class IAFullText(Crawler):
if cid in seen_ids:
continue
seen_ids.add(cid)
- note = (f"PRINTED APPEARANCE (newspaper/periodical page)" if not book else "PRINTED APPEARANCE (book reprint)")
+ note = ("SYNDICATE ADVERTISEMENT in Editor & Publisher (normally reproduces a sample Conrad cartoon; unverified)"
+ if kind == "syndicate_ad" else
+ "PRINTED APPEARANCE (newspaper/periodical page)" if not book else "PRINTED APPEARANCE (book reprint)")
note += f" found by IA full-text OCR search {sorted(ent['variants'])}; credit line (OCR, unverified): \"{evidence}\""
rec = CartoonRecord(
canonical_id=cid, identifier=f"{ident}/{base}#n{pn}", granularity="item",
@@ -315,8 +355,9 @@ class IAFullText(Crawler):
self.ck.set("last_run", db.now())
self.ck.set("counts", self.counts)
c = self.counts
- self.notes.insert(0, f"FTS hits walked={c['hits']} distinct pages={c['pages']}; credit-line pages saved={c['credit']} "
- f"(page images hotlinkable={c['images']}, lending-library/no image={c['restricted_no_image']}); "
+ self.notes.insert(0, f"FTS hits walked={c['hits']} distinct pages={c['pages']}; credit-line pages saved={c['credit']} + "
+ f"Editor & Publisher syndicate-ad pages={c.get('syndicate_ad', 0)} "
+ f"(page image URL recorded={c['images']}, restricted or unresolved/no image={c['restricted_no_image']}); "
f"rejected: about-Conrad mentions={c['mention']}, other Conrads={c['other_conrad']}; "
f"capped query-years={c['capped']}")
if c["credit"] == 0 and c["hits"] == 0:
diff --git a/src/conrad/crawlers/newspaper_probes.py b/src/conrad/crawlers/newspaper_probes.py
index f754ecb..d790495 100644
--- a/src/conrad/crawlers/newspaper_probes.py
+++ b/src/conrad/crawlers/newspaper_probes.py
@@ -19,16 +19,22 @@ PROBES = [
("chronicling_america", "Chronicling America (loc.gov, digitized newspapers to 1963)",
"https://www.loc.gov/collections/chronicling-america/",
{"q": '"paul conrad"', "fo": "json", "c": 100, "dates": "1946/1963"}, "loc_json"),
- ("nara_artifacts", "NARA presidential-library museum collections (jfk/lbj/reagan/carter .artifacts.archives.gov)",
- "https://jfk.artifacts.archives.gov/people/6554/paul-conrad-denver-post", None, "plain"),
+ ("nara_artifacts", "JFK Library eMuseum (jfk.artifacts.archives.gov) — 2 Denver Post originals 1961-63",
+ "https://jfk.artifacts.archives.gov/people/6554/paul-conrad-denver-post/objects", None, "plain"),
+ ("lbj_artifacts", "LBJ Library eMuseum (lbj.artifacts.archives.gov) — ~11 Conrad originals 1964-68",
+ "https://lbj.artifacts.archives.gov/people/14908/paul-conrad/objects", None, "plain"),
+ ("billy_ireland_dc", "Ohio State Billy Ireland — Digital Collections catalog JSON (library.osu.edu/dc)",
+ "https://library.osu.edu/dc/catalog.json", {"q": '"paul conrad"', "per_page": 100}, "plain"),
("google_news_archive", "Google News Archive (news.google.com/newspapers)",
"https://news.google.com/newspapers?nid=conrad", None, "plain"),
]
NOTES = {
"cdnc": "robots.txt 'User-agent: * Disallow: /' (only archive.org_bot/bingbot allowed, and even they may not query)",
"colorado_newspapers": "robots.txt 'User-agent: * Disallow: /' (Veridian default); Denver Post 1950-64 not reachable",
- "nara_artifacts": "eMuseum sites list Conrad originals given to presidents (e.g. LBJ 'In the front door') but answer "
- "HTTP 403 to this crawler's UA",
+ "nara_artifacts": "robots.txt allows /people/ and /objects/ (Crawl-delay 30, Disallow /assets/) but the site answers "
+ "HTTP 403 to this crawler's descriptive UA; a browser UA is NOT substituted (that would be a bypass)",
+ "lbj_artifacts": "robots.txt allows /people/ and /objects/ (Crawl-delay 30, Disallow /assets/) but the site answers "
+ "HTTP 403 to this crawler's descriptive UA; a browser UA is NOT substituted (that would be a bypass)",
"google_news_archive": "robots.txt disallows /newspapers for generic agents",
}
diff --git a/tests/test_ia_fulltext.py b/tests/test_ia_fulltext.py
index 26d2151..f77c39e 100644
--- a/tests/test_ia_fulltext.py
+++ b/tests/test_ia_fulltext.py
@@ -122,3 +122,14 @@ def test_cartoon_title_and_more_negatives():
assert classify(["Frank Miller, Des Moines Register.\n1964 — {{{Paul Conrad}}}, Denver Post.\n1966 — Don Wright"])[0] \
== "mention"
assert classify(["Warren Christophel\n{{{Paul Conrad}}}\nMrs. Chauncey Crossgrove"])[0] == "mention"
+
+
+def test_editor_publisher_syndicate_ad():
+ from conrad.crawlers.ia_fulltext import classify_hit
+ ep = {"identifier": "sim_editor-publisher_1970-08-01_103_31"}
+ ad = ["Thought-Provoking Editorial Cartoons\n{{{PAUL CONRAD}}}\nFive cartoons a week\nLos Angeles Times Syndicate"]
+ assert classify_hit(ep, ad)[0] == "syndicate_ad"
+ # negatives: the same ad text outside E&P, and an E&P prize roster, are not syndicate ads
+ assert classify_hit({"identifier": "other_1970"}, ad)[0] == "mention"
+ roster = ["Pulitzer Prize winners\n{{{Paul Conrad}}}\nLos Angeles Times Syndicate cartoons"]
+ assert classify_hit(ep, roster)[0] == "mention"
← 92c289d docs: 5 unsent draft research-access letters (CDNC, Colorado
·
back to Paul Conrad Archive
·
auto-data-snapshot: 2026-09-25T11:23:40 (1 data files) — dat 80ae388 →