← back to Paul Conrad Archive
LOC deep enumeration: robots audit per path, more format endpoints, notes-based Conrad attribution, group-component check, bounded checkpointed neighbour-LCCN probing; tests
84113e09f4b18886cd4cdebba9e31f22bc90d21d · 2026-09-24 17:04:54 -0700 · Steve Abrams
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Files touched
M src/conrad/crawlers/loc.pyA tests/test_loc_deep.py
Diff
commit 84113e09f4b18886cd4cdebba9e31f22bc90d21d
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Sep 24 17:04:54 2026 -0700
LOC deep enumeration: robots audit per path, more format endpoints, notes-based Conrad attribution, group-component check, bounded checkpointed neighbour-LCCN probing; tests
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
src/conrad/crawlers/loc.py | 167 ++++++++++++++++++++++++++++++++++++++++-----
tests/test_loc_deep.py | 70 +++++++++++++++++++
2 files changed, 221 insertions(+), 16 deletions(-)
diff --git a/src/conrad/crawlers/loc.py b/src/conrad/crawlers/loc.py
index a049621..834441b 100644
--- a/src/conrad/crawlers/loc.py
+++ b/src/conrad/crawlers/loc.py
@@ -1,12 +1,20 @@
-"""Library of Congress Prints & Photographs.
+"""Library of Congress Prints & Photographs (deep enumeration, TK-12199 DTD pick A).
robots.txt disallows /search and /pictures/search (Crawl-delay 5), so the crawler does NOT page the
search UI. It uses (a) the documented loc.gov format endpoint /photos/?fa=contributor:...&fo=json, which
robots permits, exhausting its pagination, and (b) per-item JSON (/pictures/item/<id>/?fo=json) for every
known Conrad item id (seed + format endpoint) to verify and enrich each record. Images: metadata links only.
+
+Deep enumeration (TK-12199): every permitted JSON path is robots-checked first and recorded; the format endpoints
+(/photos/, /manuscripts/, /books/, /maps/, /collections/cartoon-drawings/) are exhausted with sp= pagination; and
+the P&P catalog's sequential control numbers are probed around every known Conrad id through the permitted
+/pictures/item/<id>/?fo=json endpoint (neighbour-LCCN probing, bounded by LOC_PROBE_WINDOW / LOC_PROBE_MAX,
+Crawl-delay 5 honoured, checkpointed so a restart resumes). /pictures/related (the "neighbors" link) and
+/pictures/search stay untouched because robots.txt disallows them.
"""
from __future__ import annotations
+import os
import re
from .. import rights
@@ -20,10 +28,22 @@ FORMAT_QUERIES = [
("photos", {"fa": "contributor:conrad, paul"}),
("photos", {"q": "paul conrad editorial cartoon"}),
("photos", {"q": "conrad, paul, 1924-2010"}),
+ ("photos", {"q": "conrad, paul"}),
("manuscripts", {"fa": "contributor:conrad, paul"}),
+ ("manuscripts", {"q": "paul conrad cartoons"}),
("books", {"fa": "contributor:conrad, paul"}),
+ ("maps", {"fa": "contributor:conrad, paul"}),
+ ("collections/cartoon-drawings", {"fa": "contributor:conrad, paul"}),
+ ("collections/cartoon-drawings", {"q": "conrad"}),
]
KNOWN_GROUPS = {"2010632868": "83 proofs of political cartoons for the Los Angeles Times (group record)"}
+# ids surfaced by the seed /pictures/search cache whose creator field is empty but whose notes name Conrad
+EXTRA_CANDIDATES = {"2010634752": "seed search cache: [Original editorial cartoon drawings], creator empty"}
+ROBOTS_PATHS = ["/photos/", "/manuscripts/", "/books/", "/maps/", "/collections/cartoon-drawings/", "/item/1/",
+ "/pictures/item/1/", "/search/", "/pictures/search/", "/pictures/related/"]
+PROBE_WINDOW = int(os.environ.get("LOC_PROBE_WINDOW", 5))
+PROBE_MAX = int(os.environ.get("LOC_PROBE_MAX", 300))
+GROUP_TITLE = re.compile(r"^\[?(proofs of|original editorial cartoon drawings|political cartoons by|cartoons\]?$)", re.I)
def _pk_from(url_or_id: str) -> str | None:
@@ -35,7 +55,8 @@ def item_record(pk: str, d: dict, provenance: str) -> CartoonRecord | None:
it = d.get("item") or {}
creators = " ".join(c.get("title", "") for c in (it.get("creators") or []) if isinstance(c, dict))
creators += " " + " ".join(it.get("contributor_names") or [])
- if not re.search(r"conrad,\s*paul", creators, re.I):
+ note_txt = " ".join((n.get("note") or "") if isinstance(n, dict) else str(n) for n in (it.get("notes") or []))
+ if not re.search(r"conrad,\s*paul", creators, re.I) and not re.search(r"creators include:[^.]*paul conrad", note_txt, re.I):
return None
title = it.get("title")
dt = parse_date(it.get("created_published_date") or it.get("created_published") or it.get("date"))
@@ -65,7 +86,10 @@ def item_record(pk: str, d: dict, provenance: str) -> CartoonRecord | None:
m = re.search(r"Copyright\s+(\d{4}),?\s+([^.]+)", rights_info)
if m:
holder = re.sub(r"^The\s+", "", m[2].strip())
- group = pk in KNOWN_GROUPS or bool(re.search(r"^\[?proofs of", title or "", re.I))
+ media = re.search(r"Media includes:\s*(\d+)\s+([A-Za-z]+)", note_txt)
+ unprocessed = (it.get("call_number") or "").lower().startswith("unprocessed")
+ group = pk in KNOWN_GROUPS or bool(GROUP_TITLE.search(title or "")) or bool(media and int(media[1]) > 1) or \
+ (unprocessed and not (d.get("resources") or []))
pub = holder if holder and re.search(r"times|post", holder, re.I) else _pub_from_call(None, dt["year"])
return CartoonRecord(
canonical_id=f"loc:{pk}", identifier=pk, granularity="folder" if group else "item", title=title,
@@ -80,7 +104,10 @@ def item_record(pk: str, d: dict, provenance: str) -> CartoonRecord | None:
access_level=rights.ONLINE_IMAGE if (img or thumb) else rights.ONLINE_METADATA,
rights_url="https://www.loc.gov/rr/print/res/rights.html", provenance=provenance,
subjects=subjects, people=sorted(set(people)),
- notes=KNOWN_GROUPS.get(pk) or ("group/proof set — not a single cartoon" if group else None),
+ notes=KNOWN_GROUPS.get(pk) or (("group/lot record — not a single cartoon"
+ + (f" ({media[1]} {media[2].lower()})" if media else "")
+ + ("; unprocessed, no digitized components" if unprocessed else ""))
+ if group else None),
)
@@ -93,11 +120,23 @@ class LOCCrawler(Crawler):
access_notes = ("Item records online; many Conrad drawings are digitized but flagged 'May be restricted: Copyright "
"... Los Angeles Times' — view at loc.gov (link-out), reproduction needs permission.")
+ def _robots_audit(self) -> dict[str, bool]:
+ out = {}
+ for path in ROBOTS_PATHS:
+ out[path] = self.http.allowed(LOC + path)
+ self.notes.append("robots.txt: " + ", ".join(f"{p}={'allow' if ok else 'DISALLOW'}" for p, ok in out.items()))
+ return out
+
def crawl(self) -> None:
cp = Checkpoint("loc")
self.books: list[dict] = []
pks: dict[str, str] = {}
+ allowed = self._robots_audit()
+ per_query: dict[str, int] = {}
for fmt, params in FORMAT_QUERIES:
+ if not allowed.get(f"/{fmt}/", True):
+ self.notes.append(f"skipped /{fmt}/ (robots)")
+ continue
page = 1
max_pages = 50 if "fa" in params else 3 # keyword queries: stop early, results get irrelevant fast
while page <= max_pages:
@@ -118,33 +157,36 @@ class LOCCrawler(Crawler):
pk = _pk_from(r.get("id") or "")
if pk:
pks.setdefault(pk, f"live:{LOC}/{fmt}/?{params}")
+ per_query[f"/{fmt}/ {params}"] = per_query.get(f"/{fmt}/ {params}", 0) + hits
if not (data.get("pagination") or {}).get("next") or ("q" in params and not hits):
break
page += 1
+ self.notes.append("format endpoints (Conrad hits): " + "; ".join(f"{k}={v}" for k, v in per_query.items()))
found_live = set(pks)
# every Conrad pk already known from the seed cache gets verified individually
for (pk,) in self.conn.execute("SELECT identifier FROM cartoon_sources WHERE source_id='seed_loc'"):
pks.setdefault(pk, "live:item-json (seed id verification)")
for pk in KNOWN_GROUPS:
pks.setdefault(pk, "live:item-json (known group record)")
+ for pk, why in EXTRA_CANDIDATES.items():
+ pks.setdefault(pk, f"live:item-json ({why})")
done = set(cp.get("done", []))
+ conrad: set[str] = set()
+ self.groups: dict[str, dict] = {}
for pk, prov in sorted(pks.items()):
- url = f"{LOC}/pictures/item/{pk}/"
- try:
- d = self.http.get_json(url, params={"fo": "json"})
- except (Blocked, Transient) as e:
- self.error(url, e)
- continue
- self.stats["pages"] += 1
- rec = item_record(pk, d, prov + f" -> {url}?fo=json")
- if rec:
- self.save(rec)
+ if self._verify(pk, prov):
+ conrad.add(pk)
done.add(pk)
if len(done) % 10 == 0:
self.conn.commit()
cp.set("done", sorted(done))
self.conn.commit()
cp.set("done", sorted(done))
+ new_from_probe = self._probe_neighbours(conrad, set(pks), cp)
+ self.notes.append(f"neighbour-LCCN probing (+/-{PROBE_WINDOW}, cap {PROBE_MAX}): {self.probe_stats}; "
+ f"new Conrad ids from probing: {sorted(new_from_probe) or 'none'}")
+ for pk, g in self.groups.items():
+ self.notes.append(f"group {pk}: {g}")
for b in self.books: # LOC catalog records for Conrad's own books -> verification of the bibliography
t = (b.get("title") or "").strip(" /")
self.conn.execute("INSERT OR IGNORE INTO collections(repository,name,identifier,url,level,notes,source_id) "
@@ -152,8 +194,101 @@ class LOCCrawler(Crawler):
b.get("id"), "book", f"date={b.get('date')}", self.source_id))
self.conn.commit()
self.notes.append(f"LOC book records by Conrad: {len(self.books)}")
- self.notes.append(f"format endpoint found {len(found_live)} Conrad ids; verified {len(done)} item JSON records; "
- f"/search + /pictures/search are robots-disallowed (not paged)")
+ self.notes.append(f"format endpoints found {len(found_live)} Conrad ids; verified {len(conrad)} Conrad item JSON "
+ f"records; /search, /pictures/search, /pictures/related are robots-disallowed (not paged)")
+
+ # ------------------------------------------------------------------ helpers
+ def _verify(self, pk: str, prov: str) -> bool:
+ url = f"{LOC}/pictures/item/{pk}/"
+ try:
+ d = self.http.get_json(url, params={"fo": "json"})
+ except (Blocked, Transient) as e:
+ self.error(url, e)
+ return False
+ self.stats["pages"] += 1
+ rec = item_record(pk, d, prov + f" -> {url}?fo=json")
+ if not rec:
+ return False
+ self.save(rec)
+ if rec.granularity == "folder":
+ self.groups[pk] = group_components(pk, d)
+ return True
+
+ def _probe_neighbours(self, conrad: set[str], known: set[str], cp: Checkpoint) -> set[str]:
+ """Probe sequential P&P control numbers around every Conrad id (permitted item endpoint only).
+ A hit extends the window from itself; misses (404 / non-Conrad) are checkpointed so restarts skip them."""
+ probed = dict(cp.get("probed", {})) # pk -> 'conrad' | 'other' | 'missing' | 'error'
+ self.probe_stats = {"requests": 0, "conrad": 0, "other": 0, "missing": 0, "error": 0, "cached_skip": 0}
+ queue = sorted(p for p in conrad if p.isdigit() and len(p) >= 8)
+ new: set[str] = set()
+ seen = set(known) | set(probed)
+ while queue and self.probe_stats["requests"] < PROBE_MAX:
+ base = queue.pop(0)
+ width = len(base)
+ for off in [o for k in range(1, PROBE_WINDOW + 1) for o in (k, -k)]:
+ cand = str(int(base) + off).zfill(width)
+ if cand in seen:
+ if probed.get(cand) == "conrad" and cand not in conrad:
+ conrad.add(cand)
+ new.add(cand)
+ self.probe_stats["cached_skip"] += 1 if cand in probed else 0
+ continue
+ if self.probe_stats["requests"] >= PROBE_MAX:
+ break
+ seen.add(cand)
+ url = f"{LOC}/pictures/item/{cand}/"
+ self.probe_stats["requests"] += 1
+ try:
+ status, body = self.http.get(url, params={"fo": "json"})
+ if status != 200:
+ probed[cand] = "missing"
+ else:
+ import json as _json
+ d = _json.loads(body)
+ rec = item_record(cand, d, f"live:neighbour-probe of {base} -> {url}?fo=json")
+ if rec:
+ self.save(rec)
+ if rec.granularity == "folder":
+ self.groups[cand] = group_components(cand, d)
+ probed[cand] = "conrad"
+ conrad.add(cand)
+ new.add(cand)
+ queue.append(cand) # extend the window from every new hit
+ else:
+ probed[cand] = "other"
+ except (Blocked, Transient, ValueError) as e:
+ probed[cand] = "error"
+ self.error(url, e)
+ self.probe_stats[probed[cand]] += 1
+ self.stats["pages"] += 1
+ if self.probe_stats["requests"] % 10 == 0:
+ self.conn.commit()
+ cp.set("probed", probed)
+ self.conn.commit()
+ cp.set("probed", probed)
+ if queue:
+ self.notes.append(f"probe budget exhausted with {len(queue)} ids still queued (raise LOC_PROBE_MAX)")
+ return new
+
+
+def group_components(pk: str, d: dict) -> dict:
+ """Are a group record's components individually addressable through permitted endpoints?"""
+ it = d.get("item") or {}
+ res = [r for r in (d.get("resources") or []) if isinstance(r, dict)]
+ files = sum(len(r.get("files") or []) for r in res)
+ rel = d.get("related") or {}
+ notes = " ".join((n.get("note") or "") for n in (it.get("notes") or []) if isinstance(n, dict))
+ m = re.search(r"Media includes:\s*(\d+)\s+([A-Za-z]+)", notes)
+ out = {"call_number": it.get("call_number"), "digitized_resources": len(res), "resource_files": files,
+ "declared_components": f"{m[1]} {m[2].lower()}" if m else None,
+ "child_item_links": [x for x in (it.get("related_items") or []) if x],
+ "neighbors_link_disallowed": bool(rel.get("neighbors"))}
+ addressable = bool(out["child_item_links"])
+ out["components_addressable"] = addressable
+ out["why"] = ("child item records linked from the group" if addressable else
+ "no child item records, no digitized component files; the only component path is the "
+ "'neighbors' link under /pictures/related/, which robots.txt disallows")
+ return out
CRAWLER = LOCCrawler
diff --git a/tests/test_loc_deep.py b/tests/test_loc_deep.py
new file mode 100644
index 0000000..aaa952e
--- /dev/null
+++ b/tests/test_loc_deep.py
@@ -0,0 +1,70 @@
+"""LOC deep enumeration (TK-12199): group handling, notes-only Conrad attribution, bounded neighbour probing."""
+import json
+
+from conrad.crawlers import loc
+from conrad.crawlers.base import Blocked
+
+from conftest import FakeHttp
+
+GROUP = {"item": {"title": "[Proofs of political cartoons for the Los Angeles Times]",
+ "call_number": "Unprocessed in PR 13 CN 1983:004 [P&P]", "creators": [], "contributor_names": [],
+ "created_published": "[between 1971 and 1980]", "related_items": [],
+ "notes": [{"note": "Creators include: Paul Conrad."}, {"note": "Gift; Paul Conrad; (PR 13 CN 1983:004)"}]},
+ "resources": [], "related": {"neighbors": "https://www.loc.gov/pictures/related/?&pk=2010632868"}}
+
+
+def _item(title, pk_creator=True):
+ return {"item": {"title": title, "creators": [{"title": "Conrad, Paul, 1924-2010"}] if pk_creator else [],
+ "created_published": "1975", "call_number": "CD 1 - Conrad, no. 1 (A size)", "notes": []},
+ "resources": [{"image": "https://tile.loc.gov/x.jpg"}]}
+
+
+def test_group_record_is_folder_and_components_not_addressable():
+ rec = loc.item_record("2010632868", GROUP, "t")
+ assert rec is not None and rec.granularity == "folder"
+ g = loc.group_components("2010632868", GROUP)
+ assert g["components_addressable"] is False and "robots.txt disallows" in g["why"]
+ assert g["neighbors_link_disallowed"] is True and g["digitized_resources"] == 0
+
+
+def test_notes_only_attribution_and_media_count():
+ d = json.loads(json.dumps(GROUP))
+ d["item"]["title"] = "[Original editorial cartoon drawings]"
+ d["item"]["notes"].append({"note": "Media includes: 21 Drawings : Ink and pencil."})
+ rec = loc.item_record("2010634752", d, "t")
+ assert rec and rec.granularity == "folder" and "21 drawings" in rec.notes
+ d["item"]["notes"] = [{"note": "Creators include: Pat Oliphant."}]
+ assert loc.item_record("2010634752", d, "t") is None # other cartoonists' lots are not Conrad
+
+
+def test_neighbour_probe_bounded_and_extends(tmpdb, monkeypatch):
+ monkeypatch.setattr(loc, "PROBE_WINDOW", 1)
+ monkeypatch.setattr(loc, "PROBE_MAX", 5)
+ routes = {
+ "/pictures/item/2016685101/": (200, json.dumps(_item("Known cartoon"))),
+ "/pictures/item/2016685102/": (200, json.dumps(_item("Neighbour hit"))),
+ "/pictures/item/2016685103/": (200, json.dumps(_item("Other man's photo", pk_creator=False))),
+ "/pictures/item/2016685100/": (404, "not found"),
+ "/pictures/item/": (404, "not found"),
+ "robots": (200, ""),
+ "/photos/": (200, json.dumps({"results": [{"id": "http://www.loc.gov/item/2016685101/",
+ "contributor": ["conrad, paul"]}], "pagination": {}})),
+ "loc.gov/": (200, json.dumps({"results": [], "pagination": {}})),
+ }
+
+ class H(FakeHttp):
+ def allowed(self, url):
+ return "/pictures/search" not in url and "/pictures/related" not in url and "/search/" not in url
+
+ monkeypatch.setattr(loc, "KNOWN_GROUPS", {})
+ monkeypatch.setattr(loc, "EXTRA_CANDIDATES", {})
+ monkeypatch.setattr(loc.Checkpoint, "__init__", lambda self, n: setattr(self, "data", {}) or setattr(self, "path", None))
+ monkeypatch.setattr(loc.Checkpoint, "set", lambda self, k, v: self.data.__setitem__(k, v))
+ c = loc.LOCCrawler(conn=tmpdb, http=H(routes))
+ res = c.run()
+ assert res["status"] == "worked"
+ ids = {r[0] for r in tmpdb.execute("SELECT identifier FROM cartoon_sources WHERE source_id='loc'")}
+ assert ids == {"2016685101", "2016685102"} # hit extended the window; non-Conrad neighbour skipped
+ assert c.probe_stats["requests"] <= 5
+ assert any("/pictures/related/=DISALLOW" in n for n in c.notes)
+ assert not any("/pictures/search" in call or "/pictures/related" in call for call in c.http.calls)
← e9c5e1c Dedupe: generic/[bracketed] title matches need identifier or
·
back to Paul Conrad Archive
·
auto-data-snapshot: 2026-09-24T17:26:27 (1 data files) — dat 0fbcd4e →