← back to Paul Conrad Archive
Google-style robots matcher (wildcards, $, longest-match, product-token groups) + tests; web_images crawler: Syracuse exhibit, dealer, Invaluable, Pop History Dig, Truthdig (32 records w/ image URLs); probes: Stanford Daily API, Truthdig wp-json
1c16566a3e2d52646d45017c6691bc4b0825d382 · 2026-09-25 11:26:08 -0700 · Steve Abrams
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SW5KHgwrh2Rr9TKHM6ndqM
Files touched
M src/conrad/crawlers/base.pyM src/conrad/crawlers/newspaper_probes.pyA src/conrad/crawlers/web_images.pyA tests/test_robots_wildcards.pyA tests/test_web_images.py
Diff
commit 1c16566a3e2d52646d45017c6691bc4b0825d382
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Fri Sep 25 11:26:08 2026 -0700
Google-style robots matcher (wildcards, $, longest-match, product-token groups) + tests; web_images crawler: Syracuse exhibit, dealer, Invaluable, Pop History Dig, Truthdig (32 records w/ image URLs); probes: Stanford Daily API, Truthdig wp-json
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SW5KHgwrh2Rr9TKHM6ndqM
---
src/conrad/crawlers/base.py | 90 ++++++++-
src/conrad/crawlers/newspaper_probes.py | 6 +
src/conrad/crawlers/web_images.py | 321 ++++++++++++++++++++++++++++++++
tests/test_robots_wildcards.py | 60 ++++++
tests/test_web_images.py | 47 +++++
5 files changed, 519 insertions(+), 5 deletions(-)
diff --git a/src/conrad/crawlers/base.py b/src/conrad/crawlers/base.py
index 9b153da..cbfeedb 100644
--- a/src/conrad/crawlers/base.py
+++ b/src/conrad/crawlers/base.py
@@ -9,7 +9,6 @@ import threading
import time
import traceback
from urllib.parse import urlparse
-from urllib.robotparser import RobotFileParser
import requests
@@ -40,6 +39,87 @@ IMAGE_MAGIC = (b"\xff\xd8\xff", b"\x89PNG", b"GIF8", b"%PDF", b"II*\x00", b"MM\x
IMAGE_URL = re.compile(r"\.(jpe?g|gif|png|tiff?|webp|bmp|jp2|svg)(?:[?#]|$)", re.I)
+class RobotRules:
+ """robots.txt matcher with Google/RFC 9309 semantics (urllib.robotparser does NOT support them):
+ * the group for the most specific matching user-agent token wins, else '*';
+ * '*' in a path matches any run of characters, a trailing '$' anchors the end;
+ * the LONGEST matching rule wins; on a tie Allow beats Disallow; no match -> allowed."""
+
+ def __init__(self):
+ self.groups: list[tuple[list[str], list[tuple[bool, str]], float | None]] = []
+
+ def parse(self, lines) -> None:
+ groups, agents, rules, delay, last_was_agent = [], [], [], None, False
+ for raw in lines:
+ line = raw.split("#", 1)[0].strip()
+ if ":" not in line:
+ continue
+ k, v = (x.strip() for x in line.split(":", 1))
+ k = k.lower()
+ if k == "user-agent":
+ if not last_was_agent and agents:
+ groups.append((agents, rules, delay))
+ agents, rules, delay = [], [], None
+ agents.append(v.lower())
+ last_was_agent = True
+ continue
+ last_was_agent = False
+ if not agents:
+ continue
+ if k in ("allow", "disallow"):
+ if v:
+ rules.append((k == "allow", v))
+ elif k == "crawl-delay":
+ try:
+ delay = float(v)
+ except ValueError:
+ pass
+ if agents:
+ groups.append((agents, rules, delay))
+ self.groups = groups
+
+ def _group(self, useragent: str):
+ # match on the PRODUCT token only ('paul-conrad-archive' of 'paul-conrad-archive/0.1 research ...'), so words in
+ # the comment part ('research', 'archive') can never select someone else's group
+ ua = (useragent.lower().split("/")[0].split() or [""])[0]
+ best, best_len = None, -1
+ for agents, rules, delay in self.groups:
+ for a in agents:
+ if a != "*" and ua.startswith(a) and len(a) > best_len:
+ best, best_len = (rules, delay), len(a)
+ if best is None:
+ best = next(((rules, delay) for agents, rules, delay in self.groups if "*" in agents), None)
+ if best is None:
+ return [], None
+ # several '*' (or same-agent) groups are merged, as Google does
+ if best_len < 0:
+ rules = [r for agents, rs, _ in self.groups if "*" in agents for r in rs]
+ return rules, best[1]
+ return best
+
+ @staticmethod
+ def _match(pattern: str, path: str) -> bool:
+ anchored = pattern.endswith("$")
+ pat = pattern[:-1] if anchored else pattern
+ rx = "".join(".*" if ch == "*" else re.escape(ch) for ch in pat)
+ return re.match(rx + ("$" if anchored else ""), path) is not None
+
+ def can_fetch(self, useragent: str, url: str) -> bool:
+ p = urlparse(url)
+ path = (p.path or "/") + (("?" + p.query) if p.query else "")
+ rules, _ = self._group(useragent)
+ best_len, allowed = -1, True
+ for allow, pat in rules:
+ if self._match(pat, path):
+ n = len(pat)
+ if n > best_len or (n == best_len and allow):
+ best_len, allowed = n, allow
+ return allowed
+
+ def crawl_delay(self, useragent: str):
+ return self._group(useragent)[1]
+
+
class Blocked(Exception):
"""Robots disallow, WAF/JS challenge, 401/403, paywall — we stop, record, and move on."""
@@ -50,7 +130,7 @@ class Transient(Exception):
_host_locks: dict[str, threading.Lock] = {}
_host_last: dict[str, float] = {}
-_robots: dict[str, RobotFileParser | None] = {}
+_robots: dict[str, RobotRules | None] = {}
_glock = threading.Lock()
@@ -75,13 +155,13 @@ class Http:
self.requests_made = 0
# ---------------- robots
- def robots(self, url: str) -> RobotFileParser | None:
+ def robots(self, url: str) -> RobotRules | None:
p = urlparse(url)
base = f"{p.scheme}://{p.netloc}"
with _glock:
if base in _robots:
return _robots[base]
- rp = RobotFileParser()
+ rp = RobotRules()
try:
r = self.s.get(base + "/robots.txt", timeout=config.TIMEOUT)
if r.status_code == 403 and "amazonaws.com" in p.netloc and "<Code>AccessDenied</Code>" in r.text:
@@ -108,7 +188,7 @@ class Http:
cd = None
if rp is not None:
try:
- cd = rp.crawl_delay(config.USER_AGENT) or rp.crawl_delay("*")
+ cd = rp.crawl_delay(config.USER_AGENT)
except Exception: # noqa: BLE001
cd = None
return max(config.REQUEST_DELAY, float(cd or 0))
diff --git a/src/conrad/crawlers/newspaper_probes.py b/src/conrad/crawlers/newspaper_probes.py
index d790495..8262846 100644
--- a/src/conrad/crawlers/newspaper_probes.py
+++ b/src/conrad/crawlers/newspaper_probes.py
@@ -25,6 +25,10 @@ PROBES = [
"https://lbj.artifacts.archives.gov/people/14908/paul-conrad/objects", None, "plain"),
("billy_ireland_dc", "Ohio State Billy Ireland — Digital Collections catalog JSON (library.osu.edu/dc)",
"https://library.osu.edu/dc/catalog.json", {"q": '"paul conrad"', "per_page": 100}, "plain"),
+ ("stanford_daily", "Stanford Daily archive search API (ran Conrad from 1964-09-28 / 1976-11-15)",
+ "https://ehabp6fuc5.execute-api.us-east-1.amazonaws.com/prod", {"q": '"Paul Conrad"', "size": 50}, "plain"),
+ ("truthdig_api", "Truthdig WordPress REST API (cartoons search)", "https://www.truthdig.com/wp-json/wp/v2/cartoons",
+ {"search": "conrad", "per_page": 100}, "plain"),
("google_news_archive", "Google News Archive (news.google.com/newspapers)",
"https://news.google.com/newspapers?nid=conrad", None, "plain"),
]
@@ -36,6 +40,8 @@ NOTES = {
"lbj_artifacts": "robots.txt allows /people/ and /objects/ (Crawl-delay 30, Disallow /assets/) but the site answers "
"HTTP 403 to this crawler's descriptive UA; a browser UA is NOT substituted (that would be a bypass)",
"google_news_archive": "robots.txt disallows /newspapers for generic agents",
+ "stanford_daily": "the API host's robots.txt answers 403 -> treated as disallowed (conservative rule)",
+ "truthdig_api": "robots.txt disallows /wp-json/ (the author page /author/paul_conrad/ is used instead, see truthdig)",
}
diff --git a/src/conrad/crawlers/web_images.py b/src/conrad/crawlers/web_images.py
new file mode 100644
index 0000000..d941299
--- /dev/null
+++ b/src/conrad/crawlers/web_images.py
@@ -0,0 +1,321 @@
+"""Public web pages that SHOW Conrad cartoons (TK-12199 image priority): one institutional exhibit and several
+secondary copies (dealer, auction, blog, magazine). Each page is fetched through the polite client (robots.txt per
+path with wildcard semantics, descriptive UA); every page is registered as its own source with an honest status.
+
+Image URLs are stored as METADATA only (never fetched). Blog / dealer / auction copies are REPRINTS: their rows carry
+acquisition_method 'secondary_citation' (dealer/auction/blog) and a note saying so, so dedupe and the viewer prefer the
+institutional record whenever the same cartoon exists elsewhere. Dates are recorded only when the page states them."""
+from __future__ import annotations
+
+import html as _html
+import json
+import re
+from datetime import datetime, timezone
+from urllib.parse import urljoin
+
+from .. import db, rights
+from ..models import CartoonRecord
+from ..normalize import parse_date, presidents_in_text
+from .base import Blocked, Crawler, Transient
+
+SYR = "https://library.syracuse.edu/extsites/cartoonists/conrad.php"
+PCG = "https://www.original-political-cartoon.com/cartoon-gallery/artists/conrad-paul-1924-2010/"
+INVALUABLE = ["https://www.invaluable.com/artist/paul-conrad-saveb5g89x/sold-at-auction-prices/",
+ "https://www.invaluable.com/artist/conrad-paul-francis-67i99nw6g1/sold-at-auction-prices/"]
+PHD = "https://pophistorydig.com/topics/paul-conrad-cartoonist/"
+TRUTHDIG = "https://www.truthdig.com/author/paul_conrad/"
+
+SOURCES = {
+ "syracuse_exhibit": ("Syracuse University Libraries — 'Cartoonists' web exhibit, Paul Conrad page", SYR, "PUBLIC_HTML"),
+ "pc_gallery": ("Political Cartoon Gallery (dealer, London) — Conrad originals for sale", PCG, "PUBLIC_HTML"),
+ "invaluable": ("Invaluable — auction results for Paul Conrad originals", INVALUABLE[0], "PUBLIC_HTML"),
+ "pophistorydig": ("The Pop History Dig — 'Paul Conrad, cartoonist' article (blog reprints)", PHD, "PUBLIC_HTML"),
+ "truthdig": ("Truthdig — Paul Conrad cartoons (2000s syndicated)", TRUTHDIG, "PUBLIC_HTML"),
+}
+MONTHS = "january|february|march|april|may|june|july|august|september|october|november|december"
+
+
+def _txt(s: str | None) -> str:
+ return re.sub(r"\s+", " ", _html.unescape(re.sub(r"<[^>]+>", " ", s or ""))).strip()
+
+
+def stated_year(text: str) -> tuple[int | None, tuple[str, str] | None]:
+ """(year, (start, end)) only when the text itself dates the cartoon; '1970s' -> decade range. Book / exhibit
+ years are the caller's problem (it passes cartoon captions only)."""
+ ys = sorted({int(y) for y in re.findall(r"\b(19[5-9]\d|200\d|2010)\b(?!s)", text)})
+ if len(ys) == 1:
+ return ys[0], (f"{ys[0]}-01-01", f"{ys[0]}-12-31")
+ m = re.search(r"\b(19[5-9]0)s\b", text)
+ if m and not ys:
+ d = int(m[1])
+ return None, (f"{d}-01-01", f"{d + 9}-12-31")
+ return None, None
+
+
+# ------------------------------------------------------------------------------------------------ parsers (pure)
+def parse_syracuse(body: str) -> list[dict]:
+ out = []
+ for tag in re.findall(r"<img\b[^>]*>", body, re.I):
+ alt = re.search(r'\balt="([^"]*)"', tag)
+ src = re.search(r'\bsrc="([^"]*)"', tag)
+ if alt and src and alt[1].strip().lower().endswith(" cartoon") and "images/" in src[1]:
+ out.append(dict(title=_html.unescape(alt[1]).strip()[:-len(" cartoon")].strip(), image=urljoin(SYR, src[1])))
+ return out
+
+
+def parse_pcg_list(body: str) -> list[str]:
+ return sorted({urljoin(PCG, u) for u in re.findall(r'href="(/cartoon-gallery/buy/[^"]+/\d+/)"', body)})
+
+
+def parse_pcg_item(body: str) -> dict:
+ t = _txt(re.sub(r"<script.*?</script>|<style.*?</style>", " ", body, flags=re.S))
+ img = re.search(r'<img\s[^>]*src="(/media/filer_public_thumbnails/[^"]+)"[^>]*\balt="([^"]*)"', body, re.S)
+ def field(name, stop):
+ m = re.search(name + r"\s+(.+?)\s+(?:" + stop + r")", t)
+ return m[1].strip() if m else None
+ pub = field("Publication", "Published|Home|Size|Medium")
+ date = field("Published", "Home|Size|Medium|Description")
+ alt = _html.unescape(img[2]).strip() if img else ""
+ page_title = re.search(r"<title>\s*(.*?)\s*-\s*Cartoon Gallery\s*</title>", body, re.S)
+ title = alt if alt and alt != "None" else (_html.unescape(page_title[1]).strip() if page_title else None)
+ return dict(title=title, image=urljoin(PCG, img[1]) if img else None,
+ publication=pub, published=date, medium=field("Medium", "Publication|Published|Home|Size"),
+ size=field("Size", "Medium|Publication|Published"), sold="SOLD" in (img[2] if img else ""))
+
+
+def _enclosing_object(body: str, pos: int) -> str:
+ """The innermost JSON object {...} around position pos (string-aware brace matching)."""
+ depth, i = 0, pos
+ while i > 0: # walk back to the unmatched '{'
+ i -= 1
+ c = body[i]
+ if c == "}":
+ depth += 1
+ elif c == "{":
+ if depth == 0:
+ break
+ depth -= 1
+ start, depth, j, in_str, esc = i, 0, i, False, False
+ while j < len(body):
+ c = body[j]
+ if in_str:
+ if esc:
+ esc = False
+ elif c == "\\":
+ esc = True
+ elif c == '"':
+ in_str = False
+ elif c == '"':
+ in_str = True
+ elif c == "{":
+ depth += 1
+ elif c == "}":
+ depth -= 1
+ if depth == 0:
+ return body[start: j + 1]
+ j += 1
+ return body[start: pos + 800]
+
+
+def parse_invaluable(body: str) -> list[dict]:
+ out = []
+ for m in re.finditer(r'"lotRef"\s*:\s*"([^"]+)"', body):
+ seg = _enclosing_object(body, m.start())
+ def g(k):
+ x = re.search(r'"%s"\s*:\s*("(?:[^"\\]|\\.)*"|[\d.]+)' % k, seg)
+ if not x:
+ return None
+ try:
+ return json.loads(x[1]) if x[1].startswith('"') else x[1]
+ except ValueError:
+ return x[1].strip('"')
+ title = _html.unescape(g("lotTitle") or "")
+ if not re.search(r"paul\s+conrad|conrad,?\s+paul\s+(?:f|1924)", title, re.I) or re.search(r"15\d\d|16\d\d", title):
+ continue
+ # cartoons only: an original drawing / political cartoon lot; prints, lithographs and portraits are not
+ if not re.search(r"political cartoon|cartoon|\bink\b|1924-2010\)\s*[\"“]", title, re.I) or re.search(r"lithograph|signed print|portrait", title, re.I):
+ continue
+ ts = g("dateTimeUTCUnix")
+ named = re.sub(r"^.*?(?:1924\s*-\s*2010\)?,?|\(political cartoon\)\s*paul conrad)\s*", "", title, flags=re.I)
+ named = re.split(r",\s*(?:ink|pen|original)\b", named, flags=re.I)[0].strip(" .,'\"\\")
+ out.append(dict(ref=m[1], title=title, named=named if len(named) > 3 else None, house=g("houseName"), lot=g("lotNumber"), photo=g("photoPath"),
+ sold=datetime.fromtimestamp(int(float(ts)), timezone.utc).date().isoformat() if ts else None))
+ uniq = {d["ref"]: d for d in out}
+ return list(uniq.values())
+
+
+def parse_pophistorydig(body: str) -> list[dict]:
+ out = []
+ for tag in re.findall(r"<img\b[^>]*>", body, re.I):
+ alt = re.search(r'\balt="([^"]*)"', tag)
+ src = re.search(r'\bsrc="([^"]*)"', tag)
+ if not (alt and src):
+ continue
+ a = _html.unescape(alt[1]).strip()
+ if "conrad" not in a.lower() or re.search(r"working on a drawing|at his desk|book|photo of", a, re.I):
+ continue
+ if not re.search(r"cartoon|caricature|caption|send up|cast|depict|slaying|conrad['’]s", a, re.I):
+ continue
+ out.append(dict(alt=a, image=urljoin(PHD, src[1])))
+ return out
+
+
+def parse_og(body: str) -> dict:
+ def meta(p):
+ m = re.search(r'<meta[^>]+(?:property|name)="%s"[^>]+content="([^"]*)"' % re.escape(p), body)
+ return _html.unescape(m[1]).strip() if m else None
+ return dict(title=meta("og:title"), image=meta("og:image"), published=meta("article:published_time"),
+ desc=meta("og:description"))
+
+
+# ------------------------------------------------------------------------------------------------ crawler
+class WebImages(Crawler):
+ source_id = "web_images"
+ name = "Public web pages showing Conrad cartoons (exhibit, dealer, auction, blog, magazine)"
+ repository = None
+ url = SYR
+ classification = "PUBLIC_HTML"
+ access_notes = "Each sub-source is registered separately; images are URL metadata only; copies flagged as reprints."
+
+ def __init__(self, *a, **kw):
+ super().__init__(*a, **kw)
+ self.per: dict[str, int] = {}
+ self.imgs: dict[str, int] = {}
+ self.status_of: dict[str, tuple[str, str]] = {}
+
+ def fetch(self, sid: str, url: str) -> str | None:
+ try:
+ st, body = self.http.get(url)
+ self.stats["pages"] += 1
+ except Blocked as e:
+ self.status_of[sid] = ("blocked", str(e))
+ self.error(url, f"BLOCKED: {e}")
+ return None
+ except Transient as e:
+ self.status_of.setdefault(sid, ("failed", str(e)))
+ self.error(url, e)
+ return None
+ if st != 200:
+ self.status_of.setdefault(sid, ("failed", f"HTTP {st} at {url}"))
+ return None
+ return body
+
+ def put(self, sid: str, rec: CartoonRecord) -> None:
+ self.stats["seen"] += 1
+ st = db.save_record(self.conn, rec, sid)
+ if st in ("added", "updated"):
+ self.stats[st] += 1
+ self.per[sid] = self.per.get(sid, 0) + 1
+ if rec.image_url:
+ self.imgs[sid] = self.imgs.get(sid, 0) + 1
+
+ def base(self, sid, cid, ident, title, url, image, method, note, **kw) -> CartoonRecord:
+ text = " ".join(filter(None, [title, kw.get("description")]))
+ return CartoonRecord(canonical_id=cid, identifier=ident, granularity="item", title=title, record_url=url,
+ image_url=image, access_level=rights.ONLINE_IMAGE if image else rights.ONLINE_METADATA,
+ rights_text=rights.COPYRIGHT_NOTE, notes=note[:1000], acquisition_method=method,
+ provenance=f"live:{sid} {db.now()[:10]}", people=presidents_in_text(text),
+ subjects=["Editorial cartoons"], **kw)
+
+ # --- sub-sources
+ def syracuse(self):
+ b = self.fetch("syracuse_exhibit", SYR)
+ for i, it in enumerate(parse_syracuse(b or ""), 1):
+ self.put("syracuse_exhibit", self.base(
+ "syracuse_exhibit", f"syrx:{re.sub(r'[^0-9a-z]+', '-', it['image'].rsplit('/', 1)[-1].lower())}",
+ it["image"].rsplit("/", 1)[-1], it["title"], SYR, it["image"], "direct_html",
+ "Syracuse University Libraries 'Cartoonists' web exhibit (Special Collections); title from the exhibit's "
+ "image alt text; undated on the page", repository="Syracuse University Libraries",
+ collection_name="Cartoonists web exhibit"))
+
+ def pcg(self):
+ b = self.fetch("pc_gallery", PCG)
+ for u in parse_pcg_list(b or ""):
+ d = self.fetch("pc_gallery", u)
+ if not d:
+ continue
+ it = parse_pcg_item(d)
+ if not it["title"]:
+ continue
+ title = re.sub(r"^SOLD\s+", "", it["title"]).strip()
+ pd = parse_date(it["published"]) if it["published"] else {"date_exact": None, "date_start": None,
+ "date_end": None, "year": None}
+ caveat = ""
+ if it["publication"] and "los angeles" in it["publication"].lower() and pd["year"] and pd["year"] < 1964:
+ caveat = " | CAVEAT: dealer states the LA Times before 1964, when Conrad was at the Denver Post"
+ self.put("pc_gallery", self.base(
+ "pc_gallery", f"pcg:{u.rstrip('/').rsplit('/', 1)[-1]}", u, title, u, it["image"], "secondary_citation",
+ f"DEALER COPY (Political Cartoon Gallery, London) of an original drawing; publication/date as stated by "
+ f"the dealer: {it['publication'] or '?'} / {it['published'] or '?'}; medium {it['medium'] or '?'}, size "
+ f"{it['size'] or '?'}{caveat}", repository="Political Cartoon Gallery (dealer)",
+ collection_name="dealer stock", publication=it["publication"], medium=it["medium"],
+ dimensions=it["size"], date_exact=pd["date_exact"], date_start=pd["date_start"],
+ date_end=pd["date_end"], year=pd["year"], date_is_estimate=not pd["date_exact"]))
+
+ def invaluable(self):
+ for page in INVALUABLE:
+ b = self.fetch("invaluable", page)
+ for it in parse_invaluable(b or ""):
+ img = f"https://image.invaluable.com/housePhotos/{it['photo']}" if it.get("photo") else None
+ self.put("invaluable", self.base(
+ "invaluable", f"inv:{it['ref']}", it["ref"],
+ it["named"] or f"[Paul Conrad original cartoon — {it['house'] or 'auction'} lot {it['lot'] or '?'}, "
+ f"sold {it['sold'] or '?'}]", page, img, "secondary_citation",
+ f"AUCTION RECORD (Invaluable) lot title '{it['title']}'"
+ f"{'' if it['named'] else '; the lot title names no cartoon'}; "
+ f"sale date {it['sold']} is NOT the cartoon's date", repository=it["house"] or "auction house",
+ collection_name="Invaluable auction results"))
+
+ def pophistorydig(self):
+ b = self.fetch("pophistorydig", PHD)
+ for it in parse_pophistorydig(b or ""):
+ y, rng = stated_year(it["alt"])
+ title = it["alt"] if len(it["alt"]) <= 160 else it["alt"][:157] + "..."
+ self.put("pophistorydig", self.base(
+ "pophistorydig", f"phd:{it['image'].rsplit('/', 1)[-1].lower()}", it["image"], title, PHD,
+ it["image"], "secondary_citation",
+ f"BLOG REPRINT (The Pop History Dig); title = the blog's image caption; "
+ f"{'year stated in caption' if y else ('decade stated in caption' if rng else 'undated in source')}",
+ repository="The Pop History Dig (blog)", collection_name="pophistorydig.com", description=it["alt"],
+ year=y, date_start=rng[0] if rng else None, date_end=rng[1] if rng else None, date_is_estimate=True))
+
+ def truthdig(self):
+ b = self.fetch("truthdig", TRUTHDIG)
+ for u in sorted(set(re.findall(r'href="(https://www\.truthdig\.com/cartoons/[^"]+)"', b or ""))):
+ d = self.fetch("truthdig", u)
+ if not d:
+ continue
+ og = parse_og(d)
+ if not og["title"]:
+ continue
+ title = re.sub(r"\s*[-|–]\s*Truthdig\s*$", "", og["title"]).strip()
+ iso = (og["published"] or "")[:10] or None
+ self.put("truthdig", self.base(
+ "truthdig", f"td:{u.rstrip('/').rsplit('/', 1)[-1]}", u, title, u, og["image"], "secondary_citation",
+ "Truthdig web publication of a syndicated Conrad cartoon (Tribune Media Services); date = Truthdig's "
+ "posting date, not necessarily first publication", repository="Truthdig", collection_name="Truthdig cartoons",
+ publication="Truthdig", date_exact=None, date_start=iso, date_end=iso,
+ year=int(iso[:4]) if iso else None, date_is_estimate=True))
+
+ def crawl(self) -> None:
+ for sid, (name, url, cls) in SOURCES.items():
+ db.upsert_source(self.conn, sid, name, url=url, classification=cls, status="running")
+ for fn in (self.syracuse, self.pcg, self.invaluable, self.pophistorydig, self.truthdig):
+ try:
+ fn()
+ except (Blocked, Transient) as e: # pragma: no cover - fetch() already catches these
+ self.error(fn.__name__, e)
+ summary = []
+ for sid, (name, url, cls) in SOURCES.items():
+ n, i = self.per.get(sid, 0), self.imgs.get(sid, 0)
+ st, why = self.status_of.get(sid, ("worked" if n else "failed", ""))
+ if n and st != "blocked":
+ st = "worked"
+ db.upsert_source(self.conn, sid, name, url=url, classification="REQUIRES_PERMISSION" if st == "blocked" else cls,
+ status=st, notes=f"records={n}; image URLs recorded={i}" + (f"; {why}" if why else ""))
+ summary.append(f"{sid}: {n} rec / {i} img ({st})")
+ self.conn.commit()
+ self.notes.append("; ".join(summary))
+
+
+CRAWLER = WebImages
diff --git a/tests/test_robots_wildcards.py b/tests/test_robots_wildcards.py
new file mode 100644
index 0000000..6c11844
--- /dev/null
+++ b/tests/test_robots_wildcards.py
@@ -0,0 +1,60 @@
+"""Google/RFC 9309 robots.txt semantics in the base client (urllib.robotparser ignores '*' and '$')."""
+from conrad import config
+from conrad.crawlers.base import RobotRules
+
+UA = config.USER_AGENT
+GALLERY = "https://www.latimes.com/nation/la-me-paul-conrad-pictures-photogallery.html"
+
+
+def rules(txt):
+ r = RobotRules()
+ r.parse(txt.strip().splitlines())
+ return r
+
+
+def test_wildcard_disallow_blocks_photogallery_for_star_group():
+ # negative case: a '*' group with /*photogallery MUST block the gallery URL
+ r = rules("User-agent: *\nDisallow: /*photogallery\n")
+ assert not r.can_fetch(UA, GALLERY)
+ assert r.can_fetch(UA, "https://www.latimes.com/archives/la-xpm-2010-sep-05-la-me-paul-conrad-20100905-story.html")
+
+
+def test_latimes_real_shape_rule_only_for_googlebot_news():
+ txt = """
+User-agent: *
+Disallow: /search
+Disallow: /*/thirdpartyservice
+Disallow: /get-galleryfragment*
+
+User-agent: Googlebot-News
+Disallow: /*photogallery
+"""
+ r = rules(txt)
+ assert r.can_fetch(UA, GALLERY) # the photogallery rule binds Googlebot-News only
+ assert not r.can_fetch("Googlebot-News", GALLERY)
+ assert not r.can_fetch(UA, "https://www.latimes.com/x/thirdpartyservice?a=1") # mid-path wildcard honoured
+ assert not r.can_fetch(UA, "https://www.latimes.com/search?q=conrad")
+
+
+def test_dollar_anchor_longest_match_and_allow_tie():
+ r = rules("User-agent: *\nDisallow: /*.pdf$\nDisallow: /archive/\nAllow: /archive/public/\nAllow: /x\nDisallow: /x\n")
+ assert not r.can_fetch(UA, "https://h.org/a/b.pdf")
+ assert r.can_fetch(UA, "https://h.org/a/b.pdf?download=1") # '$' anchors the end
+ assert not r.can_fetch(UA, "https://h.org/archive/secret")
+ assert r.can_fetch(UA, "https://h.org/archive/public/page") # longest match wins
+ assert r.can_fetch(UA, "https://h.org/x") # tie -> Allow wins
+
+
+def test_disallow_all_and_crawl_delay():
+ r = rules("User-agent: *\nDisallow: /\nCrawl-delay: 30\n\nUser-agent: archive.org_bot\nDisallow: /?a=q\n")
+ assert not r.can_fetch(UA, "https://cdnc.ucr.edu/?a=q&txq=conrad")
+ assert r.crawl_delay(UA) == 30
+ assert r.can_fetch("archive.org_bot", "https://cdnc.ucr.edu/page")
+ assert rules("").can_fetch(UA, "https://h.org/anything")
+
+
+def test_group_selected_by_product_token_only():
+ r = rules("User-agent: research\nDisallow: /\n\nUser-agent: *\nAllow: /\n")
+ assert r.can_fetch(UA, "https://h.org/page") # 'research' in our UA comment must not select that group
+ r2 = rules("User-agent: paul-conrad-archive\nDisallow: /private\n\nUser-agent: *\nDisallow: /\n")
+ assert r2.can_fetch(UA, "https://h.org/page") and not r2.can_fetch(UA, "https://h.org/private/x")
diff --git a/tests/test_web_images.py b/tests/test_web_images.py
new file mode 100644
index 0000000..f8f7a90
--- /dev/null
+++ b/tests/test_web_images.py
@@ -0,0 +1,47 @@
+"""web_images parsers (exhibit / dealer / auction / blog), incl. negative cases (TK-12199)."""
+import json
+
+from conrad.crawlers.web_images import (parse_invaluable, parse_pcg_item, parse_pophistorydig, parse_syracuse,
+ stated_year)
+
+
+def test_syracuse_alt_titles():
+ b = ('<img alt="Banner" src="images/Title-Web2.jpg"/><img alt="Violation cartoon" src="images/24.jpg"/>'
+ '<img alt="Special Collections Exhibits banner" src="../_assets/x.png"/>')
+ assert parse_syracuse(b) == [{"title": "Violation",
+ "image": "https://library.syracuse.edu/extsites/cartoonists/images/24.jpg"}]
+
+
+def test_dealer_item_fields_and_title_fallback():
+ b = ('<title>Watergate Tapes - Cartoon Gallery</title><img src="/media/filer_public_thumbnails/x/conrad1.jpg__345x300.jpg"'
+ ' alt="None" title="None"/> Description tapes Size 31cm x 33cm Medium Pen and ink Publication Los Angeles Times'
+ ' Published 9 November 1973 Home')
+ it = parse_pcg_item(b)
+ assert it["title"] == "Watergate Tapes" and it["publication"] == "Los Angeles Times"
+ assert it["published"] == "9 November 1973" and it["image"].endswith("conrad1.jpg__345x300.jpg")
+
+
+def _lot(ref, title, ts=1600000000):
+ return json.dumps({"lotTitle": title, "lotRef": ref, "photoPath": "h/1.jpg", "dateTimeUTCUnix": ts,
+ "houseName": "House", "lotNumber": "1"})
+
+
+def test_invaluable_keeps_cartoons_only():
+ b = "[" + ",".join([_lot("A", "(POLITICAL CARTOON) PAUL CONRAD"),
+ _lot("B", 'PAUL CONRAD (1924-2010) "Flag Raising At Guantanamo."'),
+ _lot("C", "Paul Conrad American 1924-2010 Lithograph Martin Luther King Signed Ltd"),
+ _lot("D", "Conrad Paul (traceable 1525-1547 Lauingen) - Christ on the Mount of Olives"),
+ _lot("E", "The Beatles, signed photograph")]) + "]"
+ got = {d["ref"]: d["named"] for d in parse_invaluable(b)}
+ assert got == {"A": None, "B": "Flag Raising At Guantanamo"} # lithograph, 16th-c. Conrad, Beatles: rejected
+
+
+def test_pophistorydig_skips_photos_and_books():
+ b = ('<img alt="Paul Conrad working on a drawing at his desk, 1970s." src="/a.jpg">'
+ '<img alt="Paul Conrad’s 1975 book, “The King and Us”" src="/b.jpg">'
+ '<img alt="In this June 1972 Paul Conrad cartoon, Democrats are peeking" src="/c.jpg">')
+ items = parse_pophistorydig(b)
+ assert [i["image"] for i in items] == ["https://pophistorydig.com/c.jpg"]
+ assert stated_year(items[0]["alt"])[0] == 1972
+ assert stated_year("a 1970s cartoon") == (None, ("1970-01-01", "1979-12-31"))
+ assert stated_year("undated") == (None, None)
← 80ae388 auto-data-snapshot: 2026-09-25T11:23:40 (1 data files) — dat
·
back to Paul Conrad Archive
·
Robots re-audit + web-image hotlink allowlist (TK-12199) 167743e →