`src/conrad/crawlers/newspaper_probes.py`

89 lines
"""Probes of free digitized-newspaper / museum archives that could show Conrad cartoons as printed (TK-12199 image
priority). Each is tested through the project's polite client (robots.txt per path, descriptive UA) and registered
with an honest status. Nothing is bypassed: a robots 'Disallow: /' or a 403 to our UA is recorded as blocked.

Sources that actually yield records live in their own crawlers (ia_fulltext, discovery/DPLA, latimes)."""
from __future__ import annotations

import json

from .. import db
from .base import Blocked, Crawler, Transient

# (source_id, name, probe url, params or None, how)
PROBES = [
    ("cdnc", "California Digital Newspaper Collection (cdnc.ucr.edu, Veridian)", "https://cdnc.ucr.edu/?a=q&txq=%22Paul+Conrad%22",
     None, "plain"),
    ("colorado_newspapers", "Colorado Historic Newspapers (Veridian)",
     "https://www.coloradohistoricnewspapers.org/?a=q&txq=%22Paul+Conrad%22", None, "plain"),
    ("chronicling_america", "Chronicling America (loc.gov, digitized newspapers to 1963)",
     "https://www.loc.gov/collections/chronicling-america/",
     {"q": '"paul conrad"', "fo": "json", "c": 100, "dates": "1946/1963"}, "loc_json"),
    ("nara_artifacts", "JFK Library eMuseum (jfk.artifacts.archives.gov) — 2 Denver Post originals 1961-63",
     "https://jfk.artifacts.archives.gov/people/6554/paul-conrad-denver-post/objects", None, "plain"),
    ("lbj_artifacts", "LBJ Library eMuseum (lbj.artifacts.archives.gov) — ~11 Conrad originals 1964-68",
     "https://lbj.artifacts.archives.gov/people/14908/paul-conrad/objects", None, "plain"),
    ("billy_ireland_dc", "Ohio State Billy Ireland — Digital Collections catalog JSON (library.osu.edu/dc)",
     "https://library.osu.edu/dc/catalog.json", {"q": '"paul conrad"', "per_page": 100}, "plain"),
    ("stanford_daily", "Stanford Daily archive search API (ran Conrad from 1964-09-28 / 1976-11-15)",
     "https://ehabp6fuc5.execute-api.us-east-1.amazonaws.com/prod", {"q": '"Paul Conrad"', "size": 50}, "plain"),
    ("truthdig_api", "Truthdig WordPress REST API (cartoons search)", "https://www.truthdig.com/wp-json/wp/v2/cartoons",
     {"search": "conrad", "per_page": 100}, "plain"),
    ("google_news_archive", "Google News Archive (news.google.com/newspapers)",
     "https://news.google.com/newspapers?nid=conrad", None, "plain"),
]
NOTES = {
    "cdnc": "robots.txt 'User-agent: * Disallow: /' (only archive.org_bot/bingbot allowed, and even they may not query)",
    "colorado_newspapers": "robots.txt 'User-agent: * Disallow: /' (Veridian default); Denver Post 1950-64 not reachable",
    "nara_artifacts": "robots.txt allows /people/ and /objects/ (Crawl-delay 30, Disallow /assets/) but the site answers "
                      "HTTP 403 to this crawler's descriptive UA; a browser UA is NOT substituted (that would be a bypass)",
    "lbj_artifacts": "robots.txt allows /people/ and /objects/ (Crawl-delay 30, Disallow /assets/) but the site answers "
                     "HTTP 403 to this crawler's descriptive UA; a browser UA is NOT substituted (that would be a bypass)",
    "google_news_archive": "robots.txt disallows /newspapers for generic agents",
    "stanford_daily": "the API host's robots.txt answers 403 -> treated as disallowed (conservative rule)",
    "truthdig_api": "robots.txt disallows /wp-json/ (the author page /author/paul_conrad/ is used instead, see truthdig)",
}


class NewspaperProbes(Crawler):
    source_id = "newspaper_probes"
    name = "Digitized-newspaper / museum archive probes"
    repository = None
    url = "https://www.loc.gov/collections/chronicling-america/"
    classification = "PUBLIC_HTML"
    access_notes = "Reachability probes only; each probed archive is registered as its own source."

    def crawl(self) -> None:
        summary = []
        for sid, name, url, params, how in PROBES:
            status, cls, note = "worked", "PUBLIC_HTML", ""
            try:
                st, body = self.http.get(url, params=params)
                self.stats["pages"] += 1
                if st != 200:
                    status, note = "failed", f"HTTP {st}"
                elif how == "loc_json":
                    d = json.loads(body)
                    n = (d.get("pagination") or {}).get("of") or 0
                    cls = "PUBLIC_API"
                    note = (f"q=\"paul conrad\" 1946-1963: {n} page hit(s); Conrad drew for the Denver Post (not in "
                            f"Chronicling America) and was syndicated only from 1964 -> 0 records")
                    if n:
                        note += "; hits: " + "; ".join(f"{r.get('date')} {str(r.get('title'))[:50]}"
                                                       for r in (d.get("results") or [])[:10])
                else:
                    note = "reachable (HTTP 200); no parser for item-level results"
            except Blocked as e:
                status, cls = "blocked", "REQUIRES_PERMISSION"
                note = f"{NOTES.get(sid, '')} ({e})".strip()
            except Transient as e:  # timeouts / 5xx after retries: unmeasured, NOT blocked
                status, cls = "failed", ("PUBLIC_API" if how == "loc_json" else "PUBLIC_HTML")
                note = f"not measured — endpoint timed out / errored after retries ({e})"
            db.upsert_source(self.conn, sid, name, url=url, classification=cls, status=status, notes=note[:1500])
            summary.append(f"{sid}={status}")
        self.conn.commit()
        self.notes.append("probes: " + ", ".join(summary))


CRAWLER = NewspaperProbes