`src/conrad/crawlers/syracuse.py`

41 lines
"""Syracuse University — Paul Conrad Cartoons (1,000+ originals, mostly 1963-1969).
Live pages are behind an AWS WAF JS challenge (HTTP 202, empty body) for non-browser clients, so they are
recorded as blocked and the seed cache (fetched earlier via r.jina.ai — see REPORT provenance caveat) is used.
No challenge bypass is attempted for new crawling."""
from __future__ import annotations

from .base import Blocked, Crawler, Transient
from .seed_import import SYR_URL, syracuse_folder_records

EMPIRE = "https://empireadc.org/search/catalog/nsyu_2642890"


class Syracuse(Crawler):
    source_id = "syracuse"
    name = "Syracuse University Special Collections — Paul Conrad Cartoons"
    repository = "Syracuse University"
    url = SYR_URL
    classification = "PHYSICAL_ARCHIVE"
    access_notes = "Reading-room access at SCRC. Finding aid is folder-level (65 folders, 388 index headings)."

    def crawl(self) -> None:
        ok = False
        for url in (SYR_URL, EMPIRE):
            try:
                st, html = self.http.get(url)
                self.stats["pages"] += 1
                if st == 200 and "B1F" in html:
                    ok = True
                    self.notes.append(f"live ok: {url}")
                else:
                    self.error(url, f"HTTP {st}")
            except (Blocked, Transient) as e:
                self.error(url, f"BLOCKED: {e}")
                self.notes.append(f"blocked: {url} ({e})")
        if not ok:
            self.status = "blocked"
            self.notes.append("using seed cache (seed_syracuse) for the 65 folder records")


CRAWLER = Syracuse