`src/conrad/crawlers/ohio_state.py`

36 lines
"""Ohio State University Billy Ireland Cartoon Library & Museum (PastPerfect Online).
The PastPerfect host returns HTTP 403 to non-browser clients (including /robots.txt), so it cannot be
crawled politely; recorded as blocked / REQUIRES_PERMISSION. No bypass is attempted."""
from __future__ import annotations

from .base import Blocked, Crawler, Transient

BASE = "https://osucartoons.pastperfectonline.com/"
SEARCHES = [BASE + "search?search_criteria=%22Conrad%2C+Paul%22&onlyimages=false",
            BASE + "bysearchterm?keyword=Conrad%2C%20Paul"]


class OhioState(Crawler):
    source_id = "ohio_state"
    name = "Ohio State University — Billy Ireland Cartoon Library & Museum"
    repository = "Ohio State University"
    url = BASE
    classification = "REQUIRES_PERMISSION"
    access_notes = ("Holds Conrad material (per OSU catalog references) but the online collections database refuses "
                    "automated access (HTTP 403). Request a metadata export from the Billy Ireland reference desk.")

    def crawl(self) -> None:
        for url in [BASE, *SEARCHES]:
            try:
                st, html = self.http.get(url)
                self.stats["pages"] += 1
                self.notes.append(f"{url}: HTTP {st}")
            except (Blocked, Transient) as e:
                self.error(url, f"BLOCKED: {e}")
                self.status = "blocked"
                self.notes.append(f"blocked: {e}")
                break


CRAWLER = OhioState