`src/conrad/crawlers/oac.py`

52 lines
"""Online Archive of California — collection-level metadata for the Huntington finding aid.
robots.txt: /search and /view are disallowed, Crawl-delay 5 (honoured); /findaid/ is allowed."""
from __future__ import annotations

import re

from bs4 import BeautifulSoup

from .base import Crawler

OAC_URL = "https://oac.cdlib.org/findaid/ark:/13030/c8z03dxd/"


def parse_oac(html: str) -> dict:
    soup = BeautifulSoup(html, "html.parser")
    text = re.sub(r"\s+", " ", soup.get_text(" "))
    out = {"title": (soup.title.string or "").strip() if soup.title else None}
    for label in ("Extent", "Date", "Collection Number", "Conditions Governing Access", "Conditions Governing Use",
                  "Physical Location", "Online items available"):
        m = re.search(re.escape(label) + r"\s*:?\s*(.{0,300}?)(?=\s(?:[A-Z][a-z]+ ){0,4}[A-Z][a-z]+\s*:|$)", text)
        if m:
            out[label.lower().replace(" ", "_")] = m.group(1).strip()
    m = re.search(r"([\d,]+)\s+(?:items|original cartoon drawings|cartoons|pieces)", text, re.I)
    out["count_phrase"] = m.group(0) if m else None
    return out


class OAC(Crawler):
    source_id = "oac"
    name = "Online Archive of California (Huntington finding aid host)"
    repository = "Huntington Library"
    url = OAC_URL
    classification = "PUBLIC_HTML"
    access_notes = "Finding aid only; no digitized Conrad items. Crawl-delay 5 honoured."

    def crawl(self) -> None:
        st, html = self.http.get(OAC_URL)
        self.stats["pages"] += 1
        if st != 200:
            self.error(OAC_URL, f"HTTP {st}")
            self.status = "failed"
            return
        info = parse_oac(html)
        self.conn.execute(
            """UPDATE collections SET notes = COALESCE(notes,'') || ? WHERE repository='Huntington Library' AND level='collection'""",
            (f" | OAC: {info}",))
        self.conn.commit()
        self.notes.append(f"OAC collection page parsed: {info.get('extent') or info.get('count_phrase')}")


CRAWLER = OAC