`src/conrad/crawlers/internet_archive.py`

102 lines
"""Internet Archive — bibliography of Conrad's books (metadata API + advancedsearch; robots permit both).
Books on archive.org are Controlled-Digital-Lending scans: we record metadata only and never borrow/open them.
Name-authority conflations (other 'Paul Conrad's tagged 1924-2010) are detected and excluded."""
from __future__ import annotations

import re

from .base import Blocked, Crawler, Transient

BIBLIOGRAPHY = [  # (title, expected year) — the books named in the spec
    ("When in the Course of Human Events", 1973),
    ("The King and Us", 1974),
    ("Pro and Conrad", 1979),
    ("Drawn and Quartered", 1985),
    ("CONartist: 30 Years with the Los Angeles Times", 1993),
    ("Drawing the Line", 1999),
    ("I, Con", None),
]
ADV = "https://archive.org/advancedsearch.php"
META = "https://archive.org/metadata/"
NOT_OUR_CONRAD = re.compile(r"larryboy|veggie|kinderklapper|gott ist|thriving church|slave bug", re.I)  # slave bug: 1975 novel, no dates on creator


def _norm(t: str) -> str:
    return re.sub(r"[^a-z0-9 ]", "", (t or "").lower())


class InternetArchive(Crawler):
    source_id = "internet_archive"
    name = "Internet Archive (books metadata)"
    repository = "Internet Archive"
    url = "https://archive.org/details/paulconraddrawin0000conr"
    classification = "PUBLIC_API"
    access_notes = "Book scans are lending-library (CDL) items; metadata only, not borrowed."

    def search(self, q: str) -> list[dict]:
        d = self.http.get_json(ADV, params={"q": q, "fl[]": ["identifier", "title", "date", "creator", "mediatype"],
                                            "rows": 100, "output": "json"})
        self.stats["pages"] += 1
        return d.get("response", {}).get("docs", [])

    def crawl(self) -> None:
        cands: dict[str, dict] = {}
        try:
            for doc in self.search('creator:("Conrad, Paul") AND mediatype:texts'):
                cands[doc["identifier"]] = doc
            for title, _ in BIBLIOGRAPHY:
                short = title.split(":")[0]
                for doc in self.search(f'title:("{short}") AND creator:(conrad)'):
                    cands[doc["identifier"]] = doc
        except (Blocked, Transient) as e:
            self.error(ADV, e)
        excluded = []
        for ident, doc in sorted(cands.items()):
            creators = doc.get("creator") or []
            creators = creators if isinstance(creators, list) else [creators]
            ctext = " ".join(creators)
            if NOT_OUR_CONRAD.search(doc.get("title", "")) or not re.search(r"conrad,\s*paul", ctext, re.I) \
                    or re.search(r"1865|funke", ctext, re.I):
                excluded.append(f"{ident} ({doc.get('title', '')[:50]}; {ctext[:60]})")
                continue
            try:
                m = self.http.get_json(META + ident)
            except (Blocked, Transient) as e:
                self.error(META + ident, e)
                continue
            self.stats["pages"] += 1
            md = m.get("metadata", {})
            title = md.get("title") or doc.get("title")
            year = int(str(md.get("date") or doc.get("date") or "0")[:4] or 0) or None
            colls = md.get("collection") or []
            colls = colls if isinstance(colls, list) else [colls]
            access = "controlled_digital_lending" if {"inlibrary", "printdisabled"} & set(colls) else "public"
            bib = next((b for b in BIBLIOGRAPHY if _norm(b[0].split(":")[0])[:14] in _norm(title)), None)
            name = bib[0] if bib else title.strip()
            isbn = md.get("isbn")
            isbn = isbn[0] if isinstance(isbn, list) else isbn
            oclc = md.get("oclc-id")
            oclc = oclc[0] if isinstance(oclc, list) else oclc
            desc = md.get("description")
            desc = " | ".join(desc) if isinstance(desc, list) else desc
            self.conn.execute(
                """INSERT INTO books(title,year,publisher,isbn,oclc,ia_identifier,ia_access,record_url,notes,verified)
                   VALUES (?,?,?,?,?,?,?,?,?,1)
                   ON CONFLICT(title) DO UPDATE SET year=COALESCE(excluded.year,year), publisher=excluded.publisher,
                     isbn=COALESCE(excluded.isbn,isbn), oclc=COALESCE(excluded.oclc,oclc),
                     ia_identifier=excluded.ia_identifier, ia_access=excluded.ia_access,
                     record_url=excluded.record_url, notes=excluded.notes, verified=1""",
                (name, year, md.get("publisher"), isbn, oclc, ident, access, f"https://archive.org/details/{ident}",
                 (f"IA title: {title}. " + (desc or ""))[:800]))
            self.stats["seen"] += 1
            self.stats["added"] += 1
        # bibliography titles not found on IA stay listed, unverified
        for title, year in BIBLIOGRAPHY:
            self.conn.execute("INSERT OR IGNORE INTO books(title,year,notes,verified) VALUES (?,?,?,0)",
                              (title, year, "listed in project brief; not found on Internet Archive"))
        self.conn.commit()
        self.notes.append(f"excluded name-authority conflations: {excluded}")


CRAWLER = InternetArchive