← back to Paul Conrad Archive
src/conrad/crawlers/secondary_citations.py
464 lines
"""Cycle 3 (DTD option C): VERIFIED item-level records from secondary sources that name a specific Conrad cartoon
with a title/caption AND a date (year minimum).
How a record gets in (and why none can come from model memory):
* every citation below names a page URL plus a verbatim QUOTE (and optionally a DATE_QUOTE) from that page;
* the crawler fetches the page live (robots.txt per path, >=2 s/host, cached, restartable) and saves the record
ONLY if every quote is found in the fetched text (normalised whitespace / quote marks). A quote that is not on
the page => the citation is rejected and logged to `errors`, never saved;
* notes carry the <=200-char quote, so each record can be re-verified by re-reading its source.
Hard rails honoured here:
* no image is fetched: `Http.get` refuses image URLs, and only HTML pages are requested (a Wayback gallery page's
<img alt="..."> TEXT is read; the image binary never is);
* Wayback (web.archive.org) is used ONLY for pages whose origin is retired/moved AND whose origin robots.txt allows
the path. Copies of pages whose live origin is WAF-blocked (nytimes.com) or robots-disallowed
(washingtonpost.com) are deliberately NOT used — that would be a bypass;
* MediaWiki API / Wikidata SPARQL are robots-disallowed for generic agents (/w/api.php, query.wikidata.org), so
only robots-permitted /wiki/ pages and Special:EntityData are read.
"""
from __future__ import annotations
import html as _html
import json
import re
from .. import db, rights
from ..models import CartoonRecord
from ..normalize import parse_date, presidents_in_text
from .base import Blocked, Checkpoint, Crawler, Transient
WIKI = "https://en.wikipedia.org/wiki/Paul_Conrad"
LCIB = "https://www.loc.gov/loc/lcib/9910/conrad.html"
LAT = "https://www.latimes.com/archives/la-xpm-2010-sep-05-la-me-paul-conrad-20100905-story.html"
CANYON = ("https://web.archive.org/web/20140819082434/http://www.canyon-news.com/artman2/publish/LifeStyleMillerTimes/"
"Paul_Conrad_A_Bitter_Appreciation_printer.php")
NCR = "http://natcath.org/NCR_Online/archives2/2001d/102601/102601j.htm"
WRMEA = "https://www.wrmea.org/1988-july/pacific-perspective-conrad-cartoons.html"
AP_NBC = "https://www.nbcnews.com/news/amp/wbna39006857"
PBS_STATE = "https://web.archive.org/web/20171101005025/http://www.pbs.org/independentlens/paulconrad/state.html"
_PBS_TS = {1: "20061201210139", 2: "20100907193806", 3: "20100908073239", 4: "20100909220206", 5: "20100908073123",
6: "20100908073128", 7: "20100908072321", 8: "20100909221123", 9: "20100910032059", 10: "20100908072326",
11: "20100908072332", 12: "20100908073244", 13: "20100908073249", 14: "20100908073131", 15: "20100908073255",
16: "20100908073138", 17: "20100908073300", 18: "20100908073305", 19: "20100908072336", 20: "20100908073310",
21: "20100908072342", 22: "20100908073315", 23: "20100910032105", 24: "20100908073142", 25: "20100908072715",
26: "20100908072722", 27: "20100908073320"}
def PBS(n: int) -> str:
"""Wayback copy (CDX-listed timestamp) of the retired PBS Independent Lens 'Conrad Gallery' page n of 27."""
return f"https://web.archive.org/web/{_PBS_TS[n]}/http://www.pbs.org:80/independentlens/paulconrad/gal_{n:02d}.html"
SITE = {WIKI: "Wikipedia, 'Paul Conrad'", LCIB: "Library of Congress Information Bulletin, Oct 1999",
LAT: "Los Angeles Times obituary, 5 Sep 2010", CANYON: "Canyon News, 3 Oct 2010 (Wayback copy)",
NCR: "National Catholic Reporter, 26 Oct 2001 (Wayback copy)",
PBS_STATE: "PBS Independent Lens, 'The State of Political Cartooning' (Wayback copy)",
WRMEA: "Washington Report on Middle East Affairs, July 1988, p. 12",
AP_NBC: "Associated Press obituary, 4 Sep 2010 (NBC News copy)"}
SITE.update({PBS(n): f"PBS Independent Lens 'Conrad Gallery' {n} of 27 (Wayback copy)" for n in _PBS_TS})
LAT_ = "Los Angeles Times"
DP_ = "Denver Post"
# Each cartoon: slug, title, caption, year|date, publication, description, cites=[(url, quote, date_quote|None)],
# optional note. `seed` names the seed_catalog record believed to be the same cartoon (title is kept identical so the
# dedupe guard can link / flag it) — informational only.
C = [
# ---------------- PBS Independent Lens "Conrad Gallery" (27 captioned cartoons, each with publication + year)
dict(slug="pbs-cover-up-2004", title="The Cover-Up", caption="The Cover-Up.", year=2004, pub=LAT_,
desc="George W. Bush re-enacting the Abu Ghraib torture scandal, standing on a box holding wires.",
cites=[(PBS(1), 'The caption reads: "The Cover-Up."', "Los Angeles Times (2004)")]),
dict(slug="pbs-liberty-wiretap-1975", title="Statue of Liberty wiretapping (binoculars and microphone)", year=1975,
pub=LAT_, desc="Statue of Liberty peering through binoculars and raising a microphone instead of a torch.",
cites=[(PBS(2), 'Conrad serves up a Nixon-era version of "liberty" with this wiretapping Statue of Liberty.',
"Los Angeles Times (1975)")]),
dict(slug="pbs-liar-liar-2003", title="Liar, Liar, Pants on Fire!", caption="Liar, Liar, Pants on Fire!", year=2003,
pub=LAT_, desc="George W. Bush running from flames reading 'Weapons of Mass Destruction'.",
cites=[(PBS(3), 'The caption reads: "Liar, Liar, Pants on Fire!"', "Los Angeles Times (2003)")]),
dict(slug="pbs-elect-bush-twice-2004", title="Elect Bush once, shame on me — Elect Bush twice, shame on you",
year=2004, pub=LAT_, desc="Two Uncle Sams; comment on George W. Bush's 2004 re-election.",
cites=[(PBS(4), 'saying "Elect Bush once, shame on me," and the other pointing out and saying "Elect Bush '
'twice, shame on you."', "Los Angeles Times (2004)")]),
dict(slug="pbs-seal-zipper-1997", title="Presidential seal with its fly undone", year=1997, pub=LAT_,
desc="Seal of the President bisected by a partially undone zipper (Bill Clinton).",
cites=[(PBS(5), "Conrad portrays the presidential seal with its fly undone.", "Los Angeles Times (1997)")]),
dict(slug="pbs-young-man-horn-1994", title="Young Man With a Horn", caption="Young Man With a Horn.", year=1994,
pub=LAT_, desc="Bill Clinton playing a saxophone that doubles as the Republican elephant's trunk.",
seed="cat:obit-clinton-horn-1994",
cites=[(PBS(6), 'The caption reads: "Young Man With a Horn."', "Los Angeles Times (1994)")]),
dict(slug="pbs-bush-oil-tycoon-1986", title="George H. W. Bush prostrate on a dollar bill before an oil tycoon",
year=1986, pub=LAT_, desc="Bush kneeling on a giant dollar bill at the feet of an oil tycoon.",
note="source year (1986) conflicts with its own 'first Gulf War' (1990-91) description",
cites=[(PBS(7), "George H. Bush prostrates himself on a U.S. dollar bill before the feet of an oil tycoon",
"Los Angeles Times (1986)")]),
dict(slug="pbs-bush-reagan-monkey-1986", title="George H. W. Bush with Reagan as a monkey on his back", year=1986,
pub=LAT_, desc="Bush carrying Reagan's baggage, a Reagan-faced chimp riding on his back.",
cites=[(PBS(8), "drew him with Reagan's hairdo and his baggage, including Bonzo the chimp, a monkey on his "
"back.", "Los Angeles Times (1986)")]),
dict(slug="pbs-kinder-gentler-1984", title="A Kinder, Gentler Nation", caption="A Kinder, Gentler Nation.",
year=1984, pub=LAT_, desc="George H. W. Bush holding a giant gun and a wire clothes hanger.",
note="source year (1984) predates the 1988 'kinder, gentler' slogan it describes",
cites=[(PBS(9), 'The caption reads: "A Kinder, Gentler Nation."', "Los Angeles Times (1984)")]),
dict(slug="pbs-out-of-the-loop-1988", title="I was out of the loop.", caption="I was out of the loop.", year=1988,
pub=LAT_, desc="George H. W. Bush in a noose made of Iran-Contra names.",
cites=[(PBS(10), 'The caption reads: "I was out of the loop."', "Los Angeles Times (1988)")]),
dict(slug="pbs-reagan-clown-1984", title="Ronald Reagan as a clown", year=1984, pub=LAT_,
desc="Reagan in clown makeup, puffy collar and clown nose.",
cites=[(PBS(11), "A cartoon of Ronald Reagan as a clown, wearing clown makeup, a puffy collar and a clown "
"nose", "Los Angeles Times (1984)")]),
dict(slug="pbs-reagan-gothic-1985", title="Reagan 'American Gothic'", caption="Ronald Reagan Gothic.", year=1985,
pub=LAT_, desc="Farmhouse with gravestones for 'Farm Communities' and 'Small Farmers' (Reaganomics).",
seed="cat:obit-reagan-gothic-1986",
cites=[(PBS(12), 'The caption reads: "Ronald Reagan Gothic."', "Los Angeles Times (1985)")]),
dict(slug="pbs-reagan-gun-needle-1988", title="Reagan shooting up with a gun in place of a needle", year=1988,
pub=LAT_, desc="Reagan injecting into his arm with a gun (Central America / arms race).",
cites=[(PBS(13), "A cartoon of Ronald Reagan shooting intravenous drugs into his arm, but using a gun in place "
"of a needle", "Los Angeles Times (1988)")]),
dict(slug="pbs-nuclear-war-head-1980", title="Removing another nuclear war head.",
caption="Removing another nuclear war head.", year=1980, pub=LAT_,
desc="A giant Ronald Reagan head strapped to the flatbed of a departing truck.",
cites=[(PBS(14), 'The caption reads: "Removing another nuclear war head."', "Los Angeles Times (1980)")]),
dict(slug="pbs-carter-lust-1976", title="Jimmy Carter lusting after the Statue of Liberty", year=1976, pub=LAT_,
desc="Carter leering at the Statue of Liberty, imagining her undressed ('lust in his heart').",
cites=[(PBS(15), "A cartoon of Jimmy Carter leering at the Statue of Liberty, with a thought balloon "
"envisioning the statue undressed", "Los Angeles Times (1976)")]),
dict(slug="pbs-carter-sisyphus-1978", title="Carter as Sisyphus pushing Sadat and Begin uphill", year=1978,
pub=LAT_, desc="Carter pushing a boulder made of Sadat and Begin up a mountain (Camp David).",
cites=[(PBS(16), "This 1978 cartoon portrays Carter attempting to push a giant boulder composed of Egyptian "
"President Anwar Sadat and Israeli Prime Minister Menachem Begin", "Los Angeles Times (1978)")]),
dict(slug="pbs-ford-gun-control-1976", title="Every right-thinking American is against gun controls!",
caption="Every right-thinking American is against gun controls!", year=1976, pub=LAT_,
desc="Gerald Ford dressed as Uncle Sam, pointing a gun.",
cites=[(PBS(17), 'The caption reads: "Every right-thinking American is against gun controls!"',
"Los Angeles Times (1976)")]),
dict(slug="pbs-nixon-cross-1977", title="Nixon nailing himself to a cross", year=1977, pub=LAT_,
desc="Nixon crucifying himself on a large cross; an Easter Sunday cartoon the Times refused to run.",
note="PBS dates it 1977 while its text says 'facing his impending impeachment'; the Times declined to run it "
"(per PBS)", seed="cat:obit-nixon-cross-1974",
cites=[(PBS(18), "Richard Nixon crucifies himself on a large cross in this Easter Sunday cartoon that the Los "
"Angeles Times refused to run", "Los Angeles Times (1977)"),
(PBS_STATE, "In one drawing, Conrad drew Nixon, in the last days of his presidency, nailing himself to "
"a cross.", None),
(WIKI, "One cartoon showed Nixon, during his last days as president, nailing himself to a cross.", None),
(LCIB, "One of Mr. Conrad's personal favorites was a cartoon of Nixon nailing himself to the cross.",
None)]),
dict(slug="pbs-spider-web-1972", title="Nixon caught in the Watergate spider web", year=1972, pub=LAT_,
desc="Nixon's face formed by a spider web spelling Ehrlichman, Haldeman, Magruder, Mitchell, Liddy.",
seed="cat:obit-spiders-web-1973",
cites=[(PBS(19), "This cartoon features Nixon caught up in spider webs spelling out the names of his aides",
"Los Angeles Times (1972)")]),
dict(slug="pbs-phone-company-1972", title="He says he's from the phone company…",
caption="He says he's from the phone company…", year=1972, pub=LAT_,
desc="Nixon as a phone repairman drilling into the wall of Democratic Headquarters (Watergate).",
seed="cat:obit-phone-company-1972",
cites=[(PBS(20), "The caption reads: \"He says he's from the phone company", "Los Angeles Times (1972)"),
(LAT, "when he drew Nixon disguised as a phone company worker boring a hole in the wall at the "
"Democratic National Committee headquarters", None)]),
dict(slug="pbs-worst-enemy-1973", title="His Own Worst Enemy", caption="His own worst enemy.", year=1973, pub=LAT_,
desc="Nixon alone at a desk overrun with sheets marked 'Enemies List'.", seed="cat:obit-worst-enemy-1973",
cites=[(PBS(21), 'The caption reads: "His own worst enemy."', "Los Angeles Times (1973)"),
(LAT, '"His own worst enemy," said one cartoon of the glowering president, hunkered over his enemies '
'list.', None)]),
dict(slug="pbs-blank-check-1965", title="LBJ signing a blank check for the Vietnam War",
caption="Will 200,000 be enough?", year=1965, pub=LAT_,
desc="Lyndon Johnson handing a general a 'Blank Check, Pay to the Order of Vietnam' signed the U.S. Congress.",
seed="cat:obit-lbj-blankcheck-1964",
cites=[(PBS(22), 'The caption reads: "Will 200,000 be enough?"', "Los Angeles Times (1965)")]),
dict(slug="pbs-nixons-the-one-1968", title="Nixon's the One!", caption="Nixon's the One!", year=1968, pub=LAT_,
desc="LBJ in bed visited by the ghosts of Vietnam past, the urban present and the Middle East future.",
cites=[(PBS(23), '"Nixon\'s the one!" was a popular campaign slogan during the 1968 presidential elections.',
"Los Angeles Times (1968)")]),
dict(slug="pbs-u2-looked-back-1963", title="It looked back!!", caption="It looked back!!", year=1963, pub=DP_,
desc="JFK with a 'U-2's Over Cuba' plane and Khrushchev with a 'Snoop Flights' plane at a door.",
cites=[(PBS(24), 'The caption reads: "It looked back!!"', "Denver Post (1963)")]),
dict(slug="pbs-conspiracy-theories-1978", title="Conspiracy Theories and the House Assassinations Committee",
year=1978, pub=DP_, desc="A man in a 'Conspiracy Theories' shirt in a crowd; JFK assassination findings.",
note="PBS credits the Denver Post for 1978, although Conrad was at the Los Angeles Times from 1964",
cites=[(PBS(25), 'A man wearing a shirt that reads "Conspiracy Theories" stands in a crowd of other men',
"Denver Post (1978)")]),
dict(slug="pbs-little-rock-1958", title="Eisenhower and Little Rock school desegregation", year=1958, pub=DP_,
desc="Eisenhower as a small dejected bookend to cobwebbed books 'Racial Integration' and 'Moral and Legal "
"Leadership'.", seed="cat:obit-littlerock-1958",
cites=[(PBS(26), "This cartoon celebrates Eisenhower's 1957 enforcement of the Supreme Court's school "
"desegregation order", "Denver Post (1958)")]),
dict(slug="pbs-death-defeats-truman-1972", title="Death Defeats Truman", year=1972, pub=LAT_,
desc="Harry Truman holding a Chicago Tribune headlined 'Death Defeats Truman'.",
cites=[(PBS(27), 'holding a copy of the Chicago Tribune with the headline "Death Defeats Truman"',
"Printed on the occasion of Harry S. Truman's death in 1972")]),
dict(slug="pbs-richard-ii-1973", title="Nixon as Shakespeare's King Richard II", year=1973, pub=LAT_,
desc="A morose Nixon quoting Richard II.", seed="cat:obit-richard-ii-1973",
cites=[(PBS_STATE, "Nixon as Richard II Paul Conrad Los Angeles Times (1973)",
"a morose Nixon quotes Shakespeare's King Richard II")]),
# ---------------- Los Angeles Times obituary (free la-xpm archive page, fetched directly)
dict(slug="lat-babe-manger-1968", title="Ye shall find the babe wrapped in swaddling clothes, lying in a manger.",
caption="Ye shall find the babe wrapped in swaddling clothes, lying in a manger.", year=1968, pub=LAT_,
desc="An African American baby on a filthy mattress in a dilapidated apartment (antipoverty).",
cites=[(LAT, "One potent antipoverty image in 1968 showed an African American baby curled on a filthy "
"mattress in a dilapidated apartment", None)]),
dict(slug="lat-baby-cross-1976", title="Forgive them, Father, for they know not what they do ...",
caption="Forgive them, Father, for they know not what they do ...", year=1976, pub=LAT_,
desc="A baby nailed to a cross (abortion).",
cites=[(LAT, '"Forgive them, Father, for they know not what they do ..." said a 1976 cartoon that showed a '
'baby nailed to a cross.', None)]),
dict(slug="lat-pope-baby-1977", title="Pope Paul VI as a baby", year=1977, pub=LAT_,
desc="Pope Paul VI holding a baby in his own image (ordination of women).", seed="cat:obit-popepaulvi-1977",
cites=[(LAT, "The cartoonist drew the pope, instead, holding a baby in his own image",
"When Pope Paul VI wrote against the ordination of women in 1977, Conrad strongly disagreed")]),
dict(slug="lat-here-lies-nixon-1994", title="Here lies Richard Nixon.", caption="Here lies Richard Nixon.",
year=1994, pub=LAT_, desc="Nixon's grave, drawn after his 1994 death.",
cites=[(LAT, 'He drew the 37th president\'s grave with the words: "Here lies Richard Nixon."',
"Others saw Nixon's 1994 death as occasion for absolution.")]),
# ---------------- LOC Information Bulletin (+ Canyon News corroboration)
dict(slug="lcib-selfportrait-1984", title="Self-portrait begging for four more years for Ronald Reagan",
caption="Four more years!", date="1984-11-07", pub=LAT_,
desc="Rare self-portrait: a jubilant Conrad at the drawing board with a buffoon Reagan, after Reagan's "
"re-election.", seed="cat:obit-reagan-selfportrait-1984",
cites=[(LCIB, "in a rare self-portrait on Nov. 7, 1984, for four more years for Ronald Reagan", None),
(CANYON, "Upon Reagan's re-election \"Four more years!\" was the caption for a cartoon showing a "
"jubilant Conrad", None)]),
# ---------------- Wikipedia (article body + its citation of the Time issue)
dict(slug="wiki-time-cover-1967", title="TIME cover: the 1968 presidential race as a horse race (jockeys weighing in)",
date="1967-04-14", pub="Time",
desc="Cover art: candidates (Johnson, R. Kennedy, Humphrey, Nixon, Reagan, Romney, Rockefeller, Percy) as "
"jockeys weighing in.",
cites=[(WIKI, "In April 1967, Conrad drew the cover for Time magazine in an issue about the potential "
"candidates for the 1968 United States presidential election", "(1967, April 14). Time, 89 (15)")]),
# ---------------- Washington Report on Middle East Affairs, July 1988 (TK-12199 cycle 4)
dict(slug="wrmea-talk-peace-1988",
title="I'm willing to talk peace, but there aren't any Palestinians to talk peace with.",
caption="I'm willing to talk peace, but there aren't any Palestinians to talk peace with.",
date="1988-04-27", pub=LAT_,
desc="A gun-wielding Israeli soldier standing atop Palestinian corpses (first intifada).",
note="year from the article's own date (WRMEA, July 1988): 'April 27' = 27 Apr 1988",
cites=[(WRMEA, "Palestinian corpses captioned: \"I'm willing to talk peace, but there aren't any Palestinians "
"to talk peace with.\"", "The struggle reached a climax April 27 with publication of a Conrad "
"cartoon")]),
# ---------------- Associated Press obituary (NBC News copy), 4 Sep 2010
dict(slug="ap-cuckoos-nest-1974", title="One flew over the cuckoo's nest", caption="One flew over the cuckoo's nest.",
year=1974, range=("1974-08-08", "1974-12-31"), pub=LAT_,
desc="Nixon's helicopter leaving the White House at his resignation.", seed="cat:obit-cuckoos-nest-1974",
note="date range = Nixon's resignation (8-9 Aug 1974) .. end of 1974; AP gives no exact date",
cites=[(AP_NBC, "Conrad drew Nixon's helicopter leaving the White House with the caption: \"One flew over the "
"cuckoo's nest.\"", "At the time of the president's resignation")]),
# ---------------- National Catholic Reporter, 26 Oct 2001
dict(slug="ncr-band-of-brothers-2001", title="Band of Brothers", year=2001, range=("2001-09-11", "2001-10-26"),
pub=LAT_, desc="Firemen at the World Trade Center, drawn in the wake of the Sept. 11 attacks.",
note="date range = 11 Sep 2001 .. NCR article date 26 Oct 2001",
cites=[(NCR, 'His "Band of Brothers" drawing of firemen at the World Trade Center has been picked up by fire '
'departments nationwide', "His cartoons done in the wake of the Sept. 11 attack on New York and "
"Washington")]),
]
# Probed sources that yield no item-level record (or are blocked) — registered so REPORT.md lists them honestly.
PROBES = [
# (source_id, name, url, classification, how)
("wikimedia_commons", "Wikimedia Commons (Category:Paul Conrad)", "https://commons.wikimedia.org/wiki/Category:Paul_Conrad",
"PUBLIC_HTML", "commons"),
("wikidata", "Wikidata (Paul Conrad, Q2059192)", "https://www.wikidata.org/wiki/Special:EntityData/Q2059192.json",
"PUBLIC_API", "wikidata"),
("pulitzer", "Pulitzer Prizes (1964, 1971, 1984)", "https://www.pulitzer.org/winners/paul-conrad", "PUBLIC_HTML",
"pulitzer"),
("huntington_exhibits", "The Huntington — exhibition / collection web pages", "https://www.huntington.org/collections/lib-msscon-1-12360",
"PUBLIC_HTML", "plain"),
("nixon_foundation", "Richard Nixon Foundation (site search 'conrad')", "https://www.nixonfoundation.org/?s=conrad",
"PUBLIC_HTML", "plain"),
("nixon_library", "Richard Nixon Presidential Library (site search 'conrad')", "https://www.nixonlibrary.gov/search?keys=conrad",
"PUBLIC_HTML", "plain"),
("nytimes_obit", "New York Times obituary, 5 Sep 2010", "https://www.nytimes.com/2010/09/05/us/politics/05conrad.html",
"REQUIRES_PERMISSION", "plain"),
("washpost_obit", "Washington Post obituary, 6 Sep 2010", "http://www.washingtonpost.com/wp-dyn/content/article/2010/09/05/AR2010090503410_pf.html",
"REQUIRES_PERMISSION", "plain"),
]
BLOCKED_NOTE = {"nytimes_obit": "live page WAF-blocked; its Wayback copy is deliberately NOT used (would bypass the block)",
"washpost_obit": "robots.txt disallows; its Wayback copy is deliberately NOT used (would bypass robots)"}
API_NOTE = ("MediaWiki API (/w/api.php), Wikidata API and query.wikidata.org SPARQL are robots.txt-disallowed for generic "
"agents -> not queried; only robots-permitted /wiki/ pages and Special:EntityData were read")
def page_text(body: str) -> str:
"""Visible text + <img alt> text of an HTML page (alt TEXT only — no image is ever requested)."""
alts = " ".join(re.findall(r'\balt="([^"]*)"', body, flags=re.I))
t = re.sub(r"<script.*?</script>|<style.*?</style>", " ", body, flags=re.S | re.I)
t = re.sub(r"<[^>]+>", " ", t)
return _html.unescape(t + " " + _html.unescape(alts))
def norm(s: str) -> str:
s = _html.unescape(s or "")
s = (s.replace("’", "'").replace("‘", "'").replace("“", '"').replace("”", '"')
.replace("…", "...").replace(" ", " ").replace("—", "-").replace("–", "-"))
s = re.sub(r"[\x80-\x9f\u200b\ufeff]", "", s) # stray C1 control chars from mis-decoded pages
s = re.sub(r"\s+", " ", s)
s = re.sub(r" ([,.;:!?)\]])", r"\1", s) # "Time , 89" (link boundary) -> "Time, 89"
return s.strip().casefold()
def quote_on_page(quote: str, text_norm: str) -> bool:
return norm(quote) in text_norm
def _dates(c: dict) -> dict:
if c.get("date"):
d = parse_date(c["date"])
return dict(date_exact=d["date_exact"], date_start=d["date_start"], date_end=d["date_end"], year=d["year"],
date_is_estimate=False)
y = int(c["year"])
lo, hi = c.get("range") or (f"{y}-01-01", f"{y}-12-31")
return dict(date_exact=None, date_start=lo, date_end=hi, year=y, date_is_estimate=True)
class SecondaryCitations(Crawler):
source_id = "secondary_web"
name = "Secondary citations — verified quotes (Wikipedia, LA Times obit, LOC bulletin, PBS gallery, NCR)"
repository = "Secondary citation"
url = WIKI
classification = "PUBLIC_HTML"
access_notes = ("Each record cites a fetched page and carries a verbatim <=200-char quote; saved only if the quote "
"is on the page. Wayback used only for retired pages whose origin robots.txt allows the path.")
def __init__(self, *a, **kw):
super().__init__(*a, **kw)
self._pages: dict[str, str | None] = {}
self.ck = Checkpoint("secondary_citations")
self.per_site: dict[str, int] = {}
self.rejected: list[str] = []
# --------------------------------------------------------------- fetching
def _wayback_origin_ok(self, url: str) -> bool:
m = re.match(r"https?://web\.archive\.org/web/\d+(?:id_)?/(https?://.+)$", url)
if not m:
return True
origin = m[1]
rp = self.http.robots(origin)
if rp is not None and (getattr(rp, "_unreadable", False) or not rp.can_fetch(self.http.s.headers["User-Agent"], origin)):
return False
return True
def text(self, url: str) -> str | None:
if url in self._pages:
return self._pages[url]
out = None
if not self._wayback_origin_ok(url):
self.error(url, "BLOCKED: Wayback copy not used — origin robots.txt disallows or is unreadable")
else:
try:
st, body = self.http.get(url)
self.stats["pages"] += 1
if st == 200:
out = norm(page_text(body))
else:
self.error(url, f"HTTP {st}")
except (Blocked, Transient) as e:
self.error(url, f"{'BLOCKED' if isinstance(e, Blocked) else 'TRANSIENT'}: {e}")
self._pages[url] = out
return out
# --------------------------------------------------------------- records
def crawl(self) -> None:
for c in C:
self.one(c)
self.probes()
self.ck.set("last_run", db.now())
sites = ", ".join(f"{k}={v}" for k, v in sorted(self.per_site.items(), key=lambda kv: -kv[1]))
self.notes.append(f"verified citations saved per site: {sites}")
if self.rejected:
self.notes.append("REJECTED (quote not on fetched page / page unavailable): " + "; ".join(self.rejected))
def one(self, c: dict) -> None:
verified = []
for url, quote, date_quote in c["cites"]:
t = self.text(url)
ok = t is not None and quote_on_page(quote, t) and (date_quote is None or quote_on_page(date_quote, t))
if ok:
verified.append((url, quote, date_quote))
else:
why = "page unavailable" if t is None else "quote not found on page"
self.rejected.append(f"{c['slug']} @ {url} ({why})")
self.error(url, f"citation rejected for {c['slug']}: {why}")
if not verified:
return
d = _dates(c)
people = presidents_in_text(" ".join(filter(None, [c["title"], c.get("caption"), c.get("desc")])))
for i, (url, quote, date_quote) in enumerate(verified):
q = quote if len(quote) <= 200 else quote[:197] + "..."
note = f'Cited by {SITE.get(url, url)}: "{q}"'
if date_quote and i == 0:
note += f' | date evidence: "{date_quote[:120]}"'
if c.get("note") and i == 0:
note += f" | caveat: {c['note']}"
rec = CartoonRecord(
canonical_id=f"sec:{c['slug']}", identifier=f"{c['slug']}@{url}", granularity="item",
title=c["title"], caption=c.get("caption"), description=c.get("desc"),
publication=c.get("pub"), rights_text=rights.COPYRIGHT_NOTE, notes=note if i == 0 else None,
repository="Secondary citation", collection_name=SITE.get(url, "secondary web page"),
record_url=url, access_level=rights.ONLINE_METADATA,
provenance=f"live:secondary_web quote-verified {db.now()[:10]}; {note}"[:1000],
acquisition_method="secondary_citation", people=people, subjects=["Editorial cartoons"], **d)
if i > 0:
# corroborating citation: attach the source row without overwriting the cartoon's fields
rec.title = rec.caption = rec.description = rec.publication = rec.notes = None
rec.date_exact = rec.date_start = rec.date_end = rec.year = None
self.save(rec)
label = SITE.get(url, url)
label = "PBS Independent Lens (Wayback)" if "pbs.org" in url else label.split(",")[0].split(" (")[0]
self.per_site[label] = self.per_site.get(label, 0) + 1
# --------------------------------------------------------------- probes
def probes(self) -> None:
for sid, name, url, cls, how in PROBES:
status, note = "worked", ""
if sid in BLOCKED_NOTE:
try:
self.http.get(url)
note = "reachable; no dated item-level citation used"
except (Blocked, Transient) as e:
status, note = "blocked", f"{BLOCKED_NOTE[sid]} ({e})"
db.upsert_source(self.conn, sid, name, url=url, classification=cls, status=status, notes=note)
continue
try:
st, body = self.http.get(url)
self.stats["pages"] += 1
except (Blocked, Transient) as e:
db.upsert_source(self.conn, sid, name, url=url, classification="REQUIRES_PERMISSION"
if isinstance(e, Blocked) else cls, status="blocked", notes=f"probe: {e}")
continue
if st != 200:
db.upsert_source(self.conn, sid, name, url=url, classification=cls, status="failed", notes=f"HTTP {st}")
continue
if how == "commons":
files = [re.sub(r"\.(jpe?g|gif|png|tiff?|webp|svg|pdf)$", "", f, flags=re.I) + " [file]"
for f in re.findall(r'class="galleryfilename[^"]*"[^>]*>([^<]+)<', body)]
n = len(files) or len(re.findall(r'title="File:', body))
note = (f"category lists {n} file(s): {', '.join(files[:5]) or '?'} — none is a Conrad cartoon; "
f"0 item records. {API_NOTE}")
elif how == "wikidata":
ent = json.loads(body)["entities"]
claims = next(iter(ent.values())).get("claims", {})
nw = len(claims.get("P800", []))
note = (f"entity read; notable-work (P800) claims={nw}; reverse 'creator = Q2059192' needs SPARQL or "
f"Special:WhatLinksHere, both robots-disallowed -> not queried; 0 item records. {API_NOTE}")
elif how == "pulitzer":
cites = []
for y in (1964, 1971, 1984):
try:
_, b2 = self.http.get(f"https://www.pulitzer.org/prize-winners-by-year/{y}")
self.stats["pages"] += 1
t2 = re.sub(r"\s+", " ", _html.unescape(re.sub(r"<[^>]+>", " ", b2)))
m = re.search(r"Paul Conrad of (?:the )?[A-Z][\w .]+?(For [^.]{0,80}\.)?(?= Finalists|\s{2}|$)", t2)
cites.append(f"{y}: " + (m[0].strip()[:110] if m else "Conrad listed"))
except (Blocked, Transient) as e:
cites.append(f"{y}: {e}")
note = ("prize citations name no single cartoon (year-of-work awards) -> 0 item records; " +
" | ".join(cites))
else:
txt = norm(page_text(body))
hits = txt.count("conrad")
note = (f"HTTP 200; 'conrad' mentions={hits}; no page naming a specific dated Conrad cartoon found "
f"-> 0 item records")
db.upsert_source(self.conn, sid, name, url=url, classification=cls, status=status, notes=note)
self.conn.commit()
CRAWLER = SecondaryCitations