← back to Paul Conrad Archive
Show institution-published cartoon images by hotlink only (allowlist + CSP), All cartoons view + Has image filter; replace blanket no-img test with allowlist tests
63f07b52e061ecb3b5421275ca27bdb425a8925f · 2026-09-24 23:08:21 -0700 · Steve Abrams
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01W5NGeZrf7GXNQrfMRCTnRc
Files touched
M README.mdM src/conrad/web/app.pyA src/conrad/web/imagehost.pyM src/conrad/web/static/app.jsM src/conrad/web/static/index.htmlM src/conrad/web/static/style.cssA tests/test_image_allowlist.pyD tests/test_no_images_negative.py
Diff
commit 63f07b52e061ecb3b5421275ca27bdb425a8925f
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Sep 24 23:08:21 2026 -0700
Show institution-published cartoon images by hotlink only (allowlist + CSP), All cartoons view + Has image filter; replace blanket no-img test with allowlist tests
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01W5NGeZrf7GXNQrfMRCTnRc
---
README.md | 17 ++--
src/conrad/web/app.py | 72 ++++++++++++--
src/conrad/web/imagehost.py | 76 ++++++++++++++
src/conrad/web/static/app.js | 26 +++--
src/conrad/web/static/index.html | 4 +-
src/conrad/web/static/style.css | 8 ++
tests/test_image_allowlist.py | 208 +++++++++++++++++++++++++++++++++++++++
tests/test_no_images_negative.py | 75 --------------
8 files changed, 389 insertions(+), 97 deletions(-)
diff --git a/README.md b/README.md
index d38ace7..e0223a5 100644
--- a/README.md
+++ b/README.md
@@ -3,12 +3,17 @@
A **metadata-only** index of Paul Conrad (1924–2010) editorial cartoons and *where each one can legally be viewed*.
Ticket: TK-12199. See `REPORT.md` for current numbers, coverage, and every provenance caveat.
-## Copyright rule (hard)
-Conrad cartoon images are research-only and are **never downloaded, stored, or displayed** — not even in this private
-viewer. `image_url` / `thumbnail_url` are kept as metadata; `cartoon_sources.local_image` has a
-`CHECK (local_image IS NULL)` constraint; the viewer shows only "View at <repository>" link-outs, and
-`tests/test_no_images_negative.py` fails if any page, script, stylesheet, or API response emits an image.
-Perceptual-hash dedupe is therefore skipped; dedupe is metadata-only.
+## Image rule (hard; revised by Steve 2026-09-24, TK-12199)
+Conrad cartoon images are **never downloaded, proxied, cached, or stored** — anywhere (server, `data/`, cache, git).
+`cartoon_sources.local_image` has a `CHECK (local_image IS NULL)` constraint and the crawler refuses image URLs,
+image content-types and image magic bytes before reading a body (`crawlers/base.py`).
+On the PRIVATE admin viewer only, a cartoon whose holding institution publishes a public image is shown by
+**hotlinking the institution's own URL** (`src/conrad/web/imagehost.py`): exact-host allowlist
+`tile.loc.gov`, `cdn.loc.gov` (Library of Congress) and `5008.sydneyplus.com` (History Colorado); anything else →
+`display_image: null`. A `Content-Security-Policy: img-src 'self' <those hosts>` header makes the browser refuse every
+other image host. Every other record stays a text card with "View at <repository>" link-outs.
+`tests/test_image_allowlist.py` enforces the allowlist (incl. a negative test proving the detector goes red when the
+allowlist is bypassed) and scans the repo for image bytes. Perceptual-hash dedupe is skipped; dedupe is metadata-only.
## Record granularity
| granularity | meaning |
diff --git a/src/conrad/web/app.py b/src/conrad/web/app.py
index c672461..b3d7d28 100644
--- a/src/conrad/web/app.py
+++ b/src/conrad/web/app.py
@@ -1,7 +1,9 @@
"""Local research viewer (FastAPI). Binds 127.0.0.1:8787 only.
-COPYRIGHT RULE: this viewer never renders an image of a Conrad cartoon — no <img>, no CSS background images, no
-image proxy. Each record shows "View at <repository>" link-outs to the holding institution instead.
+IMAGE RULE (Steve, TK-12199): images are shown ONLY where the holding institution publishes a public image, and only by
+HOTLINKING the institution's own URL (see imagehost.py allowlist). Nothing is ever downloaded, proxied, cached or stored;
+a Content-Security-Policy img-src allowlist makes the browser refuse every other image host. All other records show
+"View at <repository>" link-outs.
"""
from __future__ import annotations
@@ -17,6 +19,7 @@ from fastapi.staticfiles import StaticFiles
from .. import db
from .. import provenance as prov
+from . import imagehost
STATIC = Path(__file__).parent / "static"
@@ -67,6 +70,7 @@ def create_app() -> FastAPI:
resp = await call_next(request)
resp.headers["X-Robots-Tag"] = "noindex, nofollow"
resp.headers["Referrer-Policy"] = "no-referrer"
+ resp.headers["Content-Security-Policy"] = imagehost.CSP
return resp
application.mount("/static", StaticFiles(directory=STATIC), name="static")
@@ -92,8 +96,40 @@ def conn():
return db.connect()
-def _where(q, year, year_from, year_to, person, president, subject, publication, repository, collection, granularity):
+_IMG_SOURCES_SQL = """SELECT COALESCE(k.merged_into, k.id) AS cid, x.source_id, x.repository, x.collection_name,
+ x.record_url, x.rights_url, x.image_url, x.thumbnail_url
+ FROM cartoon_sources x JOIN cartoons k ON k.id = x.cartoon_id
+ WHERE (x.image_url IS NOT NULL OR x.thumbnail_url IS NOT NULL) {extra}
+ ORDER BY cid, x.source_id LIKE 'seed%', x.source_id"""
+
+
+def display_images(c, ids=None) -> dict:
+ """{canonical cartoon id: display_image dict} for records whose holding institution publishes an image on an
+ allowlisted host. ids=None -> every record (used by the has_image filter)."""
+ extra, args = "", []
+ if ids is not None:
+ ids = list(ids)
+ if not ids:
+ return {}
+ extra = "AND COALESCE(k.merged_into, k.id) IN (" + ",".join("?" * len(ids)) + ")"
+ args = ids
+ grouped: dict = {}
+ for r in c.execute(_IMG_SOURCES_SQL.format(extra=extra), args):
+ grouped.setdefault(r["cid"], []).append(r)
+ out = {}
+ for cid, rows in grouped.items():
+ di = imagehost.pick(rows)
+ if di:
+ out[cid] = di
+ return out
+
+
+def _where(q, year, year_from, year_to, person, president, subject, publication, repository, collection, granularity,
+ has_image=False, c=None):
w, a = ["c.merged_into IS NULL"], []
+ if has_image:
+ ids = sorted(display_images(c or conn()))
+ w.append("c.id IN (" + (",".join(str(int(i)) for i in ids) or "NULL") + ")")
if granularity and granularity != "all":
w.append("c.granularity = ?"); a.append(granularity)
if q:
@@ -131,9 +167,11 @@ def index():
def search(q: str | None = None, year: int | None = None, year_from: int | None = None, year_to: int | None = None,
person: str | None = None, president: str | None = None, subject: str | None = None,
publication: str | None = None, repository: str | None = None, collection: str | None = None,
- granularity: str = "item", sort: str = "date_desc", limit: int = Query(60, le=500), offset: int = 0):
+ granularity: str = "item", has_image: bool = False, sort: str = "date_desc",
+ limit: int = Query(60, le=500), offset: int = 0):
c = conn()
- where, args = _where(q, year, year_from, year_to, person, president, subject, publication, repository, collection, granularity)
+ where, args = _where(q, year, year_from, year_to, person, president, subject, publication, repository, collection,
+ granularity, has_image, c)
total = c.execute(f"SELECT COUNT(*) FROM cartoons c WHERE {where}", args).fetchone()[0]
rows = c.execute(f"""
SELECT c.id, c.canonical_id, c.granularity, c.title, c.caption, c.date_exact, c.date_start, c.date_end, c.year,
@@ -143,7 +181,9 @@ def search(q: str | None = None, year: int | None = None, year_from: int | None
[*args, limit, offset]).fetchall()
results = [dict(r) for r in rows]
methods = prov.methods_by_canonical(c, [r["id"] for r in results])
+ imgs = display_images(c, [r["id"] for r in results])
for r in results:
+ r["display_image"] = imgs.get(r["id"])
r["acquisition_methods"] = methods.get(r["id"], [])
r["provenance_flags"] = prov.flags(r["acquisition_methods"])
return {"total": total, "offset": offset, "results": results}
@@ -186,7 +226,7 @@ def _detail(cid: int) -> dict:
ids = [r[0] for r in c.execute("SELECT id FROM cartoons WHERE id=? OR merged_into=?", (cid, cid))]
ph = ",".join("?" * len(ids))
d = dict(row)
- # image_url / thumbnail_url are deliberately NOT sent to the browser (link-out only)
+ # raw image_url / thumbnail_url are NOT sent to the browser; only the allowlisted display_image (hotlink) is
d["sources"] = [dict(r) for r in c.execute(
f"""SELECT source_id, repository, collection_name, identifier, box, folder, page, record_url, access_level,
rights_url, provenance, acquisition_method, retrieved_at, (image_url IS NOT NULL OR thumbnail_url IS NOT NULL) AS has_online_image
@@ -198,6 +238,7 @@ def _detail(cid: int) -> dict:
FROM cartoon_links l JOIN cartoons o ON o.id = CASE WHEN l.cartoon_id IN ({ph}) THEN l.related_id ELSE l.cartoon_id END
WHERE l.cartoon_id IN ({ph}) OR l.related_id IN ({ph})""", ids * 3)]
d["merged_ids"] = [i for i in ids if i != cid]
+ d["display_image"] = display_images(c, [cid]).get(cid)
d["acquisition_methods"] = sorted({s["acquisition_method"] for s in d["sources"] if s["acquisition_method"]})
d["provenance_flags"] = prov.flags(d["acquisition_methods"])
return d
@@ -260,7 +301,19 @@ def cartoon_page(cid: int):
pbadge += (' <span class="badge warn provenance-flag" data-flag="' + prov.FLAG_SECONDARY_ONLY
+ '">known only from a secondary citation — not seen in a holding repository</span>')
title = _e(d["title"] or "[untitled / not cataloged]")
- body = PAGE.format(
+ figure = ""
+ di = d["display_image"]
+ if di and imagehost.is_allowed(di["url"]):
+ rights = (' · <a rel="noopener noreferrer" target="_blank" href="' + _e(di["rights_url"]) + '">rights</a>'
+ if di["rights_url"] else "")
+ rec = (' · <a rel="noopener noreferrer" target="_blank" href="' + _e(di["record_url"]) + '">View at '
+ + _e(di["repository"]) + " ↗</a>") if di["record_url"] else ""
+ figure = ('<figure class="hero"><img src="' + _e(di["url"]) + '" alt="' + title
+ + '" loading="lazy" referrerpolicy="no-referrer" decoding="async">'
+ + '<figcaption class="credit small">Image: ' + _e(di["credit"])
+ + " (displayed from the institution’s site; not copied here)" + rec + rights
+ + "</figcaption></figure>")
+ body = PAGE.format(figure=figure,
head_title=_e(d["title"] or d["canonical_id"]), title=title, gran=_e(GRANULARITY_LABEL[d["granularity"]]),
date=_e(date), est=est + pbadge, pub=_e(d["publication"] or "publication unknown"), canonical=_e(d["canonical_id"]),
caption=caption, desc=desc, people=people, subjects=subjects, medium=_e(d["medium"] or "—"),
@@ -273,9 +326,10 @@ PAGE = """<!doctype html><html lang="en"><head><meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1"><meta name="robots" content="noindex,nofollow">
<title>{head_title} — Conrad Archive</title><link rel="stylesheet" href="/static/style.css"></head>
<body><header class="top"><a href="/" class="brand">Paul Conrad Master Archive</a>
-<span class="muted">metadata only · no images reproduced</span></header>
+<span class="muted">images only where the holding institution publishes one (shown from its site, never copied)</span></header>
<main class="detail"><p><a href="/">← back to search</a></p>
<h1>{title}</h1>
+{figure}
<p class="meta"><span class="badge">{gran}</span> <b>{date}</b>{est} · {pub} · <span class="muted">{canonical}</span></p>
{caption}{desc}
<dl><dt>People</dt><dd>{people}</dd><dt>Subjects</dt><dd>{subjects}</dd><dt>Medium</dt><dd>{medium}</dd>
@@ -283,7 +337,7 @@ PAGE = """<!doctype html><html lang="en"><head><meta charset="utf-8">
<h2>Where to see it ({n_sources} source records)</h2>
<div class="tablewrap"><table><thead><tr><th>Repository</th><th>Collection</th><th>Identifier</th><th>Box</th>
<th>Folder</th><th>Access</th><th>Link-out</th><th>Provenance</th></tr></thead><tbody>{rows}</tbody></table></div>
-<p class="muted small">Images are never displayed here (copyright). Follow the link-out to view the work at the holding repository.</p>
+<p class="muted small">Images are never copied or stored here. Where the holding institution publishes one it is displayed straight from the institution's site; otherwise follow the link-out.</p>
<h2>Related records</h2><ul>{links}</ul>
<p class="muted small">record created <span title="{created}">{created}</span> · updated {updated}</p>
</main></body></html>"""
diff --git a/src/conrad/web/imagehost.py b/src/conrad/web/imagehost.py
new file mode 100644
index 0000000..45b3e70
--- /dev/null
+++ b/src/conrad/web/imagehost.py
@@ -0,0 +1,76 @@
+"""Display-image policy (Steve, TK-12199): on the PRIVATE admin viewer, a cartoon may be SHOWN only when the holding
+institution itself publishes a public image of it, and only by HOTLINKING that institution's own URL.
+
+Nothing here ever fetches, proxies, caches or stores image bytes — it only rewrites/validates URL strings that the
+crawlers already recorded. The browser loads the image straight from the institution, and the Content-Security-Policy
+built from the same allowlist makes the browser itself refuse any other image host.
+"""
+from __future__ import annotations
+
+import re
+from urllib.parse import urlparse, urlunparse
+
+# Exact hosts (no wildcards) — also the CSP img-src list. Determined from the stored URLs:
+# tile.loc.gov / cdn.loc.gov -> Library of Congress Prints & Photographs image services
+# 5008.sydneyplus.com -> History Colorado's collections portal (ViewImage.aspx derivative)
+ALLOWED_IMAGE_HOSTS = ("tile.loc.gov", "cdn.loc.gov", "5008.sydneyplus.com")
+LOC_HOSTS = ("tile.loc.gov", "cdn.loc.gov")
+
+CSP = "img-src 'self' " + " ".join("https://" + h for h in ALLOWED_IMAGE_HOSTS)
+
+_LOC_THUMB = re.compile(r"_150px\.jpg$", re.I)
+_LOC_IMAGE = re.compile(r"/service/pnp/.+\.(jpe?g|gif|png)$", re.I)
+
+
+def display_url(url: str | None) -> str | None:
+ """Return the https URL to show for a stored image/thumbnail URL, or None if it is not on the allowlist.
+
+ LOC: the stored value is the 150px thumbnail; LOC's standard medium "reference" derivative is <stem>r.jpg
+ (present in the cached item JSON for every LOC record we hold). History Colorado: its ViewImage.aspx derivative.
+ Item PAGES (www.loc.gov/pictures/item/...) are not images and are never returned.
+ """
+ if not url or not isinstance(url, str):
+ return None
+ try:
+ p = urlparse(url.strip())
+ except ValueError:
+ return None
+ host = (p.hostname or "").lower()
+ if p.scheme not in ("http", "https") or host not in ALLOWED_IMAGE_HOSTS or p.username or p.password or p.port:
+ return None
+ if host in LOC_HOSTS:
+ if not _LOC_IMAGE.search(p.path):
+ return None
+ path = _LOC_THUMB.sub("r.jpg", p.path)
+ return urlunparse(("https", host, path, "", "", ""))
+ if host == "5008.sydneyplus.com":
+ if not p.path.lower().endswith("/viewimage.aspx"):
+ return None
+ return urlunparse(("https", host, p.path, "", p.query, ""))
+ return None # pragma: no cover — every allowlisted host is handled above
+
+
+def is_allowed(url: str | None) -> bool:
+ """True iff url is an https URL on an allowlisted image host (used by tests + defence in depth)."""
+ if not url:
+ return False
+ p = urlparse(url)
+ return p.scheme == "https" and (p.hostname or "").lower() in ALLOWED_IMAGE_HOSTS
+
+
+def pick(sources) -> dict | None:
+ """First source (in the given order) that yields an allowlisted display URL -> display_image dict."""
+ for s in sources:
+ for key in ("image_url", "thumbnail_url"):
+ u = display_url(s[key])
+ if u and is_allowed(u):
+ repo = s["repository"] or "holding institution"
+ coll = s["collection_name"]
+ return {
+ "url": u,
+ "repository": repo,
+ "credit": repo + ((" — " + coll) if coll and coll != "via DPLA" else ""),
+ "record_url": s["record_url"],
+ "rights_url": s["rights_url"],
+ }
+ return None
diff --git a/src/conrad/web/static/app.js b/src/conrad/web/static/app.js
index 566cd84..62c46b4 100644
--- a/src/conrad/web/static/app.js
+++ b/src/conrad/web/static/app.js
@@ -1,5 +1,6 @@
-// Paul Conrad Archive viewer. COPYRIGHT RULE: never render an image of a Conrad cartoon.
-// Cards and detail pages show text + "View at <repository>" link-outs only.
+// Paul Conrad Archive viewer. IMAGE RULE (TK-12199): an image is shown ONLY when the API supplies a display_image —
+// an allowlisted URL on the holding institution's own server (hotlinked, never copied). The page CSP img-src enforces
+// the same allowlist in the browser. Every other card is text + a "View at <repository>" link-out.
(function () {
const $ = (id) => document.getElementById(id);
const FILTERS = ["q", "granularity", "year_from", "year_to", "president", "person", "subject", "publication", "repository", "collection"];
@@ -7,6 +8,7 @@
get(k, d) { try { const v = localStorage.getItem("conrad." + k); return v === null ? d : v; } catch (e) { return d; } },
set(k, v) { try { localStorage.setItem("conrad." + k, v); } catch (e) { /* private mode */ } },
};
+ const PAGE = 500; // "All cartoons" = every item-level record in one page (318 today); Load more covers the rest
let offset = 0, total = 0, year = null;
function esc(s) {
@@ -21,6 +23,7 @@
const p = new URLSearchParams();
FILTERS.forEach((f) => { const v = $(f).value.trim(); if (v) p.set(f, v); });
if (year) p.set("year", year);
+ if ($("has_image").checked) p.set("has_image", "true");
p.set("sort", $("sort").value);
return p;
}
@@ -46,7 +49,10 @@
function card(r) {
const date = r.date_exact || (r.date_start ? `${r.date_start} – ${r.date_end}` : "undated");
const link = r.record_url ? `<a class="out" rel="noopener noreferrer" target="_blank" href="${esc(r.record_url)}">View at ${esc((r.repos || "").split(",")[0])} ↗</a>` : "";
- return `<article class="card g-${esc(r.granularity)}">
+ const di = r.display_image;
+ const fig = di && /^https:\/\//.test(di.url) ? `<figure class="thumb"><a href="/cartoon/${r.id}"><img src="${esc(di.url)}" alt="${esc(r.title || "[untitled]")}" loading="lazy" referrerpolicy="no-referrer" decoding="async"></a>
+ <figcaption class="credit">Image: ${esc(di.credit)}${di.rights_url ? ` · <a rel="noopener noreferrer" target="_blank" href="${esc(di.rights_url)}">rights</a>` : ""}</figcaption></figure>` : "";
+ return `<article class="card g-${esc(r.granularity)}${fig ? " has-img" : ""}">${fig}
<a class="title" href="/cartoon/${r.id}">${esc(r.title || "[untitled / not cataloged]")}</a>
<div class="meta"><b>${esc(date)}</b>${r.date_is_estimate ? ' <span class="badge warn">est.</span>' : ""}${(r.provenance_flags || []).includes("pending_direct_verification") ? ' <span class="badge warn" title="acquired via third-party reader past a bot challenge — pending direct verification">reader-bypass</span>' : ""}${(r.provenance_flags || []).includes("secondary_citation_only") ? ' <span class="badge warn" title="known only from a secondary citation">secondary</span>' : ""} · ${esc(r.publication || "")}</div>
${r.people ? `<div class="people">${esc(r.people)}</div>` : ""}
@@ -56,12 +62,13 @@
}
async function search(append) {
if (!append) offset = 0;
- const p = params(); p.set("offset", offset); p.set("limit", 60);
+ const p = params(); p.set("offset", offset); p.set("limit", PAGE);
const d = await (await fetch(`/api/search?${p}`)).json();
total = d.total;
$("grid").innerHTML = (append ? $("grid").innerHTML : "") + d.results.map(card).join("");
offset += d.results.length;
- $("count").textContent = `${total.toLocaleString()} records${year ? " in " + year : ""}`;
+ const withImg = $("grid").querySelectorAll(".card.has-img").length;
+ $("count").textContent = `${total.toLocaleString()} records${year ? " in " + year : ""} · ${withImg} shown with an image from the holding institution`;
$("more").hidden = offset >= total;
}
function density(v) { document.documentElement.style.setProperty("--cols", v); store.set("density", v); }
@@ -71,6 +78,7 @@
$("density").value = store.get("density", "3");
density($("density").value);
$("granularity").value = store.get("granularity", "item");
+ $("has_image").checked = store.get("has_image", "0") === "1";
const url = new URLSearchParams(location.search);
loadFacets().then(() => {
FILTERS.forEach((f) => { if (url.get(f)) $(f).value = url.get(f); });
@@ -83,7 +91,13 @@
FILTERS.filter((f) => f !== "q" && f !== "granularity").forEach((f) => $(f).addEventListener("change", () => search(false)));
let t; $("q").addEventListener("input", () => { clearTimeout(t); t = setTimeout(() => search(false), 250); });
$("more").addEventListener("click", () => search(true));
- $("reset").addEventListener("click", () => { FILTERS.forEach((f) => { if (f !== "granularity") $(f).value = ""; }); year = null; loadFacets(); search(false); });
+ $("has_image").addEventListener("change", () => { store.set("has_image", $("has_image").checked ? "1" : "0"); search(false); });
+ $("all").addEventListener("click", () => { // the default landing: every item-level cartoon, no filters
+ FILTERS.forEach((f) => { $(f).value = ""; }); $("granularity").value = "item"; store.set("granularity", "item");
+ $("has_image").checked = false; store.set("has_image", "0"); year = null;
+ loadFacets().then(() => search(false));
+ });
+ $("reset").addEventListener("click", () => { FILTERS.forEach((f) => { if (f !== "granularity") $(f).value = ""; }); $("has_image").checked = false; store.set("has_image", "0"); year = null; loadFacets(); search(false); });
$("timeline").addEventListener("click", (e) => {
const b = e.target.closest(".bar"); if (!b) return;
year = year == b.dataset.year ? null : b.dataset.year;
diff --git a/src/conrad/web/static/index.html b/src/conrad/web/static/index.html
index bd9e072..7e36944 100644
--- a/src/conrad/web/static/index.html
+++ b/src/conrad/web/static/index.html
@@ -10,10 +10,12 @@
<body>
<header class="top">
<a href="/" class="brand">Paul Conrad Master Archive</a>
- <span class="muted">1924–2010 · metadata index · images are never reproduced — follow “View at” link-outs</span>
+ <nav class="tabs" aria-label="Views"><button id="all" type="button" class="tab on">All cartoons</button></nav>
+ <span class="muted">1924–2010 · images appear only where the holding institution publishes one (shown from its site, never copied) — otherwise follow “View at” link-outs</span>
</header>
<main class="layout">
<aside class="filters" aria-label="Filters">
+ <label class="check"><input id="has_image" type="checkbox"> Has image (from the holding institution)</label>
<label>Search <input id="q" type="search" placeholder="title, caption, person, subject…" autocomplete="off"></label>
<label>Record level
<select id="granularity">
diff --git a/src/conrad/web/static/style.css b/src/conrad/web/static/style.css
index 9674752..73553ca 100644
--- a/src/conrad/web/static/style.css
+++ b/src/conrad/web/static/style.css
@@ -37,3 +37,11 @@ td, th { border-bottom:1px solid var(--line); padding:6px; text-align:left; vert
blockquote { border-left:3px solid var(--accent); margin:0; padding:4px 12px; font-style:italic; }
@media (max-width: 760px) { .layout { grid-template-columns:1fr; padding:12px 16px; } .grid { grid-template-columns:repeat(min(var(--cols),2), minmax(0,1fr)); } }
@media (max-width: 420px) { .grid { grid-template-columns:1fr; } .detail dl { grid-template-columns:1fr; } }
+.tabs { display:flex; gap:6px; } .tab.on { border-color:var(--accent); color:var(--accent); font-weight:bold; }
+.filters label.check { flex-direction:row; align-items:center; gap:6px; color:var(--fg); }
+.card .thumb { margin:0 0 4px; }
+.card .thumb img { display:block; width:100%; height:auto; max-height:320px; object-fit:contain; background:var(--bg); border:1px solid var(--line); border-radius:4px; }
+.credit { font-size:.75rem; color:var(--muted); margin-top:3px; overflow-wrap:anywhere; }
+.detail .hero { margin:8px 0 12px; }
+.detail .hero img { display:block; max-width:100%; height:auto; max-height:80vh; border:1px solid var(--line); border-radius:4px; background:var(--card); }
+.detail .hero .credit { font-size:.85rem; }
diff --git a/tests/test_image_allowlist.py b/tests/test_image_allowlist.py
new file mode 100644
index 0000000..7a29619
--- /dev/null
+++ b/tests/test_image_allowlist.py
@@ -0,0 +1,208 @@
+"""IMAGE ALLOWLIST tests (replaces the old blanket no-<img> test — Steve's TK-12199 decision).
+
+Rule: an image may appear on the private viewer ONLY as a hotlink to the holding institution's own public image, on an
+exact-host allowlist; nothing is ever downloaded, proxied, cached or stored. These tests assert:
+ * every <img src> in rendered HTML and every display_image in the API is on the allowlist
+ * a record whose stored image lives on a NON-allowlisted host (or is an item PAGE) yields display_image = null and
+ never leaks the raw URL
+ * the CSP header carries img-src with exactly the allowlisted hosts
+ * the detector itself goes RED if the allowlist check is bypassed (negative test, CLAUDE.md TK-11431 rule 3)
+ * no image bytes exist anywhere under the repo (data/, cache, DB) and local_image stays NULL
+The crawler's image-refusal rail is covered by tests/test_image_rail_content.py (still required to pass).
+"""
+import json
+import re
+import sqlite3
+from pathlib import Path
+
+import pytest
+from fastapi.testclient import TestClient
+
+from conrad import config, db
+from conrad.crawlers import base
+from conrad.models import CartoonRecord
+from conrad.web import imagehost
+
+LOC_THUMB = "https://tile.loc.gov/storage-services/service/pnp/acd/2a07000/2a07600/2a07662_150px.jpg"
+LOC_MEDIUM = "https://tile.loc.gov/storage-services/service/pnp/acd/2a07000/2a07600/2a07662r.jpg"
+HC_THUMB = ("http://5008.sydneyplus.com/HistoryColorado_ArgusNet_Final/ViewImage.aspx?template=Image&field=DerivedIma"
+ "&hash=0A0B7B209A21EFFA1037D98354ACC011")
+EVIL = "https://evil-image-cdn.example.com/conrad/nixon.jpg"
+PAGE_NOT_IMAGE = "https://www.loc.gov/pictures/item/2016680206/"
+EXPECTED_CSP = "img-src 'self' https://tile.loc.gov https://cdn.loc.gov https://5008.sydneyplus.com"
+IMG_SRC = re.compile(r"<img\b[^>]*\bsrc=\"([^\"]*)\"", re.I)
+CSS_URL = re.compile(r"url\(\s*['\"]?([^'\")]+)", re.I)
+MAGIC = base.IMAGE_MAGIC
+
+
+def _rec(n, title, image_url=None, thumb=None, repo="Library of Congress"):
+ return CartoonRecord(canonical_id=f"t:{n}", identifier=str(n), title=title, year=1973, date_start="1973-01-01",
+ date_end="1973-12-31", repository=repo, record_url=f"https://www.loc.gov/pictures/item/{n}/",
+ image_url=image_url, thumbnail_url=thumb, access_level="online_image",
+ people=["Nixon, Richard M."])
+
+
+@pytest.fixture
+def client(tmpdb, monkeypatch):
+ monkeypatch.delenv("CONRAD_BASIC_USER", raising=False)
+ monkeypatch.delenv("CONRAD_REQUIRE_AUTH", raising=False)
+ db.upsert_source(tmpdb, "t", "test")
+ db.save_record(tmpdb, _rec(1, "Allowed LOC cartoon", thumb=LOC_THUMB), "t")
+ db.save_record(tmpdb, _rec(2, "Allowed History Colorado cartoon", thumb=HC_THUMB, repo="History Colorado"), "t")
+ db.save_record(tmpdb, _rec(3, "Evil host cartoon", image_url=EVIL, thumb=EVIL), "t")
+ db.save_record(tmpdb, _rec(4, "Item page not image cartoon", image_url=PAGE_NOT_IMAGE), "t")
+ db.save_record(tmpdb, _rec(5, "Text only cartoon"), "t")
+ tmpdb.commit()
+ from conrad.web.app import create_app
+ return TestClient(create_app())
+
+
+def _ids(client):
+ rs = client.get("/api/search?q=cartoon&limit=500").json()["results"]
+ return {r["title"]: r for r in rs}
+
+
+def assert_images_allowlisted(text: str, where: str):
+ """The detector: every <img src>, CSS url() image and display_image URL must be on the allowlist, and neither a
+ non-allowlisted stored URL nor a raw item page may ever reach the browser."""
+ for src in IMG_SRC.findall(text):
+ if "${" in src: # a JS template slot, filled only from display_image (checked via the API below)
+ continue
+ assert imagehost.is_allowed(src.replace("&", "&")), f"non-allowlisted <img src> {src!r} in {where}"
+ for u in CSS_URL.findall(text):
+ assert u.startswith("/") or imagehost.is_allowed(u), f"non-allowlisted css url() {u!r} in {where}"
+ assert EVIL not in text, f"non-allowlisted image URL leaked in {where}"
+ try:
+ payload = json.loads(text)
+ except ValueError:
+ return
+ stack = [payload]
+ while stack:
+ x = stack.pop()
+ if isinstance(x, dict):
+ if x.get("display_image"):
+ assert imagehost.is_allowed(x["display_image"]["url"]), f"display_image off-allowlist in {where}"
+ stack.extend(x.values())
+ elif isinstance(x, list):
+ stack.extend(x)
+
+
+def test_display_image_only_for_allowlisted_hosts(client):
+ by = _ids(client)
+ assert by["Allowed LOC cartoon"]["display_image"]["url"] == LOC_MEDIUM # 150px thumb -> LOC medium 'r' derivative
+ hc = by["Allowed History Colorado cartoon"]["display_image"]["url"]
+ assert hc.startswith("https://5008.sydneyplus.com/") and "ViewImage.aspx" in hc # upgraded to https
+ assert by["Evil host cartoon"]["display_image"] is None
+ assert by["Item page not image cartoon"]["display_image"] is None
+ assert by["Text only cartoon"]["display_image"] is None
+ for t in ("Evil host cartoon", "Item page not image cartoon", "Text only cartoon"):
+ d = client.get(f"/api/cartoon/{by[t]['id']}").json()
+ assert d["display_image"] is None
+ page = client.get(f"/cartoon/{by[t]['id']}").text
+ assert not IMG_SRC.search(page), f"{t}: detail page rendered an <img>"
+ assert "View at Library of Congress" in page # the link-out survives
+
+
+def test_every_rendered_image_is_allowlisted(client):
+ by = _ids(client)
+ paths = ["/", "/static/app.js", "/static/style.css", "/static/index.html", "/api/search?q=cartoon&limit=500"]
+ paths += [f"/cartoon/{r['id']}" for r in by.values()] + [f"/api/cartoon/{r['id']}" for r in by.values()]
+ for p in paths:
+ r = client.get(p)
+ assert r.status_code == 200, p
+ assert_images_allowlisted(r.text, p)
+ page = client.get(f"/cartoon/{by['Allowed LOC cartoon']['id']}").text
+ srcs = IMG_SRC.findall(page)
+ assert srcs == [LOC_MEDIUM]
+ assert 'referrerpolicy="no-referrer"' in page and 'loading="lazy"' in page and 'alt="Allowed LOC cartoon"' in page
+ assert "Image: Library of Congress" in page # credit line
+
+
+def test_has_image_filter(client):
+ rs = client.get("/api/search?q=cartoon&has_image=true&limit=500").json()
+ assert sorted(r["title"] for r in rs["results"]) == ["Allowed History Colorado cartoon", "Allowed LOC cartoon"]
+ assert rs["total"] == 2
+
+
+def test_csp_header_exact_hosts(client):
+ assert imagehost.CSP == EXPECTED_CSP
+ for p in ["/", "/api/search", "/static/app.js", "/robots.txt"]:
+ h = client.get(p).headers
+ assert h["content-security-policy"] == EXPECTED_CSP, p
+ assert h["x-robots-tag"] == "noindex, nofollow"
+
+
+def test_detector_goes_red_when_allowlist_bypassed(client, monkeypatch):
+ """Negative test: with the host check removed, the evil record gets a display_image and the detector MUST fail."""
+ def permissive(url):
+ return url if url and url.startswith("http") and "/pictures/item/" not in url else None
+ monkeypatch.setattr(imagehost, "display_url", permissive)
+ monkeypatch.setattr(imagehost, "is_allowed", lambda u: bool(u)) # the defence-in-depth check removed too
+ leaked = client.get("/api/search?q=cartoon&limit=500").text
+ assert EVIL in leaked # the fault really was injected
+ monkeypatch.undo()
+ with pytest.raises(AssertionError):
+ assert_images_allowlisted(leaked, "bypassed /api/search")
+
+
+@pytest.mark.parametrize("url", [EVIL, PAGE_NOT_IMAGE, "https://tile.loc.gov.evil.com/service/pnp/x/1_150px.jpg",
+ "https://user:pw@tile.loc.gov/service/pnp/x/1_150px.jpg",
+ "javascript:alert(1)", "data:image/png;base64,AAAA", "ftp://tile.loc.gov/service/pnp/a.jpg",
+ "https://5008.sydneyplus.com/HistoryColorado_ArgusNet_Final/Portal/Portal.aspx?x=1"])
+def test_display_url_rejects(url):
+ assert imagehost.display_url(url) is None
+
+
+def test_local_image_column_is_always_null(tmpdb):
+ db.upsert_source(tmpdb, "x", "x")
+ db.save_record(tmpdb, CartoonRecord(canonical_id="x:1", identifier="1", image_url=LOC_THUMB), "x")
+ with pytest.raises(sqlite3.IntegrityError):
+ tmpdb.execute("UPDATE cartoon_sources SET local_image='data/img/1.jpg'")
+ assert tmpdb.execute("SELECT COUNT(*) FROM cartoon_sources WHERE local_image IS NOT NULL").fetchone()[0] == 0
+
+
+SKIP_DIRS = {".git", ".venv", "node_modules", "__pycache__", ".pytest_cache"}
+IMG_EXTS = {".jpg", ".jpeg", ".png", ".gif", ".tif", ".tiff", ".webp", ".bmp", ".jp2", ".heic", ".avif"}
+
+
+def _repo_files():
+ for p in config.ROOT.rglob("*"):
+ if p.is_file() and not (SKIP_DIRS & set(p.relative_to(config.ROOT).parts)):
+ yield p
+
+
+def test_no_image_bytes_anywhere_in_repo():
+ offenders = []
+ for p in _repo_files():
+ if p.suffix.lower() in IMG_EXTS:
+ offenders.append(f"{p}: image extension")
+ continue
+ with open(p, "rb") as fh:
+ head = fh.read(16)
+ if head.startswith(MAGIC) and p.suffix.lower() not in {".pdf"}:
+ offenders.append(f"{p}: image magic bytes")
+ if p.suffix == ".json" and "cache" in p.parts: # cached HTTP bodies: never an image body, never inline data
+ try:
+ body = json.loads(p.read_text(errors="replace")).get("body", "")
+ except (ValueError, AttributeError):
+ body = ""
+ if isinstance(body, str) and (body[:8].encode("latin-1", "replace").startswith(MAGIC)
+ or "data:image/" in body[:2000]):
+ offenders.append(f"{p}: cached image body")
+ assert offenders == [], offenders[:10]
+ real = config.ROOT / "data" / "conrad.db"
+ if real.exists():
+ c = sqlite3.connect(f"file:{real}?mode=ro", uri=True)
+ assert c.execute("SELECT COUNT(*) FROM cartoon_sources WHERE local_image IS NOT NULL").fetchone()[0] == 0
+ for (t,) in c.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'"):
+ cols = [r[1] for r in c.execute(f"PRAGMA table_info('{t}')")]
+ for col in cols:
+ n = c.execute(f"SELECT COUNT(*) FROM \"{t}\" WHERE typeof(\"{col}\")='blob'").fetchone()[0]
+ assert n == 0, f"BLOB stored in {t}.{col}"
+
+
+def test_crawler_image_rail_intact():
+ """The crawler must still refuse image URLs before any I/O (the hotlink decision does not loosen it)."""
+ for u in (LOC_THUMB, LOC_MEDIUM, EVIL):
+ with pytest.raises(base.Blocked, match="copyright rail"):
+ base.Http(use_cache=False).get(u, check_robots=False)
diff --git a/tests/test_no_images_negative.py b/tests/test_no_images_negative.py
deleted file mode 100644
index 8f2a47a..0000000
--- a/tests/test_no_images_negative.py
+++ /dev/null
@@ -1,75 +0,0 @@
-"""NEGATIVE tests for the copyright rule: the viewer must never emit an <img> (or any image-loading construct)
-pointing at a Conrad image, and nothing may ever be written to a local_image path."""
-import re
-import sqlite3
-from pathlib import Path
-
-import pytest
-from fastapi.testclient import TestClient
-
-from conrad import config, db
-from conrad.models import CartoonRecord
-
-IMG_URL = "https://tile.loc.gov/storage-services/service/pnp/acd/2a0/00000/fake_conrad_cartoon.jpg"
-THUMB = "https://tile.loc.gov/storage-services/service/pnp/acd/fake_thumb_150px.gif"
-FORBIDDEN = [re.compile(p, re.I) for p in (r"<img\b", r"<picture\b", r"<source\b[^>]*srcset", r"new\s+Image\s*\(",
- r"createElement\(\s*['\"]img", r"background(-image)?\s*:[^;]*url\(",
- r"\.(jpe?g|gif|png|tiff?|webp)\b(?![^<]*</a>)")]
-
-
-@pytest.fixture
-def client(tmpdb, monkeypatch):
- monkeypatch.delenv("CONRAD_BASIC_USER", raising=False)
- monkeypatch.delenv("CONRAD_REQUIRE_AUTH", raising=False)
- db.upsert_source(tmpdb, "loc", "LOC")
- rec = CartoonRecord(canonical_id="loc:1", identifier="1", title="Nixon test cartoon", year=1973,
- date_start="1973-01-01", date_end="1973-12-31", repository="Library of Congress",
- record_url="https://www.loc.gov/pictures/item/1/", image_url=IMG_URL, thumbnail_url=THUMB,
- access_level="online_image", people=["Nixon, Richard M."])
- db.save_record(tmpdb, rec, "loc")
- tmpdb.commit()
- from conrad.web.app import create_app
- return TestClient(create_app())
-
-
-def _assert_clean(text: str, where: str):
- assert IMG_URL not in text and THUMB not in text, f"image URL leaked in {where}"
- for p in FORBIDDEN:
- assert not p.search(text), f"forbidden image construct {p.pattern!r} in {where}"
-
-
-def test_pages_and_static_never_render_images(client):
- cid = client.get("/api/search?q=nixon").json()["results"][0]["id"]
- for path in ["/", f"/cartoon/{cid}", "/static/app.js", "/static/style.css", "/static/index.html"]:
- r = client.get(path)
- assert r.status_code == 200
- _assert_clean(r.text, path)
- detail = client.get(f"/cartoon/{cid}").text
- assert "View at Library of Congress" in detail # link-out instead of an image
-
-
-def test_api_never_ships_image_urls_to_browser(client):
- s = client.get("/api/search?q=nixon")
- _assert_clean(s.text, "/api/search")
- cid = s.json()["results"][0]["id"]
- d = client.get(f"/api/cartoon/{cid}")
- _assert_clean(d.text, "/api/cartoon")
- assert d.json()["sources"][0]["has_online_image"] == 1 # the fact is shown, the image is not
-
-
-def test_local_image_column_is_always_null(tmpdb):
- db.upsert_source(tmpdb, "x", "x")
- db.save_record(tmpdb, CartoonRecord(canonical_id="x:1", identifier="1", image_url=IMG_URL), "x")
- with pytest.raises(sqlite3.IntegrityError):
- tmpdb.execute("UPDATE cartoon_sources SET local_image='data/img/1.jpg'")
- assert tmpdb.execute("SELECT COUNT(*) FROM cartoon_sources WHERE local_image IS NOT NULL").fetchone()[0] == 0
-
-
-def test_no_image_files_in_project_data():
- exts = {".jpg", ".jpeg", ".png", ".gif", ".tif", ".tiff", ".webp"}
- found = [p for p in Path(config.DATA).rglob("*") if p.suffix.lower() in exts]
- assert found == [], f"image files written under data/: {found[:5]}"
- real = config.ROOT / "data" / "conrad.db"
- if real.exists():
- c = sqlite3.connect(real)
- assert c.execute("SELECT COUNT(*) FROM cartoon_sources WHERE local_image IS NOT NULL").fetchone()[0] == 0
← ddb5417 Draft (unsent) data-request letters to Syracuse, Wichita Sta
·
back to Paul Conrad Archive
·
Reserve card image height before lazy load so filtered grids fb7c43e →