← back to Paul Conrad Archive
Add acquisition_method provenance field: idempotent migration + backfill, API/search flags, exports, detail-page reader-bypass badge, REPORT counts, tests
88b34d26bd681b1499fb90550909fd81db818415 · 2026-09-24 17:01:41 -0700 · Steve Abrams
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Files touched
M scripts/report.pyM src/conrad/db.pyM src/conrad/exports.pyM src/conrad/models.pyA src/conrad/provenance.pyM src/conrad/web/app.pyM src/conrad/web/static/app.jsA tests/test_provenance.py
Diff
commit 88b34d26bd681b1499fb90550909fd81db818415
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Sep 24 17:01:41 2026 -0700
Add acquisition_method provenance field: idempotent migration + backfill, API/search flags, exports, detail-page reader-bypass badge, REPORT counts, tests
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
scripts/report.py | 9 +++
src/conrad/db.py | 20 ++++++-
src/conrad/exports.py | 12 ++--
src/conrad/models.py | 1 +
src/conrad/provenance.py | 85 ++++++++++++++++++++++++++++
src/conrad/web/app.py | 27 +++++++--
src/conrad/web/static/app.js | 2 +-
tests/test_provenance.py | 128 +++++++++++++++++++++++++++++++++++++++++++
8 files changed, 274 insertions(+), 10 deletions(-)
diff --git a/scripts/report.py b/scripts/report.py
index 2dad701..32a8ea4 100644
--- a/scripts/report.py
+++ b/scripts/report.py
@@ -45,6 +45,11 @@ S = {
WHERE x.access_level='archive_visit')"""),
"local_images_stored": q("SELECT COUNT(*) FROM cartoon_sources WHERE local_image IS NOT NULL"),
}
+S["source_rows_by_acquisition_method"] = dict(conn.execute(
+ "SELECT COALESCE(acquisition_method,'NULL'), COUNT(*) FROM cartoon_sources GROUP BY 1 ORDER BY 2 DESC").fetchall())
+_canon = [r for r in exports.canonical_rows(conn) if r["granularity"] == "item"]
+S["item_level_pending_direct_verification"] = sum("pending_direct_verification" in r["provenance_flags"] for r in _canon)
+S["item_level_secondary_citation_only"] = sum("secondary_citation_only" in r["provenance_flags"] for r in _canon)
table = exports.per_year(conn)
S["years_with_no_item_records"] = [y for y, t in table.items() if not t["item"]]
S["years_with_no_records_at_all"] = [y for y, t in table.items() if not (t["item"] or t["box_range"] or t["folder"])]
@@ -99,6 +104,10 @@ rows = [
("RECORDS REQUIRING ARCHIVE ACCESS (canonical, no online image)",
f"{S['canonical_requiring_archive_access']} (item-level: {S['archive_access_items']}; rest are box-range slots / folders)"),
("Local images stored (must be 0)", S["local_images_stored"]),
+ ("Source rows per acquisition_method", ", ".join(f"{k}={v}" for k, v in S["source_rows_by_acquisition_method"].items())),
+ ("Item-level canonical records acquired ONLY via reader bypass (pending direct verification)",
+ S["item_level_pending_direct_verification"]),
+ ("Item-level canonical records known ONLY from a secondary citation", S["item_level_secondary_citation_only"]),
]
for k, v in rows:
A(f"| {k} | {v} |")
diff --git a/src/conrad/db.py b/src/conrad/db.py
index a4fcf76..4c2da79 100644
--- a/src/conrad/db.py
+++ b/src/conrad/db.py
@@ -46,6 +46,8 @@ CREATE TABLE IF NOT EXISTS cartoon_sources (
record_url TEXT, image_url TEXT, thumbnail_url TEXT,
local_image TEXT CHECK (local_image IS NULL), -- copyright rule: images are NEVER stored locally
access_level TEXT, rights_url TEXT, provenance TEXT, retrieved_at TEXT,
+ acquisition_method TEXT CHECK (acquisition_method IS NULL OR acquisition_method IN
+ ('direct_api','direct_html','seed_direct','seed_via_reader_bypass','secondary_citation')),
UNIQUE (source_id, identifier)
);
CREATE INDEX IF NOT EXISTS ix_cs_cartoon ON cartoon_sources(cartoon_id);
@@ -97,9 +99,22 @@ def connect(path: Path | str | None = None) -> sqlite3.Connection:
def init_db(conn: sqlite3.Connection) -> None:
conn.executescript(SCHEMA)
+ migrate(conn)
conn.commit()
+def migrate(conn: sqlite3.Connection) -> None:
+ """Idempotent in-place migrations for databases created by an older SCHEMA."""
+ cols = {r[1] for r in conn.execute("PRAGMA table_info(cartoon_sources)")}
+ if "acquisition_method" not in cols:
+ conn.execute("""ALTER TABLE cartoon_sources ADD COLUMN acquisition_method TEXT CHECK (acquisition_method IS NULL
+ OR acquisition_method IN ('direct_api','direct_html','seed_direct','seed_via_reader_bypass',
+ 'secondary_citation'))""")
+ conn.execute("CREATE INDEX IF NOT EXISTS ix_cs_method ON cartoon_sources(acquisition_method)")
+ from .provenance import backfill
+ backfill(conn) # only rows still NULL
+
+
# ---------------------------------------------------------------- sources
def upsert_source(conn, id: str, name: str, *, repository=None, url=None, classification=None,
status=None, access_notes=None, notes=None) -> None:
@@ -120,7 +135,7 @@ CARTOON_FIELDS = ["granularity", "date_exact", "date_start", "date_end", "year",
"caption", "description", "publication", "syndicate", "medium", "dimensions", "signed_name",
"rights_text", "copyright_holder", "notes"]
SOURCE_FIELDS = ["repository", "collection_name", "box", "folder", "page", "record_url", "image_url",
- "thumbnail_url", "access_level", "rights_url", "provenance"]
+ "thumbnail_url", "access_level", "rights_url", "provenance", "acquisition_method"]
def _id_for(conn, table: str, name: str) -> int:
@@ -157,6 +172,9 @@ def save_record(conn, rec, source_id: str) -> str:
status = "unchanged"
# source row
svals = {f: getattr(rec, f) for f in SOURCE_FIELDS}
+ if not svals["acquisition_method"]:
+ from .provenance import derive
+ svals["acquisition_method"] = derive(source_id, rec.provenance)
ex = conn.execute("SELECT id FROM cartoon_sources WHERE source_id=? AND identifier=?",
(source_id, rec.identifier)).fetchone()
if ex:
diff --git a/src/conrad/exports.py b/src/conrad/exports.py
index 68122ee..3d54246 100644
--- a/src/conrad/exports.py
+++ b/src/conrad/exports.py
@@ -7,6 +7,7 @@ from collections import defaultdict
from pathlib import Path
from . import config, db
+from .provenance import flags as provenance_flags
from .rights import PUBLIC_IMAGE_LEVELS, REQUIRES_ARCHIVE
REPO_COLS = [("Huntington", "Huntington Library"), ("Syracuse", "Syracuse University"), ("LOC", "Library of Congress"),
@@ -45,6 +46,8 @@ def canonical_rows(conn) -> list[dict]:
r["people"] = sorted({x for i in ids for x in ppl[i]})
r["repositories"] = sorted({s["repository"] for s in r["sources"] if s["repository"]})
r["merged_ids"] = [i for i in ids if i != r["id"]]
+ r["acquisition_methods"] = sorted({s.get("acquisition_method") for s in r["sources"] if s.get("acquisition_method")})
+ r["provenance_flags"] = provenance_flags(r["acquisition_methods"])
return rows
@@ -55,23 +58,24 @@ def export_all(conn) -> dict:
counts["cartoons.csv"] = _w(ex / "cartoons.csv",
["id", "canonical_id", "granularity", "title", "date_exact", "date_start", "date_end", "year", "date_is_estimate",
"publication", "repositories", "identifiers", "record_urls", "public_image_links", "people", "subjects",
- "rights_text", "notes"],
+ "rights_text", "notes", "acquisition_methods", "provenance_flags"],
([r["id"], r["canonical_id"], r["granularity"], r["title"], r["date_exact"], r["date_start"], r["date_end"],
r["year"], r["date_is_estimate"], r["publication"], "; ".join(r["repositories"]),
"; ".join(f"{s['source_id']}:{s['identifier']}" for s in r["sources"]),
"; ".join(sorted({s["record_url"] for s in r["sources"] if s["record_url"]})),
"; ".join(sorted({s["image_url"] or s["thumbnail_url"] for s in r["sources"]
if s["access_level"] in PUBLIC_IMAGE_LEVELS and (s["image_url"] or s["thumbnail_url"])})),
- "; ".join(r["people"]), "; ".join(r["subjects"]), r["rights_text"], r["notes"]] for r in rows))
+ "; ".join(r["people"]), "; ".join(r["subjects"]), r["rights_text"], r["notes"],
+ "; ".join(r["acquisition_methods"]), "; ".join(r["provenance_flags"])] for r in rows))
(ex / "cartoons.json").write_text(json.dumps(rows, ensure_ascii=False, indent=1, default=str))
counts["cartoons.json"] = len(rows)
counts["sources.csv"] = _w(ex / "sources.csv",
["cartoon_id", "canonical_id", "source_id", "repository", "collection_name", "identifier", "box", "folder",
"page", "record_url", "image_url", "thumbnail_url", "local_image", "access_level", "rights_url", "provenance",
- "retrieved_at"],
+ "retrieved_at", "acquisition_method"],
conn.execute("""SELECT cs.cartoon_id, c.canonical_id, cs.source_id, cs.repository, cs.collection_name, cs.identifier,
cs.box, cs.folder, cs.page, cs.record_url, cs.image_url, cs.thumbnail_url, cs.local_image,
- cs.access_level, cs.rights_url, cs.provenance, cs.retrieved_at
+ cs.access_level, cs.rights_url, cs.provenance, cs.retrieved_at, cs.acquisition_method
FROM cartoon_sources cs JOIN cartoons c ON c.id=cs.cartoon_id ORDER BY cs.source_id, cs.identifier"""))
counts["duplicates.csv"] = _w(ex / "duplicates.csv",
["relation", "score", "a_id", "a_canonical", "a_title", "a_date", "b_id", "b_canonical", "b_title", "b_date", "detail",
diff --git a/src/conrad/models.py b/src/conrad/models.py
index 2493424..f5f06f8 100644
--- a/src/conrad/models.py
+++ b/src/conrad/models.py
@@ -37,5 +37,6 @@ class CartoonRecord:
access_level: str | None = None
rights_url: str | None = None
provenance: str | None = None
+ acquisition_method: str | None = None # see conrad.provenance.METHODS; derived from source_id when None
subjects: list[str] = field(default_factory=list)
people: list[str] = field(default_factory=list)
diff --git a/src/conrad/provenance.py b/src/conrad/provenance.py
new file mode 100644
index 0000000..6a5403b
--- /dev/null
+++ b/src/conrad/provenance.py
@@ -0,0 +1,85 @@
+"""Acquisition provenance — HOW each cartoon_sources row was obtained (Cody must-fix, TK-12199).
+
+`acquisition_method` is independent of `provenance` (the free-text trail): it is a closed vocabulary that
+lets exports, the API and the viewer flag records whose acquisition path is weaker than a direct fetch.
+
+ direct_api fetched live by this project from a documented JSON/XML API (e.g. loc.gov ?fo=json)
+ direct_html fetched live by this project from a public HTML/XML document (e.g. the Huntington EAD)
+ seed_direct seed corpus (TK-12179) whose raw cache was fetched directly from the repository
+ seed_via_reader_bypass seed corpus fetched through a third-party reader (r.jina.ai) past an AWS WAF / bot
+ challenge — every seed_wichita and seed_syracuse row. Pending direct verification.
+ secondary_citation not from a holding repository at all: a secondary source (obituary, article, book)
+"""
+from __future__ import annotations
+
+METHODS = ("direct_api", "direct_html", "seed_direct", "seed_via_reader_bypass", "secondary_citation")
+DIRECT = {"direct_api", "direct_html"}
+
+# seed sources whose raw cache was fetched via r.jina.ai past a WAF challenge (TK-12179 research/raw/{wsu,syr.md})
+READER_BYPASS_SOURCES = {"seed_wichita", "seed_syracuse"}
+SECONDARY_SOURCES = {"seed_catalog"}
+# live crawlers that talk to a documented JSON/XML API
+API_SOURCES = {"loc", "dpla", "smithsonian", "internet_archive"}
+
+BADGE_TEXT = "acquired via third-party reader past a bot challenge — pending direct verification"
+FLAG_BYPASS = "acquired_via_reader_bypass"
+FLAG_BYPASS_UNVERIFIED = "pending_direct_verification"
+FLAG_SECONDARY_ONLY = "secondary_citation_only"
+
+
+def derive(source_id: str, provenance: str | None = None) -> str:
+ """Deterministic method for a cartoon_sources row from its source_id (+ provenance text as tie-breaker)."""
+ sid = (source_id or "").strip()
+ prov = (provenance or "").lower()
+ if sid in READER_BYPASS_SOURCES or "r.jina.ai" in prov:
+ return "seed_via_reader_bypass" # never label a reader-proxied fetch as direct, whatever the source
+ if sid in SECONDARY_SOURCES:
+ return "secondary_citation"
+ if sid.startswith("seed"):
+ return "seed_direct"
+ if sid in API_SOURCES or "fo=json" in prov or "/api" in prov:
+ return "direct_api"
+ return "direct_html"
+
+
+def flags(methods) -> list[str]:
+ """Item-level provenance flags from the set of acquisition methods of every source row behind a cartoon."""
+ ms = {m for m in methods if m}
+ out = []
+ if "seed_via_reader_bypass" in ms:
+ out.append(FLAG_BYPASS)
+ if not ms & DIRECT:
+ out.append(FLAG_BYPASS_UNVERIFIED)
+ if ms and ms <= {"secondary_citation"}:
+ out.append(FLAG_SECONDARY_ONLY)
+ return out
+
+
+def backfill(conn, overwrite: bool = False) -> int:
+ """Fill acquisition_method on every row (only NULL rows unless overwrite). Idempotent. Returns rows written."""
+ where = "" if overwrite else "WHERE acquisition_method IS NULL"
+ rows = conn.execute(f"SELECT id, source_id, provenance FROM cartoon_sources {where}").fetchall()
+ conn.executemany("UPDATE cartoon_sources SET acquisition_method=? WHERE id=?",
+ [(derive(r[1], r[2]), r[0]) for r in rows])
+ conn.commit()
+ return len(rows)
+
+
+def methods_by_canonical(conn, ids=None) -> dict[int, list[str]]:
+ """{canonical cartoon id: sorted distinct methods across it and everything merged into it}."""
+ from collections import defaultdict
+ sql = """SELECT COALESCE(k.merged_into,k.id), x.acquisition_method FROM cartoon_sources x
+ JOIN cartoons k ON k.id=x.cartoon_id"""
+ args: list = []
+ if ids is not None:
+ ids = list(ids)
+ if not ids:
+ return {}
+ ph = ",".join("?" * len(ids))
+ sql += f" WHERE COALESCE(k.merged_into,k.id) IN ({ph})"
+ args = ids
+ out: dict[int, set] = defaultdict(set)
+ for canon, m in conn.execute(sql, args):
+ if m:
+ out[canon].add(m)
+ return {k: sorted(v) for k, v in out.items()}
diff --git a/src/conrad/web/app.py b/src/conrad/web/app.py
index 01cb101..9f95ab5 100644
--- a/src/conrad/web/app.py
+++ b/src/conrad/web/app.py
@@ -16,6 +16,7 @@ from fastapi.responses import FileResponse, HTMLResponse, PlainTextResponse, Res
from fastapi.staticfiles import StaticFiles
from .. import db
+from .. import provenance as prov
STATIC = Path(__file__).parent / "static"
@@ -140,7 +141,12 @@ def search(q: str | None = None, year: int | None = None, year_from: int | None
FROM cartoons c LEFT JOIN cartoon_index ix ON ix.id = c.id
WHERE {where} ORDER BY {SORTS.get(sort, SORTS['date_desc'])} LIMIT ? OFFSET ?""",
[*args, limit, offset]).fetchall()
- return {"total": total, "offset": offset, "results": [dict(r) for r in rows]}
+ results = [dict(r) for r in rows]
+ methods = prov.methods_by_canonical(c, [r["id"] for r in results])
+ for r in results:
+ r["acquisition_methods"] = methods.get(r["id"], [])
+ r["provenance_flags"] = prov.flags(r["acquisition_methods"])
+ return {"total": total, "offset": offset, "results": results}
@router.get("/api/facets")
@@ -183,7 +189,7 @@ def _detail(cid: int) -> dict:
# image_url / thumbnail_url are deliberately NOT sent to the browser (link-out only)
d["sources"] = [dict(r) for r in c.execute(
f"""SELECT source_id, repository, collection_name, identifier, box, folder, page, record_url, access_level,
- rights_url, provenance, retrieved_at, (image_url IS NOT NULL OR thumbnail_url IS NOT NULL) AS has_online_image
+ rights_url, provenance, acquisition_method, retrieved_at, (image_url IS NOT NULL OR thumbnail_url IS NOT NULL) AS has_online_image
FROM cartoon_sources WHERE cartoon_id IN ({ph}) ORDER BY source_id LIKE 'seed%', source_id""", ids)]
d["people"] = [r[0] for r in c.execute(f"SELECT DISTINCT p.name FROM cartoon_people cp JOIN people p ON p.id=cp.person_id WHERE cp.cartoon_id IN ({ph}) ORDER BY 1", ids)]
d["subjects"] = [r[0] for r in c.execute(f"SELECT DISTINCT s.name FROM cartoon_subjects cs JOIN subjects s ON s.id=cs.subject_id WHERE cs.cartoon_id IN ({ph}) ORDER BY 1", ids)]
@@ -192,6 +198,8 @@ def _detail(cid: int) -> dict:
FROM cartoon_links l JOIN cartoons o ON o.id = CASE WHEN l.cartoon_id IN ({ph}) THEN l.related_id ELSE l.cartoon_id END
WHERE l.cartoon_id IN ({ph}) OR l.related_id IN ({ph})""", ids * 3)]
d["merged_ids"] = [i for i in ids if i != cid]
+ d["acquisition_methods"] = sorted({s["acquisition_method"] for s in d["sources"] if s["acquisition_method"]})
+ d["provenance_flags"] = prov.flags(d["acquisition_methods"])
return d
@@ -213,7 +221,9 @@ def _source_row(s: dict) -> str:
link += ' · <a rel="noopener noreferrer" target="_blank" href="' + _e(s["rights_url"]) + '">rights</a>'
online = ' <span class="badge">image online at repository</span>' if s["has_online_image"] else ""
cells = [_e(s["repository"]), _e(s["collection_name"]), _e(s["identifier"]), _e(s["box"]), _e(s["folder"]),
- '<span class="badge">' + _e(s["access_level"]) + "</span>" + online, link,
+ '<span class="badge">' + _e(s["access_level"]) + "</span>" + online
+ + ' <span class="badge' + (' warn' if s.get("acquisition_method") == "seed_via_reader_bypass" else "") + '">'
+ + _e(s.get("acquisition_method")) + "</span>", link,
'<span class="muted small">' + _e(s["provenance"]) + "</span>"]
return "<tr>" + "".join("<td>" + c + "</td>" for c in cells) + "</tr>"
@@ -240,10 +250,19 @@ def cartoon_page(cid: int):
holder = (" (copyright: " + _e(d["copyright_holder"]) + ")") if d["copyright_holder"] else ""
links = "".join(_link_row(lk) for lk in d["links"][:50]) or '<li class="muted">none</li>'
rows = "".join(_source_row(s) for s in d["sources"])
+ pflags = d["provenance_flags"]
+ pbadge = ""
+ if prov.FLAG_BYPASS in pflags:
+ txt = prov.BADGE_TEXT if prov.FLAG_BYPASS_UNVERIFIED in pflags else \
+ "acquired via third-party reader past a bot challenge — also verified by a direct fetch"
+ pbadge += ' <span class="badge warn provenance-flag" data-flag="' + prov.FLAG_BYPASS + '">' + _e(txt) + "</span>"
+ if prov.FLAG_SECONDARY_ONLY in pflags:
+ pbadge += (' <span class="badge warn provenance-flag" data-flag="' + prov.FLAG_SECONDARY_ONLY
+ + '">known only from a secondary citation — not seen in a holding repository</span>')
title = _e(d["title"] or "[untitled / not cataloged]")
body = PAGE.format(
head_title=_e(d["title"] or d["canonical_id"]), title=title, gran=_e(GRANULARITY_LABEL[d["granularity"]]),
- date=_e(date), est=est, pub=_e(d["publication"] or "publication unknown"), canonical=_e(d["canonical_id"]),
+ date=_e(date), est=est + pbadge, pub=_e(d["publication"] or "publication unknown"), canonical=_e(d["canonical_id"]),
caption=caption, desc=desc, people=people, subjects=subjects, medium=_e(d["medium"] or "—"),
rights=_e(d["rights_text"] or "—") + holder, notes=_e(d["notes"] or "—"), n_sources=len(d["sources"]),
rows=rows, links=links, created=_e(d["created_at"]), updated=_e(d["updated_at"]))
diff --git a/src/conrad/web/static/app.js b/src/conrad/web/static/app.js
index 65bcdd9..566cd84 100644
--- a/src/conrad/web/static/app.js
+++ b/src/conrad/web/static/app.js
@@ -48,7 +48,7 @@
const link = r.record_url ? `<a class="out" rel="noopener noreferrer" target="_blank" href="${esc(r.record_url)}">View at ${esc((r.repos || "").split(",")[0])} ↗</a>` : "";
return `<article class="card g-${esc(r.granularity)}">
<a class="title" href="/cartoon/${r.id}">${esc(r.title || "[untitled / not cataloged]")}</a>
- <div class="meta"><b>${esc(date)}</b>${r.date_is_estimate ? ' <span class="badge warn">est.</span>' : ""} · ${esc(r.publication || "")}</div>
+ <div class="meta"><b>${esc(date)}</b>${r.date_is_estimate ? ' <span class="badge warn">est.</span>' : ""}${(r.provenance_flags || []).includes("pending_direct_verification") ? ' <span class="badge warn" title="acquired via third-party reader past a bot challenge — pending direct verification">reader-bypass</span>' : ""}${(r.provenance_flags || []).includes("secondary_citation_only") ? ' <span class="badge warn" title="known only from a secondary citation">secondary</span>' : ""} · ${esc(r.publication || "")}</div>
${r.people ? `<div class="people">${esc(r.people)}</div>` : ""}
<div class="repos">${esc(r.repos || "")} · ${r.n_sources} source record${r.n_sources == 1 ? "" : "s"}</div>
<div class="foot">${link}<span class="when" title="${esc(r.created_at)}">🕓 ${esc(fmtDate(r.created_at))}</span></div>
diff --git a/tests/test_provenance.py b/tests/test_provenance.py
new file mode 100644
index 0000000..d43a755
--- /dev/null
+++ b/tests/test_provenance.py
@@ -0,0 +1,128 @@
+"""acquisition_method provenance field (Cody must-fix, TK-12199): migration, backfill, API, exports, viewer badge."""
+import csv
+import json
+import sqlite3
+
+import pytest
+from fastapi.testclient import TestClient
+
+from conrad import config, db, exports, provenance
+from conrad.models import CartoonRecord
+
+SEEDS = [ # (source_id, canonical prefix, provenance, granularity)
+ ("seed_wichita", "wsu", "seed:research/raw/wsu/112075.md", "item"),
+ ("seed_syracuse", "syr", "seed:research/corpus.json#syracuse", "folder"),
+ ("seed_loc", "loc", "seed:research/raw/loc-page-1.json", "item"),
+ ("seed_catalog", "cat", "seed:data/cartoons.json", "item"),
+ ("loc", "loc", "live:item-json -> https://www.loc.gov/pictures/item/9/?fo=json", "item"),
+ ("huntington", "hunt", "live:https://cinco-prd.s3.amazonaws.com/media/ead/conrad.xml", "box_range"),
+]
+
+
+def _populate(conn):
+ for i, (sid, pre, prov, gran) in enumerate(SEEDS):
+ db.upsert_source(conn, sid, sid)
+ rec = CartoonRecord(canonical_id=f"{pre}:{i}", identifier=str(i), granularity=gran, title=f"Cartoon {sid}",
+ year=1970, date_start="1970-01-01", date_end="1970-12-31", repository=f"Repo {sid}",
+ record_url=f"https://example.org/{sid}/{i}", provenance=prov)
+ db.save_record(conn, rec, sid)
+ conn.commit()
+
+
+def test_derive_rules():
+ assert provenance.derive("seed_wichita") == "seed_via_reader_bypass"
+ assert provenance.derive("seed_syracuse") == "seed_via_reader_bypass"
+ assert provenance.derive("seed_loc") == "seed_direct"
+ assert provenance.derive("seed_huntington") == "seed_direct"
+ assert provenance.derive("seed_catalog") == "secondary_citation"
+ assert provenance.derive("loc") == "direct_api"
+ assert provenance.derive("huntington", "live:https://cinco-prd.s3.amazonaws.com/media/ead/conrad.xml") == "direct_html"
+ # a reader-proxied fetch is never labelled direct, whatever source claims it
+ assert provenance.derive("wichita", "live:https://r.jina.ai/https://archivesspace.wichita.edu/x") == "seed_via_reader_bypass"
+
+
+def test_save_record_sets_method_and_constraint(tmpdb):
+ _populate(tmpdb)
+ got = dict(tmpdb.execute("SELECT source_id, acquisition_method FROM cartoon_sources").fetchall())
+ assert got["seed_wichita"] == got["seed_syracuse"] == "seed_via_reader_bypass"
+ assert got["loc"] == "direct_api" and got["seed_catalog"] == "secondary_citation"
+ with pytest.raises(sqlite3.IntegrityError): # closed vocabulary
+ tmpdb.execute("UPDATE cartoon_sources SET acquisition_method='scraped_somehow' WHERE source_id='loc'")
+
+
+def test_migration_is_idempotent_on_old_db(tmp_path):
+ """A DB created by the pre-TK-12199 schema (no column) gains + backfills it; re-running changes nothing."""
+ p = tmp_path / "old.db"
+ old_schema = db.SCHEMA.replace(
+ """ acquisition_method TEXT CHECK (acquisition_method IS NULL OR acquisition_method IN
+ ('direct_api','direct_html','seed_direct','seed_via_reader_bypass','secondary_citation')),
+""", "")
+ assert "acquisition_method" not in old_schema
+ conn = db.connect(p)
+ conn.executescript(old_schema)
+ for i, (sid, pre, prov, gran) in enumerate(SEEDS):
+ conn.execute("INSERT INTO sources(id,name) VALUES (?,?)", (sid, sid))
+ cid = conn.execute("INSERT INTO cartoons(canonical_id,granularity,created_at,updated_at) VALUES (?,?,'x','x')",
+ (f"{pre}:{i}", gran)).lastrowid
+ conn.execute("INSERT INTO cartoon_sources(cartoon_id,source_id,identifier,provenance) VALUES (?,?,?,?)",
+ (cid, sid, str(i), prov))
+ conn.commit()
+ db.init_db(conn)
+ first = conn.execute("SELECT source_id, acquisition_method FROM cartoon_sources ORDER BY id").fetchall()
+ db.init_db(conn) # second run: no error, no change
+ second = conn.execute("SELECT source_id, acquisition_method FROM cartoon_sources ORDER BY id").fetchall()
+ assert [tuple(r) for r in first] == [tuple(r) for r in second]
+ assert conn.execute("SELECT COUNT(*) FROM cartoon_sources WHERE acquisition_method IS NULL").fetchone()[0] == 0
+ assert dict(first)["seed_wichita"] == "seed_via_reader_bypass"
+
+
+def test_api_exposes_method_and_flags(tmpdb, monkeypatch):
+ monkeypatch.delenv("CONRAD_BASIC_USER", raising=False)
+ monkeypatch.delenv("CONRAD_REQUIRE_AUTH", raising=False)
+ _populate(tmpdb)
+ from conrad.web.app import create_app
+ c = TestClient(create_app())
+ res = c.get("/api/search", params={"granularity": "all", "limit": 50}).json()["results"]
+ by = {r["canonical_id"].split(":")[0]: r for r in res}
+ assert by["wsu"]["acquisition_methods"] == ["seed_via_reader_bypass"]
+ assert "pending_direct_verification" in by["wsu"]["provenance_flags"]
+ assert "secondary_citation_only" in by["cat"]["provenance_flags"]
+ wsu_id = by["wsu"]["id"]
+ d = c.get(f"/api/cartoon/{wsu_id}").json()
+ assert d["sources"][0]["acquisition_method"] == "seed_via_reader_bypass"
+ assert "acquired_via_reader_bypass" in d["provenance_flags"]
+ page = c.get(f"/cartoon/{wsu_id}").text
+ assert provenance.BADGE_TEXT in page and 'data-flag="acquired_via_reader_bypass"' in page
+ # a directly fetched record carries no bypass badge
+ loc_ids = [r["id"] for r in res if r["canonical_id"].startswith("loc:")]
+ for i in loc_ids:
+ assert provenance.BADGE_TEXT not in c.get(f"/cartoon/{i}").text
+
+
+def test_exports_include_method(tmpdb, tmp_path, monkeypatch):
+ monkeypatch.setattr(config, "EXPORT_DIR", tmp_path)
+ monkeypatch.setattr(config, "SOURCES_JSON", tmp_path / "sources.json")
+ _populate(tmpdb)
+ exports.export_all(tmpdb)
+ src = list(csv.DictReader((tmp_path / "sources.csv").open()))
+ assert src and all(r["acquisition_method"] for r in src)
+ assert {r["acquisition_method"] for r in src if r["source_id"] in ("seed_wichita", "seed_syracuse")} == \
+ {"seed_via_reader_bypass"}
+ cart = list(csv.DictReader((tmp_path / "cartoons.csv").open()))
+ assert "acquisition_methods" in cart[0] and "provenance_flags" in cart[0]
+ wsu = next(r for r in cart if r["canonical_id"].startswith("wsu:"))
+ assert wsu["acquisition_methods"] == "seed_via_reader_bypass" and "pending_direct_verification" in wsu["provenance_flags"]
+ js = json.loads((tmp_path / "cartoons.json").read_text())
+ assert all("provenance_flags" in r and all("acquisition_method" in s for s in r["sources"]) for r in js)
+
+
+@pytest.mark.skipif(not (config.ROOT / "data/conrad.db").exists(), reason="project DB not built")
+def test_live_db_every_row_has_method_and_bypass_rows_flagged():
+ conn = db.connect(config.ROOT / "data/conrad.db")
+ db.init_db(conn)
+ assert conn.execute("SELECT COUNT(*) FROM cartoon_sources WHERE acquisition_method IS NULL").fetchone()[0] == 0
+ bad = conn.execute("""SELECT COUNT(*) FROM cartoon_sources WHERE source_id IN ('seed_wichita','seed_syracuse')
+ AND acquisition_method != 'seed_via_reader_bypass'""").fetchone()[0]
+ assert bad == 0
+ assert conn.execute("SELECT COUNT(*) FROM cartoon_sources WHERE source_id IN ('seed_wichita','seed_syracuse')"
+ ).fetchone()[0] > 0
← 9144a7e Ignore SQLite WAL/SHM sidecar files
·
back to Paul Conrad Archive
·
Dedupe: generic/[bracketed] title matches need identifier or e9c5e1c →