Copyright rail layer 2: refuse image/PDF responses by Content-Type or magic bytes before body read; negative test proven red without guard

e4fc5517fc9062344b4c71bd64de20c06c766052 · 2026-09-24 18:10:34 -0700 · Steve Abrams

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

Files touched

M src/conrad/crawlers/base.py
A tests/test_image_rail_content.py

Diff

commit e4fc5517fc9062344b4c71bd64de20c06c766052
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Thu Sep 24 18:10:34 2026 -0700

    Copyright rail layer 2: refuse image/PDF responses by Content-Type or magic bytes before body read; negative test proven red without guard
    
    Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 src/conrad/crawlers/base.py      | 16 +++++++++++-
 tests/test_image_rail_content.py | 53 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 68 insertions(+), 1 deletion(-)

diff --git a/src/conrad/crawlers/base.py b/src/conrad/crawlers/base.py
index b09eee7..03e1c06 100644
--- a/src/conrad/crawlers/base.py
+++ b/src/conrad/crawlers/base.py
@@ -35,6 +35,8 @@ log = logging.getLogger("conrad.http")
 
 
 # copyright rail: this project never requests an image (Conrad cartoons are in copyright) — refused before any I/O
+BINARY_CTYPES = ("image/", "application/pdf", "video/", "application/octet-stream")
+IMAGE_MAGIC = (b"\xff\xd8\xff", b"\x89PNG", b"GIF8", b"%PDF", b"II*\x00", b"MM\x00*", b"RIFF")
 IMAGE_URL = re.compile(r"\.(jpe?g|gif|png|tiff?|webp|bmp|jp2|svg)(?:[?#]|$)", re.I)
 
 
@@ -155,7 +157,19 @@ class Http:
                 time.sleep(wait)
             t0 = time.time()
             try:
-                r = self.s.get(url, params=secret_params, timeout=config.TIMEOUT)
+                r = self.s.get(url, params=secret_params, timeout=config.TIMEOUT, stream=True)
+                # copyright rail, second layer: an image/scan served from an extensionless URL is
+                # refused on its Content-Type or magic bytes BEFORE the body is read or cached.
+                ctype = r.headers.get("content-type", "").lower()
+                if ctype.startswith(BINARY_CTYPES):
+                    r.close()
+                    raise Blocked(f"copyright rail: {ctype or 'binary'} response refused, body never read: {url}")
+                head = next(r.iter_content(16), b"")
+                if head.startswith(IMAGE_MAGIC):
+                    r.close()
+                    raise Blocked(f"copyright rail: image bytes refused, body never read: {url}")
+                r._content = head + b"".join(r.iter_content(65536))
+                r._content_consumed = True
                 log.info("GET %s -> %s (%.1fs, %d bytes)", url, r.status_code, time.time() - t0, len(r.content))
             except requests.RequestException as e:
                 log.warning("GET %s failed: %s", url, type(e).__name__)
diff --git a/tests/test_image_rail_content.py b/tests/test_image_rail_content.py
new file mode 100644
index 0000000..8d8f449
--- /dev/null
+++ b/tests/test_image_rail_content.py
@@ -0,0 +1,53 @@
+"""Copyright rail, layer 2: images/scans served from EXTENSIONLESS URLs are refused on
+Content-Type or magic bytes before the body is read, and nothing is cached."""
+import threading
+from http.server import BaseHTTPRequestHandler, HTTPServer
+
+import pytest
+
+from conrad.crawlers import base
+
+JPEG = b"\xff\xd8\xff\xe0" + b"\x00" * 64
+ROUTES = {
+    "/picture": ("image/jpeg", JPEG),              # honest image type, no extension
+    "/disguised": ("text/html", JPEG),             # lies about its type
+    "/scan": ("application/pdf", b"%PDF-1.4 x"),   # newspaper scan
+    "/page": ("text/html; charset=utf-8", b"<html>Conrad cartoon caption</html>"),
+}
+
+
+class H(BaseHTTPRequestHandler):
+    def do_GET(self):
+        ctype, body = ROUTES[self.path]
+        self.send_response(200)
+        self.send_header("Content-Type", ctype)
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+    def log_message(self, *a):
+        pass
+
+
+@pytest.fixture()
+def server(monkeypatch, tmp_path):
+    monkeypatch.setattr(base.config, "REQUEST_DELAY", 0)
+    monkeypatch.setattr(base.config, "CACHE_DIR", tmp_path)
+    srv = HTTPServer(("127.0.0.1", 0), H)
+    threading.Thread(target=srv.serve_forever, daemon=True).start()
+    yield f"http://127.0.0.1:{srv.server_port}", tmp_path
+    srv.shutdown()
+
+
+@pytest.mark.parametrize("path", ["/picture", "/disguised", "/scan"])
+def test_binary_refused_and_not_cached(server, path):
+    url, cache = server
+    with pytest.raises(base.Blocked, match="copyright rail"):
+        base.Http().get(url + path, check_robots=False)
+    assert not any((cache / "http").iterdir()), "refused response must never reach the cache"
+
+
+def test_html_still_fetched(server):
+    url, _ = server
+    status, body = base.Http().get(url + "/page", check_robots=False)
+    assert status == 200 and "Conrad cartoon caption" in body

← 5ed3bca Cycle 3 data refresh: dedupe/exports/REPORT after live 10-sa · back to Paul Conrad Archive · Fix PEP701 nested-quote f-strings (broke Kamatera py3.10); a f12cde5 →