crawl_pd: use Wikimedia thumbnail URLs (iiurlwidth=1280) instead of full-res — Commons rate-limits original-res downloads, thumbnails are uncapped · re-running wikimedia crawl with limit 300

c63100cc4b943faf520b0afb6093b890c48983fd · 2026-05-11 20:36:38 -0700 · Steve

Files touched

M scripts/crawl_pd.py

Diff

commit c63100cc4b943faf520b0afb6093b890c48983fd
Author: Steve <steve@designerwallcoverings.com>
Date:   Mon May 11 20:36:38 2026 -0700

    crawl_pd: use Wikimedia thumbnail URLs (iiurlwidth=1280) instead of full-res — Commons rate-limits original-res downloads, thumbnails are uncapped · re-running wikimedia crawl with limit 300
---
 scripts/crawl_pd.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/scripts/crawl_pd.py b/scripts/crawl_pd.py
index 6d9678a..9b010ed 100644
--- a/scripts/crawl_pd.py
+++ b/scripts/crawl_pd.py
@@ -117,8 +117,8 @@ def crawl_wikimedia(limit=None, polite_s=0.5):
                 title = m['title']  # e.g. 'File:foo.jpg'
                 if not title.startswith('File:'):
                     continue
-                # Fetch imageinfo
-                ii_url = f'https://commons.wikimedia.org/w/api.php?action=query&titles={quote(title)}&prop=imageinfo&iiprop=url|extmetadata|size|mime&format=json'
+                # Fetch imageinfo (request 1280px thumbnail to avoid wikimedia.org rate limits on full-res)
+                ii_url = f'https://commons.wikimedia.org/w/api.php?action=query&titles={quote(title)}&prop=imageinfo&iiprop=url|extmetadata|size|mime&iiurlwidth=1280&format=json'
                 try:
                     ii = http_get_json(ii_url)
                 except Exception as e:
@@ -147,9 +147,10 @@ def crawl_wikimedia(limit=None, polite_s=0.5):
                 desc_clean = re.sub(r'<[^>]+>', '', desc).strip()
                 tags = [t.strip() for t in cats.split('|') if t.strip()][:30]
 
-                img_url = info.get('url')
-                width = info.get('width')
-                height = info.get('height')
+                # Prefer thumbnail URL over full-res — Wikimedia rate-limits full-res hard.
+                img_url = info.get('thumburl') or info.get('url')
+                width = info.get('thumbwidth') or info.get('width')
+                height = info.get('thumbheight') or info.get('height')
                 bytes_ = info.get('size')
 
                 # Download

← 4bb293e fix(kamatera): wrap psql with sudo -u postgres on Linux · back to Wallco Ai · edit modal: large + live preview image at top + Adjust tab d ffcec80 →