harvest infra: fix WDQS 504 timeout on the 25-museum query — now batched (3 museums/query) with retry+backoff, robust for every future harvest; register 5 Amsterdam/Dutch museums (Van Gogh Museum, Stedelijk, Rembrandt House, Amsterdam Museum, Mauritshuis) for when their Wikidata P195 coverage improves. NOTE: P195 itemization for these is currently sparse (+6 net artists, no marquee retags) — a real Amsterdam haul needs a Rijksmuseum-roster source (TK-10181)

e4eec832566987912042699cf75301b9865e0236 · 2026-08-03 20:44:41 -0700 · Steve Abrams

Files touched

M scripts/fetch-artists.mjs

Diff

commit e4eec832566987912042699cf75301b9865e0236
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Mon Aug 3 20:44:41 2026 -0700

    harvest infra: fix WDQS 504 timeout on the 25-museum query — now batched (3 museums/query) with retry+backoff, robust for every future harvest; register 5 Amsterdam/Dutch museums (Van Gogh Museum, Stedelijk, Rembrandt House, Amsterdam Museum, Mauritshuis) for when their Wikidata P195 coverage improves. NOTE: P195 itemization for these is currently sparse (+6 net artists, no marquee retags) — a real Amsterdam haul needs a Rijksmuseum-roster source (TK-10181)
---
 scripts/fetch-artists.mjs | 62 ++++++++++++++++++++++++++++++++++-------------
 1 file changed, 45 insertions(+), 17 deletions(-)

diff --git a/scripts/fetch-artists.mjs b/scripts/fetch-artists.mjs
index fee9e35..5dca7bf 100644
--- a/scripts/fetch-artists.mjs
+++ b/scripts/fetch-artists.mjs
@@ -46,15 +46,25 @@ const MUSEUMS = {
   Q657415: 'Cleveland Museum of Art',
   Q49133: 'Museum of Fine Arts Boston',
   Q510324: 'Philadelphia Museum of Art',
+  // Amsterdam / Dutch expansion (Steve 2026-08-03: "more from Amsterdam
+  // museums like the Rijks"). Rijksmuseum already above. QIDs verified same day.
+  Q224124: 'Van Gogh Museum',
+  Q924335: 'Stedelijk Museum Amsterdam',
+  Q277316: 'Rembrandt House Museum',
+  Q1820897: 'Amsterdam Museum',
+  Q221092: 'Mauritshuis',
 };
-const VALUES = Object.keys(MUSEUMS).map(q => `wd:${q}`).join(' ');
-
-const QUERY = `
+// Per-museum-batch query. The all-museums-in-one-VALUES query 504s on WDQS now
+// (too heavy across 25 museums), so we query in small batches with retry/backoff
+// and union the rows (dedup happens in the union step below).
+function queryFor(museumQids) {
+  const vals = museumQids.map(q => `wd:${q}`).join(' ');
+  return `
 SELECT ?person ?personLabel ?sig ?dod ?links
        (GROUP_CONCAT(DISTINCT ?museumLabel; separator="|") AS ?museums)
        (GROUP_CONCAT(DISTINCT ?occLabel; separator="|") AS ?occs)
 WHERE {
-  VALUES ?museum { ${VALUES} }
+  VALUES ?museum { ${vals} }
   ?work wdt:P170 ?person ; wdt:P195 ?museum .
   ?person wdt:P109 ?sig .
   ?person wdt:P570 ?dod .
@@ -66,22 +76,40 @@ WHERE {
   SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
 }
 GROUP BY ?person ?personLabel ?sig ?dod ?links
-ORDER BY DESC(?links)
-`;
-
-async function sparql(q) {
+ORDER BY DESC(?links)`;
+}
+const sleep = ms => new Promise(r => setTimeout(r, ms));
+async function sparql(q, tries = 5) {
   const url = `${ENDPOINT}?format=json&query=${encodeURIComponent(q)}`;
-  const res = await fetch(url, {
-    headers: {
-      'Accept': 'application/sparql-results+json',
-      'User-Agent': 'CelebritySignatures/1.0 (designerwallcoverings.com; PD-signature artist catalog build)',
-    },
-  });
-  if (!res.ok) throw new Error(`SPARQL ${res.status}: ${(await res.text()).slice(0, 300)}`);
-  return (await res.json()).results.bindings;
+  for (let i = 1; ; i++) {
+    let res;
+    try {
+      res = await fetch(url, {
+        headers: {
+          'Accept': 'application/sparql-results+json',
+          'User-Agent': 'CelebritySignatures/1.0 (designerwallcoverings.com; PD-signature artist catalog build)',
+        },
+      });
+    } catch (e) { if (i > tries) throw e; await sleep(8000 * i); continue; }
+    if (res.ok) return (await res.json()).results.bindings;
+    if ((res.status === 504 || res.status === 429 || res.status >= 500) && i <= tries) {
+      process.stdout.write(`(WDQS ${res.status}; backoff ${8 * i}s, try ${i}/${tries}) `);
+      await sleep(8000 * i); continue;
+    }
+    throw new Error(`SPARQL ${res.status}: ${(await res.text()).slice(0, 200)}`);
+  }
 }
 
-const rows = await sparql(QUERY);
+const ALL_QIDS = Object.keys(MUSEUMS);
+const rows = [];
+for (let i = 0; i < ALL_QIDS.length; i += 3) {
+  const batch = ALL_QIDS.slice(i, i + 3);
+  process.stdout.write(`[museums ${i + 1}-${i + batch.length}/${ALL_QIDS.length}] ${batch.map(q => MUSEUMS[q]).join(', ')} … `);
+  const r = await sparql(queryFor(batch));
+  rows.push(...r);
+  console.log(`${r.length} rows (total ${rows.length})`);
+  await sleep(1500);
+}
 console.log(`raw rows: ${rows.length}`);
 
 // One record per person; Special:FilePath URL is directly usable as an <img> src.

← 36e1cbe yoloforever C3 fast-follow (Cody SHIP-IT verified real): can · back to CelebritySignatures · Amsterdam haul via P6379: +110 net Artists (1262→1372), site 4e31653 →