← back to Dw Yolo Loop
robots/sitemap fleet canary: crawl-directive + sitemap health (read-only)
30b9b80172c07cd87c19e0bec24c484486956938 · 2026-06-16 08:48:26 -0700 · Steve Abrams
Cycle-27 read-only canary (uncovered SEO/indexing gap). Per core fleet host,
GETs /robots.txt + /sitemap.xml: flags a Googlebot-effective Disallow:/ (UA
precedence-aware), missing/invalid/empty/stale sitemaps, sitemap-not-in-robots.
Result: NO site-wide block anywhere (good); 3 real FAILs (philipperomano 404,
novasuede serves HTML-as-sitemap, raffia parked/broken), 7 no-lastmod WARNs.
Hardened mid-cycle: blocksGooglebot() honors UA-group precedence + relative
Sitemap: URLs resolved against base (killed 2 false positives before reporting).
Files touched
A scripts/robots-sitemap/robots-sitemap-canary.mjs
Diff
commit 30b9b80172c07cd87c19e0bec24c484486956938
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Tue Jun 16 08:48:26 2026 -0700
robots/sitemap fleet canary: crawl-directive + sitemap health (read-only)
Cycle-27 read-only canary (uncovered SEO/indexing gap). Per core fleet host,
GETs /robots.txt + /sitemap.xml: flags a Googlebot-effective Disallow:/ (UA
precedence-aware), missing/invalid/empty/stale sitemaps, sitemap-not-in-robots.
Result: NO site-wide block anywhere (good); 3 real FAILs (philipperomano 404,
novasuede serves HTML-as-sitemap, raffia parked/broken), 7 no-lastmod WARNs.
Hardened mid-cycle: blocksGooglebot() honors UA-group precedence + relative
Sitemap: URLs resolved against base (killed 2 false positives before reporting).
---
scripts/robots-sitemap/robots-sitemap-canary.mjs | 135 +++++++++++++++++++++++
1 file changed, 135 insertions(+)
diff --git a/scripts/robots-sitemap/robots-sitemap-canary.mjs b/scripts/robots-sitemap/robots-sitemap-canary.mjs
new file mode 100644
index 0000000..7f40471
--- /dev/null
+++ b/scripts/robots-sitemap/robots-sitemap-canary.mjs
@@ -0,0 +1,135 @@
+// robots-sitemap-canary — read-only crawl-directive + sitemap health across the
+// core DW web fleet. A single accidental site-wide `Disallow: /` or a missing/
+// stale sitemap silently suppresses organic indexing fleet-wide — invisible to
+// uptime/TLS canaries (the site is UP and the cert is valid; Google just stops
+// indexing it). Nothing else checks this. Pure read-only GET of /robots.txt +
+// /sitemap.xml per host.
+//
+// node robots-sitemap-canary.mjs [--stale 120] [--hosts a.com,b.com]
+// READ-ONLY (HTTP GET only, no writes). $0.
+import fs from 'node:fs';
+
+const args = process.argv.slice(2);
+const STALE = parseInt(args.find((_,i,a)=>a[i-1]==='--stale')||'120',10)||120; // days; sitemap lastmod older than this = WARN
+const HOSTS = (args.find((_,i,a)=>a[i-1]==='--hosts') || [
+ 'www.designerwallcoverings.com','apartmentwallpaper.com','philipperomano.com',
+ 'novasuede.com','wallco.ai','architecturalwallcoverings.com','thesetdecorator.com',
+ 'corkwallcovering.com','silkwallpaper.com','linenwallpaper.com',
+ 'raffiawallpaper.com','glitterwallpaper.com','hospitalitywallcoverings.com','grasscloth.com',
+].join(',')).split(',').map(h=>h.trim()).filter(Boolean);
+const OUT=`${process.env.HOME}/.claude/yolo-queue/robots-sitemap-2026-06-16.json`;
+const MD =`${process.env.HOME}/.claude/yolo-queue/robots-sitemap-2026-06-16.md`;
+
+async function get(url){
+ const ctrl=new AbortController(); const t=setTimeout(()=>ctrl.abort(),12000);
+ try{
+ const r=await fetch(url,{signal:ctrl.signal,redirect:'follow',headers:{'User-Agent':'DW-robots-sitemap-canary/1.0'}});
+ const body=await r.text();
+ return { status:r.status, body, finalUrl:r.url };
+ }catch(e){ return { status:0, body:'', err:String(e.message).slice(0,40) }; }
+ finally{ clearTimeout(t); }
+}
+
+// Is GOOGLEBOT blocked site-wide? robots.txt precedence: a crawler obeys the
+// MOST SPECIFIC matching User-agent group, so an explicit `User-agent: Googlebot`
+// group overrides `User-agent: *`. We evaluate from Googlebot's perspective —
+// that's the only thing that matters for indexing. (Avoids the false positive
+// where a trailing `User-agent: * / Disallow: /` coexists with an explicit
+// Googlebot allow group, e.g. legacy/parked robots.txt files.)
+function blocksGooglebot(robots){
+ const lines=robots.split(/\r?\n/).map(l=>l.replace(/#.*$/,'').trim()).filter(Boolean);
+ // collect Disallow directives per user-agent group
+ const groups=new Map(); // ua(lowercased) -> array of disallow values
+ let cur=[];
+ for(const l of lines){
+ const m=l.match(/^user-agent:\s*(.+)$/i);
+ if(m){ const ua=m[1].trim().toLowerCase(); if(!groups.has(ua)) groups.set(ua,[]); cur=groups.get(ua); continue; }
+ const d=l.match(/^disallow:\s*(.*)$/i);
+ if(d && cur) cur.push(d[1].trim());
+ }
+ // Googlebot uses its own group if present, else the * group.
+ const eff = groups.has('googlebot') ? groups.get('googlebot') : (groups.get('*')||[]);
+ return eff.some(v=>v==='/'); // Disallow: / → entire site blocked for Googlebot
+}
+function sitemapsFromRobots(robots){
+ return [...robots.matchAll(/^sitemap:\s*(\S+)/gim)].map(m=>m[1].trim());
+}
+function newestLastmod(xml){
+ const mods=[...xml.matchAll(/<lastmod>([^<]+)<\/lastmod>/gi)].map(m=>Date.parse(m[1].trim())).filter(n=>!isNaN(n));
+ if(!mods.length) return null;
+ return Math.max(...mods);
+}
+
+async function checkHost(host){
+ const base=`https://${host}`;
+ const r={ host, flags:[] };
+ // --- robots.txt ---
+ const rob=await get(`${base}/robots.txt`);
+ r.robots_status=rob.status;
+ if(rob.status!==200){ r.flags.push('robots_missing'); }
+ else {
+ r.robots_blocks_all = blocksGooglebot(rob.body);
+ if(r.robots_blocks_all) r.flags.push('ROBOTS_BLOCKS_GOOGLEBOT');
+ // resolve declared Sitemap: URLs against base (they may be relative, e.g. "/sitemap.xml")
+ r.robots_sitemaps = sitemapsFromRobots(rob.body).map(u=>{ try{ return new URL(u, base).href; }catch{ return null; } }).filter(Boolean);
+ r.robots_declares_sitemap = r.robots_sitemaps.length>0;
+ }
+ // --- sitemap.xml (prefer one declared in robots, else default) ---
+ const smUrl = (r.robots_sitemaps && r.robots_sitemaps[0]) || `${base}/sitemap.xml`;
+ r.sitemap_url=smUrl;
+ const sm=await get(smUrl);
+ r.sitemap_status=sm.status;
+ if(sm.status!==200){ r.flags.push('sitemap_missing'); }
+ else {
+ const xml=sm.body;
+ const isXml=/^\s*(<\?xml|<urlset|<sitemapindex)/i.test(xml);
+ r.sitemap_valid_xml=isXml;
+ r.sitemap_is_index=/<sitemapindex/i.test(xml);
+ r.sitemap_entries=(xml.match(/<loc>/gi)||[]).length;
+ if(!isXml){ r.flags.push('sitemap_invalid_xml'); }
+ else {
+ if(r.sitemap_entries===0) r.flags.push('sitemap_empty');
+ const nm=newestLastmod(xml);
+ if(nm){ const days=Math.floor((Date.now()-nm)/86400000); r.sitemap_newest_lastmod_days=days; if(days>STALE) r.flags.push('sitemap_stale'); }
+ else if(!r.sitemap_is_index){ r.flags.push('sitemap_no_lastmod'); }
+ }
+ if(r.robots_status===200 && !r.robots_declares_sitemap) r.flags.push('sitemap_not_in_robots');
+ }
+ // --- verdict ---
+ const hard=['ROBOTS_BLOCKS_GOOGLEBOT','sitemap_missing','sitemap_invalid_xml','sitemap_empty'];
+ const unreachable = rob.status===0 && sm.status===0;
+ if(unreachable) r.verdict='UNKNOWN';
+ else if(r.flags.some(f=>hard.includes(f))) r.verdict='FAIL';
+ else if(r.flags.length) r.verdict='WARN';
+ else r.verdict='PASS';
+ return r;
+}
+
+(async()=>{
+ const results=[];
+ for(const h of HOSTS) results.push(await checkHost(h));
+ const fail=results.filter(r=>r.verdict==='FAIL');
+ const warn=results.filter(r=>r.verdict==='WARN');
+ const unk=results.filter(r=>r.verdict==='UNKNOWN');
+ const verdict = fail.length?'FAIL':warn.length?'WARN':'PASS';
+
+ const report={ generated_at:new Date().toISOString(), checked:results.length, stale_days:STALE,
+ verdict, fail:fail.length, warn:warn.length, unknown:unk.length, results };
+ fs.writeFileSync(OUT, JSON.stringify(report,null,2));
+
+ const emoji=verdict==='FAIL'?'🔴':verdict==='WARN'?'🟠':'🟢';
+ let md=`# robots.txt / sitemap.xml fleet canary — ${new Date().toISOString().slice(0,16)}\n\n`;
+ md+=`Checked **${results.length}** core fleet hosts. **READ-ONLY (HTTP GET only), \$0.** Stale threshold: sitemap newest lastmod > ${STALE}d.\n\n`;
+ md+=`## ${emoji} ${verdict} — ${fail.length} FAIL · ${warn.length} WARN · ${unk.length} UNKNOWN\n\n`;
+ md+=`| Host | Verdict | robots | blocks-googlebot | sitemap | entries | newest lastmod | flags |\n|---|:--:|:--:|:--:|:--:|---:|---:|---|\n`;
+ for(const r of results){
+ md+=`| ${r.host} | ${r.verdict} | ${r.robots_status||'—'} | ${r.robots_blocks_all?'🔴 YES':'no'} | ${r.sitemap_status||'—'}${r.sitemap_is_index?' (idx)':''} | ${r.sitemap_entries??'—'} | ${r.sitemap_newest_lastmod_days!=null?r.sitemap_newest_lastmod_days+'d':'—'} | ${r.flags.join(', ')||'—'} |\n`;
+ }
+ md+=`\n_Flags: ROBOTS_BLOCKS_GOOGLEBOT = Googlebot's effective robots group (its own if present, else \`User-agent: *\`) has \`Disallow: /\` — suppresses ALL Google indexing (hard fail). sitemap_missing/invalid/empty = hard fail. sitemap_stale/no_lastmod/not_in_robots = warn. (idx) = sitemapindex (child sitemaps carry the lastmods). Relative \`Sitemap:\` URLs in robots are resolved against the host base. UNKNOWN = host unreachable (parked/off-fleet)._\n`;
+ fs.writeFileSync(MD,md);
+
+ console.log(`[robots-sitemap] ${emoji} ${verdict} · checked=${results.length} · FAIL=${fail.length} WARN=${warn.length} UNKNOWN=${unk.length}`);
+ for(const r of [...fail,...warn]) console.log(` ${r.verdict==='FAIL'?'🔴':'🟠'} ${r.host}: ${r.flags.join(', ')}`);
+ console.log(`Report: ${MD}`);
+ process.exit(verdict==='FAIL'?2:0);
+})().catch(e=>{console.error('FATAL',e.message);process.exit(1);});
← acaeef7 metafield-coverage audit: repped-vendor specs/images/color c
·
back to Dw Yolo Loop
·
metafield-coverage: harden specs check w/ global.width + pil d9c8874 →