`scripts/robots-sitemap/robots-sitemap-canary.mjs`

136 lines
// robots-sitemap-canary — read-only crawl-directive + sitemap health across the
// core DW web fleet. A single accidental site-wide `Disallow: /` or a missing/
// stale sitemap silently suppresses organic indexing fleet-wide — invisible to
// uptime/TLS canaries (the site is UP and the cert is valid; Google just stops
// indexing it). Nothing else checks this. Pure read-only GET of /robots.txt +
// /sitemap.xml per host.
//
//   node robots-sitemap-canary.mjs [--stale 120] [--hosts a.com,b.com]
// READ-ONLY (HTTP GET only, no writes). $0.
import fs from 'node:fs';

const args = process.argv.slice(2);
const STALE = parseInt(args.find((_,i,a)=>a[i-1]==='--stale')||'120',10)||120; // days; sitemap lastmod older than this = WARN
const HOSTS = (args.find((_,i,a)=>a[i-1]==='--hosts') || [
  'www.designerwallcoverings.com','apartmentwallpaper.com','philipperomano.com',
  'novasuede.com','wallco.ai','architecturalwallcoverings.com','thesetdecorator.com',
  'corkwallcovering.com','silkwallpaper.com','linenwallpaper.com',
  'raffiawallpaper.com','glitterwallpaper.com','hospitalitywallcoverings.com','grasscloth.com',
].join(',')).split(',').map(h=>h.trim()).filter(Boolean);
const OUT=`${process.env.HOME}/.claude/yolo-queue/robots-sitemap-2026-06-16.json`;
const MD =`${process.env.HOME}/.claude/yolo-queue/robots-sitemap-2026-06-16.md`;

async function get(url){
  const ctrl=new AbortController(); const t=setTimeout(()=>ctrl.abort(),12000);
  try{
    const r=await fetch(url,{signal:ctrl.signal,redirect:'follow',headers:{'User-Agent':'DW-robots-sitemap-canary/1.0'}});
    const body=await r.text();
    return { status:r.status, body, finalUrl:r.url };
  }catch(e){ return { status:0, body:'', err:String(e.message).slice(0,40) }; }
  finally{ clearTimeout(t); }
}

// Is GOOGLEBOT blocked site-wide? robots.txt precedence: a crawler obeys the
// MOST SPECIFIC matching User-agent group, so an explicit `User-agent: Googlebot`
// group overrides `User-agent: *`. We evaluate from Googlebot's perspective —
// that's the only thing that matters for indexing. (Avoids the false positive
// where a trailing `User-agent: * / Disallow: /` coexists with an explicit
// Googlebot allow group, e.g. legacy/parked robots.txt files.)
function blocksGooglebot(robots){
  const lines=robots.split(/\r?\n/).map(l=>l.replace(/#.*$/,'').trim()).filter(Boolean);
  // collect Disallow directives per user-agent group
  const groups=new Map(); // ua(lowercased) -> array of disallow values
  let cur=[];
  for(const l of lines){
    const m=l.match(/^user-agent:\s*(.+)$/i);
    if(m){ const ua=m[1].trim().toLowerCase(); if(!groups.has(ua)) groups.set(ua,[]); cur=groups.get(ua); continue; }
    const d=l.match(/^disallow:\s*(.*)$/i);
    if(d && cur) cur.push(d[1].trim());
  }
  // Googlebot uses its own group if present, else the * group.
  const eff = groups.has('googlebot') ? groups.get('googlebot') : (groups.get('*')||[]);
  return eff.some(v=>v==='/'); // Disallow: /  → entire site blocked for Googlebot
}
function sitemapsFromRobots(robots){
  return [...robots.matchAll(/^sitemap:\s*(\S+)/gim)].map(m=>m[1].trim());
}
function newestLastmod(xml){
  const mods=[...xml.matchAll(/<lastmod>([^<]+)<\/lastmod>/gi)].map(m=>Date.parse(m[1].trim())).filter(n=>!isNaN(n));
  if(!mods.length) return null;
  return Math.max(...mods);
}

async function checkHost(host){
  const base=`https://${host}`;
  const r={ host, flags:[] };
  // --- robots.txt ---
  const rob=await get(`${base}/robots.txt`);
  r.robots_status=rob.status;
  if(rob.status!==200){ r.flags.push('robots_missing'); }
  else {
    r.robots_blocks_all = blocksGooglebot(rob.body);
    if(r.robots_blocks_all) r.flags.push('ROBOTS_BLOCKS_GOOGLEBOT');
    // resolve declared Sitemap: URLs against base (they may be relative, e.g. "/sitemap.xml")
    r.robots_sitemaps = sitemapsFromRobots(rob.body).map(u=>{ try{ return new URL(u, base).href; }catch{ return null; } }).filter(Boolean);
    r.robots_declares_sitemap = r.robots_sitemaps.length>0;
  }
  // --- sitemap.xml (prefer one declared in robots, else default) ---
  const smUrl = (r.robots_sitemaps && r.robots_sitemaps[0]) || `${base}/sitemap.xml`;
  r.sitemap_url=smUrl;
  const sm=await get(smUrl);
  r.sitemap_status=sm.status;
  if(sm.status!==200){ r.flags.push('sitemap_missing'); }
  else {
    const xml=sm.body;
    const isXml=/^\s*(<\?xml|<urlset|<sitemapindex)/i.test(xml);
    r.sitemap_valid_xml=isXml;
    r.sitemap_is_index=/<sitemapindex/i.test(xml);
    r.sitemap_entries=(xml.match(/<loc>/gi)||[]).length;
    if(!isXml){ r.flags.push('sitemap_invalid_xml'); }
    else {
      if(r.sitemap_entries===0) r.flags.push('sitemap_empty');
      const nm=newestLastmod(xml);
      if(nm){ const days=Math.floor((Date.now()-nm)/86400000); r.sitemap_newest_lastmod_days=days; if(days>STALE) r.flags.push('sitemap_stale'); }
      else if(!r.sitemap_is_index){ r.flags.push('sitemap_no_lastmod'); }
    }
    if(r.robots_status===200 && !r.robots_declares_sitemap) r.flags.push('sitemap_not_in_robots');
  }
  // --- verdict ---
  const hard=['ROBOTS_BLOCKS_GOOGLEBOT','sitemap_missing','sitemap_invalid_xml','sitemap_empty'];
  const unreachable = rob.status===0 && sm.status===0;
  if(unreachable) r.verdict='UNKNOWN';
  else if(r.flags.some(f=>hard.includes(f))) r.verdict='FAIL';
  else if(r.flags.length) r.verdict='WARN';
  else r.verdict='PASS';
  return r;
}

(async()=>{
  const results=[];
  for(const h of HOSTS) results.push(await checkHost(h));
  const fail=results.filter(r=>r.verdict==='FAIL');
  const warn=results.filter(r=>r.verdict==='WARN');
  const unk=results.filter(r=>r.verdict==='UNKNOWN');
  const verdict = fail.length?'FAIL':warn.length?'WARN':'PASS';

  const report={ generated_at:new Date().toISOString(), checked:results.length, stale_days:STALE,
    verdict, fail:fail.length, warn:warn.length, unknown:unk.length, results };
  fs.writeFileSync(OUT, JSON.stringify(report,null,2));

  const emoji=verdict==='FAIL'?'🔴':verdict==='WARN'?'🟠':'🟢';
  let md=`# robots.txt / sitemap.xml fleet canary — ${new Date().toISOString().slice(0,16)}\n\n`;
  md+=`Checked **${results.length}** core fleet hosts. **READ-ONLY (HTTP GET only), \$0.** Stale threshold: sitemap newest lastmod > ${STALE}d.\n\n`;
  md+=`## ${emoji} ${verdict} — ${fail.length} FAIL · ${warn.length} WARN · ${unk.length} UNKNOWN\n\n`;
  md+=`| Host | Verdict | robots | blocks-googlebot | sitemap | entries | newest lastmod | flags |\n|---|:--:|:--:|:--:|:--:|---:|---:|---|\n`;
  for(const r of results){
    md+=`| ${r.host} | ${r.verdict} | ${r.robots_status||'—'} | ${r.robots_blocks_all?'🔴 YES':'no'} | ${r.sitemap_status||'—'}${r.sitemap_is_index?' (idx)':''} | ${r.sitemap_entries??'—'} | ${r.sitemap_newest_lastmod_days!=null?r.sitemap_newest_lastmod_days+'d':'—'} | ${r.flags.join(', ')||'—'} |\n`;
  }
  md+=`\n_Flags: ROBOTS_BLOCKS_GOOGLEBOT = Googlebot's effective robots group (its own if present, else \`User-agent: *\`) has \`Disallow: /\` — suppresses ALL Google indexing (hard fail). sitemap_missing/invalid/empty = hard fail. sitemap_stale/no_lastmod/not_in_robots = warn. (idx) = sitemapindex (child sitemaps carry the lastmods). Relative \`Sitemap:\` URLs in robots are resolved against the host base. UNKNOWN = host unreachable (parked/off-fleet)._\n`;
  fs.writeFileSync(MD,md);

  console.log(`[robots-sitemap] ${emoji} ${verdict} · checked=${results.length} · FAIL=${fail.length} WARN=${warn.length} UNKNOWN=${unk.length}`);
  for(const r of [...fail,...warn]) console.log(`  ${r.verdict==='FAIL'?'🔴':'🟠'} ${r.host}: ${r.flags.join(', ')}`);
  console.log(`Report: ${MD}`);
  process.exit(verdict==='FAIL'?2:0);
})().catch(e=>{console.error('FATAL',e.message);process.exit(1);});