seo: /sitemap.xml lists home + 4 project pages + Last-Modified; robots.txt advertises it + 5 smoke

230a1f7432c26418aa0f8a40f8b0476d845c5cdb · 2026-05-13 13:20:01 -0700 · SteveStudio2

Files touched

M server.js
M test/smoke.js

Diff

commit 230a1f7432c26418aa0f8a40f8b0476d845c5cdb
Author: SteveStudio2 <stevestudio2@SteveStacStudio.lan>
Date:   Wed May 13 13:20:01 2026 -0700

    seo: /sitemap.xml lists home + 4 project pages + Last-Modified; robots.txt advertises it + 5 smoke
---
 server.js     | 22 +++++++++++++++++++++-
 test/smoke.js | 10 +++++++++-
 2 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/server.js b/server.js
index e76861f..af60ee3 100644
--- a/server.js
+++ b/server.js
@@ -217,7 +217,27 @@ app.use('/css', express.static(path.join(__dirname, 'public/css'), { maxAge: '1h
 // robots.txt — let everything be crawled. /api/* is JSON discovery, public.
 app.get('/robots.txt', (_req, res) => {
   res.setHeader('Cache-Control', 'public, max-age=86400');
-  res.type('text/plain').send('User-agent: *\nAllow: /\n');
+  res.type('text/plain').send('User-agent: *\nAllow: /\nSitemap: https://projects.agentabrams.com/sitemap.xml\n');
+});
+
+// sitemap.xml — home + every /p/:slug. Commit-detail pages are excluded; with
+// 200+ commits across the fleet they'd dwarf the discoverable surface without
+// adding crawler value. Last-Modified mirrors the most recent commit per repo.
+app.get('/sitemap.xml', async (_req, res, next) => {
+  try {
+    const base = 'https://projects.agentabrams.com';
+    const today = new Date().toISOString().slice(0, 10);
+    const urls = [`<url><loc>${base}/</loc><lastmod>${today}</lastmod><changefreq>daily</changefreq><priority>1.0</priority></url>`];
+    for (const slug of Object.keys(PROJECTS)) {
+      const bundle = await getProjectBundle(slug, PROJECTS[slug]).catch(() => null);
+      const lastmod = (bundle && bundle.commits[0]) ? bundle.commits[0].dateISO.slice(0, 10) : today;
+      urls.push(`<url><loc>${base}/p/${slug}</loc><lastmod>${lastmod}</lastmod><changefreq>daily</changefreq><priority>0.8</priority></url>`);
+    }
+    const xml = `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n  ${urls.join('\n  ')}\n</urlset>\n`;
+    res.setHeader('Cache-Control', 'public, max-age=21600, stale-while-revalidate=86400');
+    res.setHeader('Last-Modified', new Date().toUTCString());
+    res.type('application/xml').send(xml);
+  } catch (e) { next(e); }
 });
 
 app.get('/healthz', (_req, res) => {
diff --git a/test/smoke.js b/test/smoke.js
index 39fe1cb..73c932b 100644
--- a/test/smoke.js
+++ b/test/smoke.js
@@ -105,10 +105,18 @@ function check(name, cond, hint = '') {
   r = await fetch('/p/nope-not-a-project');
   check('unknown project: 404',    r.status === 404);
 
-  // 8. robots.txt — public crawlable
+  // 8. robots.txt — public crawlable + sitemap referenced
   r = await fetch('/robots.txt');
   check('robots: 200',             r.status === 200);
   check('robots: allow all',       /User-agent: \*/.test(r.body));
+  check('robots: sitemap line',    /Sitemap: .*\/sitemap.xml/.test(r.body));
+
+  // 9. sitemap.xml — home + every project page
+  r = await fetch('/sitemap.xml');
+  check('sitemap: 200',            r.status === 200);
+  check('sitemap: urlset',         /<urlset/.test(r.body));
+  check('sitemap: has /p/butlr',   /<loc>[^<]*\/p\/butlr<\/loc>/.test(r.body));
+  check('sitemap: Last-Modified',  /GMT$/.test(r.headers['last-modified'] || ''));
 
   // Report
   if (process.argv.includes('--json')) {

← 5c8c27d seo: canonical + og:title/desc/url + twitter:card on every s · back to build-pages · seo: WebSite + ItemList JSON-LD on home + 2 smoke 2218b5e →