← back to Ventura Claw Leads
yolo tick 4: /robots.txt + /sitemap.xml — Google indexing infra
713a5b463e74eb5c2818b062defc79b94f6ec8f0 · 2026-05-06 17:39:50 -0700 · Steve Abrams
robots.txt: User-agent: * + Allow: / + Disallow on admin/api/webhooks/
unsubscribe/claim paths + Sitemap pointer.
sitemap.xml: 45 URLs at v0.1 — 7 static (/, /find, /map, /for-businesses,
/about, /privacy, /terms) + 8 vertical filter pages (/find?vertical=food etc.,
useful long-tail SEO landing pages) + every active business profile (lastmod
from businesses.updated_at so Google revisits when listings change). Cap at
50,000 URLs (Google limit).
Cache-Control: 1 hour. Generates on-demand from a single SQL query so it
stays current without a cron.
XML validates cleanly. Live test on prod 2026-05-06: /robots.txt → 200,
/sitemap.xml → 200 with 45 valid <url> entries. 46/46 tests still pass.
Files touched
Diff
commit 713a5b463e74eb5c2818b062defc79b94f6ec8f0
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Wed May 6 17:39:50 2026 -0700
yolo tick 4: /robots.txt + /sitemap.xml — Google indexing infra
robots.txt: User-agent: * + Allow: / + Disallow on admin/api/webhooks/
unsubscribe/claim paths + Sitemap pointer.
sitemap.xml: 45 URLs at v0.1 — 7 static (/, /find, /map, /for-businesses,
/about, /privacy, /terms) + 8 vertical filter pages (/find?vertical=food etc.,
useful long-tail SEO landing pages) + every active business profile (lastmod
from businesses.updated_at so Google revisits when listings change). Cap at
50,000 URLs (Google limit).
Cache-Control: 1 hour. Generates on-demand from a single SQL query so it
stays current without a cron.
XML validates cleanly. Live test on prod 2026-05-06: /robots.txt → 200,
/sitemap.xml → 200 with 45 valid <url> entries. 46/46 tests still pass.
---
routes/public.js | 49 +++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 49 insertions(+)
diff --git a/routes/public.js b/routes/public.js
index 8cc454c..5374312 100644
--- a/routes/public.js
+++ b/routes/public.js
@@ -227,6 +227,55 @@ router.post('/unsubscribe', async (req, res, next) => {
} catch (err) { next(err); }
});
+// SEO: robots.txt + sitemap.xml. Sitemap lists every active business +
+// the static pages, with lastmod from businesses.updated_at so Google's
+// crawler revisits when listings change.
+router.get('/robots.txt', (req, res) => {
+ const url = (process.env.PUBLIC_URL || 'https://leads.venturaclaw.com').replace(/\/+$/, '');
+ res.type('text/plain').send(
+ `User-agent: *\nAllow: /\nDisallow: /admin\nDisallow: /api/\nDisallow: /webhooks/\nDisallow: /unsubscribe\nDisallow: /claim/\n\nSitemap: ${url}/sitemap.xml\n`
+ );
+});
+
+router.get('/sitemap.xml', async (req, res, next) => {
+ try {
+ const baseUrl = (process.env.PUBLIC_URL || 'https://leads.venturaclaw.com').replace(/\/+$/, '');
+ const businesses = await db.many(`
+ SELECT slug, updated_at, vertical
+ FROM businesses
+ WHERE status = 'active'
+ ORDER BY updated_at DESC
+ LIMIT 50000
+ `);
+ const escape = (s) => String(s).replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'"').replace(/'/g,''');
+ const today = new Date().toISOString().slice(0, 10);
+ const staticPages = [
+ { loc: '/', changefreq: 'daily', priority: '1.0' },
+ { loc: '/find', changefreq: 'daily', priority: '0.9' },
+ { loc: '/map', changefreq: 'daily', priority: '0.8' },
+ { loc: '/for-businesses', changefreq: 'weekly', priority: '0.7' },
+ { loc: '/about', changefreq: 'monthly', priority: '0.5' },
+ { loc: '/privacy', changefreq: 'yearly', priority: '0.3' },
+ { loc: '/terms', changefreq: 'yearly', priority: '0.3' }
+ ];
+ // One <url> per vertical filter — useful long-tail landing pages.
+ const verticalPages = VERTICALS.map(v => ({
+ loc: `/find?vertical=${v.key}`, changefreq: 'daily', priority: '0.7'
+ }));
+
+ let xml = '<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n';
+ for (const p of [...staticPages, ...verticalPages]) {
+ xml += ` <url><loc>${baseUrl}${escape(p.loc)}</loc><lastmod>${today}</lastmod><changefreq>${p.changefreq}</changefreq><priority>${p.priority}</priority></url>\n`;
+ }
+ for (const b of businesses) {
+ const lastmod = (b.updated_at instanceof Date ? b.updated_at : new Date(b.updated_at)).toISOString().slice(0, 10);
+ xml += ` <url><loc>${baseUrl}/business/${escape(b.slug)}</loc><lastmod>${lastmod}</lastmod><changefreq>weekly</changefreq><priority>0.6</priority></url>\n`;
+ }
+ xml += '</urlset>\n';
+ res.set('Cache-Control', 'public, max-age=3600').type('application/xml').send(xml);
+ } catch (err) { next(err); }
+});
+
router.get('/healthz', async (req, res) => {
try {
await db.query('SELECT 1');
← e42956b yolo tick 3: 46-test npm test suite covering auth, complianc
·
back to Ventura Claw Leads
·
yolo tick 5: sort + density slider on /find (per global stan 20a0eb6 →