← back to Animals
yolo: Generate sitemap.xml + robots.txt
51d1f1330205aebfac3063554f9c284a931a7402 · 2026-05-09 10:37:54 -0700 · animal-yolo
Task: yolo.directory-sitemap
Prompt: In ~/Projects/animals: generate a /sitemap.xml route that lists every breed, every business detail page, every shows event, every breed gallery page. Cache at agent level — regenerate every 6h. Add /r
Files touched
M agents/animal-yolo/state.jsonM src/server/index.jsM src/server/render.jsA src/server/sitemap.js
Diff
commit 51d1f1330205aebfac3063554f9c284a931a7402
Author: animal-yolo <steve@designerwallcoverings.com>
Date: Sat May 9 10:37:54 2026 -0700
yolo: Generate sitemap.xml + robots.txt
Task: yolo.directory-sitemap
Prompt: In ~/Projects/animals: generate a /sitemap.xml route that lists every breed, every business detail page, every shows event, every breed gallery page. Cache at agent level — regenerate every 6h. Add /r
---
agents/animal-yolo/state.json | 4 +-
src/server/index.js | 111 ++++++++++---------------
src/server/render.js | 44 ++++++++++
src/server/sitemap.js | 188 ++++++++++++++++++++++++++++++++++++++++++
4 files changed, 276 insertions(+), 71 deletions(-)
diff --git a/agents/animal-yolo/state.json b/agents/animal-yolo/state.json
index c971f82..67c7f33 100644
--- a/agents/animal-yolo/state.json
+++ b/agents/animal-yolo/state.json
@@ -1,5 +1,5 @@
{
- "cursor": 10,
- "runs": 10,
+ "cursor": 11,
+ "runs": 11,
"started_at": "2026-05-08T07:01:22.386Z"
}
\ No newline at end of file
diff --git a/src/server/index.js b/src/server/index.js
index 3214d3b..d1d9fa0 100644
--- a/src/server/index.js
+++ b/src/server/index.js
@@ -24,7 +24,8 @@ import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { pool, many, one } from '../lib/db.js';
import { log } from '../lib/log.js';
-import { renderHome, renderBreedsIndex, renderBreed, renderCategory, renderBusiness, renderCityMap, renderLeadForm, renderAdmin, renderNotFound, renderShows, renderPhotographer, renderPhotographersIndex, renderAbout, renderFlagQueue } from './render.js';
+import { renderHome, renderBreedsIndex, renderBreed, renderCategory, renderBusiness, renderCityMap, renderLeadForm, renderAdmin, renderNotFound, renderShows, renderEvent, renderPhotographer, renderPhotographersIndex, renderAbout, renderFlagQueue } from './render.js';
+import { getSitemapXml, getImageSitemapXml } from './sitemap.js';
import { pickAdForPage, recordImpression, recordClick } from '../monetize/ad_engine.js';
import { matchLead } from '../monetize/match_engine.js';
import { community } from './community.js';
@@ -586,89 +587,46 @@ app.get('/api/random-photo', async (_req, res) => {
// XML sitemap — every breed page + every photographer profile + the static
// pages. Kept under 50k URLs (Google's per-sitemap limit) which we're nowhere
// near. Returns text/xml so crawlers index it properly.
+// /sitemap.xml — covers every breed, business, event, city, photographer.
+// Built once and cached in-process for 6h to keep these queries off the hot
+// path. Pinned to PUBLIC_BASE_URL to defend against host-header injection.
app.get('/sitemap.xml', async (_req, res) => {
- // SECURITY: was using req.headers.host raw — Host-header injection let
- // attackers poison the sitemap to point crawlers at evil.com. Pin to env
- // PUBLIC_BASE_URL with a hardcoded fallback.
const base = process.env.PUBLIC_BASE_URL || 'https://animalsdirectory.com';
- const breeds = await many(`SELECT slug FROM breeds ORDER BY slug`);
- const photogs = await many(`
- SELECT lower(regexp_replace(author, '[^a-zA-Z0-9]+', '-', 'g')) AS slug,
- COUNT(*) AS n
- FROM breed_images
- WHERE author IS NOT NULL AND length(author) > 1
- AND lower(author) NOT LIKE '%unknown%'
- AND dead_at IS NULL
- GROUP BY 1
- HAVING COUNT(*) >= 3
- ORDER BY 2 DESC
- LIMIT 5000`);
- const urls = [
- { loc: '/', priority: '1.0' },
- { loc: '/breeds', priority: '0.9' },
- { loc: '/photographers', priority: '0.7' },
- { loc: '/dog-park', priority: '0.7' },
- { loc: '/dog-park/breed-avatars', priority: '0.6' },
- { loc: '/shows', priority: '0.6' },
- ...breeds.map(b => ({ loc: `/breeds/${b.slug}`, priority: '0.8' })),
- ...photogs.map(p => ({ loc: `/photographer/${p.slug.replace(/^-+|-+$/g, '')}`, priority: '0.5' })),
- ];
- const xml = `<?xml version="1.0" encoding="UTF-8"?>
-<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-${urls.map(u => ` <url><loc>${base}${u.loc}</loc><priority>${u.priority}</priority></url>`).join('\n')}
-</urlset>`;
- res.set('content-type', 'application/xml; charset=utf-8').send(xml);
+ try {
+ const xml = await getSitemapXml(base);
+ res.set('content-type', 'application/xml; charset=utf-8')
+ .set('cache-control', 'public, max-age=3600')
+ .send(xml);
+ } catch (err) {
+ log.error('sitemap serve failed', { err: err.message });
+ res.status(503).set('retry-after', '60').send('Sitemap temporarily unavailable');
+ }
});
// Image sitemap — Google indexes these to give us photo-search visibility.
// One <url> per breed page with up to 12 child <image:image> entries.
app.get('/sitemap-images.xml', async (_req, res) => {
- // SECURITY: was using req.headers.host raw — Host-header injection let
- // attackers poison the sitemap to point crawlers at evil.com. Pin to env
- // PUBLIC_BASE_URL with a hardcoded fallback.
const base = process.env.PUBLIC_BASE_URL || 'https://animalsdirectory.com';
- const rows = await many(`
- SELECT b.slug, b.common_name,
- json_agg(json_build_object(
- 'url', bi.image_url, 'title', bi.title, 'license', bi.license_url
- ) ORDER BY bi.is_hero DESC, bi.width DESC NULLS LAST) FILTER (WHERE bi.id IS NOT NULL) AS images
- FROM breeds b
- LEFT JOIN LATERAL (
- SELECT id, image_url, title, license_url, is_hero, width
- FROM breed_images
- WHERE breed_id = b.id AND dead_at IS NULL
- ORDER BY is_hero DESC, width DESC NULLS LAST
- LIMIT 12
- ) bi ON TRUE
- GROUP BY b.id, b.slug, b.common_name
- HAVING COUNT(bi.id) > 0`);
- const xmlEsc = s => String(s || '').replace(/[&<>'"]/g, c => ({'&':'&','<':'<','>':'>',"'":''','"':'"'}[c]));
- const xml = `<?xml version="1.0" encoding="UTF-8"?>
-<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
- xmlns:image="http://www.google.com/schemas/sitemap-image/1.1">
-${rows.map(r => ` <url>
- <loc>${base}/breeds/${r.slug}</loc>
-${(r.images || []).map(im => ` <image:image>
- <image:loc>${xmlEsc(im.url)}</image:loc>
- ${im.title ? `<image:title>${xmlEsc(im.title)}</image:title>` : ''}
- ${im.license ? `<image:license>${xmlEsc(im.license)}</image:license>` : ''}
- </image:image>`).join('\n')}
- </url>`).join('\n')}
-</urlset>`;
- res.set('content-type', 'application/xml; charset=utf-8').send(xml);
+ try {
+ const xml = await getImageSitemapXml(base);
+ res.set('content-type', 'application/xml; charset=utf-8')
+ .set('cache-control', 'public, max-age=3600')
+ .send(xml);
+ } catch (err) {
+ log.error('image sitemap serve failed', { err: err.message });
+ res.status(503).set('retry-after', '60').send('Sitemap temporarily unavailable');
+ }
});
-// robots.txt — points crawlers at our sitemaps + protects /admin
+// robots.txt — points crawlers at our sitemaps. Allow everything except
+// admin tooling and the JSON API surface (those have no SEO value and the
+// API-quota burn from accidental crawler indexing isn't worth it).
app.get('/robots.txt', (_req, res) => {
- // SECURITY: was using req.headers.host raw — Host-header injection let
- // attackers poison the sitemap to point crawlers at evil.com. Pin to env
- // PUBLIC_BASE_URL with a hardcoded fallback.
const base = process.env.PUBLIC_BASE_URL || 'https://animalsdirectory.com';
res.type('text/plain').send(
`User-agent: *
Allow: /
-Disallow: /admin
-Disallow: /my-pets
+Disallow: /admin/
Disallow: /api/
Sitemap: ${base}/sitemap.xml
@@ -1346,6 +1304,21 @@ app.get('/shows', async (req, res) => {
res.send(renderShows({ events, filters: { state, kind }, ad }));
});
+// Per-event detail. Each event gets its own indexable URL so Google can rank
+// us for "{event title} {city}" / "{breed} show in {state}" long-tail queries.
+app.get('/shows/:id', async (req, res) => {
+ const id = parseInt(req.params.id, 10);
+ if (!Number.isFinite(id)) return res.status(404).send(renderNotFound('Event not found'));
+ const event = await one(`
+ SELECT id, kind, title, organizer, sponsor_org, venue, address, city, state, zip,
+ latitude, longitude, start_at, end_at, url, source_url, description,
+ breeds_focus, status
+ FROM events
+ WHERE id = $1`, [id]);
+ if (!event) return res.status(404).send(renderNotFound('Event not found'));
+ res.send(renderEvent({ event }));
+});
+
// City map ("Everything pet-related in {city}, {state}")
app.get('/in/:cityState', async (req, res) => {
const m = req.params.cityState.match(/^(.+)-([A-Za-z]{2})$/);
diff --git a/src/server/render.js b/src/server/render.js
index c2b2924..f26566d 100644
--- a/src/server/render.js
+++ b/src/server/render.js
@@ -1230,3 +1230,47 @@ ${points.length ? `
})();
</script>` : ''}` + footer() + '</body></html>';
}
+
+// Per-event detail page — gives Google one indexable URL per event so the
+// long-tail "{event title} {city}" / "{breed} show in {state}" queries can
+// land here instead of on a generic /shows listing.
+export function renderEvent({ event }) {
+ const fmt = (d) => new Date(d).toLocaleDateString('en-US', {
+ weekday:'long', month:'long', day:'numeric', year:'numeric'
+ });
+ const fmtTime = (d) => new Date(d).toLocaleTimeString('en-US', {
+ hour:'numeric', minute:'2-digit'
+ });
+ const kindLabel = KIND_LABEL[event.kind] || event.kind;
+ const cityState = [event.city, event.state].filter(Boolean).join(', ');
+ const cancelled = event.status === 'cancelled';
+ const completed = event.status === 'completed' ||
+ (event.end_at && new Date(event.end_at) < new Date());
+
+ const title = `${event.title}${cityState ? ' — ' + cityState : ''}`;
+ const robotsMeta = cancelled
+ ? '<meta name="robots" content="noindex,follow">'
+ : '';
+
+ return head(title, robotsMeta) + nav() + `
+<main class="container">
+ <p class="muted"><a href="/shows">← All events</a>${event.state ? ` · <a href="/shows?state=${esc(event.state)}">${esc(event.state)} events</a>` : ''}</p>
+ <h1>${esc(event.title)}</h1>
+ ${cancelled ? '<p class="empty"><strong>This event has been cancelled.</strong></p>' : ''}
+ ${completed && !cancelled ? '<p class="muted"><em>This event has already taken place.</em></p>' : ''}
+ <p><span class="kind">${esc(kindLabel)}</span></p>
+ <ul class="event-meta" style="list-style:none;padding:0;line-height:1.8">
+ <li><strong>When:</strong> ${esc(fmt(event.start_at))} · ${esc(fmtTime(event.start_at))}${event.end_at && new Date(event.end_at).getTime() !== new Date(event.start_at).getTime() ? ` — through ${esc(fmt(event.end_at))}` : ''}</li>
+ ${event.venue ? `<li><strong>Venue:</strong> ${esc(event.venue)}</li>` : ''}
+ ${event.address ? `<li><strong>Address:</strong> ${esc(event.address)}${cityState ? ', ' + esc(cityState) : ''}${event.zip ? ' ' + esc(event.zip) : ''}</li>` : (cityState ? `<li><strong>Location:</strong> ${esc(cityState)}${event.zip ? ' ' + esc(event.zip) : ''}</li>` : '')}
+ ${event.organizer ? `<li><strong>Organizer:</strong> ${esc(event.organizer)}</li>` : ''}
+ ${event.sponsor_org ? `<li><strong>Sponsor:</strong> ${esc(event.sponsor_org)}</li>` : ''}
+ </ul>
+ ${event.description ? `<p>${esc(event.description)}</p>` : ''}
+ ${event.url ? `<p><a class="cta" href="${esc(event.url)}" target="_blank" rel="nofollow noopener">Registration / details ↗</a></p>` : ''}
+ ${event.breeds_focus && event.breeds_focus.length ? `
+ <h3>Featured breeds</h3>
+ <p>${event.breeds_focus.map(s => `<a href="/breeds/${esc(s)}">${esc(s.replace(/_/g, ' '))}</a>`).join(' · ')}</p>` : ''}
+</main>
+` + footer() + '</body></html>';
+}
diff --git a/src/server/sitemap.js b/src/server/sitemap.js
new file mode 100644
index 0000000..d7649c8
--- /dev/null
+++ b/src/server/sitemap.js
@@ -0,0 +1,188 @@
+// Sitemap builder + in-process cache.
+//
+// Regenerated every 6h (TTL). Concurrent requests during a build share the
+// same in-flight promise (no thundering herd). On build failure, we serve
+// the previous good copy if we have one; otherwise the request errors.
+//
+// Coverage:
+// - static pages (home, /breeds, /shows, etc.)
+// - one entry per directory category (/vets, /groomers, ...)
+// - one entry per breed (/breeds/:slug) — gallery is part of the breed page
+// - one entry per business (/clinic/:id) excluding opt-outs
+// - one entry per recent/upcoming event (/shows/:id)
+// - one entry per city with ≥2 businesses (/in/{city}-{ST}) — the long-tail
+// "[city] [state] vet/groomer/etc" SEO target
+// - one entry per photographer with ≥3 images (existing behavior)
+//
+// Pinned to PUBLIC_BASE_URL to defend against host-header injection.
+
+import { many } from '../lib/db.js';
+import { log } from '../lib/log.js';
+
+const SIX_HOURS_MS = 6 * 60 * 60 * 1000;
+
+const CATEGORIES = [
+ 'vets','emergency-vets','specialty-hospitals','groomers','pet-stores',
+ 'shelters','rescues','pounds','breeders','trainers',
+ 'boarding','kennels','daycare','dog-parks',
+];
+
+const STATIC_PAGES = [
+ ['/', '1.0'],
+ ['/breeds', '0.9'],
+ ['/photographers', '0.7'],
+ ['/dog-park', '0.7'],
+ ['/dog-park/breed-avatars', '0.6'],
+ ['/shows', '0.8'],
+ ['/about', '0.5'],
+];
+
+const cache = {
+ sitemap: { xml: null, expires: 0, building: null },
+ images: { xml: null, expires: 0, building: null },
+};
+
+function xmlEsc(s) {
+ return String(s || '').replace(/[&<>'"]/g, c =>
+ ({ '&':'&','<':'<','>':'>',"'":''','"':'"' }[c]));
+}
+
+// Mirror the /in/:cityState route's parser (regex /^(.+)-([A-Za-z]{2})$/ +
+// `city = m[1].replace(/-/g,' ')`). We only emit cities whose names are pure
+// `[A-Za-z0-9 ]` so the slug round-trips cleanly back to the stored city.
+function citySlug(city, state) {
+ return city.trim().replace(/\s+/g, '-') + '-' + state.toUpperCase();
+}
+
+function isoDate(d) {
+ if (!d) return null;
+ try { return new Date(d).toISOString().slice(0, 10); }
+ catch { return null; }
+}
+
+async function buildSitemapXml(base) {
+ const [breeds, businesses, events, cities, photogs] = await Promise.all([
+ many(`SELECT slug, updated_at FROM breeds ORDER BY slug`),
+ many(`SELECT id, updated_at FROM businesses
+ WHERE opt_out_flag = FALSE
+ ORDER BY id`),
+ many(`SELECT id FROM events
+ WHERE status != 'cancelled'
+ AND start_at >= NOW() - INTERVAL '90 days'
+ ORDER BY start_at DESC
+ LIMIT 20000`),
+ many(`SELECT LOWER(city) AS city, state
+ FROM businesses
+ WHERE opt_out_flag = FALSE
+ AND city IS NOT NULL AND state IS NOT NULL
+ AND city ~ '^[A-Za-z][A-Za-z0-9 ]*$'
+ GROUP BY 1, 2
+ HAVING COUNT(*) >= 2`),
+ many(`SELECT lower(regexp_replace(author, '[^a-zA-Z0-9]+', '-', 'g')) AS slug
+ FROM breed_images
+ WHERE author IS NOT NULL AND length(author) > 1
+ AND lower(author) NOT LIKE '%unknown%'
+ AND dead_at IS NULL
+ GROUP BY 1
+ HAVING COUNT(*) >= 3
+ LIMIT 5000`),
+ ]);
+
+ const lines = [];
+ lines.push('<?xml version="1.0" encoding="UTF-8"?>');
+ lines.push('<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">');
+
+ const url = (loc, priority, lastmod) => {
+ let s = ` <url><loc>${xmlEsc(base + loc)}</loc>`;
+ if (lastmod) s += `<lastmod>${lastmod}</lastmod>`;
+ if (priority) s += `<priority>${priority}</priority>`;
+ s += '</url>';
+ lines.push(s);
+ };
+
+ for (const [loc, p] of STATIC_PAGES) url(loc, p);
+ for (const cat of CATEGORIES) url('/' + cat, '0.7');
+ for (const b of breeds) url(`/breeds/${b.slug}`, '0.8', isoDate(b.updated_at));
+ for (const biz of businesses) url(`/clinic/${biz.id}`, '0.6', isoDate(biz.updated_at));
+ for (const e of events) url(`/shows/${e.id}`, '0.6');
+ for (const c of cities) url(`/in/${citySlug(c.city, c.state)}`, '0.6');
+ for (const p of photogs) {
+ const slug = p.slug.replace(/^-+|-+$/g, '');
+ if (slug) url(`/photographer/${slug}`, '0.5');
+ }
+
+ lines.push('</urlset>');
+ return lines.join('\n');
+}
+
+async function buildImageSitemapXml(base) {
+ const rows = await many(`
+ SELECT b.slug, b.common_name,
+ json_agg(json_build_object(
+ 'url', bi.image_url, 'title', bi.title, 'license', bi.license_url
+ ) ORDER BY bi.is_hero DESC, bi.width DESC NULLS LAST)
+ FILTER (WHERE bi.id IS NOT NULL) AS images
+ FROM breeds b
+ LEFT JOIN LATERAL (
+ SELECT id, image_url, title, license_url, is_hero, width
+ FROM breed_images
+ WHERE breed_id = b.id AND dead_at IS NULL
+ ORDER BY is_hero DESC, width DESC NULLS LAST
+ LIMIT 12
+ ) bi ON TRUE
+ GROUP BY b.id, b.slug, b.common_name
+ HAVING COUNT(bi.id) > 0`);
+
+ const lines = [];
+ lines.push('<?xml version="1.0" encoding="UTF-8"?>');
+ lines.push('<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"');
+ lines.push(' xmlns:image="http://www.google.com/schemas/sitemap-image/1.1">');
+ for (const r of rows) {
+ lines.push(' <url>');
+ lines.push(` <loc>${xmlEsc(base + '/breeds/' + r.slug)}</loc>`);
+ for (const im of (r.images || [])) {
+ lines.push(' <image:image>');
+ lines.push(` <image:loc>${xmlEsc(im.url)}</image:loc>`);
+ if (im.title) lines.push(` <image:title>${xmlEsc(im.title)}</image:title>`);
+ if (im.license) lines.push(` <image:license>${xmlEsc(im.license)}</image:license>`);
+ lines.push(' </image:image>');
+ }
+ lines.push(' </url>');
+ }
+ lines.push('</urlset>');
+ return lines.join('\n');
+}
+
+async function getCached(slot, builder, base) {
+ if (slot.xml && slot.expires > Date.now()) return slot.xml;
+ if (slot.building) return slot.building;
+ slot.building = (async () => {
+ try {
+ const xml = await builder(base);
+ slot.xml = xml;
+ slot.expires = Date.now() + SIX_HOURS_MS;
+ return xml;
+ } catch (err) {
+ log.error('sitemap build failed', { err: err.message });
+ if (slot.xml) return slot.xml;
+ throw err;
+ } finally {
+ slot.building = null;
+ }
+ })();
+ return slot.building;
+}
+
+export function getSitemapXml(base) {
+ return getCached(cache.sitemap, buildSitemapXml, base);
+}
+
+export function getImageSitemapXml(base) {
+ return getCached(cache.images, buildImageSitemapXml, base);
+}
+
+// Test hook — lets a manual rebuild script blow away the cache.
+export function _resetCache() {
+ cache.sitemap = { xml: null, expires: 0, building: null };
+ cache.images = { xml: null, expires: 0, building: null };
+}
← 856153b yolo: Wire Stripe Checkout for upgrade orders
·
back to Animals
·
yolo: Business owners can claim their listing ed86357 →