asseeninmovies: GET /sitemap.xml + /robots.txt — XML sitemap with 6 static pages + top-10k movies (popularity-ranked) + top-10k people + all verified SPOTTED anchors; 10-min in-memory cache; robots disallows /spotted/queue + /api/. Closes the asim-vs-sod sitemap parity gap.

bbaae4b5d2af9c01caa185f3785320db8e090ed0 · 2026-05-12 18:14:32 -0700 · SteveStudio2

Files touched

M server.js

Diff

commit bbaae4b5d2af9c01caa185f3785320db8e090ed0
Author: SteveStudio2 <stevestudio2@SteveStacStudio.lan>
Date:   Tue May 12 18:14:32 2026 -0700

    asseeninmovies: GET /sitemap.xml + /robots.txt — XML sitemap with 6 static pages + top-10k movies (popularity-ranked) + top-10k people + all verified SPOTTED anchors; 10-min in-memory cache; robots disallows /spotted/queue + /api/. Closes the asim-vs-sod sitemap parity gap.
---
 server.js | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)

diff --git a/server.js b/server.js
index 5fa4af1..151f175 100644
--- a/server.js
+++ b/server.js
@@ -1491,6 +1491,65 @@ app.get('/', async (_req, res) => {
 </body></html>`);
 });
 
+// ────────────────────────────────────────────────────────────────────
+// /sitemap.xml + /robots.txt — SEO surface. asseeninmovies has 290k movies
+// and 61k people; we surface the static pages, then a popularity-capped
+// slice of detail pages (top ~10k each by popularity / credit count) plus
+// all verified SPOTTED pages. Sitemap protocol max is 50,000 URLs per
+// file — we sit well under. Cached in-memory for 10 min so 290k count(*)
+// doesn't hit on every crawler probe.
+// ────────────────────────────────────────────────────────────────────
+let _sitemapCache = { at: 0, xml: null };
+const SITEMAP_TTL_MS = 10 * 60 * 1000;
+app.get('/sitemap.xml', async (_req, res) => {
+  const now = Date.now();
+  if (_sitemapCache.xml && (now - _sitemapCache.at) < SITEMAP_TTL_MS) {
+    return res.type('application/xml').send(_sitemapCache.xml);
+  }
+  try {
+    const base = process.env.ASIM_PUBLIC_URL || 'https://asseeninmovies.com';
+    const today = new Date().toISOString().slice(0, 10);
+    const STATIC = [
+      ['/',               '1.0', 'weekly'],
+      ['/movies',         '0.9', 'weekly'],
+      ['/people',         '0.9', 'weekly'],
+      ['/spotted',        '0.9', 'daily'],
+      ['/spotted/submit', '0.4', 'monthly'],
+      ['/search',         '0.5', 'weekly'],
+    ];
+    const [moviesR, peopleR, spottedR] = await Promise.all([
+      pool.query(`SELECT id FROM asim_movies WHERE poster_url IS NOT NULL ORDER BY popularity DESC NULLS LAST, id DESC LIMIT 10000`),
+      pool.query(`SELECT id FROM asim_people WHERE headshot_url IS NOT NULL ORDER BY id LIMIT 10000`),
+      pool.query(`SELECT id, movie_id FROM asim_spotted WHERE verified_at IS NOT NULL AND (verified_by IS NULL OR verified_by NOT LIKE 'rejected%')`)
+    ]);
+    const urls = [];
+    for (const [path, pri, cf] of STATIC) {
+      urls.push(`<url><loc>${base}${path}</loc><lastmod>${today}</lastmod><changefreq>${cf}</changefreq><priority>${pri}</priority></url>`);
+    }
+    for (const m of moviesR.rows) {
+      urls.push(`<url><loc>${base}/movie/${m.id}</loc><lastmod>${today}</lastmod><changefreq>monthly</changefreq><priority>0.6</priority></url>`);
+    }
+    for (const p of peopleR.rows) {
+      urls.push(`<url><loc>${base}/person/${p.id}</loc><lastmod>${today}</lastmod><changefreq>monthly</changefreq><priority>0.5</priority></url>`);
+    }
+    // Each verified spotted lives under its movie's page — surface the movie URL
+    // anchored to #spotted-<id> so crawlers can discover them individually.
+    for (const s of spottedR.rows) {
+      urls.push(`<url><loc>${base}/movie/${s.movie_id}#spotted-${s.id}</loc><lastmod>${today}</lastmod><changefreq>weekly</changefreq><priority>0.7</priority></url>`);
+    }
+    const xml = `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n  ${urls.join('\n  ')}\n</urlset>`;
+    _sitemapCache = { at: now, xml };
+    res.type('application/xml').send(xml);
+  } catch (e) {
+    res.status(500).type('text/plain').send(`error: ${e.message}`);
+  }
+});
+
+app.get('/robots.txt', (_req, res) => {
+  const base = process.env.ASIM_PUBLIC_URL || 'https://asseeninmovies.com';
+  res.type('text/plain').send(`User-agent: *\nAllow: /\nDisallow: /spotted/queue\nDisallow: /api/\nSitemap: ${base}/sitemap.xml\n`);
+});
+
 app.listen(PORT, '127.0.0.1', async () => {
   await ensureSchema();
   console.log(`asseeninmovies → http://127.0.0.1:${PORT}`);

← 08671ef asseeninmovies: /movie/:id + /person/:id — parallelize credi · back to AsSeenInMovies · asseeninmovies: SEO surface — inline SVG favicon (data: URI, b59fa66 →