`public/robots.txt`

28 lines
User-agent: *
Allow: /

# Important pages for crawling
Allow: /api/chat
Allow: /messages
Allow: /profile
Allow: /terms
Allow: /privacy

# Block sensitive API endpoints
Disallow: /api/auth/
Disallow: /api/user-profile
Disallow: /api/delete-account
Disallow: /api/send-welcome-email
Disallow: /api/upload-bubbe

# Block development and test files
Disallow: /tests/
Disallow: /_next/
Disallow: /node_modules/

# Sitemap location
Sitemap: https://bubbe.ai/sitemap.xml
Sitemap: http://45.61.58.125:3011/sitemap.xml

# Crawl-delay for respectful crawling
Crawl-delay: 1