`public/robots.txt`

51 lines
# Robots.txt for Omega Watch Price History
# Optimized for search engine crawling

User-agent: *
Allow: /
Allow: /watches
Allow: /api/watches
Allow: /api/statistics
Allow: /api/collections
Disallow: /api/admin/
Disallow: /api/watchlist/
Disallow: /api/backup
Disallow: /api/restore

# Sitemap location
Sitemap: http://45.61.58.125:7600/sitemap.xml

# Crawl delay to prevent server overload
Crawl-delay: 1

# Specific rules for major search engines
User-agent: Googlebot
Allow: /
Crawl-delay: 0.5

User-agent: Googlebot-Image
Allow: /
Allow: /assets/
Allow: /images/

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 1

# Block bad bots
User-agent: AhrefsBot
Crawl-delay: 5

User-agent: SemrushBot
Crawl-delay: 5

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /