`tests/fixtures/latimes_robots_20260925.txt`

93 lines
User-agent: *
Disallow: /search
Disallow: /recipes/search
Disallow: /changebrowser
Disallow: /thirdpartyservice
Disallow: /*/thirdpartyservice
Disallow: /config/
Disallow: /*/config
Disallow: /dzcfg
Disallow: /*/dzcfg
Disallow: /*/test-template
Disallow: /test/
Disallow: /includes/
Disallow: /_comments/
Disallow: /deeplinkid/
Disallow: /searchsuggest
Disallow: /*/searchsuggest
Disallow: /svgimageproc
Disallow: /hive/
Disallow: /get-galleryfragment*
Disallow: /gs-*

User-agent: Googlebot-News
Disallow: /*-ugc-*.html
Disallow: /about/
Disallow: /cbp-*.html
Disallow: /*/cbp-*.html
Disallow: /bp/*
Disallow: /*-adstory.html
Disallow: /*sns-ap
Disallow: /*photogallery
Disallow: /brandpublishing/
Disallow: /*/brandpublishing/
Disallow: /shopping/
Disallow: /*/shopping/
Disallow: /paid-posts/
Disallow: /*/paid-posts/
Disallow: /search
Disallow: /*/search
Disallow: /changebrowser
Disallow: /thirdpartyservice
Disallow: /*/thirdpartyservice
Disallow: /config/
Disallow: /*/config
Disallow: /dzcfg
Disallow: /*/dzcfg
Disallow: /*/test-template
Disallow: /test/
Disallow: /_comments/
Disallow: /deeplinkid/
Disallow: /searchsuggest
Disallow: /*/searchsuggest
Disallow: /svgimageproc
Disallow: /hive/
Disallow: /get-galleryfragment*
Disallow: /sponsored/
Disallow: /*/sponsored/

User-agent: Twitterbot
Allow: /

User-agent: Mediapartners-Google
Allow: /

User-agent: AmazonAdBot
Allow: /

User-agent: Bytespider
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: discobot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

Sitemap: https://www.latimes.com/sitemaps/sitemap.xml
Sitemap: https://www.latimes.com/news-sitemap.xml