`tests/test_robots_wildcards.py`

61 lines
"""Google/RFC 9309 robots.txt semantics in the base client (urllib.robotparser ignores '*' and '$')."""
from conrad import config
from conrad.crawlers.base import RobotRules

UA = config.USER_AGENT
GALLERY = "https://www.latimes.com/nation/la-me-paul-conrad-pictures-photogallery.html"


def rules(txt):
    r = RobotRules()
    r.parse(txt.strip().splitlines())
    return r


def test_wildcard_disallow_blocks_photogallery_for_star_group():
    # negative case: a '*' group with /*photogallery MUST block the gallery URL
    r = rules("User-agent: *\nDisallow: /*photogallery\n")
    assert not r.can_fetch(UA, GALLERY)
    assert r.can_fetch(UA, "https://www.latimes.com/archives/la-xpm-2010-sep-05-la-me-paul-conrad-20100905-story.html")


def test_latimes_real_shape_rule_only_for_googlebot_news():
    txt = """
User-agent: *
Disallow: /search
Disallow: /*/thirdpartyservice
Disallow: /get-galleryfragment*

User-agent: Googlebot-News
Disallow: /*photogallery
"""
    r = rules(txt)
    assert r.can_fetch(UA, GALLERY)                       # the photogallery rule binds Googlebot-News only
    assert not r.can_fetch("Googlebot-News", GALLERY)
    assert not r.can_fetch(UA, "https://www.latimes.com/x/thirdpartyservice?a=1")  # mid-path wildcard honoured
    assert not r.can_fetch(UA, "https://www.latimes.com/search?q=conrad")


def test_dollar_anchor_longest_match_and_allow_tie():
    r = rules("User-agent: *\nDisallow: /*.pdf$\nDisallow: /archive/\nAllow: /archive/public/\nAllow: /x\nDisallow: /x\n")
    assert not r.can_fetch(UA, "https://h.org/a/b.pdf")
    assert r.can_fetch(UA, "https://h.org/a/b.pdf?download=1")    # '$' anchors the end
    assert not r.can_fetch(UA, "https://h.org/archive/secret")
    assert r.can_fetch(UA, "https://h.org/archive/public/page")    # longest match wins
    assert r.can_fetch(UA, "https://h.org/x")                      # tie -> Allow wins


def test_disallow_all_and_crawl_delay():
    r = rules("User-agent: *\nDisallow: /\nCrawl-delay: 30\n\nUser-agent: archive.org_bot\nDisallow: /?a=q\n")
    assert not r.can_fetch(UA, "https://cdnc.ucr.edu/?a=q&txq=conrad")
    assert r.crawl_delay(UA) == 30
    assert r.can_fetch("archive.org_bot", "https://cdnc.ucr.edu/page")
    assert rules("").can_fetch(UA, "https://h.org/anything")


def test_group_selected_by_product_token_only():
    r = rules("User-agent: research\nDisallow: /\n\nUser-agent: *\nAllow: /\n")
    assert r.can_fetch(UA, "https://h.org/page")   # 'research' in our UA comment must not select that group
    r2 = rules("User-agent: paul-conrad-archive\nDisallow: /private\n\nUser-agent: *\nDisallow: /\n")
    assert r2.can_fetch(UA, "https://h.org/page") and not r2.can_fetch(UA, "https://h.org/private/x")