https://github.com/python/cpython/commit/05f0b61e6d67621c88c0cf0d900181dc1d32ca0c commit: 05f0b61e6d67621c88c0cf0d900181dc1d32ca0c branch: 3.13 author: Miss Islington (bot) <[email protected]> committer: serhiy-storchaka <[email protected]> date: 2026-07-17T06:12:42Z summary:
[3.13] gh-153404: Silently ignore non-decimal digits in robots.txt Crawl-delay and Request-rate (GH-153405) (GH-153832) str.isdigit() returned True for non-decimal Unicode digits such as U+00B2 SUPERSCRIPT TWO, which int() then rejected with ValueError. (cherry picked from commit 597ec2d7e5f8bbd5a1eb9e6780444cae46cec3ea) Co-authored-by: tonghuaroot (童话) <[email protected]> files: A Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst M Lib/test/test_robotparser.py M Lib/urllib/robotparser.py diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 620e579c6f21518..415d9a76077552c 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -188,6 +188,8 @@ def test_request_rate(self): parsed_request_rate.seconds, self.request_rate.seconds ) + else: + self.assertIsNone(parsed_request_rate) class EmptyFileTest(BaseRequestRateTest, unittest.TestCase): @@ -246,6 +248,32 @@ class InvalidCrawlDelayTest(BaseRobotTest, unittest.TestCase): bad = [] +class NonDecimalDigitsTest(BaseRequestRateTest, unittest.TestCase): + # Non-decimal Unicode digits pass str.isdigit() but int() rejects + # them, so the directive must be silently ignored, not raise. + robots_txt = """\ +User-Agent: * +Disallow: /tmp/ +Crawl-delay: ² +Request-rate: ²/5 + """ + good = ['/foo.html'] + bad = ['/tmp/'] + crawl_delay = None + request_rate = None + + +class NonDecimalDenominatorTest(BaseRequestRateTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Disallow: /tmp/ +Request-rate: 5/² + """ + good = ['/foo.html'] + request_rate = None + bad = ['/tmp/'] + + class AnotherInvalidRequestRateTest(BaseRobotTest, unittest.TestCase): # also test that Allow and Diasallow works well with each other robots_txt = """\ diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 469af30d0d7aa8c..3e25b6c73863410 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -143,15 +143,15 @@ def parse(self, lines): # before trying to convert to int we need to make # sure that robots.txt has valid syntax otherwise # it will crash - if line[1].strip().isdigit(): + if line[1].strip().isdecimal(): entry.delay = int(line[1]) state = 2 elif line[0] == "request-rate": if state != 0: numbers = line[1].split('/') # check if all values are sane - if (len(numbers) == 2 and numbers[0].strip().isdigit() - and numbers[1].strip().isdigit()): + if (len(numbers) == 2 and numbers[0].strip().isdecimal() + and numbers[1].strip().isdecimal()): entry.req_rate = RequestRate(int(numbers[0]), int(numbers[1])) state = 2 elif line[0] == "sitemap": diff --git a/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst b/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst new file mode 100644 index 000000000000000..0746d3f12149d52 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst @@ -0,0 +1,4 @@ +:class:`urllib.robotparser.RobotFileParser` now silently ignores a +``Crawl-delay`` or ``Request-rate`` value written with non-decimal digits +(such as ``U+00B2 SUPERSCRIPT TWO``) instead of raising :exc:`ValueError` +and aborting the parse of the whole ``robots.txt`` file. _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
