https://github.com/python/cpython/commit/597ec2d7e5f8bbd5a1eb9e6780444cae46cec3ea
commit: 597ec2d7e5f8bbd5a1eb9e6780444cae46cec3ea
branch: main
author: tonghuaroot (童话) <[email protected]>
committer: serhiy-storchaka <[email protected]>
date: 2026-07-17T05:54:46Z
summary:
gh-153404: Silently ignore non-decimal digits in robots.txt Crawl-delay and
Request-rate (GH-153405)
str.isdigit() returned True for non-decimal Unicode digits such as
U+00B2 SUPERSCRIPT TWO, which int() then rejected with ValueError.
files:
A Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst
M Lib/test/test_robotparser.py
M Lib/urllib/robotparser.py
diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py
index 725c6e3b09e1d04..9693cfc259286d4 100644
--- a/Lib/test/test_robotparser.py
+++ b/Lib/test/test_robotparser.py
@@ -188,6 +188,8 @@ def test_request_rate(self):
parsed_request_rate.seconds,
self.request_rate.seconds
)
+ else:
+ self.assertIsNone(parsed_request_rate)
class EmptyFileTest(BaseRequestRateTest, unittest.TestCase):
@@ -246,6 +248,32 @@ class InvalidCrawlDelayTest(BaseRobotTest,
unittest.TestCase):
bad = []
+class NonDecimalDigitsTest(BaseRequestRateTest, unittest.TestCase):
+ # Non-decimal Unicode digits pass str.isdigit() but int() rejects
+ # them, so the directive must be silently ignored, not raise.
+ robots_txt = """\
+User-Agent: *
+Disallow: /tmp/
+Crawl-delay: ²
+Request-rate: ²/5
+ """
+ good = ['/foo.html']
+ bad = ['/tmp/']
+ crawl_delay = None
+ request_rate = None
+
+
+class NonDecimalDenominatorTest(BaseRequestRateTest, unittest.TestCase):
+ robots_txt = """\
+User-agent: *
+Disallow: /tmp/
+Request-rate: 5/²
+ """
+ good = ['/foo.html']
+ request_rate = None
+ bad = ['/tmp/']
+
+
class AnotherInvalidRequestRateTest(BaseRobotTest, unittest.TestCase):
# also test that Allow and Diasallow works well with each other
robots_txt = """\
diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py
index 61772d90e2d53bf..8d0311d96f5e0b4 100644
--- a/Lib/urllib/robotparser.py
+++ b/Lib/urllib/robotparser.py
@@ -148,15 +148,15 @@ def parse(self, lines):
# before trying to convert to int we need to make
# sure that robots.txt has valid syntax otherwise
# it will crash
- if line[1].strip().isdigit():
+ if line[1].strip().isdecimal():
entry.delay = int(line[1])
state = 2
elif line[0] == "request-rate":
if state != 0:
numbers = line[1].split('/')
# check if all values are sane
- if (len(numbers) == 2 and numbers[0].strip().isdigit()
- and numbers[1].strip().isdigit()):
+ if (len(numbers) == 2 and
numbers[0].strip().isdecimal()
+ and numbers[1].strip().isdecimal()):
entry.req_rate = RequestRate(int(numbers[0]),
int(numbers[1]))
state = 2
elif line[0] == "sitemap":
diff --git
a/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst
b/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst
new file mode 100644
index 000000000000000..0746d3f12149d52
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst
@@ -0,0 +1,4 @@
+:class:`urllib.robotparser.RobotFileParser` now silently ignores a
+``Crawl-delay`` or ``Request-rate`` value written with non-decimal digits
+(such as ``U+00B2 SUPERSCRIPT TWO``) instead of raising :exc:`ValueError`
+and aborting the parse of the whole ``robots.txt`` file.
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
