https://github.com/python/cpython/commit/a624878b25582ed018ae0ea347bb672f5704e1a1 commit: a624878b25582ed018ae0ea347bb672f5704e1a1 branch: 3.14 author: Miss Islington (bot) <[email protected]> committer: serhiy-storchaka <[email protected]> date: 2026-07-05T11:39:40Z summary:
[3.14] gh-79638: Restore "Treat an unreachable robots.txt as disallow all" (GH-152525) (GH-153108) This change was accidentally reverted by f0daba1652c (gh-106693, GH-149514), which only intended to revert the ob_sval change. The tests were already restored by GH-149569. (cherry picked from commit 70100b9ea0b43a95226eee45652a5735bb91e9c7) Co-authored-by: Serhiy Storchaka <[email protected]> Co-authored-by: Claude Opus 4.8 (1M context) <[email protected]> files: A Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst M Lib/urllib/robotparser.py diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index e70eae800367840..0c3e5d928909358 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -65,9 +65,17 @@ def read(self): f = urllib.request.urlopen(self.url) except urllib.error.HTTPError as err: if err.code in (401, 403): + # If access to robot.txt has the status Unauthorized/Forbidden, + # then most likely this applies to the entire site. self.disallow_all = True - elif err.code >= 400 and err.code < 500: + elif 400 <= err.code < 500: + # RFC 9309, Section 2.3.1.3: the crawler MAY access any + # resources on the server. self.allow_all = True + elif 500 <= err.code < 600: + # RFC 9309, Section 2.3.1.4: the crawler MUST assume + # complete disallow. + self.disallow_all = True err.close() else: raw = f.read() diff --git a/Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst b/Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst new file mode 100644 index 000000000000000..bd9fff0bc2e31b6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst @@ -0,0 +1,2 @@ +Disallow all access in :mod:`urllib.robotparser` if the ``robots.txt`` file +is unreachable due to server or network errors. _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
