https://github.com/python/cpython/commit/d35cfee409db4eceeaa389c206e4a18eeb6eaf9d commit: d35cfee409db4eceeaa389c206e4a18eeb6eaf9d branch: 3.13 author: Miss Islington (bot) <[email protected]> committer: serhiy-storchaka <[email protected]> date: 2026-07-05T11:37:12Z summary:
[3.13] gh-79638: Restore "Treat an unreachable robots.txt as disallow all" (GH-152525) (GH-153109) This change was accidentally reverted by f0daba1652c (gh-106693, GH-149514), which only intended to revert the ob_sval change. The tests were already restored by GH-149569. (cherry picked from commit 70100b9ea0b43a95226eee45652a5735bb91e9c7) Co-authored-by: Serhiy Storchaka <[email protected]> Co-authored-by: Claude Opus 4.8 (1M context) <[email protected]> files: A Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst M Lib/urllib/robotparser.py diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index d055fd4e41baf37..469af30d0d7aa8c 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -65,9 +65,17 @@ def read(self): f = urllib.request.urlopen(self.url) except urllib.error.HTTPError as err: if err.code in (401, 403): + # If access to robot.txt has the status Unauthorized/Forbidden, + # then most likely this applies to the entire site. self.disallow_all = True - elif err.code >= 400 and err.code < 500: + elif 400 <= err.code < 500: + # RFC 9309, Section 2.3.1.3: the crawler MAY access any + # resources on the server. self.allow_all = True + elif 500 <= err.code < 600: + # RFC 9309, Section 2.3.1.4: the crawler MUST assume + # complete disallow. + self.disallow_all = True err.close() else: raw = f.read() diff --git a/Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst b/Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst new file mode 100644 index 000000000000000..bd9fff0bc2e31b6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst @@ -0,0 +1,2 @@ +Disallow all access in :mod:`urllib.robotparser` if the ``robots.txt`` file +is unreachable due to server or network errors. _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
