https://github.com/python/cpython/commit/cdcf228f6ab764fcd8237231f0a473b9f80586ad commit: cdcf228f6ab764fcd8237231f0a473b9f80586ad branch: main author: Andrew James <[email protected]> committer: orsenthil <[email protected]> date: 2026-07-05T08:57:09-07:00 summary:
gh-60055: Allow passing a Request instance for RobotParser URLs (#103753) * Allow passing a Request instance for the url parameter * 📜🤖 Added by blurb_it. * gh-60055: rebase onto main and document Request support * Fix the lint warnings and CI Errors. * Addressing RobotFileParser review comments. * Update Doc/library/urllib.robotparser.rst Co-authored-by: Serhiy Storchaka <[email protected]> * Address additional Review Comments. * Changed versionchanged to next. --------- Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Senthil Kumaran <[email protected]> Co-authored-by: Serhiy Storchaka <[email protected]> files: A Misc/NEWS.d/next/Library/2023-04-24-11-12-00.gh-issue-60055.UjP6aX.rst M Doc/library/urllib.robotparser.rst M Lib/test/test_robotparser.py M Lib/urllib/robotparser.py diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index 1fa7fc13baa539a..3e84f86d4dd383a 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -24,11 +24,18 @@ structure of :file:`robots.txt` files, see :rfc:`9309`. .. class:: RobotFileParser(url='') This class provides methods to read, parse and answer questions about the - :file:`robots.txt` file at *url*. + :file:`robots.txt` file at *url* or a :class:`urllib.request.Request` object. + + .. versionchanged:: next + *url* parameter can be a :class:`urllib.request.Request` object. .. method:: set_url(url) - Sets the URL referring to a :file:`robots.txt` file. + Sets the URL referring to a :file:`robots.txt` file or a + :class:`urllib.request.Request` object. + + .. versionchanged:: next + *url* parameter can be a :class:`urllib.request.Request` object. .. method:: read() @@ -102,3 +109,17 @@ class:: True >>> rp.can_fetch("*", "http://www.pythontest.net/no-robots-here/";) False + + +The following example demonstrates use of a :class:`urllib.request.Request` +object with additional user-agent headers populated:: + + >>> import urllib.robotparser + >>> import urllib.request + >>> rp = urllib.robotparser.RobotFileParser() + >>> rp.set_url(urllib.request.Request("http://www.pythontest.net/robots.txt";, headers={"User-Agent": "IsraBot"})) + >>> rp.read() + >>> rp.can_fetch("*", "http://www.pythontest.net/";) + True + >>> rp.can_fetch("*", "http://www.pythontest.net/no-robots-here/";) + False diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index cd1477037e94b74..725c6e3b09e1d04 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -773,6 +773,37 @@ def testServiceUnavailable(self): self.assertFalse(parser.can_fetch("*", url + '/path/file.html')) +class UserAgentSiteTestCase(BaseLocalNetworkTestCase, unittest.TestCase): + + class RobotHandler(BaseHTTPRequestHandler): + def do_GET(self): + if self.headers.get('User-Agent').startswith('Python-urllib'): + self.send_error(403, "Forbidden access") + else: + self.send_response(200) + self.end_headers() + self.wfile.write(b"User-agent: *\nDisallow:") + + def log_message(self, format, *args): + pass + + def testUserAgentFilteringSite(self): + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + file_url = url + "/document" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(robots_url) + parser.read() + self.assertTrue(parser.disallow_all) + self.assertFalse(parser.can_fetch("*", file_url)) + parser = urllib.robotparser.RobotFileParser() + parser.set_url(urllib.request.Request(robots_url, headers={'User-Agent': 'cybermapper'})) + parser.read() + self.assertFalse(parser.disallow_all) + self.assertTrue(parser.can_fetch("*", file_url)) + + @support.requires_working_socket() class NetworkTestCase(unittest.TestCase): diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 0c3e5d928909358..61772d90e2d53bf 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -55,8 +55,13 @@ def modified(self): self.last_checked = time.time() def set_url(self, url): - """Sets the URL referring to a robots.txt file.""" + """Sets the URL referring to a robots.txt file. + can be a string or a Request object. + """ self.url = url + + if isinstance(url, urllib.request.Request): + url = url.full_url self.host, self.path = urllib.parse.urlsplit(url)[1:3] def read(self): diff --git a/Misc/NEWS.d/next/Library/2023-04-24-11-12-00.gh-issue-60055.UjP6aX.rst b/Misc/NEWS.d/next/Library/2023-04-24-11-12-00.gh-issue-60055.UjP6aX.rst new file mode 100644 index 000000000000000..58fface7caded53 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-24-11-12-00.gh-issue-60055.UjP6aX.rst @@ -0,0 +1 @@ +Let ``urllib.robotparser.RobotFileParser`` accept a ``urllib.request.Request`` object as well as a url string when setting a robots.txt url. _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
