John Vandenberg has uploaded a new change for review.
https://gerrit.wikimedia.org/r/172023
Change subject: Asyncronous HTTP requests
......................................................................
Asyncronous HTTP requests
pywikibot.comms.http.request now returns the ThreadedHttp object,
which now provides some backwards compatibility to the basestring
object which was previously returned.
Added http methods to perform requests without any error handling
and with asyncronous callbacks.
Change-Id: I4f71b39d6402abefed5841d4ea67a7ac47cc46a2
---
M pywikibot/comms/http.py
M pywikibot/comms/threadedhttp.py
M pywikibot/data/wikidataquery.py
M pywikibot/version.py
M pywikibot/weblib.py
M tests/http_tests.py
6 files changed, 160 insertions(+), 33 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core
refs/changes/23/172023/1
diff --git a/pywikibot/comms/http.py b/pywikibot/comms/http.py
index 660e597..1211337 100644
--- a/pywikibot/comms/http.py
+++ b/pywikibot/comms/http.py
@@ -243,11 +243,10 @@
format_string = kwargs.setdefault("headers", {}).get("user-agent")
kwargs["headers"]["user-agent"] = user_agent(site, format_string)
- request = threadedhttp.HttpRequest(baseuri, *args, **kwargs)
- http_queue.put(request)
- while not request.lock.acquire(False):
- time.sleep(0.1)
+ return request_normal(baseuri, *args, **kwargs)
+
+def http_error_handling_callback(request):
# TODO: do some error correcting stuff
if isinstance(request.data, SSLHandshakeError):
if SSL_CERT_VERIFY_FAILED_MSG in str(request.data):
@@ -258,21 +257,53 @@
raise request.data
if request.data[0].status == 504:
- raise Server504Error("Server %s timed out" % site.hostname())
+ raise Server504Error("Server %s timed out" % request.hostname)
# HTTP status 207 is also a success status for Webdav FINDPROP,
# used by the version module.
- if request.data[0].status not in (200, 207):
+ if request.status not in (200, 207):
pywikibot.warning(u"Http response status %(status)s"
- % {'status': request.data[0].status})
+ % {'status': request.status})
- pos = request.data[0]['content-type'].find('charset=')
- if pos >= 0:
- pos += len('charset=')
- encoding = request.data[0]['content-type'][pos:]
- else:
- encoding = 'ascii'
- # Don't warn, many pages don't contain one
- pywikibot.log(u"Http response doesn't contain a charset.")
- return request.data[1].decode(encoding)
+def request_async(*args, **kwargs):
+ """
+ Non-blocking threaded HTTP request with callback.
+
+ The callback and error handler run in the HTTP thread,
+ which means exceptions are not able to be caught.
+
+ The kwargs may contain a 'callbacks' (plural), in which
+ case the error handler and 'callback' (singural) are executed
+ first.
+ """
+ callback = kwargs.pop('callback', None)
+ error_handling = kwargs.pop('error_handling', None)
+
+ callbacks = []
+ if error_handling:
+ callbacks.append(http_error_handling_callback)
+ if callback:
+ callbacks.append(callback)
+
+ kwargs['callbacks'] = callbacks + kwargs.get('callbacks', [])
+
+ format_string = kwargs.setdefault("headers", {}).get("user-agent")
+ if not format_string or '{' in format_string:
+ kwargs["headers"]["user-agent"] = user_agent(None, format_string)
+
+ request = threadedhttp.HttpRequest(*args, **kwargs)
+ http_queue.put(request)
+ return request
+
+
+def request_raw(*args, **kwargs):
+ request = request_async(*args, **kwargs)
+ request._join() # wait for it
+ return request
+
+
+def request_normal(*args, **kwargs):
+ request = request_raw(*args, **kwargs)
+ http_error_handling_callback(request)
+ return request
diff --git a/pywikibot/comms/threadedhttp.py b/pywikibot/comms/threadedhttp.py
index 32e55bd..864c9d1 100644
--- a/pywikibot/comms/threadedhttp.py
+++ b/pywikibot/comms/threadedhttp.py
@@ -21,19 +21,24 @@
__docformat__ = 'epytext'
# standard python libraries
-import sys
import re
+import sys
import threading
if sys.version_info[0] > 2:
from http import cookiejar as cookielib
- from urllib.parse import splittype, splithost, unquote
+ from urllib.parse import splittype, splithost, unquote, urlparse
+ unicode = str
else:
import cookielib
+ import urlparse
from urllib import splittype, splithost, unquote
import pywikibot
+
from pywikibot import config
+
+from pywikibot.tools import UnicodeMixin, deprecated
_logger = "comm.threadedhttp"
@@ -300,7 +305,7 @@
response, content)
-class HttpRequest(object):
+class HttpRequest(UnicodeMixin):
"""Object wrapper for HTTP requests that need to block origin thread.
@@ -329,17 +334,108 @@
* an exception
"""
- def __init__(self, *args, **kwargs):
+ def __init__(self, uri, method="GET", body=None, headers=None,
+ callbacks=None, **kwargs):
"""
Constructor.
See C{Http.request} for parameters.
"""
- self.args = args
+ self.uri = uri
+ self.method = method
+ self.body = body
+ self.headers = headers
+
+ self.callbacks = callbacks
+
+ self.args = [uri, method, body, headers]
self.kwargs = kwargs
- self.data = None
+
+ self._parsed_uri = None
+ self._data = None
self.lock = threading.Semaphore(0)
+ super(HttpRequest, self).__init__()
+
+ def _join(self):
+ self.lock.acquire(True)
+
+ @property
+ def data(self):
+ if not self._data:
+ self._join()
+
+ assert(self._data)
+ return self._data
+
+ @data.setter
+ def data(self, value):
+ self._data = value
+
+ if self.callbacks:
+ for callback in self.callbacks:
+ callback(self)
+
+ @property
+ def exception(self):
+ if isinstance(self.data, Exception):
+ return self.data
+
+ @property
+ def response_headers(self):
+ if not self.exception:
+ return self.data[0]
+
+ @property
+ def raw(self):
+ if not self.exception:
+ return self.data[1]
+
+ @property
+ def parsed_uri(self):
+ if not self._parsed_uri:
+ self._parsed_uri = urlparse(self.uri)
+ return self._parsed_uri
+
+ @property
+ def hostname(self):
+ return self.parsed_uri.netloc
+
+ @property
+ def status(self):
+ return self.response_headers.status
+
+ @property
+ def encoding(self):
+ pos = self.response_headers['content-type'].find('charset=')
+ if pos >= 0:
+ pos += len('charset=')
+ encoding = self.response_headers['content-type'][pos:]
+ else:
+ encoding = 'ascii'
+ # Don't warn, many pages don't contain one
+ pywikibot.log(u"Http response doesn't contain a charset.")
+
+ return encoding
+
+ def decode(self, encoding):
+ return self.raw.decode(encoding)
+
+ @property
+ def content(self):
+ return self.raw.decode(self.encoding)
+
+ def __unicode__(self):
+ return self.content
+
+ def __bytes__(self):
+ return self.raw
+
+ @deprecated
+ def __contains__(self, member):
+ if member in unicode(self):
+ return True
+
class HttpProcessor(threading.Thread):
diff --git a/pywikibot/data/wikidataquery.py b/pywikibot/data/wikidataquery.py
index c3eaf35..6fe3864 100644
--- a/pywikibot/data/wikidataquery.py
+++ b/pywikibot/data/wikidataquery.py
@@ -564,7 +564,7 @@
raise
try:
- data = json.loads(resp)
+ data = json.loads(resp.content)
except ValueError:
pywikibot.warning(u"Data received from host but no JSON could be
decoded")
raise pywikibot.ServerError
diff --git a/pywikibot/version.py b/pywikibot/version.py
index e82f0c5..596699b 100644
--- a/pywikibot/version.py
+++ b/pywikibot/version.py
@@ -244,8 +244,8 @@
url = repo or 'https://git.wikimedia.org/feed/pywikibot/core'
hsh = None
- buf = http.request(site=None, uri=url)
- buf = buf.split('\r\n')
+ r = http.request_normal(url)
+ buf = r.content.split('\r\n')
try:
hsh = buf[13].split('/')[5][:-1]
except Exception as e:
diff --git a/pywikibot/weblib.py b/pywikibot/weblib.py
index f682fec..ba36e92 100644
--- a/pywikibot/weblib.py
+++ b/pywikibot/weblib.py
@@ -38,7 +38,7 @@
uri = uri + urlencode(query)
jsontext = http.request(uri=uri, site=None)
if "closest" in jsontext:
- data = json.loads(jsontext)
+ data = json.loads(jsontext.content)
return data['archived_snapshots']['closest']['url']
else:
return None
@@ -67,7 +67,7 @@
uri = uri + urlencode(query)
xmltext = http.request(uri=uri, site=None)
if "success" in xmltext:
- data = ET.fromstring(xmltext)
+ data = ET.fromstring(xmltext.content)
return data.find('.//webcite_url').text
else:
return None
diff --git a/tests/http_tests.py b/tests/http_tests.py
index 28bb87a..9ef7b3d 100644
--- a/tests/http_tests.py
+++ b/tests/http_tests.py
@@ -27,14 +27,14 @@
def test_http(self):
"""Test http request function."""
r = http.request(site=None, uri='http://www.wikipedia.org/')
- self.assertIsInstance(r, unicode)
- self.assertIn('<html lang="mul"', r)
+ self.assertIn('<html lang="mul"', unicode(r))
+ self.assertIn('<html lang="mul"', r.raw)
def test_https(self):
"""Test http request function using https."""
r = http.request(site=None, uri='https://www.wikiquote.org/')
- self.assertIsInstance(r, unicode)
- self.assertIn('<html lang="mul"', r)
+ self.assertIn('<html lang="mul"', unicode(r))
+ self.assertIn('<html lang="mul"', r.raw)
def test_https_cert_error(self):
"""Test http request function fails on ssl bad certificate."""
@@ -53,8 +53,8 @@
r = http.request(site=None,
uri='https://en.vikidia.org/wiki/Main_Page',
disable_ssl_certificate_validation=True)
- self.assertIsInstance(r, unicode)
- self.assertIn('<title>Vikidia</title>', r)
+ self.assertIn('<title>Vikidia</title>', unicode(r))
+ self.assertIn('<title>Vikidia</title>', r.raw)
def test_https_cert_invalid(self):
"""Verify certificate is bad."""
--
To view, visit https://gerrit.wikimedia.org/r/172023
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I4f71b39d6402abefed5841d4ea67a7ac47cc46a2
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: John Vandenberg <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
