[MediaWiki-commits] [Gerrit] mediawiki/vagrant[master]: Log when HTTP status codes from Mediawiki and Thumbor are di...
jenkins-bot has submitted this change and it was merged.
Change subject: Log when HTTP status codes from Mediawiki and Thumbor are
different
..
Log when HTTP status codes from Mediawiki and Thumbor are different
Bug: T147918
Change-Id: Idd80104bead130dbe9363e4f4f2aa202e78846e6
---
M puppet/modules/swift/files/SwiftMedia/wmf/rewrite.py
1 file changed, 30 insertions(+), 4 deletions(-)
Approvals:
Gilles: Looks good to me, approved
jenkins-bot: Verified
diff --git a/puppet/modules/swift/files/SwiftMedia/wmf/rewrite.py
b/puppet/modules/swift/files/SwiftMedia/wmf/rewrite.py
index 2e81747..4d051ac 100644
--- a/puppet/modules/swift/files/SwiftMedia/wmf/rewrite.py
+++ b/puppet/modules/swift/files/SwiftMedia/wmf/rewrite.py
@@ -44,12 +44,34 @@
# converted and only the path sent back (eg en.wikipedia/thumb).
self.backend_url_format = conf['backend_url_format'].strip() # asis,
sitelang
+def collectHttpStatusCodes(self, url, thumbor_thread, mediawiki_code):
+self.logger.debug("Mediawiki: %d %s" % (mediawiki_code, url))
+
+if thumbor_thread is None:
+return
+
+try:
+# Waits for Thumbor if it took longer than Mediawiki to process
the image
+# Otherwise returns/throws exceptions immediately
+thumbor_result = thumbor_thread.wait()
+code = thumbor_result.getcode()
+except urllib2.HTTPError, error:
+code = error.code
+except urllib2.URLError, error:
+code = 503
+
+self.logger.debug("Thumbor: %d %s" % (code, url))
+
+if code != mediawiki_code:
+self.logger.warn("HTTP status code mismatch. Mediawiki: %d
Thumbor: %d URL: %s" % (mediawiki_code, code, url))
+
def handle404(self, reqorig, url, container, obj):
"""
Return a webob.Response which fetches the thumbnail from the thumb
host and returns it. Note also that the thumb host might write it out
to Swift so it won't 404 next time.
"""
+original_request_url = reqorig.url
# go to the thumb media store for unknown files
reqorig.host = self.thumbhost
# upload doesn't like our User-agent, otherwise we could call it
@@ -85,7 +107,7 @@
# Thumbor never needs URL mangling and it needs a different host
if self.thumborhost:
-thumbor_reqorig = reqorig
+thumbor_reqorig = reqorig.copy()
thumbor_reqorig.host = self.thumborhost
thumbor_urlobj = list(urlparse.urlsplit(thumbor_reqorig.url))
thumbor_urlobj[2] = urllib2.quote(thumbor_urlobj[2], '%/')
@@ -130,16 +152,18 @@
else:
self.logger.warn("no sitelang match on encodedurl: %s" %
encodedurl)
+thumbor_thread = None
+
# call thumbor first, otherwise if Mediawiki image scalers return
an error,
# thumbor doesn't get a change to try to generate that thumbnail
if self.thumborhost:
if not self.thumbor_wiki_list or '-'.join((proj, lang)) in
self.thumbor_wiki_list:
-# call Thumbor blindly, don't look at the result
-eventlet.spawn_n(thumbor_opener.open, thumbor_encodedurl)
+# call Thumbor but don't wait for the result
+thumbor_thread = eventlet.spawn(thumbor_opener.open,
thumbor_encodedurl)
# ok, call the encoded url
upcopy = opener.open(encodedurl)
-self.logger.debug("Mediawiki: %d %s" % (upcopy.getcode(),
encodedurl))
+self.collectHttpStatusCodes(original_request_url, thumbor_thread,
upcopy.getcode())
except urllib2.HTTPError, error:
# copy the urllib2 HTTPError into a webob HTTPError class as-is
@@ -156,11 +180,13 @@
headers=error.hdrs.items())
resp = CopiedHTTPError()
+self.collectHttpStatusCodes(original_request_url, thumbor_thread,
resp.code)
return resp
except urllib2.URLError, error:
msg = 'There was a problem while contacting the image scaler: %s'
% \
error.reason
resp = webob.exc.HTTPServiceUnavailable(msg)
+self.collectHttpStatusCodes(original_request_url, thumbor_thread,
resp.code)
return resp
# get the Content-Type.
--
To view, visit https://gerrit.wikimedia.org/r/315647
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Idd80104bead130dbe9363e4f4f2aa202e78846e6
Gerrit-PatchSet: 2
Gerrit-Project: mediawiki/vagrant
Gerrit-Branch: master
Gerrit-Owner: Gilles
Gerrit-Reviewer: Gilles
Gerrit-Reviewer: jenkins-bot <>
___
MediaWiki-commits mailing list
MediaWiki-commits@lis
[MediaWiki-commits] [Gerrit] mediawiki/vagrant[master]: Log when HTTP status codes from Mediawiki and Thumbor are di...
Gilles has uploaded a new change for review.
https://gerrit.wikimedia.org/r/315647
Change subject: Log when HTTP status codes from Mediawiki and Thumbor are
different
..
Log when HTTP status codes from Mediawiki and Thumbor are different
Refs T147918
Change-Id: Idd80104bead130dbe9363e4f4f2aa202e78846e6
---
M puppet/modules/swift/files/SwiftMedia/wmf/rewrite.py
1 file changed, 30 insertions(+), 4 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/vagrant
refs/changes/47/315647/1
diff --git a/puppet/modules/swift/files/SwiftMedia/wmf/rewrite.py
b/puppet/modules/swift/files/SwiftMedia/wmf/rewrite.py
index 2e81747..4d051ac 100644
--- a/puppet/modules/swift/files/SwiftMedia/wmf/rewrite.py
+++ b/puppet/modules/swift/files/SwiftMedia/wmf/rewrite.py
@@ -44,12 +44,34 @@
# converted and only the path sent back (eg en.wikipedia/thumb).
self.backend_url_format = conf['backend_url_format'].strip() # asis,
sitelang
+def collectHttpStatusCodes(self, url, thumbor_thread, mediawiki_code):
+self.logger.debug("Mediawiki: %d %s" % (mediawiki_code, url))
+
+if thumbor_thread is None:
+return
+
+try:
+# Waits for Thumbor if it took longer than Mediawiki to process
the image
+# Otherwise returns/throws exceptions immediately
+thumbor_result = thumbor_thread.wait()
+code = thumbor_result.getcode()
+except urllib2.HTTPError, error:
+code = error.code
+except urllib2.URLError, error:
+code = 503
+
+self.logger.debug("Thumbor: %d %s" % (code, url))
+
+if code != mediawiki_code:
+self.logger.warn("HTTP status code mismatch. Mediawiki: %d
Thumbor: %d URL: %s" % (mediawiki_code, code, url))
+
def handle404(self, reqorig, url, container, obj):
"""
Return a webob.Response which fetches the thumbnail from the thumb
host and returns it. Note also that the thumb host might write it out
to Swift so it won't 404 next time.
"""
+original_request_url = reqorig.url
# go to the thumb media store for unknown files
reqorig.host = self.thumbhost
# upload doesn't like our User-agent, otherwise we could call it
@@ -85,7 +107,7 @@
# Thumbor never needs URL mangling and it needs a different host
if self.thumborhost:
-thumbor_reqorig = reqorig
+thumbor_reqorig = reqorig.copy()
thumbor_reqorig.host = self.thumborhost
thumbor_urlobj = list(urlparse.urlsplit(thumbor_reqorig.url))
thumbor_urlobj[2] = urllib2.quote(thumbor_urlobj[2], '%/')
@@ -130,16 +152,18 @@
else:
self.logger.warn("no sitelang match on encodedurl: %s" %
encodedurl)
+thumbor_thread = None
+
# call thumbor first, otherwise if Mediawiki image scalers return
an error,
# thumbor doesn't get a change to try to generate that thumbnail
if self.thumborhost:
if not self.thumbor_wiki_list or '-'.join((proj, lang)) in
self.thumbor_wiki_list:
-# call Thumbor blindly, don't look at the result
-eventlet.spawn_n(thumbor_opener.open, thumbor_encodedurl)
+# call Thumbor but don't wait for the result
+thumbor_thread = eventlet.spawn(thumbor_opener.open,
thumbor_encodedurl)
# ok, call the encoded url
upcopy = opener.open(encodedurl)
-self.logger.debug("Mediawiki: %d %s" % (upcopy.getcode(),
encodedurl))
+self.collectHttpStatusCodes(original_request_url, thumbor_thread,
upcopy.getcode())
except urllib2.HTTPError, error:
# copy the urllib2 HTTPError into a webob HTTPError class as-is
@@ -156,11 +180,13 @@
headers=error.hdrs.items())
resp = CopiedHTTPError()
+self.collectHttpStatusCodes(original_request_url, thumbor_thread,
resp.code)
return resp
except urllib2.URLError, error:
msg = 'There was a problem while contacting the image scaler: %s'
% \
error.reason
resp = webob.exc.HTTPServiceUnavailable(msg)
+self.collectHttpStatusCodes(original_request_url, thumbor_thread,
resp.code)
return resp
# get the Content-Type.
--
To view, visit https://gerrit.wikimedia.org/r/315647
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Idd80104bead130dbe9363e4f4f2aa202e78846e6
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/vagrant
Gerrit-Branch: master
Gerrit-Owner: Gilles
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedi
