Gilles has uploaded a new change for review.

  https://gerrit.wikimedia.org/r/315647

Change subject: Log when HTTP status codes from Mediawiki and Thumbor are 
different
......................................................................

Log when HTTP status codes from Mediawiki and Thumbor are different

Refs T147918

Change-Id: Idd80104bead130dbe9363e4f4f2aa202e78846e6
---
M puppet/modules/swift/files/SwiftMedia/wmf/rewrite.py
1 file changed, 30 insertions(+), 4 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/vagrant 
refs/changes/47/315647/1

diff --git a/puppet/modules/swift/files/SwiftMedia/wmf/rewrite.py 
b/puppet/modules/swift/files/SwiftMedia/wmf/rewrite.py
index 2e81747..4d051ac 100644
--- a/puppet/modules/swift/files/SwiftMedia/wmf/rewrite.py
+++ b/puppet/modules/swift/files/SwiftMedia/wmf/rewrite.py
@@ -44,12 +44,34 @@
         # converted  and only the path sent back (eg en.wikipedia/thumb).
         self.backend_url_format = conf['backend_url_format'].strip()  # asis, 
sitelang
 
+    def collectHttpStatusCodes(self, url, thumbor_thread, mediawiki_code):
+        self.logger.debug("Mediawiki: %d %s" % (mediawiki_code, url))
+
+        if thumbor_thread is None:
+            return
+
+        try:
+            # Waits for Thumbor if it took longer than Mediawiki to process 
the image
+            # Otherwise returns/throws exceptions immediately
+            thumbor_result = thumbor_thread.wait()
+            code = thumbor_result.getcode()
+        except urllib2.HTTPError, error:
+            code = error.code
+        except urllib2.URLError, error:
+            code = 503
+
+        self.logger.debug("Thumbor: %d %s" % (code, url))
+
+        if code != mediawiki_code:
+            self.logger.warn("HTTP status code mismatch. Mediawiki: %d 
Thumbor: %d URL: %s" % (mediawiki_code, code, url))
+
     def handle404(self, reqorig, url, container, obj):
         """
         Return a webob.Response which fetches the thumbnail from the thumb
         host and returns it. Note also that the thumb host might write it out
         to Swift so it won't 404 next time.
         """
+        original_request_url = reqorig.url
         # go to the thumb media store for unknown files
         reqorig.host = self.thumbhost
         # upload doesn't like our User-agent, otherwise we could call it
@@ -85,7 +107,7 @@
 
             # Thumbor never needs URL mangling and it needs a different host
             if self.thumborhost:
-                thumbor_reqorig = reqorig
+                thumbor_reqorig = reqorig.copy()
                 thumbor_reqorig.host = self.thumborhost
                 thumbor_urlobj = list(urlparse.urlsplit(thumbor_reqorig.url))
                 thumbor_urlobj[2] = urllib2.quote(thumbor_urlobj[2], '%/')
@@ -130,16 +152,18 @@
                 else:
                     self.logger.warn("no sitelang match on encodedurl: %s" % 
encodedurl)
 
+            thumbor_thread = None
+
             # call thumbor first, otherwise if Mediawiki image scalers return 
an error,
             # thumbor doesn't get a change to try to generate that thumbnail
             if self.thumborhost:
                 if not self.thumbor_wiki_list or '-'.join((proj, lang)) in 
self.thumbor_wiki_list:
-                    # call Thumbor blindly, don't look at the result
-                    eventlet.spawn_n(thumbor_opener.open, thumbor_encodedurl)
+                    # call Thumbor but don't wait for the result
+                    thumbor_thread = eventlet.spawn(thumbor_opener.open, 
thumbor_encodedurl)
 
             # ok, call the encoded url
             upcopy = opener.open(encodedurl)
-            self.logger.debug("Mediawiki: %d %s" % (upcopy.getcode(), 
encodedurl))
+            self.collectHttpStatusCodes(original_request_url, thumbor_thread, 
upcopy.getcode())
         except urllib2.HTTPError, error:
             # copy the urllib2 HTTPError into a webob HTTPError class as-is
 
@@ -156,11 +180,13 @@
                         headers=error.hdrs.items())
 
             resp = CopiedHTTPError()
+            self.collectHttpStatusCodes(original_request_url, thumbor_thread, 
resp.code)
             return resp
         except urllib2.URLError, error:
             msg = 'There was a problem while contacting the image scaler: %s' 
% \
                   error.reason
             resp = webob.exc.HTTPServiceUnavailable(msg)
+            self.collectHttpStatusCodes(original_request_url, thumbor_thread, 
resp.code)
             return resp
 
         # get the Content-Type.

-- 
To view, visit https://gerrit.wikimedia.org/r/315647
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Idd80104bead130dbe9363e4f4f2aa202e78846e6
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/vagrant
Gerrit-Branch: master
Gerrit-Owner: Gilles <gdu...@wikimedia.org>

_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

Reply via email to