Andrew Bogott has uploaded a new change for review.
https://gerrit.wikimedia.org/r/61889
Change subject: Pep8 cleanup
......................................................................
Pep8 cleanup
Change-Id: I71c8af8db5e1d48a52e953f43e64041007e44ebb
---
M files/swift/SwiftMedia/wmf/rewrite.py
1 file changed, 153 insertions(+), 90 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/89/61889/1
diff --git a/files/swift/SwiftMedia/wmf/rewrite.py
b/files/swift/SwiftMedia/wmf/rewrite.py
index ef30cd8..2716c8f 100644
--- a/files/swift/SwiftMedia/wmf/rewrite.py
+++ b/files/swift/SwiftMedia/wmf/rewrite.py
@@ -13,12 +13,14 @@
from swift.common.utils import get_logger
from swift.common.wsgi import WSGIContext
+
class DumbRedirectHandler(urllib2.HTTPRedirectHandler):
def http_error_301(self, req, fp, code, msg, headers):
return None
def http_error_302(self, req, fp, code, msg, headers):
return None
+
class _WMFRewriteContext(WSGIContext):
"""
@@ -37,13 +39,19 @@
self.thumbhost = conf['thumbhost'].strip()
self.user_agent = conf['user_agent'].strip()
self.bind_port = conf['bind_port'].strip()
- self.shard_containers = conf['shard_containers'].strip() #all, some,
none
+ # all, some, none:
+ self.shard_containers = conf['shard_containers'].strip()
if (self.shard_containers == 'some'):
- # if we're supposed to shard some containers, get a cleaned list
of the containers to shard
- self.shard_container_list =
striplist(conf['shard_container_list'].split(','))
- # this parameter controls whether URLs sent to the thumbhost are sent
as is (eg. upload/proj/lang/) or with the site/lang
+ # if we're supposed to shard some containers,
+ #get a cleaned list of the containers to shard
+ self.shard_container_list = striplist(
+ conf['shard_container_list'].split(','))
+ # this parameter controls whether URLs sent to the thumbhost are sent
+ # as is (eg. upload/proj/lang/) or with the site/lang
# converted and only the path sent back (eg en.wikipedia/thumb).
- self.backend_url_format = conf['backend_url_format'].strip() #'asis',
'sitelang'
+
+ # 'asis', 'sitelang':
+ self.backend_url_format = conf['backend_url_format'].strip()
def handle404(self, reqorig, url, container, obj):
"""
@@ -61,13 +69,15 @@
# Pass on certain headers from the caller squid to the scalers
opener.addheaders = []
if reqorig.headers.get('User-Agent') != None:
- opener.addheaders.append(('User-Agent',
reqorig.headers.get('User-Agent')))
+ opener.addheaders.append(('User-Agent',
+ reqorig.headers.get('User-Agent')))
else:
opener.addheaders.append(('User-Agent', self.user_agent))
for header_to_pass in ['X-Forwarded-For', 'X-Forwarded-Proto',
'Accept', 'Accept-Encoding', 'X-Original-URI']:
- if reqorig.headers.get( header_to_pass ) != None:
- opener.addheaders.append((header_to_pass, reqorig.headers.get(
header_to_pass )))
+ if reqorig.headers.get(header_to_pass) != None:
+ opener.addheaders.append((header_to_pass,
+ reqorig.headers.get(header_to_pass)))
# At least in theory, we shouldn't be handing out links to originals
# that we don't have (or in the case of thumbs, can't generate).
# However, someone may have a formerly valid link to a file, so we
@@ -80,43 +90,63 @@
encodedurl = urlparse.urlunsplit(urlobj)
# if sitelang, we're supposed to mangle the URL so that
- #
http://upload.wikimedia.org/wikipedia/commons/thumb/a/a2/Little_kitten_.jpg/330px-Little_kitten_.jpg
- # changes to
http://commons.wikipedia.org/thumb/a/a2/Little_kitten_.jpg/330px-Little_kitten_.jpg
+ # http://upload.wikimedia.org/wikipedia/commons/thumb/
+ # a/a2/Little_kitten_.jpg/330px-Little_kitten_.jpg
+ # changes to http://commons.wikipedia.org/thumb/
+ # a/a2/Little_kitten_.jpg/330px-Little_kitten_.jpg
if self.backend_url_format == 'sitelang':
- match =
re.match(r'^http://(?P<host>[^/]+)/(?P<proj>[^-/]+)/(?P<lang>[^/]+)/thumb/(?P<path>.+)',
encodedurl)
+
+ re = (r'^http://(?P<host>[^/]+)/(?P<proj>[^-/]+)'
+ '/(?P<lang>[^/]+)/thumb/(?P<path>.+)')
+ match = re.match(re, encodedurl)
if match:
proj = match.group('proj')
lang = match.group('lang')
- # and here are all the legacy special cases, imported from
thumb_handler.php
+ # and here are all the legacy special cases,
+ # imported from thumb_handler.php
if(proj == 'wikipedia'):
- if(lang in ['meta', 'commons', 'internal', 'grants',
'wikimania2006']):
+ if(lang in ['meta', 'commons', 'internal',
+ 'grants', 'wikimania2006']):
proj = 'wikimedia'
if(lang in ['mediawiki']):
lang = 'www'
proj = 'mediawiki'
hostname = '%s.%s.org' % (lang, proj)
if(proj == 'wikipedia' and lang == 'sources'):
- #yay special case
+ # yay special case
hostname = 'wikisource.org'
- # ok, replace the URL with just the part starting with
thumb/
- # take off the first two parts of the path (eg
/wikipedia/commons/); make sure the string starts with a /
- encodedurl = 'http://%s/w/thumb_handler.php/%s' %
(hostname, match.group('path'))
- # add in the X-Original-URI with the swift got (minus the
hostname)
- opener.addheaders.append(('X-Original-URI',
list(urlparse.urlsplit(reqorig.url))[2]))
+ # ok, replace the URL with just part starting with thumb/
+ # take off the first two parts of the path
+ # (eg /wikipedia/commons/); make sure the string
+ # starts with a /
+ encodedurl = ('http://%s/w/thumb_handler.php/%s' %
+ (hostname, match.group('path')))
+ # add in the X-Original-URI with the swift got
+ # (minus the hostname)
+ opener.addheaders.append(('X-Original-URI',
+ list(urlparse.urlsplit(reqorig.url))[2]))
else:
- # ASSERT this code should never be hit since only thumbs
should call the 404 handler
- self.logger.warn("non-thumb in 404 handler! encodedurl =
%s" % encodedurl)
+ # ASSERT this code should never be hit since
+ # only thumbs should call the 404 handler
+ self.logger.warn("non-thumb in 404 handler! "
+ "encodedurl = %s" % encodedurl)
resp = webob.exc.HTTPNotFound('Unexpected error')
return resp
else:
- # log the result of the match here to test and make sure it's
sane before enabling the config
- match =
re.match(r'^http://(?P<host>[^/]+)/(?P<proj>[^-/]+)/(?P<lang>[^/]+)/thumb/(?P<path>.+)',
encodedurl)
+ # log the result of the match here to test
+ # and make sure it's sane before enabling the config
+ match = re.match(r'^http://(?P<host>[^/]+)/(?P<proj>[^-/]+)'
+ '/(?P<lang>[^/]+)/thumb/(?P<path>.+)',
+ encodedurl)
if match:
proj = match.group('proj')
lang = match.group('lang')
- self.logger.warn("sitelang match has proj %s lang %s
encodedurl %s" % (proj, lang, encodedurl))
+ self.logger.warn("sitelang match has proj %s lang %s "
+ "encodedurl %s" %
+ (proj, lang, encodedurl))
else:
- self.logger.warn("no sitelang match on encodedurl: %s" %
encodedurl)
+ self.logger.warn("no sitelang match on encodedurl: %s" %
+ encodedurl)
# ok, call the encoded url
upcopy = opener.open(encodedurl)
@@ -138,8 +168,8 @@
resp = CopiedHTTPError()
return resp
except urllib2.URLError, error:
- msg = 'There was a problem while contacting the image scaler: %s'
% \
- error.reason
+ msg = ('There was a problem while contacting '
+ 'the image scaler: %s' % error.reason)
resp = webob.exc.HTTPServiceUnavailable(msg)
return resp
@@ -155,7 +185,8 @@
resp = webob.Response(app_iter=upcopy, content_type=c_t)
# add in the headers if we've got them
- for header in ['Content-Length', 'Content-Disposition',
'Last-Modified', 'Accept-Ranges']:
+ for header in ['Content-Length', 'Content-Disposition',
+ 'Last-Modified', 'Accept-Ranges']:
if(uinfo.getheader(header)):
resp.headers.add(header, uinfo.getheader(header))
@@ -167,15 +198,18 @@
def handle_request(self, env, start_response):
req = webob.Request(env)
- # Double (or triple, etc.) slashes in the URL should be ignored;
collapse them. fixes bug 32864
- req.path_info = re.sub( r'/{2,}', '/', req.path_info )
+ # Double (or triple, etc.) slashes in the URL should be ignored;
+ # collapse them. fixes bug 32864
+ req.path_info = re.sub(r'/{2,}', '/', req.path_info)
# Keep a copy of the original request so we can ask the scalers for it
reqorig = req.copy()
- # Containers have 5 components: project, language, repo, zone, and
shard.
- # If there's no zone in the URL, the zone is assumed to be 'public'
(for b/c).
- # Shard is optional (and configurable), and is only used for large
containers.
+ # Containers have 5 parts: project, language, repo, zone, and shard.
+ # If there's no zone in the URL, the zone is assumed to be
+ # 'public' (for b/c).
+ # Shard is optional (and configurable), and is only
+ # used for large containers.
#
# Projects are wikipedia, wikinews, etc.
# Languages are en, de, fr, commons, etc.
@@ -191,65 +225,79 @@
#
# Rewrite wiki-global URLs of these forms:
# (a) http://upload.wikimedia.org/math/<relpath>
- # =>
http://msfe/v1/AUTH_<hash>/global-data-math-render/<relpath>
+ # => http://msfe/v1/AUTH_<hash>/global-data-math-render/<relpath>
# (b) http://upload.wikimedia.org/<proj>/<lang>/math/<relpath> (legacy)
- # =>
http://msfe/v1/AUTH_<hash>/global-data-math-render/<relpath>
+ # => http://msfe/v1/AUTH_<hash>/global-data-math-render/<relpath>
#
# Rewrite wiki-relative URLs of these forms:
# (a) http://upload.wikimedia.org/<proj>/<lang>/<relpath>
- # =>
http://msfe/v1/AUTH_<hash>/<proj>-<lang>-local-public/<relpath>
+ # => http://msfe/v1/AUTH_<hash>/<proj>-<lang>-local-public/<relpath>
# (b) http://upload.wikimedia.org/<proj>/<lang>/archive/<relpath>
- # =>
http://msfe/v1/AUTH_<hash>/<proj>-<lang>-local-public/archive/<relpath>
+ # => http://msfe/v1/AUTH_<hash>/<proj>-<lang>-local-public/
+ # archive/<relpath>
# (c) http://upload.wikimedia.org/<proj>/<lang>/thumb/<relpath>
- # =>
http://msfe/v1/AUTH_<hash>/<proj>-<lang>-local-thumb/<relpath>
+ # => http://msfe/v1/AUTH_<hash>/<proj>-<lang>-local-thumb/<relpath>
# (d) http://upload.wikimedia.org/<proj>/<lang>/thumb/archive/<relpath>
- # =>
http://msfe/v1/AUTH_<hash>/<proj>-<lang>-local-thumb/archive/<relpath>
+ # => http://msfe/v1/AUTH_<hash>/<proj>-<lang>-local-thumb/
+ # archive/<relpath>
# (e) http://upload.wikimedia.org/<proj>/<lang>/thumb/temp/<relpath>
- # =>
http://msfe/v1/AUTH_<hash>/<proj>-<lang>-local-thumb/temp/<relpath>
+ # => http://msfe/v1/AUTH_<hash>/<proj>-<lang>-local-thumb/
+ # temp/<relpath>
# (f) http://upload.wikimedia.org/<proj>/<lang>/temp/<relpath>
- # =>
http://msfe/v1/AUTH_<hash>/<proj>-<lang>-local-temp/<relpath>
+ # => http://msfe/v1/AUTH_<hash>/<proj>-<lang>-local-temp/<relpath>
# (g) http://upload.wikimedia.org/<proj>/<lang>/transcoded/<relpath>
- # =>
http://msfe/v1/AUTH_<hash>/<proj>-<lang>-local-transcoded/<relpath>
+ # => http://msfe/v1/AUTH_<hash>/<proj>-<lang>-local-transcoded/
+ # <relpath>
# (h) http://upload.wikimedia.org/<proj>/<lang>/timeline/<relpath>
- # =>
http://msfe/v1/AUTH_<hash>/<proj>-<lang>-timeline-render/<relpath>
+ # => http://msfe/v1/AUTH_<hash>/<proj>-<lang>-timeline-render/
+ # <relpath>
# regular uploads
- match =
re.match(r'^/(?P<proj>[^/]+)/(?P<lang>[^/]+)/((?P<zone>transcoded|thumb|temp)/)?(?P<path>((temp|archive)/)?[0-9a-f]/(?P<shard>[0-9a-f]{2})/.+)$',
req.path)
+ match = re.match(r'^/(?P<proj>[^/]+)/(?P<lang>[^/]+)/'
+ '((?P<zone>transcoded|thumb|temp)/)'
+ '?(?P<path>((temp|archive)/)?[0-9a-f]'
+ '/(?P<shard>[0-9a-f]{2})/.+)$', req.path)
if match:
- proj = match.group('proj')
- lang = match.group('lang')
- repo = 'local' # the upload repo name is "local"
+ proj = match.group('proj')
+ lang = match.group('lang')
+ repo = 'local' # the upload repo name is "local"
# Get the repo zone (if not provided that means "public")
- zone = (match.group('zone') if match.group('zone') else 'public')
+ zone = (match.group('zone') if match.group('zone') else 'public')
# Get the object path relative to the zone (and thus container)
- obj = match.group('path') # e.g. "archive/a/ab/..."
+ obj = match.group('path') # e.g. "archive/a/ab/..."
shard = match.group('shard')
# timeline renderings
if match is None:
# /wikipedia/en/timeline/a876297c277d80dfd826e1f23dbfea3f.png
- match =
re.match(r'^/(?P<proj>[^/]+)/(?P<lang>[^/]+)/(?P<repo>timeline)/(?P<path>.+)$',
req.path)
+ match = re.match(r'^/(?P<proj>[^/]+)/(?P<lang>[^/]+)'
+ '/(?P<repo>timeline)/(?P<path>.+)$', req.path)
if match:
- proj = match.group('proj') # wikipedia
- lang = match.group('lang') # en
- repo = match.group('repo') # timeline
- zone = 'render'
- obj = match.group('path') #
a876297c277d80dfd826e1f23dbfea3f.png
+ proj = match.group('proj') # wikipedia
+ lang = match.group('lang') # en
+ repo = match.group('repo') # timeline
+ zone = 'render'
+ # a876297c277d80dfd826e1f23dbfea3f.png:
+ obj = match.group('path')
shard = ''
# math renderings
if match is None:
# /math/c/9/f/c9f2055dadfb49853eff822a453d9ceb.png
- # /wikipedia/en/math/c/9/f/c9f2055dadfb49853eff822a453d9ceb.png
(legacy)
- match =
re.match(r'^(/(?P<proj>[^/]+)/(?P<lang>[^/]+))?/(?P<repo>math)/(?P<path>(?P<shard1>[0-9a-f])/(?P<shard2>[0-9a-f])/.+)$',
req.path)
+ # /wikipedia/en/math/c/9/f/c9f2055dadfb49853eff822a453d9ceb.png
+ # (legacy)
+ match = re.match(r'^(/(?P<proj>[^/]+)/(?P<lang>[^/]+))'
+ '?/(?P<repo>math)/(?P<path>(?P<shard1>[0-9a-f])'
+ '/(?P<shard2>[0-9a-f])/.+)$', req.path)
if match:
- proj = 'global'
- lang = 'data'
- repo = match.group('repo') # math
- zone = 'render'
- obj = match.group('path') #
c/9/f/c9f2055dadfb49853eff822a453d9ceb.png
- shard = match.group('shard1') + match.group('shard2') # c9
+ proj = 'global'
+ lang = 'data'
+ repo = match.group('repo') # math
+ zone = 'render'
+ # c/9/f/c9f2055dadfb49853eff822a453d9ceb.png:
+ obj = match.group('path')
+ shard = match.group('shard1') + match.group('shard2') # c9
# score renderings
if match is None:
@@ -257,17 +305,18 @@
# /score/override-midi/8/i/8i9pzt87wtpy45lpz1rox8wusjkt7ki.ogg
match = re.match(r'^/(?P<repo>score)/(?P<path>.+)$', req.path)
if match:
- proj = 'global'
- lang = 'data'
- repo = match.group('repo') # score
- zone = 'render'
- obj = match.group('path') #
j/q/jqn99bwy8777srpv45hxjoiu24f0636/jqn99bwy.png
+ proj = 'global'
+ lang = 'data'
+ repo = match.group('repo') # score
+ zone = 'render'
+ # j/q/jqn99bwy8777srpv45hxjoiu24f0636/jqn99bwy.png:
+ obj = match.group('path')
shard = ''
if match is None:
match = re.match(r'^/monitoring/(?P<what>.+)$', req.path)
if match:
- what = match.group('what')
+ what = match.group('what')
if what == 'frontend':
headers = {'Content-Type': 'application/octet-stream'}
resp = webob.Response(headers=headers, body="OK\n")
@@ -279,9 +328,11 @@
status = self._get_status_int()
headers = self._response_headers
- resp = webob.Response(status=status, headers=headers,
app_iter=app_iter)
+ resp = webob.Response(status=status, headers=headers,
+ app_iter=app_iter)
else:
- resp = webob.exc.HTTPNotFound('Monitoring type not found
"%s"' % (req.path))
+ resp = webob.exc.HTTPNotFound('Monitoring type '
+ 'not found "%s"' % (req.path))
return resp(env, start_response)
if match is None:
@@ -289,7 +340,7 @@
# /index.html /favicon.ico /robots.txt etc.
# serve from a default "root" container
if match:
- path = match.group('path')
+ path = match.group('path')
if not path:
path = 'index.html'
@@ -300,17 +351,21 @@
status = self._get_status_int()
headers = self._response_headers
- resp = webob.Response(status=status, headers=headers,
app_iter=app_iter)
+ resp = webob.Response(status=status, headers=headers,
+ app_iter=app_iter)
return resp(env, start_response)
# Internally rewrite the URL based on the regex it matched...
if match:
- # Get the per-project "conceptual" container name, e.g.
"<proj><lang><repo><zone>"
- container = "%s-%s-%s-%s" % (proj, lang, repo, zone) #02/#03
- # Add 2-digit shard to the container if it is supposed to be
sharded.
- # We may thus have an "actual" container name like
"<proj><lang><repo><zone>.<shard>"
- if ( (self.shard_containers == 'all') or \
- ((self.shard_containers == 'some') and (container in
self.shard_container_list)) ):
+ # Get the per-project "conceptual" container name, e.g.
+ # "<proj><lang><repo><zone>"
+ container = "%s-%s-%s-%s" % (proj, lang, repo, zone) # 02/#03
+ # Add 2-digit shard to the container if supposed to be sharded.
+ # We may thus have an "actual" container name like
+ # "<proj><lang><repo><zone>.<shard>"
+ if ((self.shard_containers == 'all') or
+ ((self.shard_containers == 'some') and
+ (container in self.shard_container_list))):
container += ".%s" % shard
# Save a url with just the account name in it.
@@ -319,12 +374,13 @@
req.host = '127.0.0.1:%s' % port
url = req.url[:]
# Create a path to our object's name.
- req.path_info = "/v1/%s/%s/%s" % (self.account, container,
urllib2.unquote(obj))
+ req.path_info = "/v1/%s/%s/%s" % (self.account,
+ container, urllib2.unquote(obj))
#self.logger.warn("new path is %s" % req.path_info)
# do_start_response just remembers what it got called with,
# because our 404 handler will generate a different response.
- app_iter = self._app_call(env) #01
+ app_iter = self._app_call(env) # 01
status = self._get_status_int()
headers = self._response_headers
@@ -332,24 +388,32 @@
# We have it! Just return it as usual.
#headers['X-Swift-Proxy']= `headers`
return webob.Response(status=status, headers=headers,
- app_iter=app_iter)(env, start_response) #01a
- elif status == 404: #4
- # only send thumbs to the 404 handler; just return a 404 for
everything else.
+ app_iter=app_iter)(env, start_response) # 01a
+ elif status == 404: # 4
+ # only send thumbs to the 404 handler;
+ # just return a 404 for everything else.
if repo == 'local' and zone == 'thumb':
resp = self.handle404(reqorig, url, container, obj)
return resp(env, start_response)
else:
- resp = webob.exc.HTTPNotFound('File not found: %s' %
req.path)
+ resp = webob.exc.HTTPNotFound('File not found: %s' %
+ req.path)
return resp(env, start_response)
elif status == 401:
- # if the Storage URL is invalid or has expired we'll get this
error.
- resp = webob.exc.HTTPUnauthorized('Token may have timed out')
#05
+ # if the Storage URL is invalid or has expired
+ # we'll get this error.
+ # 05:
+ resp = webob.exc.HTTPUnauthorized('Token may have timed out')
return resp(env, start_response)
else:
- resp = webob.exc.HTTPNotImplemented('Unknown Status: %s' %
(status)) #10
+ # 10:
+ resp = webob.exc.HTTPNotImplemented('Unknown Status: %s' %
+ (status))
return resp(env, start_response)
else:
- resp = webob.exc.HTTPNotFound('Regexp failed to match URI: "%s"' %
(req.path)) #11
+ # 11:
+ resp = webob.exc.HTTPNotFound('Regexp failed to match URI: "%s"' %
+ (req.path))
return resp(env, start_response)
@@ -383,4 +447,3 @@
return wmfrewrite_filter
# vim: set expandtab tabstop=4 shiftwidth=4 autoindent:
-
--
To view, visit https://gerrit.wikimedia.org/r/61889
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I71c8af8db5e1d48a52e953f43e64041007e44ebb
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Andrew Bogott <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
