Filippo Giunchedi has submitted this change and it was merged.
Change subject: swift: Fix PEP8 violations
......................................................................
swift: Fix PEP8 violations
* E126 continuation line over-indented for hanging indent
* E265 block comment should start with '# '
* E501 line too long (n > 100 characters)
* F401 'x' imported but unused
* F841 local variable 'x' is assigned to but never used
Change-Id: I3015e27fa9801eaf6d4f91588e5c459e54e3b90a
---
M modules/swift/files/SwiftMedia/wmf/cors.py
M modules/swift/files/SwiftMedia/wmf/rewrite.py
M modules/swift/files/SwiftMedia/wmf/rewrite_integration_test.py
M modules/swift/files/swift-account-stats
M modules/swift/files/swift-container-stats
5 files changed, 84 insertions(+), 61 deletions(-)
Approvals:
Ori.livneh: Looks good to me, but someone else must approve
Andrew Bogott: Looks good to me, but someone else must approve
Filippo Giunchedi: Verified; Looks good to me, approved
Hashar: Looks good to me, but someone else must approve
diff --git a/modules/swift/files/SwiftMedia/wmf/cors.py
b/modules/swift/files/SwiftMedia/wmf/cors.py
index afd2f53..10a81c6 100644
--- a/modules/swift/files/SwiftMedia/wmf/cors.py
+++ b/modules/swift/files/SwiftMedia/wmf/cors.py
@@ -1,5 +1,6 @@
class CORS(object):
"""Poor man's Wildcard CORS support for Swift <= 1.7.4"""
+
def __init__(self, app):
self.app = app
diff --git a/modules/swift/files/SwiftMedia/wmf/rewrite.py
b/modules/swift/files/SwiftMedia/wmf/rewrite.py
index 3b520f3..9673b0e 100644
--- a/modules/swift/files/SwiftMedia/wmf/rewrite.py
+++ b/modules/swift/files/SwiftMedia/wmf/rewrite.py
@@ -8,13 +8,13 @@
import webob.exc
import re
from eventlet.green import urllib2
-import time
import urlparse
from swift.common.utils import get_logger
from swift.common.wsgi import WSGIContext
class DumbRedirectHandler(urllib2.HTTPRedirectHandler):
+
def http_error_301(self, req, fp, code, msg, headers):
return None
@@ -36,9 +36,11 @@
self.thumbhost = conf['thumbhost'].strip()
self.user_agent = conf['user_agent'].strip()
self.bind_port = conf['bind_port'].strip()
- self.shard_container_list = [item.strip() for item in
conf['shard_container_list'].split(',')]
- # this parameter controls whether URLs sent to the thumbhost are sent
as is (eg. upload/proj/lang/) or with the site/lang
- # converted and only the path sent back (eg en.wikipedia/thumb).
+ self.shard_container_list = [
+ item.strip() for item in conf['shard_container_list'].split(',')]
+ # this parameter controls whether URLs sent to the thumbhost are sent
+ # as is (eg. upload/proj/lang/) or with the site/lang converted and
+ # only the path sent back (eg en.wikipedia/thumb).
self.backend_url_format = conf['backend_url_format'].strip() # asis,
sitelang
def handle404(self, reqorig, url, container, obj):
@@ -76,10 +78,13 @@
encodedurl = urlparse.urlunsplit(urlobj)
# if sitelang, we're supposed to mangle the URL so that
- #
http://upload.wikimedia.org/wikipedia/commons/thumb/a/a2/Little_kitten_.jpg/330px-Little_kitten_.jpg
- # changes to
http://commons.wikipedia.org/w/thumb_handler.php/a/a2/Little_kitten_.jpg/330px-Little_kitten_.jpg
+ #
http://upload.wm.o/wikipedia/commons/thumb/a/a2/Foo_.jpg/330px-Foo_.jpg
+ # changes to
+ #
http://commons.wp.o/w/thumb_handler.php/a/a2/Foo_.jpg/330px-Foo_.jpg
if self.backend_url_format == 'sitelang':
- match =
re.match(r'^http://(?P<host>[^/]+)/(?P<proj>[^-/]+)/(?P<lang>[^/]+)/thumb/(?P<path>.+)',
encodedurl)
+ match = re.match(
+
r'^http://(?P<host>[^/]+)/(?P<proj>[^-/]+)/(?P<lang>[^/]+)/thumb/(?P<path>.+)',
+ encodedurl)
if match:
proj = match.group('proj')
lang = match.group('lang')
@@ -92,25 +97,35 @@
proj = 'mediawiki'
hostname = '%s.%s.org' % (lang, proj)
if(proj == 'wikipedia' and lang == 'sources'):
- #yay special case
+ # yay special case
hostname = 'wikisource.org'
# ok, replace the URL with just the part starting with
thumb/
- # take off the first two parts of the path (eg
/wikipedia/commons/); make sure the string starts with a /
- encodedurl = 'http://%s/w/thumb_handler.php/%s' %
(hostname, match.group('path'))
+ # take off the first two parts of the path
+ # (eg /wikipedia/commons/); make sure the string starts
+ # with a /
+ encodedurl = 'http://%s/w/thumb_handler.php/%s' % (
+ hostname, match.group('path'))
# add in the X-Original-URI with the swift got (minus the
hostname)
- opener.addheaders.append(('X-Original-URI',
list(urlparse.urlsplit(reqorig.url))[2]))
+ opener.addheaders.append(
+ ('X-Original-URI',
list(urlparse.urlsplit(reqorig.url))[2]))
else:
- # ASSERT this code should never be hit since only thumbs
should call the 404 handler
+ # ASSERT this code should never be hit since only thumbs
+ # should call the 404 handler
self.logger.warn("non-thumb in 404 handler! encodedurl =
%s" % encodedurl)
resp = webob.exc.HTTPNotFound('Unexpected error')
return resp
else:
- # log the result of the match here to test and make sure it's
sane before enabling the config
- match =
re.match(r'^http://(?P<host>[^/]+)/(?P<proj>[^-/]+)/(?P<lang>[^/]+)/thumb/(?P<path>.+)',
encodedurl)
+ # log the result of the match here to test and make sure it's
+ # sane before enabling the config
+ match = re.match(
+
r'^http://(?P<host>[^/]+)/(?P<proj>[^-/]+)/(?P<lang>[^/]+)/thumb/(?P<path>.+)',
+ encodedurl)
if match:
proj = match.group('proj')
lang = match.group('lang')
- self.logger.warn("sitelang match has proj %s lang %s
encodedurl %s" % (proj, lang, encodedurl))
+ self.logger.warn(
+ "sitelang match has proj %s lang %s encodedurl %s" % (
+ proj, lang, encodedurl))
else:
self.logger.warn("no sitelang match on encodedurl: %s" %
encodedurl)
@@ -142,12 +157,6 @@
# get the Content-Type.
uinfo = upcopy.info()
c_t = uinfo.gettype()
- content_length = uinfo.getheader('Content-Length', None)
- # sometimes Last-Modified isn't present; use now() when that happens.
- try:
- last_modified = time.mktime(uinfo.getdate('Last-Modified'))
- except TypeError:
- last_modified = time.mktime(time.localtime())
resp = webob.Response(app_iter=upcopy, content_type=c_t)
# add in the headers if we've got them
@@ -163,7 +172,7 @@
def handle_request(self, env, start_response):
try:
return self._handle_request(env, start_response)
- except UnicodeDecodeError as e:
+ except UnicodeDecodeError:
self.logger.exception('Failed to decode request %r', env)
resp = webob.exc.HTTPBadRequest('Failed to decode request')
return resp(env, start_response)
@@ -171,7 +180,8 @@
def _handle_request(self, env, start_response):
req = webob.Request(env)
- # Double (or triple, etc.) slashes in the URL should be ignored;
collapse them. fixes T34864
+ # Double (or triple, etc.) slashes in the URL should be ignored;
+ # collapse them. fixes T34864
req.path_info = re.sub(r'/{2,}', '/', req.path_info)
# Keep a copy of the original request so we can ask the scalers for it
@@ -216,7 +226,11 @@
# =>
http://msfe/v1/AUTH_<hash>/<proj>-<lang>-timeline-render/<relpath>
# regular uploads
- match =
re.match(r'^/(?P<proj>[^/]+)/(?P<lang>[^/]+)/((?P<zone>transcoded|thumb)/)?(?P<path>((temp|archive)/)?[0-9a-f]/(?P<shard>[0-9a-f]{2})/.+)$',
req.path)
+ match = re.match(
+ (r'^/(?P<proj>[^/]+)/(?P<lang>[^/]+)/'
+ r'((?P<zone>transcoded|thumb)/)?'
+
r'(?P<path>((temp|archive)/)?[0-9a-f]/(?P<shard>[0-9a-f]{2})/.+)$'),
+ req.path)
if match:
proj = match.group('proj')
lang = match.group('lang')
@@ -230,7 +244,9 @@
# timeline renderings
if match is None:
# /wikipedia/en/timeline/a876297c277d80dfd826e1f23dbfea3f.png
- match =
re.match(r'^/(?P<proj>[^/]+)/(?P<lang>[^/]+)/(?P<repo>timeline)/(?P<path>.+)$',
req.path)
+ match = re.match(
+
r'^/(?P<proj>[^/]+)/(?P<lang>[^/]+)/(?P<repo>timeline)/(?P<path>.+)$',
+ req.path)
if match:
proj = match.group('proj') # wikipedia
lang = match.group('lang') # en
@@ -243,7 +259,10 @@
if match is None:
# /math/c/9/f/c9f2055dadfb49853eff822a453d9ceb.png
# /wikipedia/en/math/c/9/f/c9f2055dadfb49853eff822a453d9ceb.png
(legacy)
- match =
re.match(r'^(/(?P<proj>[^/]+)/(?P<lang>[^/]+))?/(?P<repo>math)/(?P<path>(?P<shard1>[0-9a-f])/(?P<shard2>[0-9a-f])/.+)$',
req.path)
+ match = re.match(
+ (r'^(/(?P<proj>[^/]+)/(?P<lang>[^/]+))?/(?P<repo>math)/'
+ r'(?P<path>(?P<shard1>[0-9a-f])/(?P<shard2>[0-9a-f])/.+)$'),
+ req.path)
if match:
proj = 'global'
@@ -321,7 +340,7 @@
url = req.url[:]
# Create a path to our object's name.
req.path_info = "/v1/%s/%s/%s" % (self.account, container,
urllib2.unquote(obj))
- #self.logger.warn("new path is %s" % req.path_info)
+ # self.logger.warn("new path is %s" % req.path_info)
# do_start_response just remembers what it got called with,
# because our 404 handler will generate a different response.
@@ -331,7 +350,7 @@
if 200 <= status < 300 or status == 304:
# We have it! Just return it as usual.
- #headers['X-Swift-Proxy']= `headers`
+ # headers['X-Swift-Proxy']= `headers`
return webob.Response(status=status, headers=headers,
app_iter=app_iter)(env, start_response)
elif status == 404:
@@ -355,6 +374,7 @@
class WMFRewrite(object):
+
def __init__(self, app, conf):
self.app = app
self.conf = conf
diff --git a/modules/swift/files/SwiftMedia/wmf/rewrite_integration_test.py
b/modules/swift/files/SwiftMedia/wmf/rewrite_integration_test.py
index e5e5546..0ccfd55 100644
--- a/modules/swift/files/SwiftMedia/wmf/rewrite_integration_test.py
+++ b/modules/swift/files/SwiftMedia/wmf/rewrite_integration_test.py
@@ -9,8 +9,6 @@
# * an existing original image
import unittest
-import urllib
-
import requests
@@ -26,7 +24,9 @@
original_path_swift =
'/v1/AUTH_mw/wikipedia-commons-local-public.02/Berlin_2014_077.JPG'
original_size = 1433230
thumb_path =
'/wikipedia/commons/thumb/0/02/Berlin_2014_077.JPG/666px-Berlin_2014_077.JPG'
- thumb_path_swift =
'/v1/AUTH_mw/wikipedia-commons-local-thumb.02/0/02/Berlin_2014_077.JPG/666px-Berlin_2014_077.JPG'
+ thumb_path_swift = (
+ '/v1/AUTH_mw/wikipedia-commons-local-thumb.02/0/02/'
+ 'Berlin_2014_077.JPG/666px-Berlin_2014_077.JPG')
thumb_size = 70195
def test_remapping_originals(self):
@@ -61,7 +61,9 @@
self.assertEquals(r.status_code, 404)
def test_nonexist_thumb(self):
- r =
requests.get('http://localhost/wikipedia/commons/thumb/non_existant',
headers=self.headers)
+ r = requests.get(
+ 'http://localhost/wikipedia/commons/thumb/non_existant',
+ headers=self.headers)
self.assertEquals(r.status_code, 404)
diff --git a/modules/swift/files/swift-account-stats
b/modules/swift/files/swift-account-stats
index bb556fa..14d8526 100755
--- a/modules/swift/files/swift-account-stats
+++ b/modules/swift/files/swift-account-stats
@@ -19,23 +19,23 @@
def main():
parser = argparse.ArgumentParser(description="Print swift account
statistics")
parser.add_argument('-A', '--auth', dest='auth',
- default=os.environ.get('ST_AUTH', None),
- help='URL for obtaining an auth token')
+ default=os.environ.get('ST_AUTH', None),
+ help='URL for obtaining an auth token')
parser.add_argument('-U', '--user', dest='user',
- default=os.environ.get('ST_USER', None),
- help='User name for obtaining an auth token')
+ default=os.environ.get('ST_USER', None),
+ help='User name for obtaining an auth token')
parser.add_argument('-K', '--key', dest='key',
- default=os.environ.get('ST_KEY', None),
- help='Key for obtaining an auth token')
+ default=os.environ.get('ST_KEY', None),
+ help='Key for obtaining an auth token')
parser.add_argument('--prefix', dest='prefix',
- default='',
- help='Prefix to use with the metrics')
+ default='',
+ help='Prefix to use with the metrics')
parser.add_argument('--statsd-host', dest='statsd_host',
- default='', metavar="HOST",
- help='Send metrics to this statsd host as well')
+ default='', metavar="HOST",
+ help='Send metrics to this statsd host as well')
parser.add_argument('--statsd-port', dest='statsd_port',
- default='8125', metavar="PORT", type=int,
- help='Send metrics to this statsd port')
+ default='8125', metavar="PORT", type=int,
+ help='Send metrics to this statsd port')
args = parser.parse_args()
if None in (args.auth, args.user, args.key):
diff --git a/modules/swift/files/swift-container-stats
b/modules/swift/files/swift-container-stats
index fcbbd60..29a7d4a 100755
--- a/modules/swift/files/swift-container-stats
+++ b/modules/swift/files/swift-container-stats
@@ -42,29 +42,29 @@
def main():
parser = argparse.ArgumentParser(description="Print swift container
statistics")
parser.add_argument('-A', '--auth', dest='auth',
- default=os.environ.get('ST_AUTH', None),
- help='URL for obtaining an auth token')
+ default=os.environ.get('ST_AUTH', None),
+ help='URL for obtaining an auth token')
parser.add_argument('-U', '--user', dest='user',
- default=os.environ.get('ST_USER', None),
- help='User name for obtaining an auth token')
+ default=os.environ.get('ST_USER', None),
+ help='User name for obtaining an auth token')
parser.add_argument('-K', '--key', dest='key',
- default=os.environ.get('ST_KEY', None),
- help='Key for obtaining an auth token')
+ default=os.environ.get('ST_KEY', None),
+ help='Key for obtaining an auth token')
parser.add_argument('--prefix', dest='prefix',
- default='',
- help='Prefix to use when reporting metrics')
+ default='',
+ help='Prefix to use when reporting metrics')
parser.add_argument('--container-set', dest='container_set',
- default='', choices=CONTAINER_SETS.keys(),
- help='Report aggregated container statistics from this set')
+ default='', choices=CONTAINER_SETS.keys(),
+ help='Report aggregated container statistics from this
set')
parser.add_argument('--ignore-unknown', dest='ignore_unknown',
- default=False, action='store_true',
- help='Do not report unknown containers')
+ default=False, action='store_true',
+ help='Do not report unknown containers')
parser.add_argument('--statsd-host', dest='statsd_host',
- default='', metavar="HOST",
- help='Send metrics to this statsd host as well')
+ default='', metavar="HOST",
+ help='Send metrics to this statsd host as well')
parser.add_argument('--statsd-port', dest='statsd_port',
- default='8125', metavar="PORT", type=int,
- help='Send metrics to this statsd port')
+ default='8125', metavar="PORT", type=int,
+ help='Send metrics to this statsd port')
args = parser.parse_args()
if None in (args.auth, args.user, args.key):
@@ -89,7 +89,7 @@
continue
bucket_stats = container_stats.setdefault(
- bucket, {'bytes': 0, 'objects': 0})
+ bucket, {'bytes': 0, 'objects': 0})
bucket_stats['bytes'] += container['bytes']
bucket_stats['objects'] += container['count']
--
To view, visit https://gerrit.wikimedia.org/r/291173
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I3015e27fa9801eaf6d4f91588e5c459e54e3b90a
Gerrit-PatchSet: 4
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: BryanDavis <[email protected]>
Gerrit-Reviewer: Andrew Bogott <[email protected]>
Gerrit-Reviewer: Filippo Giunchedi <[email protected]>
Gerrit-Reviewer: Hashar <[email protected]>
Gerrit-Reviewer: Ori.livneh <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
