Hello community, here is the log from the commit of package openstack-swift for openSUSE:Factory checked in at 2013-09-27 17:55:35 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/openstack-swift (Old) and /work/SRC/openSUSE:Factory/.openstack-swift.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "openstack-swift" Changes: -------- --- /work/SRC/openSUSE:Factory/openstack-swift/openstack-swift.changes 2013-09-25 14:33:34.000000000 +0200 +++ /work/SRC/openSUSE:Factory/.openstack-swift.new/openstack-swift.changes 2013-09-27 17:55:36.000000000 +0200 @@ -1,0 +2,14 @@ +Thu Sep 26 16:46:22 UTC 2013 - [email protected] + +- Update to version 1.9.2.175.gc1f9f66+git.1380213982.c1f9f66: + + update SLO delete error handling + + Update SAIO doc to have double proxy-logging in pipeline. + + Fix internal swift.source tracking. + +-------------------------------------------------------------------- +Wed Sep 25 00:00:12 UTC 2013 - [email protected] + +- Update to version 1.9.2.170.gbb3f965+git.1380067212.bb3f965: + + Log x-copy-from when it could be useful + +-------------------------------------------------------------------- ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ openstack-swift-doc.spec ++++++ --- /var/tmp/diff_new_pack.go70Ej/_old 2013-09-27 17:55:37.000000000 +0200 +++ /var/tmp/diff_new_pack.go70Ej/_new 2013-09-27 17:55:37.000000000 +0200 @@ -19,7 +19,7 @@ %define component swift Name: openstack-%{component}-doc -Version: 1.9.2.168.g10bb74a+git.1379722790.10bb74a +Version: 1.9.2.175.gc1f9f66+git.1380213982.c1f9f66 Release: 0 Summary: OpenStack Storage (Swift) - Documentation License: Apache-2.0 @@ -47,7 +47,7 @@ This package contains documentation files for openstack-swift. %prep -%setup -q -n swift-1.9.2.168.g10bb74a +%setup -q -n swift-1.9.2.175.gc1f9f66 sed -i "s/\r//" LICENSE # Fix wrong-file-end-of-line-encoding warning %openstack_cleanup_prep ++++++ openstack-swift.spec ++++++ --- /var/tmp/diff_new_pack.go70Ej/_old 2013-09-27 17:55:37.000000000 +0200 +++ /var/tmp/diff_new_pack.go70Ej/_new 2013-09-27 17:55:37.000000000 +0200 @@ -21,7 +21,7 @@ %define username openstack-%{component} Name: openstack-%{component} -Version: 1.9.2.168.g10bb74a+git.1379722790.10bb74a +Version: 1.9.2.175.gc1f9f66+git.1380213982.c1f9f66 Release: 0 Summary: OpenStack Storage (Swift) License: Apache-2.0 @@ -171,7 +171,7 @@ of OpenStack Swift. %prep -%setup -q -n swift-1.9.2.168.g10bb74a +%setup -q -n swift-1.9.2.175.gc1f9f66 sed -i "s/\r//" LICENSE # Fix wrong-file-end-of-line-encoding warning tar xvf %{SOURCE21} %openstack_cleanup_prep ++++++ swift-master.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/swift-1.9.2.168.g10bb74a/PKG-INFO new/swift-1.9.2.175.gc1f9f66/PKG-INFO --- old/swift-1.9.2.168.g10bb74a/PKG-INFO 2013-09-20 21:19:28.000000000 +0200 +++ new/swift-1.9.2.175.gc1f9f66/PKG-INFO 2013-09-26 02:29:46.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: swift -Version: 1.9.2.168.g10bb74a +Version: 1.9.2.175.gc1f9f66 Summary: OpenStack Object Storage Home-page: http://www.openstack.org/ Author: OpenStack diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/swift-1.9.2.168.g10bb74a/doc/source/development_saio.rst new/swift-1.9.2.175.gc1f9f66/doc/source/development_saio.rst --- old/swift-1.9.2.168.g10bb74a/doc/source/development_saio.rst 2013-09-20 21:18:41.000000000 +0200 +++ new/swift-1.9.2.175.gc1f9f66/doc/source/development_saio.rst 2013-09-26 02:29:13.000000000 +0200 @@ -306,7 +306,8 @@ eventlet_debug = true [pipeline:main] - pipeline = healthcheck cache tempauth proxy-logging proxy-server + # Yes, proxy-logging appears twice. This is not a mistake. + pipeline = healthcheck proxy-logging cache tempauth proxy-logging proxy-server [app:proxy-server] use = egg:swift#proxy diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/swift-1.9.2.168.g10bb74a/swift/common/middleware/bulk.py new/swift-1.9.2.175.gc1f9f66/swift/common/middleware/bulk.py --- old/swift-1.9.2.168.g10bb74a/swift/common/middleware/bulk.py 2013-09-20 21:18:44.000000000 +0200 +++ new/swift-1.9.2.175.gc1f9f66/swift/common/middleware/bulk.py 2013-09-26 02:29:13.000000000 +0200 @@ -190,6 +190,8 @@ conf.get('max_containers_per_extraction', 10000)) self.max_failed_extractions = int( conf.get('max_failed_extractions', 1000)) + self.max_failed_deletes = int( + conf.get('max_failed_deletes', 1000)) self.max_deletes_per_request = int( conf.get('max_deletes_per_request', 10000)) self.yield_frequency = int(conf.get('yield_frequency', 60)) @@ -239,7 +241,8 @@ while data_remaining: if '\n' in line: obj_to_delete, line = line.split('\n', 1) - objs_to_delete.append(unquote(obj_to_delete)) + objs_to_delete.append( + {'name': unquote(obj_to_delete)}) else: data = req.body_file.read(MAX_PATH_LENGTH) if data: @@ -247,7 +250,8 @@ else: data_remaining = False if line.strip(): - objs_to_delete.append(unquote(line)) + objs_to_delete.append( + {'name': unquote(line)}) if len(objs_to_delete) > self.max_deletes_per_request: raise HTTPRequestEntityTooLarge( 'Maximum Bulk Deletes: %d per request' % @@ -304,13 +308,22 @@ separator = '\r\n\r\n' last_yield = time() yield ' ' - obj_to_delete = obj_to_delete.strip() - if not obj_to_delete: + obj_name = obj_to_delete['name'].strip() + if not obj_name: + continue + if len(failed_files) >= self.max_failed_deletes: + raise HTTPBadRequest('Max delete failures exceeded') + if obj_to_delete.get('error'): + if obj_to_delete['error']['code'] == HTTP_NOT_FOUND: + resp_dict['Number Not Found'] += 1 + else: + failed_files.append([quote(obj_name), + obj_to_delete['error']['message']]) continue delete_path = '/'.join(['', vrs, account, - obj_to_delete.lstrip('/')]) + obj_name.lstrip('/')]) if not check_utf8(delete_path): - failed_files.append([quote(obj_to_delete), + failed_files.append([quote(obj_name), HTTPPreconditionFailed().status]) continue new_env = req.environ.copy() @@ -327,13 +340,12 @@ elif resp.status_int == HTTP_NOT_FOUND: resp_dict['Number Not Found'] += 1 elif resp.status_int == HTTP_UNAUTHORIZED: - failed_files.append([quote(obj_to_delete), + failed_files.append([quote(obj_name), HTTPUnauthorized().status]) - raise HTTPUnauthorized(request=req) else: if resp.status_int // 100 == 5: failed_file_response_type = HTTPBadGateway - failed_files.append([quote(obj_to_delete), resp.status]) + failed_files.append([quote(obj_name), resp.status]) if failed_files: resp_dict['Response Status'] = \ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/swift-1.9.2.168.g10bb74a/swift/common/middleware/slo.py new/swift-1.9.2.175.gc1f9f66/swift/common/middleware/slo.py --- old/swift-1.9.2.168.g10bb74a/swift/common/middleware/slo.py 2013-09-20 21:18:44.000000000 +0200 +++ new/swift-1.9.2.175.gc1f9f66/swift/common/middleware/slo.py 2013-09-26 02:29:13.000000000 +0200 @@ -141,10 +141,11 @@ from hashlib import md5 from swift.common.swob import Request, HTTPBadRequest, HTTPServerError, \ HTTPMethodNotAllowed, HTTPRequestEntityTooLarge, HTTPLengthRequired, \ - HTTPOk, HTTPPreconditionFailed, HTTPException + HTTPOk, HTTPPreconditionFailed, HTTPException, HTTPNotFound, \ + HTTPUnauthorized from swift.common.utils import json, get_logger, config_true_value from swift.common.constraints import check_utf8, MAX_BUFFERED_SLO_SEGMENTS -from swift.common.http import HTTP_NOT_FOUND +from swift.common.http import HTTP_NOT_FOUND, HTTP_UNAUTHORIZED from swift.common.wsgi import WSGIContext from swift.common.middleware.bulk import get_response_body, \ ACCEPTABLE_FORMATS, Bulk @@ -216,8 +217,7 @@ 1024 * 1024 * 2)) self.min_segment_size = int(self.conf.get('min_segment_size', 1024 * 1024)) - self.bulk_deleter = Bulk( - app, {'max_deletes_per_request': self.max_manifest_segments}) + self.bulk_deleter = Bulk(app, {}) def handle_multipart_put(self, req, start_response): """ @@ -333,66 +333,91 @@ A generator function to be used to delete all the segments and sub-segments referenced in a manifest. - :raises HTTPBadRequest: on sub manifest not manifest anymore or - on too many buffered sub segments - :raises HTTPServerError: on unable to load manifest + :params req: a swob.Request with an SLO manifest in path + :raises HTTPPreconditionFailed: on invalid UTF8 in request path + :raises HTTPBadRequest: on too many buffered sub segments and + on invalid SLO manifest path """ + if not check_utf8(req.path_info): + raise HTTPPreconditionFailed( + request=req, body='Invalid UTF8 or contains NULL') try: vrs, account, container, obj = req.split_path(4, 4, True) except ValueError: - raise HTTPBadRequest('Not a SLO manifest') - sub_segments = [{ + raise HTTPBadRequest('Invalid SLO manifiest path') + + segments = [{ 'sub_slo': True, 'name': ('/%s/%s' % (container, obj)).decode('utf-8')}] - while sub_segments: - if len(sub_segments) > MAX_BUFFERED_SLO_SEGMENTS: + while segments: + if len(segments) > MAX_BUFFERED_SLO_SEGMENTS: raise HTTPBadRequest( 'Too many buffered slo segments to delete.') - seg_data = sub_segments.pop(0) + seg_data = segments.pop(0) if seg_data.get('sub_slo'): - new_env = req.environ.copy() - new_env['REQUEST_METHOD'] = 'GET' - del(new_env['wsgi.input']) - new_env['QUERY_STRING'] = 'multipart-manifest=get' - new_env['CONTENT_LENGTH'] = 0 - new_env['HTTP_USER_AGENT'] = \ - '%s MultipartDELETE' % new_env.get('HTTP_USER_AGENT') - new_env['swift.source'] = 'SLO' - new_env['PATH_INFO'] = ( - '/%s/%s/%s' % ( - vrs, account, - seg_data['name'].lstrip('/'))).encode('utf-8') - sub_resp = Request.blank('', new_env).get_response(self.app) - if sub_resp.is_success: - try: - # if its still a SLO, load its segments - if config_true_value( - sub_resp.headers.get('X-Static-Large-Object')): - sub_segments.extend(json.loads(sub_resp.body)) - except ValueError: - raise HTTPServerError('Unable to load SLO manifest') - # add sub-manifest back to be deleted after sub segments - # (even if obj is not a SLO) - seg_data['sub_slo'] = False - sub_segments.append(seg_data) - elif sub_resp.status_int != HTTP_NOT_FOUND: - # on deletes treat not found as success - raise HTTPServerError('Sub SLO unable to load.') + try: + segments.extend( + self.get_slo_segments(seg_data['name'], req)) + except HTTPException as err: + # allow bulk delete response to report errors + seg_data['error'] = {'code': err.status_int, + 'message': err.body} + + # add manifest back to be deleted after segments + seg_data['sub_slo'] = False + segments.append(seg_data) + else: + seg_data['name'] = seg_data['name'].encode('utf-8') + yield seg_data + + def get_slo_segments(self, obj_name, req): + """ + Performs a swob.Request and returns the SLO manifest's segments. + + :raises HTTPServerError: on unable to load obj_name or + on unable to load the SLO manifest data. + :raises HTTPBadRequest: on not an SLO manifest + :raises HTTPNotFound: on SLO manifest not found + :returns: SLO manifest's segments + """ + vrs, account, _junk = req.split_path(2, 3, True) + new_env = req.environ.copy() + new_env['REQUEST_METHOD'] = 'GET' + del(new_env['wsgi.input']) + new_env['QUERY_STRING'] = 'multipart-manifest=get' + new_env['CONTENT_LENGTH'] = 0 + new_env['HTTP_USER_AGENT'] = \ + '%s MultipartDELETE' % new_env.get('HTTP_USER_AGENT') + new_env['swift.source'] = 'SLO' + new_env['PATH_INFO'] = ( + '/%s/%s/%s' % ( + vrs, account, + obj_name.lstrip('/'))).encode('utf-8') + resp = Request.blank('', new_env).get_response(self.app) + + if resp.is_success: + if config_true_value(resp.headers.get('X-Static-Large-Object')): + try: + return json.loads(resp.body) + except ValueError: + raise HTTPServerError('Unable to load SLO manifest') else: - yield seg_data['name'].encode('utf-8') + raise HTTPBadRequest('Not an SLO manifest') + elif resp.status_int == HTTP_NOT_FOUND: + raise HTTPNotFound('SLO manifest not found') + elif resp.status_int == HTTP_UNAUTHORIZED: + raise HTTPUnauthorized('401 Unauthorized') + else: + raise HTTPServerError('Unable to load SLO manifest or segment.') def handle_multipart_delete(self, req): """ Will delete all the segments in the SLO manifest and then, if successful, will delete the manifest file. + :params req: a swob.Request with an obj in path - :raises HTTPServerError: on invalid manifest :returns: swob.Response whose app_iter set to Bulk.handle_delete_iter """ - if not check_utf8(req.path_info): - raise HTTPPreconditionFailed( - request=req, body='Invalid UTF8 or contains NULL') - resp = HTTPOk(request=req) out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS) if out_content_type: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/swift-1.9.2.168.g10bb74a/swift/proxy/controllers/base.py new/swift-1.9.2.175.gc1f9f66/swift/proxy/controllers/base.py --- old/swift-1.9.2.168.g10bb74a/swift/proxy/controllers/base.py 2013-09-20 21:18:44.000000000 +0200 +++ new/swift-1.9.2.175.gc1f9f66/swift/proxy/controllers/base.py 2013-09-26 02:29:13.000000000 +0200 @@ -238,7 +238,8 @@ """ (version, account, container, obj) = \ split_path(path or env['PATH_INFO'], 4, 4, True) - info = _get_object_info(app, env, account, container, obj) + info = _get_object_info(app, env, account, container, obj, + swift_source=swift_source) if not info: info = headers_to_object_info({}, 0) return info @@ -253,7 +254,8 @@ """ (version, account, container, unused) = \ split_path(env['PATH_INFO'], 3, 4, True) - info = get_info(app, env, account, container, ret_not_found=True) + info = get_info(app, env, account, container, ret_not_found=True, + swift_source=swift_source) if not info: info = headers_to_container_info({}, 0) return info @@ -268,7 +270,8 @@ """ (version, account, _junk, _junk) = \ split_path(env['PATH_INFO'], 2, 4, True) - info = get_info(app, env, account, ret_not_found=True) + info = get_info(app, env, account, ret_not_found=True, + swift_source=swift_source) if not info: info = headers_to_account_info({}, 0) if info.get('container_count') is None: @@ -421,22 +424,24 @@ return None -def _prepare_pre_auth_info_request(env, path): +def _prepare_pre_auth_info_request(env, path, swift_source): """ Prepares a pre authed request to obtain info using a HEAD. :param env: the environment used by the current request :param path: The unquoted request path + :param swift_source: value for swift.source in WSGI environment :returns: the pre authed request """ # Set the env for the pre_authed call without a query string newenv = make_pre_authed_env(env, 'HEAD', path, agent='Swift', - query_string='', swift_source='GET_INFO') + query_string='', swift_source=swift_source) # Note that Request.blank expects quoted path return Request.blank(quote(path), environ=newenv) -def get_info(app, env, account, container=None, ret_not_found=False): +def get_info(app, env, account, container=None, ret_not_found=False, + swift_source=None): """ Get the info about accounts or containers @@ -462,7 +467,8 @@ return None path += '/' + container - req = _prepare_pre_auth_info_request(env, path) + req = _prepare_pre_auth_info_request( + env, path, (swift_source or 'GET_INFO')) # Whenever we do a GET/HEAD, the GETorHEAD_base will set the info in # the environment under environ[env_key] and in memcache. We will # pick the one from environ[env_key] and use it to set the caller env @@ -478,7 +484,7 @@ return None -def _get_object_info(app, env, account, container, obj): +def _get_object_info(app, env, account, container, obj, swift_source=None): """ Get the info about object @@ -498,7 +504,7 @@ return info # Not in cached, let's try the object servers path = '/v1/%s/%s/%s' % (account, container, obj) - req = _prepare_pre_auth_info_request(env, path) + req = _prepare_pre_auth_info_request(env, path, swift_source) # Whenever we do a GET/HEAD, the GETorHEAD_base will set the info in # the environment under environ[env_key]. We will # pick the one from environ[env_key] and use it to set the caller env diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/swift-1.9.2.168.g10bb74a/swift/proxy/controllers/obj.py new/swift-1.9.2.175.gc1f9f66/swift/proxy/controllers/obj.py --- old/swift-1.9.2.168.g10bb74a/swift/proxy/controllers/obj.py 2013-09-20 21:18:44.000000000 +0200 +++ new/swift-1.9.2.175.gc1f9f66/swift/proxy/controllers/obj.py 2013-09-26 02:29:13.000000000 +0200 @@ -965,6 +965,9 @@ source_header = req.headers.get('X-Copy-From') source_resp = None if source_header: + if req.environ.get('swift.orig_req_method', req.method) != 'POST': + req.environ.setdefault('swift.log_info', []).append( + 'x-copy-from:%s' % source_header) source_header = unquote(source_header) acct = req.path_info.split('/', 2)[1] if isinstance(acct, unicode): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/swift-1.9.2.168.g10bb74a/swift.egg-info/PKG-INFO new/swift-1.9.2.175.gc1f9f66/swift.egg-info/PKG-INFO --- old/swift-1.9.2.168.g10bb74a/swift.egg-info/PKG-INFO 2013-09-20 21:19:28.000000000 +0200 +++ new/swift-1.9.2.175.gc1f9f66/swift.egg-info/PKG-INFO 2013-09-26 02:29:46.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: swift -Version: 1.9.2.168.g10bb74a +Version: 1.9.2.175.gc1f9f66 Summary: OpenStack Object Storage Home-page: http://www.openstack.org/ Author: OpenStack diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/swift-1.9.2.168.g10bb74a/test/unit/common/middleware/test_bulk.py new/swift-1.9.2.175.gc1f9f66/test/unit/common/middleware/test_bulk.py --- old/swift-1.9.2.168.g10bb74a/test/unit/common/middleware/test_bulk.py 2013-09-20 21:18:44.000000000 +0200 +++ new/swift-1.9.2.175.gc1f9f66/test/unit/common/middleware/test_bulk.py 2013-09-26 02:29:13.000000000 +0200 @@ -23,6 +23,7 @@ from mock import patch from swift.common.middleware import bulk from swift.common.swob import Request, Response, HTTPException +from swift.common.http import HTTP_NOT_FOUND, HTTP_UNAUTHORIZED from swift.common.utils import json @@ -35,6 +36,8 @@ def __call__(self, env, start_response): self.calls += 1 if env['PATH_INFO'].startswith('/unauth/'): + if env['PATH_INFO'].endswith('/c/f_ok'): + return Response(status='204 No Content')(env, start_response) return Response(status=401)(env, start_response) if env['PATH_INFO'].startswith('/create_cont/'): if env['REQUEST_METHOD'] == 'HEAD': @@ -493,6 +496,29 @@ req, out_content_type=out_content_type)) return resp_body + def test_bulk_delete_uses_predefined_object_errors(self): + req = Request.blank('/delete_works/AUTH_Acc') + objs_to_delete = [ + {'name': '/c/file_a'}, + {'name': '/c/file_b', 'error': {'code': HTTP_NOT_FOUND, + 'message': 'not found'}}, + {'name': '/c/file_c', 'error': {'code': HTTP_UNAUTHORIZED, + 'message': 'unauthorized'}}, + {'name': '/c/file_d'}] + resp_body = ''.join(self.bulk.handle_delete_iter( + req, objs_to_delete=objs_to_delete, + out_content_type='application/json')) + self.assertEquals( + self.app.delete_paths, ['/delete_works/AUTH_Acc/c/file_a', + '/delete_works/AUTH_Acc/c/file_d']) + self.assertEquals(self.app.calls, 2) + resp_data = json.loads(resp_body) + self.assertEquals(resp_data['Response Status'], '400 Bad Request') + self.assertEquals(resp_data['Number Deleted'], 2) + self.assertEquals(resp_data['Number Not Found'], 1) + self.assertEquals(resp_data['Errors'], + [['/c/file_c', 'unauthorized']]) + def test_bulk_delete_works(self): req = Request.blank('/delete_works/AUTH_Acc', body='/c/f\n/c/f404', headers={'Accept': 'application/json'}) @@ -538,13 +564,14 @@ req.method = 'DELETE' with patch.object(self.bulk, 'max_deletes_per_request', 2): results = self.bulk.get_objs_to_delete(req) - self.assertEquals(results, ['1\r', '2\r']) + self.assertEquals(results, [{'name': '1\r'}, {'name': '2\r'}]) with patch.object(bulk, 'MAX_PATH_LENGTH', 2): results = [] req.environ['wsgi.input'] = StringIO('1\n2\n3') results = self.bulk.get_objs_to_delete(req) - self.assertEquals(results, ['1', '2', '3']) + self.assertEquals(results, + [{'name': '1'}, {'name': '2'}, {'name': '3'}]) with patch.object(self.bulk, 'max_deletes_per_request', 9): with patch.object(bulk, 'MAX_PATH_LENGTH', 1): @@ -611,14 +638,15 @@ self.assertTrue('400 Bad Request' in resp_body) def test_bulk_delete_unauth(self): - req = Request.blank('/unauth/AUTH_acc/', body='/c/f\n/c/f2\n', + req = Request.blank('/unauth/AUTH_acc/', body='/c/f\n/c/f_ok\n', headers={'Accept': 'application/json'}) req.method = 'DELETE' resp_body = self.handle_delete_and_iter(req) - self.assertEquals(self.app.calls, 1) + self.assertEquals(self.app.calls, 2) resp_data = json.loads(resp_body) self.assertEquals(resp_data['Errors'], [['/c/f', '401 Unauthorized']]) - self.assertEquals(resp_data['Response Status'], '401 Unauthorized') + self.assertEquals(resp_data['Response Status'], '400 Bad Request') + self.assertEquals(resp_data['Number Deleted'], 1) def test_bulk_delete_500_resp(self): req = Request.blank('/broke/AUTH_acc/', body='/c/f\nc/f2\n', @@ -667,5 +695,21 @@ resp_body = self.handle_delete_and_iter(req) self.assertTrue('400 Bad Request' in resp_body) + def test_bulk_delete_max_failures(self): + req = Request.blank('/unauth/AUTH_Acc', body='/c/f1\n/c/f2\n/c/f3', + headers={'Accept': 'application/json'}) + req.method = 'DELETE' + with patch.object(self.bulk, 'max_failed_deletes', 2): + resp_body = self.handle_delete_and_iter(req) + self.assertEquals(self.app.calls, 2) + resp_data = json.loads(resp_body) + self.assertEquals(resp_data['Response Status'], '400 Bad Request') + self.assertEquals(resp_data['Response Body'], + 'Max delete failures exceeded') + self.assertEquals(resp_data['Errors'], + [['/c/f1', '401 Unauthorized'], + ['/c/f2', '401 Unauthorized']]) + + if __name__ == '__main__': unittest.main() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/swift-1.9.2.168.g10bb74a/test/unit/common/middleware/test_slo.py new/swift-1.9.2.175.gc1f9f66/test/unit/common/middleware/test_slo.py --- old/swift-1.9.2.168.g10bb74a/test/unit/common/middleware/test_slo.py 2013-09-20 21:18:44.000000000 +0200 +++ new/swift-1.9.2.175.gc1f9f66/test/unit/common/middleware/test_slo.py 2013-09-26 02:29:13.000000000 +0200 @@ -81,9 +81,18 @@ return Response(status=200, body='lalala')(env, start_response) if env['PATH_INFO'].startswith('/test_delete_404/'): + good_data = json.dumps( + [{'name': '/c/a_1', 'hash': 'a', 'bytes': '1'}, + {'name': '/d/b_2', 'hash': 'b', 'bytes': '2'}]) self.req_method_paths.append((env['REQUEST_METHOD'], env['PATH_INFO'])) - return Response(status=404)(env, start_response) + if env['PATH_INFO'].endswith('/c/man_404'): + return Response(status=404)(env, start_response) + if env['PATH_INFO'].endswith('/c/a_1'): + return Response(status=404)(env, start_response) + return Response(status=200, + headers={'X-Static-Large-Object': 'True'}, + body=good_data)(env, start_response) if env['PATH_INFO'].startswith('/test_delete/'): good_data = json.dumps( @@ -115,6 +124,21 @@ headers={'X-Static-Large-Object': 'True'}, body=good_data)(env, start_response) + if env['PATH_INFO'].startswith('/test_delete_nested_404/'): + good_data = json.dumps( + [{'name': '/a/a_1', 'hash': 'a', 'bytes': '1'}, + {'name': '/a/sub_nest', 'hash': 'a', 'bytes': '2', + 'sub_slo': True}, + {'name': '/d/d_3', 'hash': 'b', 'bytes': '3'}]) + self.req_method_paths.append((env['REQUEST_METHOD'], + env['PATH_INFO'])) + if 'sub_nest' in env['PATH_INFO']: + return Response(status=404)(env, start_response) + else: + return Response(status=200, + headers={'X-Static-Large-Object': 'True'}, + body=good_data)(env, start_response) + if env['PATH_INFO'].startswith('/test_delete_bad_json/'): self.req_method_paths.append((env['REQUEST_METHOD'], env['PATH_INFO'])) @@ -127,13 +151,13 @@ env['PATH_INFO'])) return Response(status=200, body='')(env, start_response) - if env['PATH_INFO'].startswith('/test_delete_bad/'): + if env['PATH_INFO'].startswith('/test_delete_401/'): good_data = json.dumps( [{'name': '/c/a_1', 'hash': 'a', 'bytes': '1'}, {'name': '/d/b_2', 'hash': 'b', 'bytes': '2'}]) self.req_method_paths.append((env['REQUEST_METHOD'], env['PATH_INFO'])) - if env['PATH_INFO'].endswith('/c/a_1'): + if env['PATH_INFO'].endswith('/d/b_2'): return Response(status=401)(env, start_response) return Response(status=200, headers={'X-Static-Large-Object': 'True'}, @@ -368,13 +392,38 @@ def test_handle_multipart_delete_whole_404(self): req = Request.blank( - '/test_delete_404/A/c/man?multipart-manifest=delete', - environ={'REQUEST_METHOD': 'DELETE'}) + '/test_delete_404/A/c/man_404?multipart-manifest=delete', + environ={'REQUEST_METHOD': 'DELETE', + 'HTTP_ACCEPT': 'application/json'}) app_iter = self.slo(req.environ, fake_start_response) - list(app_iter) # iterate through whole response + app_iter = list(app_iter) # iterate through whole response + resp_data = json.loads(app_iter[0]) self.assertEquals(self.app.calls, 1) self.assertEquals(self.app.req_method_paths, - [('GET', '/test_delete_404/A/c/man')]) + [('GET', '/test_delete_404/A/c/man_404')]) + self.assertEquals(resp_data['Response Status'], '200 OK') + self.assertEquals(resp_data['Response Body'], '') + self.assertEquals(resp_data['Number Deleted'], 0) + self.assertEquals(resp_data['Number Not Found'], 1) + self.assertEquals(resp_data['Errors'], []) + + def test_handle_multipart_delete_segment_404(self): + req = Request.blank( + '/test_delete_404/A/c/man?multipart-manifest=delete', + environ={'REQUEST_METHOD': 'DELETE', + 'HTTP_ACCEPT': 'application/json'}) + app_iter = self.slo(req.environ, fake_start_response) + app_iter = list(app_iter) # iterate through whole response + resp_data = json.loads(app_iter[0]) + self.assertEquals(self.app.calls, 4) + self.assertEquals(self.app.req_method_paths, + [('GET', '/test_delete_404/A/c/man'), + ('DELETE', '/test_delete_404/A/c/a_1'), + ('DELETE', '/test_delete_404/A/d/b_2'), + ('DELETE', '/test_delete_404/A/c/man')]) + self.assertEquals(resp_data['Response Status'], '200 OK') + self.assertEquals(resp_data['Number Deleted'], 2) + self.assertEquals(resp_data['Number Not Found'], 1) def test_handle_multipart_delete_whole(self): req = Request.blank( @@ -407,9 +456,28 @@ ('DELETE', '/test_delete_nested/A/d/d_3'), ('DELETE', '/test_delete_nested/A/c/man')])) + def test_handle_multipart_delete_nested_404(self): + req = Request.blank( + '/test_delete_nested_404/A/c/man?multipart-manifest=delete', + environ={'REQUEST_METHOD': 'DELETE', + 'HTTP_ACCEPT': 'application/json'}) + app_iter = self.slo(req.environ, fake_start_response) + app_iter = list(app_iter) # iterate through whole response + resp_data = json.loads(app_iter[0]) + self.assertEquals(self.app.calls, 5) + self.assertEquals(self.app.req_method_paths, + [('GET', '/test_delete_nested_404/A/c/man'), + ('DELETE', '/test_delete_nested_404/A/a/a_1'), + ('GET', '/test_delete_nested_404/A/a/sub_nest'), + ('DELETE', '/test_delete_nested_404/A/d/d_3'), + ('DELETE', '/test_delete_nested_404/A/c/man')]) + self.assertEquals(resp_data['Response Status'], '200 OK') + self.assertEquals(resp_data['Response Body'], '') + self.assertEquals(resp_data['Number Deleted'], 3) + self.assertEquals(resp_data['Number Not Found'], 1) + self.assertEquals(resp_data['Errors'], []) + def test_handle_multipart_delete_not_a_manifest(self): - # when trying to delete a SLO and its not an SLO, just go ahead - # and delete it req = Request.blank( '/test_delete_bad_man/A/c/man?multipart-manifest=delete', environ={'REQUEST_METHOD': 'DELETE', @@ -417,11 +485,15 @@ app_iter = self.slo(req.environ, fake_start_response) app_iter = list(app_iter) # iterate through whole response resp_data = json.loads(app_iter[0]) - self.assertEquals(self.app.calls, 2) + self.assertEquals(self.app.calls, 1) self.assertEquals(self.app.req_method_paths, - [('GET', '/test_delete_bad_man/A/c/man'), - ('DELETE', '/test_delete_bad_man/A/c/man')]) - self.assertEquals(resp_data['Response Status'], '200 OK') + [('GET', '/test_delete_bad_man/A/c/man')]) + self.assertEquals(resp_data['Response Status'], '400 Bad Request') + self.assertEquals(resp_data['Response Body'], '') + self.assertEquals(resp_data['Number Deleted'], 0) + self.assertEquals(resp_data['Number Not Found'], 0) + self.assertEquals(resp_data['Errors'], + [['/c/man', 'Not an SLO manifest']]) def test_handle_multipart_delete_bad_json(self): req = Request.blank( @@ -434,18 +506,33 @@ self.assertEquals(self.app.calls, 1) self.assertEquals(self.app.req_method_paths, [('GET', '/test_delete_bad_json/A/c/man')]) - self.assertEquals(resp_data["Response Status"], "500 Internal Error") + self.assertEquals(resp_data['Response Status'], '400 Bad Request') + self.assertEquals(resp_data['Response Body'], '') + self.assertEquals(resp_data['Number Deleted'], 0) + self.assertEquals(resp_data['Number Not Found'], 0) + self.assertEquals(resp_data['Errors'], + [['/c/man', 'Unable to load SLO manifest']]) - def test_handle_multipart_delete_whole_bad(self): + def test_handle_multipart_delete_401(self): req = Request.blank( - '/test_delete_bad/A/c/man?multipart-manifest=delete', - environ={'REQUEST_METHOD': 'DELETE'}) + '/test_delete_401/A/c/man?multipart-manifest=delete', + environ={'REQUEST_METHOD': 'DELETE', + 'HTTP_ACCEPT': 'application/json'}) app_iter = self.slo(req.environ, fake_start_response) - list(app_iter) # iterate through whole response - self.assertEquals(self.app.calls, 2) + app_iter = list(app_iter) # iterate through whole response + resp_data = json.loads(app_iter[0]) + self.assertEquals(self.app.calls, 4) self.assertEquals(self.app.req_method_paths, - [('GET', '/test_delete_bad/A/c/man'), - ('DELETE', '/test_delete_bad/A/c/a_1')]) + [('GET', '/test_delete_401/A/c/man'), + ('DELETE', '/test_delete_401/A/c/a_1'), + ('DELETE', '/test_delete_401/A/d/b_2'), + ('DELETE', '/test_delete_401/A/c/man')]) + self.assertEquals(resp_data['Response Status'], '400 Bad Request') + self.assertEquals(resp_data['Response Body'], '') + self.assertEquals(resp_data['Number Deleted'], 2) + self.assertEquals(resp_data['Number Not Found'], 0) + self.assertEquals(resp_data['Errors'], + [['/d/b_2', '401 Unauthorized']]) if __name__ == '__main__': unittest.main() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/swift-1.9.2.168.g10bb74a/test/unit/proxy/controllers/test_base.py new/swift-1.9.2.175.gc1f9f66/test/unit/proxy/controllers/test_base.py --- old/swift-1.9.2.168.g10bb74a/test/unit/proxy/controllers/test_base.py 2013-09-20 21:18:44.000000000 +0200 +++ new/swift-1.9.2.175.gc1f9f66/test/unit/proxy/controllers/test_base.py 2013-09-26 02:29:13.000000000 +0200 @@ -48,19 +48,23 @@ class FakeRequest(object): - def __init__(self, env, path): + def __init__(self, env, path, swift_source=None): self.environ = env (version, account, container, obj) = split_path(path, 2, 4, True) self.account = account self.container = container self.obj = obj if obj: + stype = 'object' self.headers = {'content-length': 5555, 'content-type': 'text/plain'} else: stype = container and 'container' or 'account' self.headers = {'x-%s-object-count' % (stype): 1000, 'x-%s-bytes-used' % (stype): 6666} + if swift_source: + meta = 'x-%s-meta-fakerequest-swift-source' % stype + self.headers[meta] = swift_source def get_response(self, app): return FakeResponse(self.headers, self.environ, self.account, @@ -127,7 +131,7 @@ self.assertEquals(info_a['bytes'], 6666) self.assertEquals(info_a['total_object_count'], 1000) # Make sure the env cache is set - self.assertEquals(env, {'swift.account/a': info_a}) + self.assertEquals(env.get('swift.account/a'), info_a) # Do an env cached call to account info_a = get_info(None, env, 'a') @@ -136,7 +140,7 @@ self.assertEquals(info_a['bytes'], 6666) self.assertEquals(info_a['total_object_count'], 1000) # Make sure the env cache is set - self.assertEquals(env, {'swift.account/a': info_a}) + self.assertEquals(env.get('swift.account/a'), info_a) # This time do env cached call to account and non cached to container with patch('swift.proxy.controllers.base.' @@ -147,8 +151,8 @@ self.assertEquals(info_c['bytes'], 6666) self.assertEquals(info_c['object_count'], 1000) # Make sure the env cache is set - self.assertEquals(env['swift.account/a'], info_a) - self.assertEquals(env['swift.container/a/c'], info_c) + self.assertEquals(env.get('swift.account/a'), info_a) + self.assertEquals(env.get('swift.container/a/c'), info_c) # This time do a non cached call to account than non cached to # container @@ -161,8 +165,8 @@ self.assertEquals(info_c['bytes'], 6666) self.assertEquals(info_c['object_count'], 1000) # Make sure the env cache is set - self.assertEquals(env['swift.account/a'], info_a) - self.assertEquals(env['swift.container/a/c'], info_c) + self.assertEquals(env.get('swift.account/a'), info_a) + self.assertEquals(env.get('swift.container/a/c'), info_c) # This time do an env cached call to container while account is not # cached @@ -173,7 +177,7 @@ self.assertEquals(info_c['bytes'], 6666) self.assertEquals(info_c['object_count'], 1000) # Make sure the env cache is set and account still not cached - self.assertEquals(env, {'swift.container/a/c': info_c}) + self.assertEquals(env.get('swift.container/a/c'), info_c) # Do a non cached call to account not found with ret_not_found env = {} @@ -189,7 +193,7 @@ self.assertEquals(info_a['bytes'], 6666) self.assertEquals(info_a['total_object_count'], 1000) # Make sure the env cache is set - self.assertEquals(env, {'swift.account/a': info_a}) + self.assertEquals(env.get('swift.account/a'), info_a) # Do a cached call to account not found with ret_not_found info_a = get_info(None, env, 'a', ret_not_found=True) @@ -198,7 +202,7 @@ self.assertEquals(info_a['bytes'], 6666) self.assertEquals(info_a['total_object_count'], 1000) # Make sure the env cache is set - self.assertEquals(env, {'swift.account/a': info_a}) + self.assertEquals(env.get('swift.account/a'), info_a) # Do a non cached call to account not found without ret_not_found env = {} @@ -219,6 +223,21 @@ self.assertEquals(info_a, None) self.assertEquals(env['swift.account/a']['status'], 404) + def test_get_container_info_swift_source(self): + req = Request.blank("/v1/a/c", environ={'swift.cache': FakeCache({})}) + with patch('swift.proxy.controllers.base.' + '_prepare_pre_auth_info_request', FakeRequest): + resp = get_container_info(req.environ, 'app', swift_source='MC') + self.assertEquals(resp['meta']['fakerequest-swift-source'], 'MC') + + def test_get_object_info_swift_source(self): + req = Request.blank("/v1/a/c/o", + environ={'swift.cache': FakeCache({})}) + with patch('swift.proxy.controllers.base.' + '_prepare_pre_auth_info_request', FakeRequest): + resp = get_object_info(req.environ, 'app', swift_source='LU') + self.assertEquals(resp['meta']['fakerequest-swift-source'], 'LU') + def test_get_container_info_no_cache(self): req = Request.blank("/v1/AUTH_account/cont", environ={'swift.cache': FakeCache({})}) @@ -250,6 +269,13 @@ resp = get_container_info(req.environ, 'xxx') self.assertEquals(resp['bytes'], 3867) + def test_get_account_info_swift_source(self): + req = Request.blank("/v1/a", environ={'swift.cache': FakeCache({})}) + with patch('swift.proxy.controllers.base.' + '_prepare_pre_auth_info_request', FakeRequest): + resp = get_account_info(req.environ, 'a', swift_source='MC') + self.assertEquals(resp['meta']['fakerequest-swift-source'], 'MC') + def test_get_account_info_no_cache(self): req = Request.blank("/v1/AUTH_account", environ={'swift.cache': FakeCache({})}) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/swift-1.9.2.168.g10bb74a/test/unit/proxy/controllers/test_obj.py new/swift-1.9.2.175.gc1f9f66/test/unit/proxy/controllers/test_obj.py --- old/swift-1.9.2.168.g10bb74a/test/unit/proxy/controllers/test_obj.py 2013-09-20 21:18:44.000000000 +0200 +++ new/swift-1.9.2.175.gc1f9f66/test/unit/proxy/controllers/test_obj.py 2013-09-26 02:29:13.000000000 +0200 @@ -17,6 +17,8 @@ import unittest from contextlib import contextmanager +import mock + import swift from swift.proxy import server as proxy_server from test.unit import FakeRing, FakeMemcache, fake_http_connect @@ -85,5 +87,36 @@ res = controller._connect_put_node(nodes, '', '', {}, ('', '')) self.assertTrue(res is None) + +class TestObjController(unittest.TestCase): + + def test_PUT_log_info(self): + # mock out enough to get to the area of the code we want to test + with mock.patch('swift.proxy.controllers.obj.check_object_creation', + mock.MagicMock(return_value=None)): + app = mock.MagicMock() + app.container_ring.get_nodes.return_value = (1, [2]) + app.object_ring.get_nodes.return_value = (1, [2]) + controller = proxy_server.ObjectController(app, 'a', 'c', 'o') + controller.container_info = mock.MagicMock(return_value={ + 'partition': 1, + 'nodes': [{}], + 'write_acl': None, + 'sync_key': None, + 'versions': None}) + # and now test that we add the header to log_info + req = swift.common.swob.Request.blank('/v1/a/c/o') + req.headers['x-copy-from'] = 'somewhere' + controller.PUT(req) + self.assertEquals( + req.environ.get('swift.log_info'), ['x-copy-from:somewhere']) + # and then check that we don't do that for originating POSTs + req = swift.common.swob.Request.blank('/v1/a/c/o') + req.method = 'POST' + req.headers['x-copy-from'] = 'elsewhere' + controller.PUT(req) + self.assertEquals(req.environ.get('swift.log_info'), None) + + if __name__ == '__main__': unittest.main() ++++++ swift-test-configs-0.0.0.tar.bz2 ++++++ -- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
