Repository: libcloud Updated Branches: refs/heads/trunk 95338d89a -> 99d6bce74
Initial commit of W.I.P. implementation of Backblaze B2 storage driver. Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/03e52b83 Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/03e52b83 Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/03e52b83 Branch: refs/heads/trunk Commit: 03e52b83aa75d18e1e86802a8202d4ccfb4147c8 Parents: ae5b069 Author: Tomaz Muraus <[email protected]> Authored: Sat Dec 19 17:59:44 2015 +0100 Committer: Tomaz Muraus <[email protected]> Committed: Sun Jan 10 16:54:37 2016 +0100 ---------------------------------------------------------------------- libcloud/storage/drivers/backblaze_b2.py | 324 ++++++++++++++++++++++++++ libcloud/storage/providers.py | 2 + libcloud/storage/types.py | 1 + 3 files changed, 327 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/03e52b83/libcloud/storage/drivers/backblaze_b2.py ---------------------------------------------------------------------- diff --git a/libcloud/storage/drivers/backblaze_b2.py b/libcloud/storage/drivers/backblaze_b2.py new file mode 100644 index 0000000..2be6ffc --- /dev/null +++ b/libcloud/storage/drivers/backblaze_b2.py @@ -0,0 +1,324 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Driver for Backblaze B2 service. +""" + +import base64 + +try: + import simplejson as json +except ImportError: + import json + +from libcloud.utils.py3 import b +from libcloud.utils.py3 import httplib +from libcloud.utils.py3 import urlparse + +from libcloud.common.base import ConnectionUserAndKey +from libcloud.common.base import JsonResponse +from libcloud.common.types import InvalidCredsError +from libcloud.storage.providers import Provider +from libcloud.storage.base import Object, Container, StorageDriver + +__all__ = [ + 'BackblazeB2StorageDriver', + + 'BackblazeB2Connection', + 'BackblazeB2AuthConnection' +] + +AUTH_API_HOST = 'api.backblaze.com' +API_PATH = '/b2api/v1/' + + +class BackblazeB2Response(JsonResponse): + def success(self): + return self.status in [httplib.OK, httplib.CREATED, httplib.ACCEPTED] + + def parse_error(self): + status = int(self.status) + body = self.parse_body() + + if status == httplib.UNAUTHORIZED: + raise InvalidCredsError(body['message']) + + return self.body + + +class BackblazeB2AuthConnection(ConnectionUserAndKey): + host = AUTH_API_HOST + secure = True + responseCls = BackblazeB2Response + + def __init__(self, *args, **kwargs): + super(BackblazeB2AuthConnection, self).__init__(*args, **kwargs) + + # Those attributes are populated after authentication + self.account_id = None + self.api_url = None + self.api_host = None + self.download_url = None + self.download_host = None + self.auth_token = None + + def authenticate(self, force=False): + """ + :param force: Force authentication if if we have already obtained the + token. + :type force: ``bool`` + """ + if not self._is_authentication_needed(force=force): + return self + + headers = {} + action = 'b2_authorize_account' + auth_b64 = base64.b64encode(b('%s:%s' % (self.user_id, self.key))) + headers['Authorization'] = 'Basic %s' % (auth_b64.decode('utf-8')) + + action = API_PATH + 'b2_authorize_account' + resp = self.request(action=action, headers=headers, method='GET') + + if resp.status == httplib.OK: + self._parse_and_set_auth_info(data=resp.object) + else: + raise Exception('Failed to authenticate: %s' % (str(resp.object))) + + return self + + def _parse_and_set_auth_info(self, data): + result = {} + self.account_id = data['accountId'] + self.api_url = data['apiUrl'] + self.download_url = data['downloadUrl'] + self.auth_token = data['authorizationToken'] + + parsed_api_url = urlparse.urlparse(self.api_url) + self.api_host = parsed_api_url.netloc + + parsed_download_url = urlparse.urlparse(self.download_url) + self.download_host = parsed_download_url.netloc + + return result + + def _is_authentication_needed(self, force=False): + if not self.auth_token or force: + return True + + return False + + +class BackblazeB2Connection(ConnectionUserAndKey): + host = None # Note: host is set after authentication + secure = True + responseCls = BackblazeB2Response + + def __init__(self, *args, **kwargs): + super(BackblazeB2Connection, self).__init__(*args, **kwargs) + + # Stores info retrieved after authnetication (auth token, api url, + # dowload url). + self._auth_conn = BackblazeB2AuthConnection(*args, **kwargs) + + def request(self, action, params=None, data=None, headers=None, + method='GET', raw=False, include_account_id=False): + params = params or {} + headers = headers or {} + + # Lazily perform authentication + auth_conn = self._auth_conn.authenticate() + + # Set host + if raw: + # File download or upload request: + self.host = auth_conn.download_host + else: + self.host = auth_conn.api_host + + # Provide auth token + headers['Authorization'] = '%s' % (auth_conn.auth_token) + + # Include Content-Type + if not raw and data: + headers['Content-Type'] = 'application/json' + + # Include account id + if include_account_id: + if method == 'GET': + params['accountId'] = auth_conn.account_id + elif method == 'POST': + data = data or {} + data['accountId'] = auth_conn.account_id + + if not raw: + action = API_PATH + action + else: + action = '/file/' + action + + if data: + data = json.dumps(data) + + response = super(BackblazeB2Connection, self).request(action=action, + params=params, + data=data, + method=method, + headers=headers, + raw=raw) + return response + + +class BackblazeB2StorageDriver(StorageDriver): + connectionCls = BackblazeB2Connection + name = 'Backblaze B2' + website = 'https://www.backblaze.com/b2/' + type = Provider.BACKBLAZE_B2 + hash_type = 'sha1' + + def iterate_containers(self): + resp = self.connection.request(action='b2_list_buckets', + method='GET', + include_account_id=True) + containers = self._to_containers(data=resp.object) + return containers + + def iterate_container_objects(self, container): + # TODO: Support pagination + params = {'bucketId': container.extra['id']} + resp = self.connection.request(action='b2_list_file_names', + method='GET', + params=params) + objects = self._to_objects(data=resp.object, container=container) + return objects + + def create_container(self, container_name, ex_type='allPrivate'): + data = {} + data['bucketName'] = container_name + data['bucketType'] = ex_type + resp = self.connection.request(action='b2_create_bucket', + data=data, method='POST', + include_account_id=True) + container = self._to_container(item=resp.object) + return container + + def delete_container(self, container): + data = {} + data['bucketId'] = container.extra['id'] + resp = self.connection.request(action='b2_delete_bucket', + data=data, method='POST', + include_account_id=True) + return resp.status == httplib.OK + + def upload_object(self, file_path, container, object_name, extra=None, + verify_hash=True): + pass + + def download_object(self, obj, destination_path, overwrite_existing=False, + delete_on_failure=True): + # TODO: Escape name + action = obj.container.name + '/' + obj.name + response = self.connection.request(action=action, method='GET', raw=True) + + # TODO: Include metadata from response headers + return self._get_object(obj=obj, callback=self._save_object, + response=response, + callback_kwargs={ + 'obj': obj, + 'response': response.response, + 'destination_path': destination_path, + 'overwrite_existing': overwrite_existing, + 'delete_on_failure': delete_on_failure + }, + success_status_code=httplib.OK) + + def delete_object(self, obj): + data = {} + data['fileName'] = obj.name + data['fileId'] = obj.extra['fileId'] + resp = self.connection.request(action='b2_delete_file_version', + data=data, method='POST') + return resp.status == httplib.OK + + def ex_get_object(self, object_id): + params = {} + params['fileId'] = object_id + resp = self.connection.request(action='b2_get_file_info', + method='GET', + params=params) + obj = self._to_object(item=resp.object, container=None) + return obj + + def ex_hide_object(self, container_id, object_name): + data = {} + data['bucketId'] = container_id + data['fileName'] = object_name + resp = self.connection.request(action='b2_hide_file', + data=data, method='POST') + obj = self._to_object(item=resp.object, container=None) + return obj + + def ex_list_object_versions(self, container_id, ex_start_file_name=None, + ex_start_file_id=None, ex_max_file_count=None): + params = {} + params['bucketId'] = container_id + + if ex_start_file_name: + params['startFileName'] = ex_start_file_name + + if ex_start_file_id: + params['startFileId'] = ex_start_file_id + + if ex_max_file_count: + params['maxFileCount'] = ex_max_file_count + + resp = self.connection.request(action='b2_list_file_versions', + params=params, method='GET') + objects = self._to_objects(data=resp.object, container=None) + return objects + + def _to_containers(self, data): + result = [] + for item in data['buckets']: + container = self._to_container(item=item) + result.append(container) + + return result + + def _to_container(self, item): + extra = {} + extra['id'] = item['bucketId'] + extra['bucketType'] = item['bucketType'] + container = Container(name=item['bucketName'], extra=extra, + driver=self) + return container + + def _to_objects(self, data, container): + result = [] + for item in data['files']: + obj = self._to_object(item=item, container=container) + result.append(obj) + + return result + + def _to_object(self, item, container=None): + extra = {} + extra['fileId'] = item['fileId'] + extra['uploadTimestamp'] = item.get('uploadTimestamp', None) + size = item.get('size', item.get('contentLength', None)) + hash = item.get('contentSha1', None) + meta_data = item.get('fileInfo', {}) + obj = Object(name=item['fileName'], size=size, hash=hash, extra=extra, + meta_data=meta_data, container=container, driver=self) + return obj http://git-wip-us.apache.org/repos/asf/libcloud/blob/03e52b83/libcloud/storage/providers.py ---------------------------------------------------------------------- diff --git a/libcloud/storage/providers.py b/libcloud/storage/providers.py index cff34c8..1e79acf 100644 --- a/libcloud/storage/providers.py +++ b/libcloud/storage/providers.py @@ -52,6 +52,8 @@ DRIVERS = { ('libcloud.storage.drivers.ktucloud', 'KTUCloudStorageDriver'), Provider.AURORAOBJECTS: ('libcloud.storage.drivers.auroraobjects', 'AuroraObjectsStorageDriver'), + Provider.BACKBLAZE_B2: + ('libcloud.storage.drivers.backblaze_b2', 'BackblazeB2StorageDriver'), # Deprecated Provider.CLOUDFILES_US: http://git-wip-us.apache.org/repos/asf/libcloud/blob/03e52b83/libcloud/storage/types.py ---------------------------------------------------------------------- diff --git a/libcloud/storage/types.py b/libcloud/storage/types.py index 53d5973..0ac45f1 100644 --- a/libcloud/storage/types.py +++ b/libcloud/storage/types.py @@ -61,6 +61,7 @@ class Provider(object): AZURE_BLOBS = 'azure_blobs' KTUCLOUD = 'ktucloud' AURORAOBJECTS = 'auroraobjects' + BACKBLAZE_B2 = 'backblaze_b2' # Deperecated CLOUDFILES_US = 'cloudfiles_us'
