Hello community, here is the log from the commit of package azure-cli-storage for openSUSE:Factory checked in at 2018-05-13 16:03:26 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/azure-cli-storage (Old) and /work/SRC/openSUSE:Factory/.azure-cli-storage.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "azure-cli-storage" Sun May 13 16:03:26 2018 rev:3 rq:600071 version:2.0.31 Changes: -------- --- /work/SRC/openSUSE:Factory/azure-cli-storage/azure-cli-storage.changes 2018-02-14 09:32:35.397375605 +0100 +++ /work/SRC/openSUSE:Factory/.azure-cli-storage.new/azure-cli-storage.changes 2018-05-13 16:03:29.273784234 +0200 @@ -1,0 +2,8 @@ +Fri Apr 20 12:20:58 UTC 2018 - [email protected] + +- New upstream release + + Version 2.0.31 +- Move LICENSE.txt from %doc to %license section +- Update Requires from setup.py + +------------------------------------------------------------------- Old: ---- azure-cli-storage-2.0.24.tar.gz New: ---- azure-cli-storage-2.0.31.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ azure-cli-storage.spec ++++++ --- /var/tmp/diff_new_pack.HGzj1z/_old 2018-05-13 16:03:29.973758697 +0200 +++ /var/tmp/diff_new_pack.HGzj1z/_new 2018-05-13 16:03:29.977758551 +0200 @@ -17,7 +17,7 @@ Name: azure-cli-storage -Version: 2.0.24 +Version: 2.0.31 Release: 0 Summary: Microsoft Azure CLI 'storage' Command Module License: MIT @@ -32,9 +32,10 @@ BuildRequires: python3-setuptools BuildRequires: unzip Requires: azure-cli-command-modules-nspkg +Requires: azure-cli-core Requires: azure-cli-nspkg Requires: python3-azure-mgmt-storage >= 1.5.0 -Requires: python3-azure-multiapi-storage >= 0.1.7 +Requires: python3-azure-multiapi-storage >= 0.2.0 Requires: python3-azure-nspkg Conflicts: azure-cli < 2.0.0 @@ -64,7 +65,8 @@ %files %defattr(-,root,root,-) -%doc HISTORY.rst LICENSE.txt README.rst +%doc HISTORY.rst README.rst +%license LICENSE.txt %{python3_sitelib}/azure/cli/command_modules/storage %{python3_sitelib}/azure_cli_storage-*.egg-info ++++++ azure-cli-storage-2.0.24.tar.gz -> azure-cli-storage-2.0.31.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-storage-2.0.24/HISTORY.rst new/azure-cli-storage-2.0.31/HISTORY.rst --- old/azure-cli-storage-2.0.24/HISTORY.rst 2018-01-26 17:12:24.000000000 +0100 +++ new/azure-cli-storage-2.0.31/HISTORY.rst 2018-04-06 19:33:14.000000000 +0200 @@ -3,6 +3,37 @@ Release History =============== +2.0.31 +++++++ +* Better error message for malformed connection strings. +* `sdist` is now compatible with wheel 0.31.0 + +2.0.30 +++++++ +* Fix issue of upload file with size between 195GB and 200GB + +2.0.29 +++++++ +* Minor fixes. + +2.0.28 +++++++ +* Fix problems with append blob uploads ignoring condition parameters. + +2.0.27 +++++++ +* Fix issue of missing endpoint suffix in batch copy command. +* Blob batch commands no longer throw error upon failed precondition. +* Support Autorest 3.0 based SDKs + +2.0.26 +++++++ +* Enabled specifying destination-path/prefix to blobs in batch upload and copy commands. + +2.0.25 +++++++ +* Added `storage blob service-properties delete-policy` and `storage blob undelete` commands to enable soft-delete. + 2.0.24 ++++++ * `storage account update`: do not create new networkRuleSet if "default_action" arg is not provided. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-storage-2.0.24/PKG-INFO new/azure-cli-storage-2.0.31/PKG-INFO --- old/azure-cli-storage-2.0.24/PKG-INFO 2018-01-26 17:12:42.000000000 +0100 +++ new/azure-cli-storage-2.0.31/PKG-INFO 2018-04-06 19:33:45.000000000 +0200 @@ -1,11 +1,12 @@ Metadata-Version: 1.1 Name: azure-cli-storage -Version: 2.0.24 +Version: 2.0.31 Summary: Microsoft Azure Command-Line Tools Storage Command Module Home-page: https://github.com/Azure/azure-cli Author: Microsoft Corporation Author-email: [email protected] License: MIT +Description-Content-Type: UNKNOWN Description: Microsoft Azure CLI 'storage' Command Module ============================================ @@ -20,6 +21,37 @@ Release History =============== + 2.0.31 + ++++++ + * Better error message for malformed connection strings. + * `sdist` is now compatible with wheel 0.31.0 + + 2.0.30 + ++++++ + * Fix issue of upload file with size between 195GB and 200GB + + 2.0.29 + ++++++ + * Minor fixes. + + 2.0.28 + ++++++ + * Fix problems with append blob uploads ignoring condition parameters. + + 2.0.27 + ++++++ + * Fix issue of missing endpoint suffix in batch copy command. + * Blob batch commands no longer throw error upon failed precondition. + * Support Autorest 3.0 based SDKs + + 2.0.26 + ++++++ + * Enabled specifying destination-path/prefix to blobs in batch upload and copy commands. + + 2.0.25 + ++++++ + * Added `storage blob service-properties delete-policy` and `storage blob undelete` commands to enable soft-delete. + 2.0.24 ++++++ * `storage account update`: do not create new networkRuleSet if "default_action" arg is not provided. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-storage-2.0.24/azure/cli/command_modules/storage/_help.py new/azure-cli-storage-2.0.31/azure/cli/command_modules/storage/_help.py --- old/azure-cli-storage-2.0.24/azure/cli/command_modules/storage/_help.py 2018-01-26 17:12:24.000000000 +0100 +++ new/azure-cli-storage-2.0.31/azure/cli/command_modules/storage/_help.py 2018-04-06 19:33:14.000000000 +0200 @@ -16,7 +16,7 @@ short-summary: The name of the table to insert the entity into. - name: --entity -e type: list - short-summary: A space-separated list of key=value pairs. Must contain a PartitionKey and a RowKey. + short-summary: Space-separated list of key=value pairs. Must contain a PartitionKey and a RowKey. long-summary: The PartitionKey and RowKey must be unique within the table, and may be up to 64Kb in size. If using an integer value as a key, convert it to a fixed-width string which can be canonically sorted. For example, convert the integer value 1 to the string value "0000001" to ensure proper sorting. @@ -196,8 +196,7 @@ short-summary: List blobs in a given container. parameters: - name: --include - short-summary: 'Specifies additional datasets to include: (c)opy-info, (m)etadata, (s)napshots. Can be - combined.' + short-summary: 'Specifies additional datasets to include: (c)opy-info, (m)etadata, (s)napshots, (d)eleted-soft. Can be combined.' """ helps['storage blob copy'] = """ @@ -225,6 +224,19 @@ short-summary: Manage storage blob service properties. """ +helps['storage blob service-properties delete-policy'] = """ + type: group + short-summary: Manage storage blob delete-policy service properties. +""" +helps['storage blob service-properties delete-policy show'] = """ + type: command + short-summary: Show the storage blob delete-policy. +""" +helps['storage blob service-properties delete-policy update'] = """ + type: command + short-summary: Update the storage blob delete-policy. +""" + helps['storage blob set-tier'] = """ type: command short-summary: Set the block or page tiers on the blob. @@ -407,13 +419,14 @@ - name: --max-age short-summary: The maximum number of seconds the client/browser should cache a preflight response. - name: --origins - short-summary: List of origin domains that will be allowed via CORS, or '*' to allow all domains. + short-summary: Space-separated list of origin domains that will be allowed via CORS, or '*' to allow all + domains. - name: --methods - short-summary: List of HTTP methods allowed to be executed by the origin. + short-summary: Space-separated list of HTTP methods allowed to be executed by the origin. - name: --allowed-headers - short-summary: List of response headers allowed to be part of the cross-origin request. + short-summary: Space-separated list of response headers allowed to be part of the cross-origin request. - name: --exposed-headers - short-summary: List of response headers to expose to CORS clients. + short-summary: Space-separated list of response headers to expose to CORS clients. """ helps['storage cors clear'] = """ @@ -506,6 +519,9 @@ type: string short-summary: The destination of the upload operation. long-summary: The destination can be the file share URL or the share name. When the destination is the share URL, the storage account name is parsed from the URL. + - name: --destination-path + type: string + short-summary: The directory where the source data is copied to. If omitted, data is copied to the root directory. - name: --pattern type: string short-summary: The pattern used for file globbing. The supported patterns are '*', '?', '[seq', and '[!seq]'. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-storage-2.0.24/azure/cli/command_modules/storage/_params.py new/azure-cli-storage-2.0.31/azure/cli/command_modules/storage/_params.py --- old/azure-cli-storage-2.0.24/azure/cli/command_modules/storage/_params.py 2018-01-26 17:12:24.000000000 +0100 +++ new/azure-cli-storage-2.0.31/azure/cli/command_modules/storage/_params.py 2018-04-06 19:33:14.000000000 +0200 @@ -12,7 +12,7 @@ validate_included_datasets, validate_custom_domain, validate_container_public_access, validate_table_payload_format, validate_key, add_progress_callback, storage_account_key_options, process_file_download_namespace, process_metric_update_namespace, - get_char_options_validator, validate_bypass) + get_char_options_validator, validate_bypass, validate_encryption_source) def load_arguments(self, _): # pylint: disable=too-many-locals, too-many-statements @@ -113,6 +113,16 @@ arg_type=get_enum_type(['true', 'false'])) c.argument('tags', tags_type, default=None) + with self.argument_context('storage account update', arg_group='Customer managed key', min_api='2017-06-01') as c: + c.extra('encryption_key_name', help='The name of the KeyVault key', ) + c.extra('encryption_key_vault', help='The Uri of the KeyVault') + c.extra('encryption_key_version', help='The version of the KeyVault key') + c.argument('encryption_key_source', + arg_type=get_enum_type(['Microsoft.Storage', 'Microsoft.Keyvault'], 'Microsoft.Storage'), + help='The default encryption service', + validator=validate_encryption_source) + c.ignore('encryption_key_vault_properties') + for scope in ['storage account create', 'storage account update']: with self.argument_context(scope, resource_type=ResourceType.MGMT_STORAGE, min_api='2017-06-01', arg_group='Network Rule') as c: @@ -184,6 +194,7 @@ with self.argument_context('storage blob') as c: c.argument('blob_name', options_list=('--name', '-n'), arg_type=blob_name_type) + c.argument('destination_path', help='The destination path that will be appended to the blob name.') with self.argument_context('storage blob list') as c: c.argument('include', validator=validate_included_datasets) @@ -220,6 +231,11 @@ c.argument('tier', validator=blob_tier_validator) c.argument('timeout', type=int) + with self.argument_context('storage blob service-properties delete-policy update') as c: + c.argument('enable', arg_type=get_enum_type(['true', 'false']), help='Enables/disables soft-delete.') + c.argument('days_retained', type=int, + help='Number of days that soft-deleted blob will be retained. Must be in range [1,365].') + with self.argument_context('storage blob upload') as c: from ._validators import page_blob_tier_validator from .sdkutil import get_blob_types, get_blob_tier_names @@ -663,7 +679,7 @@ c.ignore('property_resolver') c.argument('entity', options_list=('--entity', '-e'), validator=validate_entity, nargs='+') c.argument('select', nargs='+', validator=validate_select, - help='Space separated list of properties to return for each entity.') + help='Space-separated list of properties to return for each entity.') with self.argument_context('storage entity insert') as c: c.argument('if_exists', arg_type=get_enum_type(['fail', 'merge', 'replace'])) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-storage-2.0.24/azure/cli/command_modules/storage/_validators.py new/azure-cli-storage-2.0.31/azure/cli/command_modules/storage/_validators.py --- old/azure-cli-storage-2.0.24/azure/cli/command_modules/storage/_validators.py 2018-01-26 17:12:24.000000000 +0100 +++ new/azure-cli-storage-2.0.31/azure/cli/command_modules/storage/_validators.py 2018-04-06 19:33:14.000000000 +0200 @@ -14,7 +14,6 @@ from azure.cli.command_modules.storage.sdkutil import get_table_data_type from azure.cli.command_modules.storage.url_quote_util import encode_for_url - storage_account_key_options = {'primary': 'key1', 'secondary': 'key2'} @@ -76,8 +75,12 @@ # if connection string supplied or in environment variables, extract account key and name if n.connection_string: conn_dict = validate_key_value_pairs(n.connection_string) - n.account_name = conn_dict['AccountName'] - n.account_key = conn_dict['AccountKey'] + n.account_name = conn_dict.get('AccountName') + n.account_key = conn_dict.get('AccountKey') + if not n.account_name or not n.account_key: + from knack.util import CLIError + raise CLIError('Connection-string: %s, is malformed. Some shell environments require the ' + 'connection string to be surrounded by quotes.' % n.connection_string) # otherwise, simply try to retrieve the remaining variables from environment variables if not n.account_name: @@ -368,17 +371,13 @@ if namespace.encryption_services: t_encryption_services, t_encryption_service = get_sdk(cmd.cli_ctx, ResourceType.MGMT_STORAGE, 'EncryptionServices', 'EncryptionService', mod='models') - services = {service: t_encryption_service(True) for service in namespace.encryption_services} + services = {service: t_encryption_service(enabled=True) for service in namespace.encryption_services} namespace.encryption_services = t_encryption_services(**services) def validate_encryption_source(cmd, namespace): ns = vars(namespace) - if namespace.encryption_key_source: - allowed_options = ['Microsoft.Storage', 'Microsoft.Keyvault'] - if namespace.encryption_key_source not in allowed_options: - raise ValueError('--encryption-key-source allows to values: {}'.format(', '.join(allowed_options))) key_name = ns.pop('encryption_key_name', None) key_version = ns.pop('encryption_key_version', None) @@ -397,7 +396,7 @@ if not KeyVaultProperties: return - kv_prop = KeyVaultProperties(key_name, key_version, key_vault_uri) + kv_prop = KeyVaultProperties(key_name=key_name, key_version=key_version, key_vault_uri=key_vault_uri) namespace.encryption_key_vault_properties = kv_prop @@ -468,11 +467,11 @@ def validate_included_datasets(cmd, namespace): if namespace.include: include = namespace.include - if set(include) - set('cms'): - help_string = '(c)opy-info (m)etadata (s)napshots' + if set(include) - set('cmsd'): + help_string = '(c)opy-info (m)etadata (s)napshots (d)eleted' raise ValueError('valid values are {} or a combination thereof.'.format(help_string)) t_blob_include = cmd.get_models('blob#Include') - namespace.include = t_blob_include('s' in include, 'm' in include, False, 'c' in include) + namespace.include = t_blob_include('s' in include, 'm' in include, False, 'c' in include, 'd' in include) def validate_key(namespace): @@ -600,13 +599,17 @@ source_key = _query_account_key(cmd.cli_ctx, source_account) if source_container: - ns['source_client'] = t_block_blob_svc(account_name=source_account, - account_key=source_key, - sas_token=source_sas) + ns['source_client'] = get_storage_data_service_client(cmd.cli_ctx, + t_block_blob_svc, + name=source_account, + key=source_key, + sas_token=source_sas) elif source_share: - ns['source_client'] = t_file_svc(account_name=source_account, - account_key=source_key, - sas_token=source_sas) + ns['source_client'] = get_storage_data_service_client(cmd.cli_ctx, + t_file_svc, + name=source_account, + key=source_key, + sas_token=source_sas) else: raise ValueError(usage_string) @@ -625,13 +628,16 @@ elif identifier.container: ns['source_container'] = identifier.container if identifier.account_name != ns.get('account_name'): - ns['source_client'] = t_block_blob_svc(account_name=identifier.account_name, - sas_token=identifier.sas_token) + ns['source_client'] = get_storage_data_service_client(cmd.cli_ctx, t_block_blob_svc, + name=identifier.account_name, + sas_token=identifier.sas_token) elif identifier.share: ns['source_share'] = identifier.share if identifier.account_name != ns.get('account_name'): - ns['source_client'] = t_file_svc(account_name=identifier.account_name, - sas_token=identifier.sas_token) + ns['source_client'] = get_storage_data_service_client(cmd.cli_ctx, + t_file_svc, + name=identifier.account_name, + sas_token=identifier.sas_token) def add_progress_callback(cmd, namespace): @@ -642,6 +648,7 @@ hook.add(message='Alive', value=current, total_val=total) if total == current: hook.end() + if not namespace.no_progress: namespace.progress_callback = _update_progress del namespace.no_progress diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-storage-2.0.24/azure/cli/command_modules/storage/commands.py new/azure-cli-storage-2.0.31/azure/cli/command_modules/storage/commands.py --- old/azure-cli-storage-2.0.24/azure/cli/command_modules/storage/commands.py 2018-01-26 17:12:24.000000000 +0100 +++ new/azure-cli-storage-2.0.31/azure/cli/command_modules/storage/commands.py 2018-04-06 19:33:14.000000000 +0200 @@ -43,7 +43,6 @@ custom_command_type=storage_account_custom_type) as g: g.command('check-name', 'check_name_availability') g.custom_command('create', 'create_storage_account', min_api='2016-01-01') - g.custom_command('create', 'create_storage_account_with_account_type', max_api='2015-06-15') g.command('delete', 'delete', confirmation=True) g.command('show', 'get_properties', exception_handler=g.get_handler_suppress_404()) g.custom_command('list', 'list_storage_accounts') @@ -104,6 +103,8 @@ g.storage_command('exists', 'exists', transform=create_boolean_result_output_transformer('exists')) g.storage_command('delete', 'delete_blob', transform=create_boolean_result_output_transformer('deleted'), table_transformer=transform_boolean_for_table) + g.storage_command('undelete', 'undelete_blob', transform=create_boolean_result_output_transformer('undeleted'), + table_transformer=transform_boolean_for_table, min_api='2017-07-29') g.storage_custom_command('set-tier', 'set_blob_tier') g.storage_custom_command('upload', 'upload_blob', @@ -139,6 +140,14 @@ min_api='2016-05-31') as g: g.storage_command('cancel', 'abort_copy_blob') + with self.command_group('storage blob service-properties delete-policy', command_type=base_blob_sdk, + min_api='2017-07-29', + custom_command_type=get_custom_sdk('blob', blob_data_service_factory)) as g: + g.storage_command('show', 'get_blob_service_properties', + transform=lambda x: getattr(x, 'delete_retention_policy', x), + exception_handler=g.get_handler_suppress_404()) + g.storage_custom_command('update', 'set_delete_policy') + with self.command_group('storage blob service-properties', command_type=base_blob_sdk) as g: g.storage_command('show', 'get_blob_service_properties', exception_handler=g.get_handler_suppress_404()) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-storage-2.0.24/azure/cli/command_modules/storage/operations/account.py new/azure-cli-storage-2.0.31/azure/cli/command_modules/storage/operations/account.py --- old/azure-cli-storage-2.0.24/azure/cli/command_modules/storage/operations/account.py 2018-01-26 17:12:24.000000000 +0100 +++ new/azure-cli-storage-2.0.31/azure/cli/command_modules/storage/operations/account.py 2018-04-06 19:33:14.000000000 +0200 @@ -15,9 +15,9 @@ cmd.get_models('StorageAccountCreateParameters', 'Kind', 'Sku', 'CustomDomain', 'AccessTier', 'Identity', 'Encryption', 'NetworkRuleSet') scf = storage_client_factory(cmd.cli_ctx) - params = StorageAccountCreateParameters(sku=Sku(sku), kind=Kind(kind), location=location, tags=tags) + params = StorageAccountCreateParameters(sku=Sku(name=sku), kind=Kind(kind), location=location, tags=tags) if custom_domain: - params.custom_domain = CustomDomain(custom_domain, None) + params.custom_domain = CustomDomain(name=custom_domain, use_sub_domain=None) if encryption_services: params.encryption = Encryption(services=encryption_services) if access_tier: @@ -37,14 +37,6 @@ return scf.storage_accounts.create(resource_group_name, account_name, params) -def create_storage_account_with_account_type(cmd, resource_group_name, account_name, account_type, location=None, - tags=None): - StorageAccountCreateParameters, AccountType = cmd.get_models('StorageAccountCreateParameters', 'AccountType') - scf = storage_client_factory(cmd.cli_ctx) - params = StorageAccountCreateParameters(location, AccountType(account_type), tags) - return scf.storage_accounts.create(resource_group_name, account_name, params) - - def list_storage_accounts(cmd, resource_group_name=None): scf = storage_client_factory(cmd.cli_ctx) if resource_group_name: @@ -96,7 +88,7 @@ 'Encryption', 'NetworkRuleSet') domain = instance.custom_domain if custom_domain is not None: - domain = CustomDomain(custom_domain) + domain = CustomDomain(name=custom_domain) if use_subdomain is not None: domain.name = use_subdomain == 'true' @@ -115,7 +107,7 @@ raise ValueError('--encryption-services is required when configure encryption key source') params = StorageAccountUpdateParameters( - sku=Sku(sku) if sku is not None else instance.sku, + sku=Sku(name=sku) if sku is not None else instance.sku, tags=tags if tags is not None else instance.tags, custom_domain=domain, encryption=encryption, @@ -163,12 +155,12 @@ VirtualNetworkRule = cmd.get_models('VirtualNetworkRule') if not rules.virtual_network_rules: rules.virtual_network_rules = [] - rules.virtual_network_rules.append(VirtualNetworkRule(subnet, action=action)) + rules.virtual_network_rules.append(VirtualNetworkRule(virtual_network_resource_id=subnet, action=action)) if ip_address: IpRule = cmd.get_models('IPRule') if not rules.ip_rules: rules.ip_rules = [] - rules.ip_rules.append(IpRule(ip_address, action=action)) + rules.ip_rules.append(IpRule(ip_address_or_range=ip_address, action=action)) StorageAccountUpdateParameters = cmd.get_models('StorageAccountUpdateParameters') params = StorageAccountUpdateParameters(network_rule_set=rules) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-storage-2.0.24/azure/cli/command_modules/storage/operations/blob.py new/azure-cli-storage-2.0.31/azure/cli/command_modules/storage/operations/blob.py --- old/azure-cli-storage-2.0.24/azure/cli/command_modules/storage/operations/blob.py 2018-01-26 17:12:24.000000000 +0100 +++ new/azure-cli-storage-2.0.31/azure/cli/command_modules/storage/operations/blob.py 2018-04-06 19:33:14.000000000 +0200 @@ -5,6 +5,7 @@ from __future__ import print_function +import os from knack.log import get_logger from azure.cli.command_modules.storage.util import (create_blob_service_from_storage_client, @@ -12,7 +13,8 @@ create_short_lived_share_sas, create_short_lived_container_sas, filter_none, collect_blobs, collect_files, - mkdir_p, guess_content_type) + mkdir_p, guess_content_type, normalize_blob_file_path, + check_precondition_success) from azure.cli.command_modules.storage.url_quote_util import encode_for_url, make_encoded_file_url_and_params @@ -27,8 +29,24 @@ raise ValueError('Blob tier is only applicable to block or page blob.') -def storage_blob_copy_batch(cmd, client, source_client, - destination_container=None, source_container=None, source_share=None, +def set_delete_policy(client, enable=None, days_retained=None): + policy = client.get_blob_service_properties().delete_retention_policy + + if enable is not None: + policy.enabled = enable == 'true' + if days_retained is not None: + policy.days = days_retained + + if policy.enabled and not policy.days: + from knack.util import CLIError + raise CLIError("must specify days-retained") + + client.set_blob_service_properties(delete_retention_policy=policy) + return client.get_blob_service_properties().delete_retention_policy + + +def storage_blob_copy_batch(cmd, client, source_client, destination_container=None, + destination_path=None, source_container=None, source_share=None, source_sas=None, pattern=None, dryrun=False): """Copy a group of blob or files to a blob container.""" logger = None @@ -57,7 +75,7 @@ if dryrun: logger.warning(' - copy blob %s', blob_name) else: - return _copy_blob_to_blob_container(client, source_client, destination_container, + return _copy_blob_to_blob_container(client, source_client, destination_container, destination_path, source_container, source_sas, blob_name) return list(filter_none(action_blob_copy(blob) for blob in collect_blobs(source_client, @@ -78,10 +96,9 @@ def action_file_copy(file_info): dir_name, file_name = file_info if dryrun: - import os.path logger.warning(' - copy file %s', os.path.join(dir_name, file_name)) else: - return _copy_file_to_blob_container(client, source_client, destination_container, + return _copy_file_to_blob_container(client, source_client, destination_container, destination_path, source_share, source_sas, dir_name, file_name) return list(filter_none(action_file_copy(file) for file in collect_files(cmd, @@ -95,8 +112,8 @@ # pylint: disable=unused-argument def storage_blob_download_batch(client, source, destination, source_container_name, pattern=None, dryrun=False, progress_callback=None, max_connections=2): + def _download_blob(blob_service, container, destination_folder, blob_name): - import os # TODO: try catch IO exception destination_path = os.path.join(destination_folder, blob_name) destination_folder = os.path.dirname(destination_path) @@ -124,13 +141,14 @@ def storage_blob_upload_batch(cmd, client, source, destination, pattern=None, # pylint: disable=too-many-locals - source_files=None, + source_files=None, destination_path=None, destination_container_name=None, blob_type=None, content_settings=None, metadata=None, validate_content=False, maxsize_condition=None, max_connections=2, lease_id=None, progress_callback=None, if_modified_since=None, if_unmodified_since=None, if_match=None, if_none_match=None, timeout=None, dryrun=False): def _create_return_result(blob_name, blob_content_settings, upload_result=None): + blob_name = normalize_blob_file_path(destination_path, blob_name) return { 'Blob': client.make_blob_url(destination_container_name, blob_name), 'Type': blob_content_settings.content_type, @@ -151,17 +169,26 @@ for src, dst in source_files or []: results.append(_create_return_result(dst, guess_content_type(src, content_settings, t_content_settings))) else: + @check_precondition_success + def _upload_blob(*args, **kwargs): + return upload_blob(*args, **kwargs) + for src, dst in source_files or []: logger.warning('uploading %s', src) guessed_content_settings = guess_content_type(src, content_settings, t_content_settings) - result = upload_blob(cmd, client, destination_container_name, dst, src, - blob_type=blob_type, content_settings=guessed_content_settings, metadata=metadata, - validate_content=validate_content, maxsize_condition=maxsize_condition, - max_connections=max_connections, lease_id=lease_id, - progress_callback=progress_callback, if_modified_since=if_modified_since, - if_unmodified_since=if_unmodified_since, if_match=if_match, - if_none_match=if_none_match, timeout=timeout) - results.append(_create_return_result(dst, guessed_content_settings, result)) + + include, result = _upload_blob(cmd, client, destination_container_name, + normalize_blob_file_path(destination_path, dst), src, + blob_type=blob_type, content_settings=guessed_content_settings, + metadata=metadata, validate_content=validate_content, + maxsize_condition=maxsize_condition, max_connections=max_connections, + lease_id=lease_id, progress_callback=progress_callback, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, if_match=if_match, + if_none_match=if_none_match, timeout=timeout) + if include: + results.append(_create_return_result(dst, guessed_content_settings, result)) + return results @@ -175,17 +202,22 @@ content_settings = guess_content_type(file_path, content_settings, t_content_settings) def upload_append_blob(): - if not client.exists(container_name, blob_name): - client.create_blob( - container_name=container_name, - blob_name=blob_name, - content_settings=content_settings, - metadata=metadata, - lease_id=lease_id, - if_modified_since=if_modified_since, - if_match=if_match, - if_none_match=if_none_match, - timeout=timeout) + check_blob_args = { + 'container_name': container_name, + 'blob_name': blob_name, + 'lease_id': lease_id, + 'if_modified_since': if_modified_since, + 'if_unmodified_since': if_unmodified_since, + 'if_match': if_match, + 'if_none_match': if_none_match, + 'timeout': timeout + } + + if client.exists(container_name, blob_name): + # used to check for the preconditions as append_blob_from_path() cannot + client.get_blob_properties(**check_blob_args) + else: + client.create_blob(content_settings=content_settings, metadata=metadata, **check_blob_args) append_blob_args = { 'container_name': container_name, @@ -203,9 +235,8 @@ return client.append_blob_from_path(**append_blob_args) def upload_block_blob(): - # increase the block size to 100MB when the file is larger than 200GB - import os.path - if os.path.isfile(file_path) and os.stat(file_path).st_size > 200 * 1024 * 1024 * 1024: + # increase the block size to 100MB when the block list will contain more than 50,000 blocks + if os.path.isfile(file_path) and os.stat(file_path).st_size > 50000 * 4 * 1024 * 1024: client.MAX_BLOCK_SIZE = 100 * 1024 * 1024 client.MAX_SINGLE_PUT_SIZE = 256 * 1024 * 1024 @@ -244,6 +275,7 @@ def storage_blob_delete_batch(client, source, source_container_name, pattern=None, lease_id=None, delete_snapshots=None, if_modified_since=None, if_unmodified_since=None, if_match=None, if_none_match=None, timeout=None, dryrun=False): + @check_precondition_success def _delete_blob(blob_name): delete_blob_args = { 'container_name': source_container_name, @@ -256,8 +288,7 @@ 'if_none_match': if_none_match, 'timeout': timeout } - response = client.delete_blob(**delete_blob_args) - return response + return client.delete_blob(**delete_blob_args) source_blobs = list(collect_blobs(client, source_container_name, pattern)) @@ -272,38 +303,37 @@ logger.warning(' - %s', blob) return [] - return [_delete_blob(blob) for blob in source_blobs] + return [result for include, result in (_delete_blob(blob) for blob in source_blobs) if include] -def _copy_blob_to_blob_container(blob_service, source_blob_service, destination_container, +def _copy_blob_to_blob_container(blob_service, source_blob_service, destination_container, destination_path, source_container, source_sas, source_blob_name): from azure.common import AzureException source_blob_url = source_blob_service.make_blob_url(source_container, encode_for_url(source_blob_name), sas_token=source_sas) - + destination_blob_name = normalize_blob_file_path(destination_path, source_blob_name) try: - blob_service.copy_blob(destination_container, source_blob_name, source_blob_url) - return blob_service.make_blob_url(destination_container, source_blob_name) + blob_service.copy_blob(destination_container, destination_blob_name, source_blob_url) + return blob_service.make_blob_url(destination_container, destination_blob_name) except AzureException: from knack.util import CLIError error_template = 'Failed to copy blob {} to container {}.' raise CLIError(error_template.format(source_blob_name, destination_container)) -def _copy_file_to_blob_container(blob_service, source_file_service, destination_container, +def _copy_file_to_blob_container(blob_service, source_file_service, destination_container, destination_path, source_share, source_sas, source_file_dir, source_file_name): from azure.common import AzureException - import os.path file_url, source_file_dir, source_file_name = \ make_encoded_file_url_and_params(source_file_service, source_share, source_file_dir, source_file_name, source_sas) - blob_name = os.path.join(source_file_dir, source_file_name) \ - if source_file_dir else source_file_name + source_path = os.path.join(source_file_dir, source_file_name) if source_file_dir else source_file_name + destination_blob_name = normalize_blob_file_path(destination_path, source_path) try: - blob_service.copy_blob(destination_container, blob_name=blob_name, copy_source=file_url) - return blob_service.make_blob_url(destination_container, blob_name) + blob_service.copy_blob(destination_container, destination_blob_name, file_url) + return blob_service.make_blob_url(destination_container, destination_blob_name) except AzureException as ex: from knack.util import CLIError error_template = 'Failed to copy file {} to container {}. {}' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-storage-2.0.24/azure/cli/command_modules/storage/operations/file.py new/azure-cli-storage-2.0.31/azure/cli/command_modules/storage/operations/file.py --- old/azure-cli-storage-2.0.24/azure/cli/command_modules/storage/operations/file.py 2018-01-26 17:12:24.000000000 +0100 +++ new/azure-cli-storage-2.0.31/azure/cli/command_modules/storage/operations/file.py 2018-04-06 19:33:14.000000000 +0200 @@ -7,6 +7,7 @@ Commands for storage file share operations """ +import os from knack.log import get_logger from azure.cli.command_modules.storage.util import (filter_none, collect_blobs, collect_files, @@ -30,12 +31,12 @@ return generator -def storage_file_upload_batch(cmd, client, destination, source, pattern=None, dryrun=False, validate_content=False, - content_settings=None, max_connections=1, metadata=None, progress_callback=None): +def storage_file_upload_batch(cmd, client, destination, source, destination_path=None, pattern=None, dryrun=False, + validate_content=False, content_settings=None, max_connections=1, metadata=None, + progress_callback=None): """ Upload local files to Azure Storage File Share in batch """ - from ..util import glob_files_locally - import os.path + from azure.cli.command_modules.storage.util import glob_files_locally, normalize_blob_file_path source_files = [c for c in glob_files_locally(source, pattern)] logger = get_logger(__name__) @@ -46,13 +47,14 @@ logger.info(' account %s', client.account_name) logger.info(' share %s', destination) logger.info(' total %d', len(source_files)) - return [{'File': client.make_file_url(destination, os.path.dirname(src), os.path.basename(dst)), + return [{'File': client.make_file_url(destination, os.path.dirname(dst) or None, os.path.basename(dst)), 'Type': guess_content_type(src, content_settings, settings_class).content_type} for src, dst in source_files] # TODO: Performance improvement # 1. Upload files in parallel def _upload_action(src, dst): + dst = normalize_blob_file_path(destination_path, dst) dir_name = os.path.dirname(dst) file_name = os.path.basename(dst) @@ -79,8 +81,7 @@ Download files from file share to local directory in batch """ - from ..util import glob_files_remotely, mkdir_p - import os.path + from azure.cli.command_modules.storage.util import glob_files_remotely, mkdir_p source_files = glob_files_remotely(cmd, client, source, pattern) @@ -181,7 +182,6 @@ def action_file_copy(file_info): dir_name, file_name = file_info if dryrun: - import os.path logger.warning(' - copy file %s', os.path.join(dir_name, file_name)) else: return _create_file_and_directory_from_file(client, source_client, destination_share, source_share, @@ -207,7 +207,7 @@ return client.delete_file(**delete_file_args) - from ..util import glob_files_remotely + from azure.cli.command_modules.storage.util import glob_files_remotely source_files = list(glob_files_remotely(cmd, client, source, pattern)) if dryrun: @@ -230,10 +230,10 @@ Copy a blob to file share and create the directory if needed. """ from azure.common import AzureException - import os.path + from azure.cli.command_modules.storage.util import normalize_blob_file_path blob_url = blob_service.make_blob_url(container, encode_for_url(blob_name), sas_token=sas) - full_path = os.path.join(destination_dir, blob_name) if destination_dir else blob_name + full_path = normalize_blob_file_path(destination_dir, blob_name) file_name = os.path.basename(full_path) dir_name = os.path.dirname(full_path) _make_directory_in_files_share(file_service, share, dir_name, existing_dirs) @@ -255,14 +255,13 @@ Copy a file from one file share to another """ from azure.common import AzureException - import os.path + from azure.cli.command_modules.storage.util import normalize_blob_file_path file_url, source_file_dir, source_file_name = make_encoded_file_url_and_params(source_file_service, source_share, source_file_dir, source_file_name, sas_token=sas) - full_path = os.path.join(destination_dir, source_file_dir, source_file_name) if destination_dir else os.path.join( - source_file_dir, source_file_name) + full_path = normalize_blob_file_path(destination_dir, os.path.join(source_file_dir, source_file_name)) file_name = os.path.basename(full_path) dir_name = os.path.dirname(full_path) _make_directory_in_files_share(file_service, share, dir_name, existing_dirs) @@ -286,7 +285,6 @@ which already exists. """ from azure.common import AzureHttpError - import os.path if not directory_path: return diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-storage-2.0.24/azure/cli/command_modules/storage/util.py new/azure-cli-storage-2.0.31/azure/cli/command_modules/storage/util.py --- old/azure-cli-storage-2.0.24/azure/cli/command_modules/storage/util.py 2018-01-26 17:12:24.000000000 +0100 +++ new/azure-cli-storage-2.0.31/azure/cli/command_modules/storage/util.py 2018-04-06 19:33:14.000000000 +0200 @@ -4,6 +4,9 @@ # -------------------------------------------------------------------------------------------- +import os + + def collect_blobs(blob_service, container, pattern=None): """ List the blobs in the given blob container, filter the blob by comparing their path to the given pattern. @@ -24,7 +27,7 @@ except NameError: blob_name = blob.name - if _match_path(pattern, blob_name): + if not pattern or _match_path(blob_name, pattern): results.append(blob_name) return results @@ -67,25 +70,19 @@ def glob_files_locally(folder_path, pattern): """glob files in local folder based on the given pattern""" - import os pattern = os.path.join(folder_path, pattern.lstrip('/')) if pattern else None - from os import walk len_folder_path = len(folder_path) + 1 - for root, _, files in walk(folder_path): + for root, _, files in os.walk(folder_path): for f in files: - from fnmatch import fnmatch full_path = os.path.join(root, f) - if pattern and fnmatch(full_path, pattern): - yield (full_path, full_path[len_folder_path:]) - elif not pattern: + if not pattern or _match_path(full_path, pattern): yield (full_path, full_path[len_folder_path:]) def glob_files_remotely(cmd, client, share_name, pattern): """glob the files in remote file share based on the given pattern""" - import os from collections import deque t_dir, t_file = cmd.get_models('file.models#Directory', 'file.models#File') @@ -94,8 +91,7 @@ current_dir = queue.pop() for f in client.list_directories_and_files(share_name, current_dir): if isinstance(f, t_file): - from fnmatch import fnmatch - if (pattern and fnmatch(os.path.join(current_dir, f.name), pattern)) or (not pattern): + if not pattern or _match_path(os.path.join(current_dir, f.name), pattern): yield current_dir, f.name elif isinstance(f, t_dir): queue.appendleft(os.path.join(current_dir, f.name)) @@ -158,7 +154,6 @@ def mkdir_p(path): import errno - import os try: os.makedirs(path) except OSError as exc: # Python <= 2.5 @@ -172,10 +167,9 @@ return not p or p.find('*') != -1 or p.find('?') != -1 or p.find('[') != -1 -def _match_path(pattern, *args): +def _match_path(path, pattern): from fnmatch import fnmatch - import os - return fnmatch(os.path.join(*args), pattern) if pattern else True + return fnmatch(path, pattern) def guess_content_type(file_path, original, settings_class): @@ -205,3 +199,30 @@ sas_token = getattr(namespace, 'sas_token', az_config.get('storage', 'sas_token', None)) return get_storage_data_service_client(cli_ctx, service_type, name, key, connection_string, sas_token) + + +def normalize_blob_file_path(path, name): + # '/' is the path separator used by blobs/files, we normalize to it + path_sep = '/' + if path: + name = path_sep.join((path, name)) + return path_sep.join(os.path.normpath(name).split(os.path.sep)).strip(path_sep) + + +def check_precondition_success(func): + def wrapper(*args, **kwargs): + from azure.common import AzureHttpError + try: + return True, func(*args, **kwargs) + except AzureHttpError as ex: + # Precondition failed error + # https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/412 + # Not modified error + # https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/304 + if ex.status_code not in [304, 412]: + raise + from knack.log import get_logger + logger = get_logger(__name__) + logger.warning('Failed precondition') + return False, None + return wrapper diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-storage-2.0.24/azure_cli_storage.egg-info/PKG-INFO new/azure-cli-storage-2.0.31/azure_cli_storage.egg-info/PKG-INFO --- old/azure-cli-storage-2.0.24/azure_cli_storage.egg-info/PKG-INFO 2018-01-26 17:12:42.000000000 +0100 +++ new/azure-cli-storage-2.0.31/azure_cli_storage.egg-info/PKG-INFO 2018-04-06 19:33:45.000000000 +0200 @@ -1,11 +1,12 @@ Metadata-Version: 1.1 Name: azure-cli-storage -Version: 2.0.24 +Version: 2.0.31 Summary: Microsoft Azure Command-Line Tools Storage Command Module Home-page: https://github.com/Azure/azure-cli Author: Microsoft Corporation Author-email: [email protected] License: MIT +Description-Content-Type: UNKNOWN Description: Microsoft Azure CLI 'storage' Command Module ============================================ @@ -20,6 +21,37 @@ Release History =============== + 2.0.31 + ++++++ + * Better error message for malformed connection strings. + * `sdist` is now compatible with wheel 0.31.0 + + 2.0.30 + ++++++ + * Fix issue of upload file with size between 195GB and 200GB + + 2.0.29 + ++++++ + * Minor fixes. + + 2.0.28 + ++++++ + * Fix problems with append blob uploads ignoring condition parameters. + + 2.0.27 + ++++++ + * Fix issue of missing endpoint suffix in batch copy command. + * Blob batch commands no longer throw error upon failed precondition. + * Support Autorest 3.0 based SDKs + + 2.0.26 + ++++++ + * Enabled specifying destination-path/prefix to blobs in batch upload and copy commands. + + 2.0.25 + ++++++ + * Added `storage blob service-properties delete-policy` and `storage blob undelete` commands to enable soft-delete. + 2.0.24 ++++++ * `storage account update`: do not create new networkRuleSet if "default_action" arg is not provided. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-storage-2.0.24/azure_cli_storage.egg-info/requires.txt new/azure-cli-storage-2.0.31/azure_cli_storage.egg-info/requires.txt --- old/azure-cli-storage-2.0.24/azure_cli_storage.egg-info/requires.txt 2018-01-26 17:12:42.000000000 +0100 +++ new/azure-cli-storage-2.0.31/azure_cli_storage.egg-info/requires.txt 2018-04-06 19:33:45.000000000 +0200 @@ -1,3 +1,3 @@ -azure-multiapi-storage==0.1.7 +azure-multiapi-storage==0.2.0 azure-mgmt-storage==1.5.0 azure-cli-core diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-storage-2.0.24/setup.py new/azure-cli-storage-2.0.31/setup.py --- old/azure-cli-storage-2.0.24/setup.py 2018-01-26 17:12:24.000000000 +0100 +++ new/azure-cli-storage-2.0.31/setup.py 2018-04-06 19:33:14.000000000 +0200 @@ -14,7 +14,7 @@ logger.warn("Wheel is not available, disabling bdist_wheel hook") cmdclass = {} -VERSION = "2.0.24" +VERSION = "2.0.31" CLASSIFIERS = [ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', @@ -30,7 +30,7 @@ ] DEPENDENCIES = [ - 'azure-multiapi-storage==0.1.7', + 'azure-multiapi-storage==0.2.0', 'azure-mgmt-storage==1.5.0', 'azure-cli-core' ]
