Hello community, here is the log from the commit of package azure-cli-core for openSUSE:Factory checked in at 2020-07-26 16:17:43 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/azure-cli-core (Old) and /work/SRC/openSUSE:Factory/.azure-cli-core.new.3592 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "azure-cli-core" Sun Jul 26 16:17:43 2020 rev:16 rq:821948 version:2.9.1 Changes: -------- --- /work/SRC/openSUSE:Factory/azure-cli-core/azure-cli-core.changes 2020-06-27 23:22:21.365778057 +0200 +++ /work/SRC/openSUSE:Factory/.azure-cli-core.new.3592/azure-cli-core.changes 2020-07-26 16:19:02.192791035 +0200 @@ -1,0 +2,9 @@ +Mon Jul 20 14:42:00 UTC 2020 - John Paul Adrian Glaubitz <adrian.glaub...@suse.com> + +- New upstream release + + Version 2.9.1 + + For detailed information about changes see the + HISTORY.txt file provided with this package +- Update Requires from setup.py + +------------------------------------------------------------------- Old: ---- azure-cli-core-2.7.0.tar.gz New: ---- azure-cli-core-2.9.1.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ azure-cli-core.spec ++++++ --- /var/tmp/diff_new_pack.I6ldk1/_old 2020-07-26 16:19:03.328792098 +0200 +++ /var/tmp/diff_new_pack.I6ldk1/_new 2020-07-26 16:19:03.332792101 +0200 @@ -17,7 +17,7 @@ Name: azure-cli-core -Version: 2.7.0 +Version: 2.9.1 Release: 0 Summary: Microsoft Azure CLI Core Module License: MIT @@ -42,10 +42,10 @@ Requires: python3-argcomplete >= 1.8 Requires: python3-azure-mgmt-core < 2.0.0 Requires: python3-azure-mgmt-core >= 1.0.0 -Requires: python3-azure-mgmt-resource < 10.0.0 -Requires: python3-azure-mgmt-resource >= 9.0.0 +Requires: python3-azure-mgmt-resource < 11.0.0 +Requires: python3-azure-mgmt-resource >= 10.0.0 Requires: python3-azure-nspkg >= 3.0.0 -Requires: python3-colorama >= 0.3.9 +Requires: python3-colorama >= 0.4.1 Requires: python3-humanfriendly < 9.0 Requires: python3-humanfriendly >= 4.7 Requires: python3-jmespath @@ -63,7 +63,7 @@ Requires: python3-pkginfo >= 1.5.0.1 Requires: python3-pyOpenSSL >= 17.1.0 Requires: python3-requests < 3.0.0 -Requires: python3-requests >= 2.20 +Requires: python3-requests >= 2.22 Requires: python3-six < 2.0.0 Requires: python3-six >= 1.12 Requires: python3-wheel >= 0.30.0 ++++++ azure-cli-core-2.7.0.tar.gz -> azure-cli-core-2.9.1.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-core-2.7.0/HISTORY.rst new/azure-cli-core-2.9.1/HISTORY.rst --- old/azure-cli-core-2.7.0/HISTORY.rst 2020-05-29 10:06:31.000000000 +0200 +++ new/azure-cli-core-2.9.1/HISTORY.rst 2020-07-16 10:09:55.000000000 +0200 @@ -3,6 +3,18 @@ Release History =============== +2.9.1 +++++++ +* Minor fixes + +2.9.0 +++++++ +* Fix get_token() issue in msi login and `expiresIn` key error in cloud shell login credentials for track 2 SDK related commands (#14187) + +2.8.0 +++++++ +* Add get_command_loader() entry to support to load customized CommandLoader (#13763) + 2.7.0 ++++++ * Enable local context for location (#13682) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-core-2.7.0/PKG-INFO new/azure-cli-core-2.9.1/PKG-INFO --- old/azure-cli-core-2.7.0/PKG-INFO 2020-05-29 10:06:40.000000000 +0200 +++ new/azure-cli-core-2.9.1/PKG-INFO 2020-07-16 10:10:08.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: azure-cli-core -Version: 2.7.0 +Version: 2.9.1 Summary: Microsoft Azure Command-Line Tools Core Module Home-page: https://github.com/Azure/azure-cli Author: Microsoft Corporation @@ -15,6 +15,18 @@ Release History =============== + 2.9.1 + ++++++ + * Minor fixes + + 2.9.0 + ++++++ + * Fix get_token() issue in msi login and `expiresIn` key error in cloud shell login credentials for track 2 SDK related commands (#14187) + + 2.8.0 + ++++++ + * Add get_command_loader() entry to support to load customized CommandLoader (#13763) + 2.7.0 ++++++ * Enable local context for location (#13682) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-core-2.7.0/azure/cli/core/__init__.py new/azure-cli-core-2.9.1/azure/cli/core/__init__.py --- old/azure-cli-core-2.7.0/azure/cli/core/__init__.py 2020-05-29 10:06:31.000000000 +0200 +++ new/azure-cli-core-2.9.1/azure/cli/core/__init__.py 2020-07-16 10:09:55.000000000 +0200 @@ -6,7 +6,7 @@ from __future__ import print_function -__version__ = "2.7.0" +__version__ = "2.9.1" import os import sys @@ -31,6 +31,12 @@ 'content_version', 'kwargs', 'client', 'no_wait'] EVENT_FAILED_EXTENSION_LOAD = 'MainLoader.OnFailedExtensionLoad' +# [Reserved, in case of future usage] +# Modules that will always be loaded. They don't expose commands but hook into CLI core. +ALWAYS_LOADED_MODULES = [] +# Extensions that will always be loaded if installed. They don't expose commands but hook into CLI core. +ALWAYS_LOADED_EXTENSIONS = ['azext_ai_examples', 'azext_ai_did_you_mean_this'] + class AzCli(CLI): @@ -42,7 +48,7 @@ register_ids_argument, register_global_subscription_argument) from azure.cli.core.cloud import get_active_cloud from azure.cli.core.commands.transform import register_global_transforms - from azure.cli.core._session import ACCOUNT, CONFIG, SESSION + from azure.cli.core._session import ACCOUNT, CONFIG, SESSION, INDEX from knack.util import ensure_dir @@ -57,6 +63,8 @@ ACCOUNT.load(os.path.join(azure_folder, 'azureProfile.json')) CONFIG.load(os.path.join(azure_folder, 'az.json')) SESSION.load(os.path.join(azure_folder, 'az.sess'), max_age=3600) + INDEX.load(os.path.join(azure_folder, 'commandIndex.json')) + self.cloud = get_active_cloud(self) logger.debug('Current cloud config:\n%s', str(self.cloud.name)) self.local_context = AzCLILocalContext(self) @@ -143,11 +151,18 @@ args_str = [] for name, value in local_context_args: args_str.append('{}: {}'.format(name, value)) - logger.warning('Command argument values saved to local context: %s', ', '.join(args_str)) + logger.warning('Your preference of %s now saved to local context. To learn more, type in `az ' + 'local-context --help`', ', '.join(args_str) + ' is' if len(args_str) == 1 else ' are') class MainCommandsLoader(CLICommandsLoader): + # Format string for pretty-print the command module table + header_mod = "%-20s %10s %9s %9s" % ("Name", "Load Time", "Groups", "Commands") + item_format_string = "%-20s %10.3f %9d %9d" + header_ext = header_mod + " Directory" + item_ext_format_string = item_format_string + " %s" + def __init__(self, cli_ctx=None): super(MainCommandsLoader, self).__init__(cli_ctx) self.cmd_to_loader_map = {} @@ -160,33 +175,48 @@ loader.command_table = self.command_table loader._update_command_definitions() # pylint: disable=protected-access - # pylint: disable=too-many-statements + # pylint: disable=too-many-statements, too-many-locals def load_command_table(self, args): from importlib import import_module import pkgutil import traceback from azure.cli.core.commands import ( - _load_module_command_loader, _load_extension_command_loader, BLACKLISTED_MODS, ExtensionCommandSource) + _load_module_command_loader, _load_extension_command_loader, BLOCKED_MODS, ExtensionCommandSource) from azure.cli.core.extension import ( get_extensions, get_extension_path, get_extension_modname) - def _update_command_table_from_modules(args): - '''Loads command table(s) - When `module_name` is specified, only commands from that module will be loaded. - If the module is not found, all commands are loaded. - ''' - installed_command_modules = [] - try: - mods_ns_pkg = import_module('azure.cli.command_modules') - installed_command_modules = [modname for _, modname, _ in - pkgutil.iter_modules(mods_ns_pkg.__path__) - if modname not in BLACKLISTED_MODS] - except ImportError as e: - logger.warning(e) + def _update_command_table_from_modules(args, command_modules=None): + """Loads command tables from modules and merge into the main command table. + + :param args: Arguments of the command. + :param list command_modules: Command modules to load, in the format like ['resource', 'profile']. + If None, will do module discovery and load all modules. + If [], only ALWAYS_LOADED_MODULES will be loaded. + Otherwise, the list will be extended using ALWAYS_LOADED_MODULES. + """ + + # As command modules are built-in, the existence of modules in ALWAYS_LOADED_MODULES is NOT checked + if command_modules is not None: + command_modules.extend(ALWAYS_LOADED_MODULES) + else: + # Perform module discovery + command_modules = [] + try: + mods_ns_pkg = import_module('azure.cli.command_modules') + command_modules = [modname for _, modname, _ in + pkgutil.iter_modules(mods_ns_pkg.__path__)] + logger.debug('Discovered command modules: %s', command_modules) + except ImportError as e: + logger.warning(e) - logger.debug('Installed command modules %s', installed_command_modules) + count = 0 cumulative_elapsed_time = 0 - for mod in [m for m in installed_command_modules if m not in BLACKLISTED_MODS]: + cumulative_group_count = 0 + cumulative_command_count = 0 + logger.debug("Loading command modules:") + logger.debug(self.header_mod) + + for mod in [m for m in command_modules if m not in BLOCKED_MODS]: try: start_time = timeit.default_timer() module_command_table, module_group_table = _load_module_command_loader(self, args, mod) @@ -194,9 +224,14 @@ cmd.command_source = mod self.command_table.update(module_command_table) self.command_group_table.update(module_group_table) + elapsed_time = timeit.default_timer() - start_time - logger.debug("Loaded module '%s' in %.3f seconds.", mod, elapsed_time) + logger.debug(self.item_format_string, mod, elapsed_time, + len(module_group_table), len(module_command_table)) + count += 1 cumulative_elapsed_time += elapsed_time + cumulative_group_count += len(module_group_table) + cumulative_command_count += len(module_command_table) except Exception as ex: # pylint: disable=broad-except # Changing this error message requires updating CI script that checks for failed # module loading. @@ -205,14 +240,21 @@ telemetry.set_exception(exception=ex, fault_type='module-load-error-' + mod, summary='Error loading module: {}'.format(mod)) logger.debug(traceback.format_exc()) - logger.debug("Loaded all modules in %.3f seconds. " - "(note: there's always an overhead with the first module loaded)", - cumulative_elapsed_time) - - def _update_command_table_from_extensions(ext_suppressions): - - from azure.cli.core.extension.operations import check_version_compatibility - + # Summary line + logger.debug(self.item_format_string, + "Total ({})".format(count), cumulative_elapsed_time, + cumulative_group_count, cumulative_command_count) + + def _update_command_table_from_extensions(ext_suppressions, extension_modname=None): + """Loads command tables from extensions and merge into the main command table. + + :param ext_suppressions: Extension suppression information. + :param extension_modname: Command modules to load, in the format like ['azext_timeseriesinsights']. + If None, will do extension discovery and load all extensions. + If [], only ALWAYS_LOADED_EXTENSIONS will be loaded. + Otherwise, the list will be extended using ALWAYS_LOADED_EXTENSIONS. + If the extensions in the list are not installed, it will be skipped. + """ def _handle_extension_suppressions(extensions): filtered_extensions = [] for ext in extensions: @@ -224,13 +266,39 @@ filtered_extensions.append(ext) return filtered_extensions + def _filter_modname(extensions): + # Extension's name may not be the same as its modname. eg. name: virtual-wan, modname: azext_vwan + filtered_extensions = [] + for ext in extensions: + ext_mod = get_extension_modname(ext.name, ext.path) + # Filter the extensions according to the index + if ext_mod in extension_modname: + filtered_extensions.append(ext) + extension_modname.remove(ext_mod) + if extension_modname: + logger.debug("These extensions are not installed and will be skipped: %s", extension_modname) + return filtered_extensions + extensions = get_extensions() if extensions: - logger.debug("Found %s extensions: %s", len(extensions), [e.name for e in extensions]) + if extension_modname is not None: + extension_modname.extend(ALWAYS_LOADED_EXTENSIONS) + extensions = _filter_modname(extensions) allowed_extensions = _handle_extension_suppressions(extensions) module_commands = set(self.command_table.keys()) + + count = 0 + cumulative_elapsed_time = 0 + cumulative_group_count = 0 + cumulative_command_count = 0 + logger.debug("Loading extensions:") + logger.debug(self.header_ext) + for ext in allowed_extensions: try: + # Import in the `for` loop because `allowed_extensions` can be []. In such case we + # don't need to import `check_version_compatibility` at all. + from azure.cli.core.extension.operations import check_version_compatibility check_version_compatibility(ext.get_metadata()) except CLIError as ex: # issue warning and skip loading extensions that aren't compatible with the CLI core @@ -238,7 +306,6 @@ continue ext_name = ext.name ext_dir = ext.path or get_extension_path(ext_name) - logger.debug("Extensions directory: '%s'", ext_dir) sys.path.append(ext_dir) try: ext_mod = get_extension_modname(ext_name, ext_dir=ext_dir) @@ -258,13 +325,24 @@ self.command_table.update(extension_command_table) self.command_group_table.update(extension_group_table) + elapsed_time = timeit.default_timer() - start_time - logger.debug("Loaded extension '%s' in %.3f seconds.", ext_name, elapsed_time) + logger.debug(self.item_ext_format_string, ext_name, elapsed_time, + len(extension_group_table), len(extension_command_table), + ext_dir) + count += 1 + cumulative_elapsed_time += elapsed_time + cumulative_group_count += len(extension_group_table) + cumulative_command_count += len(extension_command_table) except Exception as ex: # pylint: disable=broad-except self.cli_ctx.raise_event(EVENT_FAILED_EXTENSION_LOAD, extension_name=ext_name) logger.warning("Unable to load extension '%s: %s'. Use --debug for more information.", ext_name, ex) logger.debug(traceback.format_exc()) + # Summary line + logger.debug(self.item_ext_format_string, + "Total ({})".format(count), cumulative_elapsed_time, + cumulative_group_count, cumulative_command_count, "") def _wrap_suppress_extension_func(func, ext): """ Wrapper method to handle centralization of log messages for extension filters """ @@ -295,15 +373,63 @@ res.append(sup) return res + def _roughly_parse_command(args): + # Roughly parse the command part: <az vm create> --name vm1 + # Similar to knack.invocation.CommandInvoker._rudimentary_get_command, but we don't need to bother with + # positional args + nouns = [] + for arg in args: + if arg and arg[0] != '-': + nouns.append(arg) + else: + break + return ' '.join(nouns).lower() + + # Clear the tables to make this method idempotent + self.command_group_table.clear() + self.command_table.clear() + + command_index = None + # Set fallback=False to turn off command index in case of regression + use_command_index = self.cli_ctx.config.getboolean('core', 'use_command_index', fallback=True) + if use_command_index: + command_index = CommandIndex(self.cli_ctx) + index_result = command_index.get(args) + if index_result: + index_modules, index_extensions = index_result + # Always load modules and extensions, because some of them (like those in + # ALWAYS_LOADED_EXTENSIONS) don't expose a command, but hooks into handlers in CLI core + _update_command_table_from_modules(args, index_modules) + # The index won't contain suppressed extensions + _update_command_table_from_extensions([], index_extensions) + + logger.debug("Loaded %d groups, %d commands.", len(self.command_group_table), len(self.command_table)) + # The index may be outdated. Make sure the command appears in the loaded command table + command_str = _roughly_parse_command(args) + if command_str in self.command_table: + logger.debug("Found a match in the command table for '%s'", command_str) + return self.command_table + if command_str in self.command_group_table: + logger.debug("Found a match in the command group table for '%s'", command_str) + return self.command_table + + logger.debug("Could not find a match in the command table for '%s'. The index may be outdated", + command_str) + else: + logger.debug("No module found from index for '%s'", args) + + # No module found from the index. Load all command modules and extensions + logger.debug("Loading all modules and extensions") _update_command_table_from_modules(args) - try: - ext_suppressions = _get_extension_suppressions(self.loaders) - # We always load extensions even if the appropriate module has been loaded - # as an extension could override the commands already loaded. - _update_command_table_from_extensions(ext_suppressions) - except Exception: # pylint: disable=broad-except - logger.warning("Unable to load extensions. Use --debug for more information.") - logger.debug(traceback.format_exc()) + + ext_suppressions = _get_extension_suppressions(self.loaders) + # We always load extensions even if the appropriate module has been loaded + # as an extension could override the commands already loaded. + _update_command_table_from_extensions(ext_suppressions) + logger.debug("Loaded %d groups, %d commands.", len(self.command_group_table), len(self.command_table)) + + if use_command_index: + command_index.update(self.command_table) return self.command_table @@ -348,6 +474,111 @@ loader._update_command_definitions() # pylint: disable=protected-access +class CommandIndex: + + _COMMAND_INDEX = 'commandIndex' + _COMMAND_INDEX_VERSION = 'version' + _COMMAND_INDEX_CLOUD_PROFILE = 'cloudProfile' + + def __init__(self, cli_ctx=None): + """Class to manage command index. + + :param cli_ctx: Only needed when `get` or `update` is called. + """ + from azure.cli.core._session import INDEX + self.INDEX = INDEX + if cli_ctx: + self.version = __version__ + self.cloud_profile = cli_ctx.cloud.profile + + def get(self, args): + """Get the corresponding module and extension list of a command. + + :param args: command arguments, like ['network', 'vnet', 'create', '-h'] + :return: a tuple containing a list of modules and a list of extensions. + """ + # If the command index version or cloud profile doesn't match those of the current command, + # invalidate the command index. + index_version = self.INDEX[self._COMMAND_INDEX_VERSION] + cloud_profile = self.INDEX[self._COMMAND_INDEX_CLOUD_PROFILE] + if not (index_version and index_version == self.version and + cloud_profile and cloud_profile == self.cloud_profile): + logger.debug("Command index version or cloud profile is invalid or doesn't match the current command.") + self.invalidate() + return None + + # Make sure the top-level command is provided, like `az version`. + # Skip command index for `az` or `az --help`. + if not args or args[0].startswith('-'): + return None + + # Get the top-level command, like `network` in `network vnet create -h` + top_command = args[0] + index = self.INDEX[self._COMMAND_INDEX] + # Check the command index for (command: [module]) mapping, like + # "network": ["azure.cli.command_modules.natgateway", "azure.cli.command_modules.network", "azext_firewall"] + index_modules_extensions = index.get(top_command) + + if index_modules_extensions: + # This list contains both built-in modules and extensions + index_builtin_modules = [] + index_extensions = [] + # Found modules from index + logger.debug("Modules found from index for '%s': %s", top_command, index_modules_extensions) + command_module_prefix = 'azure.cli.command_modules.' + for m in index_modules_extensions: + if m.startswith(command_module_prefix): + # The top-level command is from a command module + index_builtin_modules.append(m[len(command_module_prefix):]) + elif m.startswith('azext_'): + # The top-level command is from an extension + index_extensions.append(m) + else: + logger.warning("Unrecognized module: %s", m) + return index_builtin_modules, index_extensions + + return None + + def update(self, command_table): + """Update the command index according to the given command table. + + :param command_table: The command table built by azure.cli.core.MainCommandsLoader.load_command_table + """ + start_time = timeit.default_timer() + self.INDEX[self._COMMAND_INDEX_VERSION] = __version__ + self.INDEX[self._COMMAND_INDEX_CLOUD_PROFILE] = self.cloud_profile + from collections import defaultdict + index = defaultdict(list) + + # self.cli_ctx.invocation.commands_loader.command_table doesn't exist in DummyCli due to the lack of invocation + for command_name, command in command_table.items(): + # Get the top-level name: <vm> create + top_command = command_name.split()[0] + # Get module name, like azure.cli.command_modules.vm, azext_webapp + module_name = command.loader.__module__ + if module_name not in index[top_command]: + index[top_command].append(module_name) + elapsed_time = timeit.default_timer() - start_time + self.INDEX[self._COMMAND_INDEX] = index + logger.debug("Updated command index in %.3f seconds.", elapsed_time) + + def invalidate(self): + """Invalidate the command index. + + This function MUST be called when installing or updating extensions. Otherwise, when an extension + 1. overrides a built-in command, or + 2. extends an existing command group, + the command or command group will only be loaded from the command modules as per the stale command index, + making the newly installed extension be ignored. + + This function can be called when removing extensions. + """ + self.INDEX[self._COMMAND_INDEX_VERSION] = "" + self.INDEX[self._COMMAND_INDEX_CLOUD_PROFILE] = "" + self.INDEX[self._COMMAND_INDEX] = {} + logger.debug("Command index has been invalidated.") + + class ModExtensionSuppress(object): # pylint: disable=too-few-public-methods def __init__(self, mod_name, suppress_extension_name, suppress_up_to_version, reason=None, recommend_remove=False, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-core-2.7.0/azure/cli/core/_profile.py new/azure-cli-core-2.9.1/azure/cli/core/_profile.py --- old/azure-cli-core-2.7.0/azure/cli/core/_profile.py 2020-05-29 10:06:31.000000000 +0200 +++ new/azure-cli-core-2.9.1/azure/cli/core/_profile.py 2020-07-16 10:09:55.000000000 +0200 @@ -14,7 +14,6 @@ import string from copy import deepcopy from enum import Enum -from six.moves import BaseHTTPServer from azure.cli.core._environment import get_config_dir from azure.cli.core._session import ACCOUNT @@ -307,18 +306,18 @@ import jwt from requests import HTTPError - from msrestazure.azure_active_directory import MSIAuthentication from msrestazure.tools import is_valid_resource_id + from azure.cli.core.adal_authentication import MSIAuthenticationWrapper resource = self.cli_ctx.cloud.endpoints.active_directory_resource_id if identity_id: if is_valid_resource_id(identity_id): - msi_creds = MSIAuthentication(resource=resource, msi_res_id=identity_id) + msi_creds = MSIAuthenticationWrapper(resource=resource, msi_res_id=identity_id) identity_type = MsiAccountTypes.user_assigned_resource_id else: authenticated = False try: - msi_creds = MSIAuthentication(resource=resource, client_id=identity_id) + msi_creds = MSIAuthenticationWrapper(resource=resource, client_id=identity_id) identity_type = MsiAccountTypes.user_assigned_client_id authenticated = True except HTTPError as ex: @@ -330,7 +329,7 @@ if not authenticated: try: identity_type = MsiAccountTypes.user_assigned_object_id - msi_creds = MSIAuthentication(resource=resource, object_id=identity_id) + msi_creds = MSIAuthenticationWrapper(resource=resource, object_id=identity_id) authenticated = True except HTTPError as ex: if ex.response.reason == 'Bad Request' and ex.response.status == 400: @@ -343,7 +342,7 @@ else: identity_type = MsiAccountTypes.system_assigned - msi_creds = MSIAuthentication(resource=resource) + msi_creds = MSIAuthenticationWrapper(resource=resource) token_entry = msi_creds.token token = token_entry['access_token'] @@ -388,8 +387,8 @@ return deepcopy(consolidated) def _get_token_from_cloud_shell(self, resource): # pylint: disable=no-self-use - from msrestazure.azure_active_directory import MSIAuthentication - auth = MSIAuthentication(resource=resource) + from azure.cli.core.adal_authentication import MSIAuthenticationWrapper + auth = MSIAuthenticationWrapper(resource=resource) auth.set_token() token_entry = auth.token return (token_entry['token_type'], token_entry['access_token'], token_entry) @@ -650,9 +649,11 @@ tenant_dest, resource) else: # Service Principal + use_cert_sn_issuer = bool(account[_USER_ENTITY].get(_SERVICE_PRINCIPAL_CERT_SN_ISSUER_AUTH)) creds = self._creds_cache.retrieve_token_for_service_principal(username_or_sp_id, resource, - tenant_dest) + tenant_dest, + use_cert_sn_issuer) return (creds, None if tenant else str(account[_SUBSCRIPTION_ID]), str(tenant if tenant else account[_TENANT_ID])) @@ -772,15 +773,15 @@ @staticmethod def msi_auth_factory(cli_account_name, identity, resource): - from msrestazure.azure_active_directory import MSIAuthentication + from azure.cli.core.adal_authentication import MSIAuthenticationWrapper if cli_account_name == MsiAccountTypes.system_assigned: - return MSIAuthentication(resource=resource) + return MSIAuthenticationWrapper(resource=resource) if cli_account_name == MsiAccountTypes.user_assigned_client_id: - return MSIAuthentication(resource=resource, client_id=identity) + return MSIAuthenticationWrapper(resource=resource, client_id=identity) if cli_account_name == MsiAccountTypes.user_assigned_object_id: - return MSIAuthentication(resource=resource, object_id=identity) + return MSIAuthenticationWrapper(resource=resource, object_id=identity) if cli_account_name == MsiAccountTypes.user_assigned_resource_id: - return MSIAuthentication(resource=resource, msi_res_id=identity) + return MSIAuthenticationWrapper(resource=resource, msi_res_id=identity) raise ValueError("unrecognized msi account name '{}'".format(cli_account_name)) @@ -1132,22 +1133,25 @@ def __init__(self, password_arg_value, use_cert_sn_issuer=None): if not password_arg_value: raise CLIError('missing secret or certificate in order to ' - 'authnenticate through a service principal') + 'authenticate through a service principal') if os.path.isfile(password_arg_value): certificate_file = password_arg_value from OpenSSL.crypto import load_certificate, FILETYPE_PEM self.certificate_file = certificate_file self.public_certificate = None - with open(certificate_file, 'r') as file_reader: - self.cert_file_string = file_reader.read() - cert = load_certificate(FILETYPE_PEM, self.cert_file_string) - self.thumbprint = cert.digest("sha1").decode() - if use_cert_sn_issuer: - # low-tech but safe parsing based on - # https://github.com/libressl-portable/openbsd/blob/master/src/lib/libcrypto/pem/pem.h - match = re.search(r'\-+BEGIN CERTIFICATE.+\-+(?P<public>[^-]+)\-+END CERTIFICATE.+\-+', - self.cert_file_string, re.I) - self.public_certificate = match.group('public').strip() + try: + with open(certificate_file, 'r') as file_reader: + self.cert_file_string = file_reader.read() + cert = load_certificate(FILETYPE_PEM, self.cert_file_string) + self.thumbprint = cert.digest("sha1").decode() + if use_cert_sn_issuer: + # low-tech but safe parsing based on + # https://github.com/libressl-portable/openbsd/blob/master/src/lib/libcrypto/pem/pem.h + match = re.search(r'\-+BEGIN CERTIFICATE.+\-+(?P<public>[^-]+)\-+END CERTIFICATE.+\-+', + self.cert_file_string, re.I) + self.public_certificate = match.group('public').strip() + except UnicodeDecodeError: + raise CLIError('Invalid certificate, please use a valid PEM file.') else: self.secret = password_arg_value @@ -1171,43 +1175,43 @@ return entry -class ClientRedirectServer(BaseHTTPServer.HTTPServer): # pylint: disable=too-few-public-methods - query_params = {} - +def _get_authorization_code_worker(authority_url, resource, results): + # pylint: disable=too-many-statements + import socket + import random + import http.server -class ClientRedirectHandler(BaseHTTPServer.BaseHTTPRequestHandler): - # pylint: disable=line-too-long + class ClientRedirectServer(http.server.HTTPServer): # pylint: disable=too-few-public-methods + query_params = {} - def do_GET(self): - try: - from urllib.parse import parse_qs - except ImportError: - from urlparse import parse_qs # pylint: disable=import-error - - if self.path.endswith('/favicon.ico'): # deal with legacy IE - self.send_response(204) - return - - query = self.path.split('?', 1)[-1] - query = parse_qs(query, keep_blank_values=True) - self.server.query_params = query - - self.send_response(200) - self.send_header('Content-type', 'text/html') - self.end_headers() - - landing_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'auth_landing_pages', - 'ok.html' if 'code' in query else 'fail.html') - with open(landing_file, 'rb') as html_file: - self.wfile.write(html_file.read()) - - def log_message(self, format, *args): # pylint: disable=redefined-builtin,unused-argument,no-self-use - pass # this prevent http server from dumping messages to stdout + class ClientRedirectHandler(http.server.BaseHTTPRequestHandler): + # pylint: disable=line-too-long + def do_GET(self): + try: + from urllib.parse import parse_qs + except ImportError: + from urlparse import parse_qs # pylint: disable=import-error + + if self.path.endswith('/favicon.ico'): # deal with legacy IE + self.send_response(204) + return + + query = self.path.split('?', 1)[-1] + query = parse_qs(query, keep_blank_values=True) + self.server.query_params = query + + self.send_response(200) + self.send_header('Content-type', 'text/html') + self.end_headers() + + landing_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'auth_landing_pages', + 'ok.html' if 'code' in query else 'fail.html') + with open(landing_file, 'rb') as html_file: + self.wfile.write(html_file.read()) -def _get_authorization_code_worker(authority_url, resource, results): - import socket - import random + def log_message(self, format, *args): # pylint: disable=redefined-builtin,unused-argument,no-self-use + pass # this prevent http server from dumping messages to stdout reply_url = None diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-core-2.7.0/azure/cli/core/_session.py new/azure-cli-core-2.9.1/azure/cli/core/_session.py --- old/azure-cli-core-2.7.0/azure/cli/core/_session.py 2020-05-29 10:06:31.000000000 +0200 +++ new/azure-cli-core-2.9.1/azure/cli/core/_session.py 2020-07-16 10:09:55.000000000 +0200 @@ -105,6 +105,9 @@ # SESSION provides read-write session variables SESSION = Session() +# INDEX contains {top-level command: [command_modules and extensions]} mapping index +INDEX = Session() + # VERSIONS provides local versions and pypi versions. # DO NOT USE it to get the current version of azure-cli, # it could be lagged behind and can be used to check whether diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-core-2.7.0/azure/cli/core/adal_authentication.py new/azure-cli-core-2.9.1/azure/cli/core/adal_authentication.py --- old/azure-cli-core-2.7.0/azure/cli/core/adal_authentication.py 2020-05-29 10:06:31.000000000 +0200 +++ new/azure-cli-core-2.9.1/azure/cli/core/adal_authentication.py 2020-07-16 10:09:55.000000000 +0200 @@ -8,6 +8,7 @@ import adal from msrest.authentication import Authentication +from msrestazure.azure_active_directory import MSIAuthentication from azure.core.credentials import AccessToken from azure.cli.core.util import in_cloud_console @@ -60,8 +61,10 @@ # This method is exposed for Azure Core. def get_token(self, *scopes, **kwargs): # pylint:disable=unused-argument _, token, full_token, _ = self._get_token() - - return AccessToken(token, int(full_token['expiresIn'] + time.time())) + try: + return AccessToken(token, int(full_token['expiresIn'] + time.time())) + except KeyError: # needed to deal with differing unserialized MSI token payload + return AccessToken(token, int(full_token['expires_on'])) # This method is exposed for msrest. def signed_session(self, session=None): # pylint: disable=arguments-differ @@ -83,3 +86,10 @@ logger = get_logger(__name__) logger.warning("A Cloud Shell credential problem occurred. When you report the issue with the error " "below, please mention the hostname '%s'", socket.gethostname()) + + +class MSIAuthenticationWrapper(MSIAuthentication): + # This method is exposed for Azure Core. + def get_token(self): + self.set_token() + return AccessToken(self.token['access_token'], int(self.token['expires_on'])) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-core-2.7.0/azure/cli/core/cloud.py new/azure-cli-core-2.9.1/azure/cli/core/cloud.py --- old/azure-cli-core-2.7.0/azure/cli/core/cloud.py 2020-05-29 10:06:31.000000000 +0200 +++ new/azure-cli-core-2.9.1/azure/cli/core/cloud.py 2020-07-16 10:09:55.000000000 +0200 @@ -114,6 +114,7 @@ def __init__(self, storage_endpoint=None, + storage_sync_endpoint=None, keyvault_dns=None, sql_server_hostname=None, azure_datalake_store_file_system_endpoint=None, @@ -124,6 +125,7 @@ mariadb_server_endpoint=None): # Attribute names are significant. They are used when storing/retrieving clouds from config self.storage_endpoint = storage_endpoint + self.storage_sync_endpoint = storage_sync_endpoint self.keyvault_dns = keyvault_dns self.sql_server_hostname = sql_server_hostname self.mysql_server_endpoint = mysql_server_endpoint @@ -163,6 +165,14 @@ return graph_endpoint_mapper.get(cloud_name, None) +def _get_storage_sync_endpoint(cloud_name): + storage_sync_endpoint_mapper = { + 'AzureCloud': 'afs.azure.net', + 'AzureUSGovernment': 'afs.azure.us', + } + return storage_sync_endpoint_mapper.get(cloud_name, 'afs.azure.net') + + def _convert_arm_to_cli(arm_cloud_metadata_dict): cli_cloud_metadata_dict = {} for cloud in arm_cloud_metadata_dict: @@ -191,6 +201,7 @@ log_analytics_resource_id=arm_dict['logAnalyticsResourceId'] if 'logAnalyticsResourceId' in arm_dict else None), # pylint: disable=line-too-long suffixes=CloudSuffixes( storage_endpoint=arm_dict['suffixes']['storage'], + storage_sync_endpoint=arm_dict['suffix']['storageSyncEndpointSuffix'] if 'storageSyncEndpointSuffix' in arm_dict['suffixes'] else _get_storage_sync_endpoint(arm_dict['name']), # pylint: disable=line-too-long keyvault_dns=arm_dict['suffixes']['keyVaultDns'], sql_server_hostname=arm_dict['suffixes']['sqlServerHostname'], mysql_server_endpoint=arm_dict['suffixes']['mysqlServerEndpoint'], @@ -248,6 +259,7 @@ app_insights_telemetry_channel_resource_id='https://dc.applicationinsights.azure.com/v2/track'), suffixes=CloudSuffixes( storage_endpoint='core.windows.net', + storage_sync_endpoint='afs.azure.net', keyvault_dns='.vault.azure.net', sql_server_hostname='.database.windows.net', mysql_server_endpoint='.mysql.database.azure.com', @@ -304,6 +316,7 @@ app_insights_telemetry_channel_resource_id='https://dc.applicationinsights.us/v2/track'), suffixes=CloudSuffixes( storage_endpoint='core.usgovcloudapi.net', + storage_sync_endpoint='afs.azure.us', keyvault_dns='.vault.usgovcloudapi.net', sql_server_hostname='.database.usgovcloudapi.net', mysql_server_endpoint='.mysql.database.usgovcloudapi.net', diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-core-2.7.0/azure/cli/core/commands/__init__.py new/azure-cli-core-2.9.1/azure/cli/core/commands/__init__.py --- old/azure-cli-core-2.7.0/azure/cli/core/commands/__init__.py 2020-05-29 10:06:31.000000000 +0200 +++ new/azure-cli-core-2.9.1/azure/cli/core/commands/__init__.py 2020-07-16 10:09:55.000000000 +0200 @@ -21,7 +21,7 @@ # pylint: disable=unused-import from azure.cli.core.commands.constants import ( - BLACKLISTED_MODS, DEFAULT_QUERY_TIME_RANGE, CLI_COMMON_KWARGS, CLI_COMMAND_KWARGS, CLI_PARAM_KWARGS, + BLOCKED_MODS, DEFAULT_QUERY_TIME_RANGE, CLI_COMMON_KWARGS, CLI_COMMAND_KWARGS, CLI_PARAM_KWARGS, CLI_POSITIONAL_PARAM_KWARGS, CONFIRM_PARAM_NAME) from azure.cli.core.commands.parameters import ( AzArgumentContext, patch_arg_make_required, patch_arg_make_optional) @@ -827,11 +827,13 @@ @staticmethod def remove_additional_prop_layer(obj, converted_dic): - from msrest.serialization import Model - if isinstance(obj, Model): - # let us make sure this is the additional properties auto-generated by SDK - if ('additionalProperties' in converted_dic and isinstance(obj.additional_properties, dict)): + # Follow EAFP to flatten `additional_properties` auto-generated by SDK + # See https://docs.python.org/3/glossary.html#term-eafp + try: + if 'additionalProperties' in converted_dic and isinstance(obj.additional_properties, dict): converted_dic.update(converted_dic.pop('additionalProperties')) + except AttributeError: + pass return converted_dic def _validate_cmd_level(self, ns, cmd_validator): # pylint: disable=no-self-use @@ -843,12 +845,14 @@ pass def _validate_arg_level(self, ns, **_): # pylint: disable=no-self-use - from msrest.exceptions import ValidationError for validator in getattr(ns, '_argument_validators', []): try: validator(**self._build_kwargs(validator, ns)) - except ValidationError: - logger.debug('Validation error in %s.', str(validator)) + except Exception as ex: + # Delay the import and mimic an exception handler + from msrest.exceptions import ValidationError + if isinstance(ex, ValidationError): + logger.debug('Validation error in %s.', str(validator)) raise try: delattr(ns, '_argument_validators') @@ -1010,6 +1014,13 @@ def _load_command_loader(loader, args, name, prefix): module = import_module(prefix + name) loader_cls = getattr(module, 'COMMAND_LOADER_CLS', None) + if not loader_cls: + try: + get_command_loader = getattr(module, 'get_command_loader', None) + loader_cls = get_command_loader(loader.cli_ctx) + except (ImportError, AttributeError, TypeError): + logger.debug("Module '%s' is missing `get_command_loader` entry.", name) + command_table = {} if loader_cls: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-core-2.7.0/azure/cli/core/commands/arm.py new/azure-cli-core-2.9.1/azure/cli/core/commands/arm.py --- old/azure-cli-core-2.7.0/azure/cli/core/commands/arm.py 2020-05-29 10:06:31.000000000 +0200 +++ new/azure-cli-core-2.9.1/azure/cli/core/commands/arm.py 2020-07-16 10:09:55.000000000 +0200 @@ -179,7 +179,6 @@ def register_ids_argument(cli_ctx): from knack import events - from msrestazure.tools import parse_resource_id, is_valid_resource_id ids_metadata = {} @@ -305,6 +304,7 @@ if full_id_list: setattr(namespace, '_ids', full_id_list) + from azure.mgmt.core.tools import parse_resource_id, is_valid_resource_id for val in full_id_list: if not is_valid_resource_id(val): raise CLIError('invalid resource ID: {}'.format(val)) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-core-2.7.0/azure/cli/core/commands/constants.py new/azure-cli-core-2.9.1/azure/cli/core/commands/constants.py --- old/azure-cli-core-2.7.0/azure/cli/core/commands/constants.py 2020-05-29 10:06:31.000000000 +0200 +++ new/azure-cli-core-2.9.1/azure/cli/core/commands/constants.py 2020-07-16 10:09:55.000000000 +0200 @@ -30,11 +30,11 @@ # 1 hour in milliseconds DEFAULT_QUERY_TIME_RANGE = 3600000 -BLACKLISTED_MODS = ['context', 'shell', 'documentdb', 'component'] +BLOCKED_MODS = ['context', 'shell', 'documentdb', 'component'] -SURVEY_PROMPT = 'Please let us know how we are doing: https://aka.ms/clihats' +SURVEY_PROMPT = 'Please let us know how we are doing: https://aka.ms/azureclihats' SURVEY_PROMPT_COLOR = Fore.YELLOW + Style.BRIGHT + 'Please let us know how we are doing: ' + Fore.BLUE + \ - 'https://aka.ms/clihats' + Style.RESET_ALL + 'https://aka.ms/azureclihats' + Style.RESET_ALL UX_SURVEY_PROMPT = 'and let us know if you\'re interested in trying out our newest features: https://aka.ms/CLIUXstudy' UX_SURVEY_PROMPT_COLOR = Fore.YELLOW + Style.BRIGHT + \ 'and let us know if you\'re interested in trying out our newest features: ' \ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-core-2.7.0/azure/cli/core/decorators.py new/azure-cli-core-2.9.1/azure/cli/core/decorators.py --- old/azure-cli-core-2.7.0/azure/cli/core/decorators.py 2020-05-29 10:06:31.000000000 +0200 +++ new/azure-cli-core-2.9.1/azure/cli/core/decorators.py 2020-07-16 10:09:55.000000000 +0200 @@ -72,8 +72,9 @@ def _wrapped_func(*args, **kwargs): try: return func(*args, **kwargs) - except Exception as ex: # nopa pylint: disable=broad-except - get_logger(__name__).info('Suppress exception %s', ex) + except Exception: # nopa pylint: disable=broad-except + import traceback + get_logger(__name__).info('Suppress exception:\n%s', traceback.format_exc()) if fallback_return is not None: return fallback_return return _wrapped_func diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-core-2.7.0/azure/cli/core/extension/_index.py new/azure-cli-core-2.9.1/azure/cli/core/extension/_index.py --- old/azure-cli-core-2.7.0/azure/cli/core/extension/_index.py 2020-05-29 10:06:31.000000000 +0200 +++ new/azure-cli-core-2.9.1/azure/cli/core/extension/_index.py 2020-07-16 10:09:55.000000000 +0200 @@ -2,7 +2,6 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -import requests from knack.log import get_logger from knack.util import CLIError @@ -22,6 +21,7 @@ # pylint: disable=inconsistent-return-statements def get_index(index_url=None): + import requests from azure.cli.core.util import should_disable_connection_verify index_url = index_url or DEFAULT_INDEX_URL diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-core-2.7.0/azure/cli/core/extension/_resolve.py new/azure-cli-core-2.9.1/azure/cli/core/extension/_resolve.py --- old/azure-cli-core-2.7.0/azure/cli/core/extension/_resolve.py 2020-05-29 10:06:31.000000000 +0200 +++ new/azure-cli-core-2.9.1/azure/cli/core/extension/_resolve.py 2020-07-16 10:09:55.000000000 +0200 @@ -53,15 +53,19 @@ return filter_func -def resolve_from_index(extension_name, cur_version=None, index_url=None): +def resolve_from_index(extension_name, cur_version=None, index_url=None, target_version=None): """ Gets the download Url and digest for the matching extension + + :param cur_version: threshold verssion to filter out extensions. """ candidates = get_index_extensions(index_url=index_url).get(extension_name, []) + if not candidates: raise NoExtensionCandidatesError("No extension found with name '{}'".format(extension_name)) filters = [_is_not_platform_specific, _is_compatible_with_cli_version, _is_greater_than_cur_version(cur_version)] + for f in filters: logger.debug("Candidates %s", [c['filename'] for c in candidates]) candidates = list(filter(f, candidates)) @@ -71,7 +75,15 @@ candidates_sorted = sorted(candidates, key=lambda c: parse_version(c['metadata']['version']), reverse=True) logger.debug("Candidates %s", [c['filename'] for c in candidates_sorted]) logger.debug("Choosing the latest of the remaining candidates.") - chosen = candidates_sorted[0] + + if target_version: + try: + chosen = [c for c in candidates_sorted if c['metadata']['version'] == target_version][0] + except IndexError: + raise NoExtensionCandidatesError('Extension with version {} not found'.format(target_version)) + else: + chosen = candidates_sorted[0] + logger.debug("Chosen %s", chosen) download_url, digest = chosen.get('downloadUrl'), chosen.get('sha256Digest') if not download_url: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-core-2.7.0/azure/cli/core/extension/operations.py new/azure-cli-core-2.9.1/azure/cli/core/extension/operations.py --- old/azure-cli-core-2.7.0/azure/cli/core/extension/operations.py 2020-05-29 10:06:31.000000000 +0200 +++ new/azure-cli-core-2.9.1/azure/cli/core/extension/operations.py 2020-07-16 10:09:55.000000000 +0200 @@ -15,9 +15,9 @@ from subprocess import check_output, STDOUT, CalledProcessError from six.moves.urllib.parse import urlparse # pylint: disable=import-error -import requests from pkg_resources import parse_version +from azure.cli.core import CommandIndex from azure.cli.core.util import CLIError, reload_module from azure.cli.core.extension import (extension_exists, build_extension_path, get_extensions, get_extension_modname, get_extension, ext_compat_with_cli, @@ -63,6 +63,7 @@ def _whl_download_from_url(url_parse_result, ext_file): + import requests from azure.cli.core.util import should_disable_connection_verify url = url_parse_result.geturl() r = requests.get(url, stream=True, verify=(not should_disable_connection_verify())) @@ -105,6 +106,7 @@ tmp_dir = tempfile.mkdtemp() ext_file = os.path.join(tmp_dir, whl_filename) logger.debug('Downloading %s to %s', source, ext_file) + import requests try: cmd.cli_ctx.get_progress_controller().add(message='Downloading') _whl_download_from_url(url_parse_result, ext_file) @@ -188,8 +190,8 @@ def check_version_compatibility(azext_metadata): is_compatible, cli_core_version, min_required, max_required = ext_compat_with_cli(azext_metadata) - logger.debug("Extension compatibility result: is_compatible=%s cli_core_version=%s min_required=%s " - "max_required=%s", is_compatible, cli_core_version, min_required, max_required) + # logger.debug("Extension compatibility result: is_compatible=%s cli_core_version=%s min_required=%s " + # "max_required=%s", is_compatible, cli_core_version, min_required, max_required) if not is_compatible: min_max_msg_fmt = "The '{}' extension is not compatible with this version of the CLI.\n" \ "You have CLI core version {} and this extension " \ @@ -205,8 +207,12 @@ def add_extension(cmd, source=None, extension_name=None, index_url=None, yes=None, # pylint: disable=unused-argument - pip_extra_index_urls=None, pip_proxy=None, system=None): + pip_extra_index_urls=None, pip_proxy=None, system=None, + version=None): ext_sha256 = None + + version = None if version == 'latest' else version + if extension_name: cmd.cli_ctx.get_progress_controller().add(message='Searching') ext = None @@ -220,10 +226,16 @@ return logger.warning("Overriding development version of '%s' with production version.", extension_name) try: - source, ext_sha256 = resolve_from_index(extension_name, index_url=index_url) + source, ext_sha256 = resolve_from_index(extension_name, index_url=index_url, target_version=version) except NoExtensionCandidatesError as err: logger.debug(err) - raise CLIError("No matching extensions for '{}'. Use --debug for more information.".format(extension_name)) + + if version: + err = "No matching extensions for '{} ({})'. Use --debug for more information.".format(extension_name, version) + else: + err = "No matching extensions for '{}'. Use --debug for more information.".format(extension_name) + raise CLIError(err) + extension_name = _add_whl_ext(cmd=cmd, source=source, ext_sha256=ext_sha256, pip_extra_index_urls=pip_extra_index_urls, pip_proxy=pip_proxy, system=system) try: @@ -234,6 +246,7 @@ "Please use with discretion.", extension_name) elif extension_name and ext.preview: logger.warning("The installed extension '%s' is in preview.", extension_name) + CommandIndex().invalidate() except ExtensionNotInstalledException: pass @@ -253,6 +266,7 @@ # We call this just before we remove the extension so we can get the metadata before it is gone _augment_telemetry_with_ext_info(extension_name, ext) shutil.rmtree(ext.path, onerror=log_err) + CommandIndex().invalidate() except ExtensionNotInstalledException as e: raise CLIError(e) @@ -305,6 +319,7 @@ logger.debug('Copying %s to %s', backup_dir, extension_path) shutil.copytree(backup_dir, extension_path) raise CLIError('Failed to update. Rolled {} back to {}.'.format(extension_name, cur_version)) + CommandIndex().invalidate() except ExtensionNotInstalledException as e: raise CLIError(e) @@ -347,6 +362,24 @@ def add_extension_to_path(extension_name, ext_dir=None): ext_dir = ext_dir or get_extension(extension_name).path sys.path.append(ext_dir) + # If this path update should have made a new "azure" module available, + # extend the existing module with its path. This allows extensions to + # include (or depend on) Azure SDK modules that are not yet part of + # the CLI. This applies to both the "azure" and "azure.mgmt" namespaces, + # but ensures that modules installed by the CLI take priority. + azure_dir = os.path.join(ext_dir, "azure") + if os.path.isdir(azure_dir): + import azure + azure.__path__.append(azure_dir) + azure_mgmt_dir = os.path.join(azure_dir, "mgmt") + if os.path.isdir(azure_mgmt_dir): + try: + # Should have been imported already, so this will be quick + import azure.mgmt + except ImportError: + pass + else: + azure.mgmt.__path__.append(azure_mgmt_dir) def get_lsb_release(): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-core-2.7.0/azure/cli/core/parser.py new/azure-cli-core-2.9.1/azure/cli/core/parser.py --- old/azure-cli-core-2.7.0/azure/cli/core/parser.py 2020-05-29 10:06:31.000000000 +0200 +++ new/azure-cli-core-2.9.1/azure/cli/core/parser.py 2020-07-16 10:09:55.000000000 +0200 @@ -152,6 +152,9 @@ with CommandLoggerContext(logger): logger.error('%(prog)s: error: %(message)s', args) self.print_usage(sys.stderr) + # Manual recommendations + self._set_manual_recommendations(args['message']) + # AI recommendations failure_recovery_recommendations = self._get_failure_recovery_recommendations() self._suggestion_msg.extend(failure_recovery_recommendations) self._print_suggestion_msg(sys.stderr) @@ -179,6 +182,14 @@ argcomplete.autocomplete(self, validator=lambda c, p: c.lower().startswith(p.lower()), default_completer=lambda _: ()) + def _set_manual_recommendations(self, error_msg): + recommendations = [] + # recommendation for --query value error + if '--query' in error_msg: + recommendations.append('To learn more about [--query JMESPATH] usage in AzureCLI, ' + 'visit https://aka.ms/CLIQuery') + self._suggestion_msg.extend(recommendations) + def _get_failure_recovery_arguments(self, action=None): # Strip the leading "az " and any extraneous whitespace. command = self.prog[3:].strip() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-core-2.7.0/azure/cli/core/profiles/_shared.py new/azure-cli-core-2.9.1/azure/cli/core/profiles/_shared.py --- old/azure-cli-core-2.7.0/azure/cli/core/profiles/_shared.py 2020-05-29 10:06:31.000000000 +0200 +++ new/azure-cli-core-2.9.1/azure/cli/core/profiles/_shared.py 2020-07-16 10:09:55.000000000 +0200 @@ -131,22 +131,23 @@ AZURE_API_PROFILES = { 'latest': { ResourceType.MGMT_STORAGE: '2019-06-01', - ResourceType.MGMT_NETWORK: '2020-04-01', - ResourceType.MGMT_COMPUTE: SDKProfile('2019-07-01', { + ResourceType.MGMT_NETWORK: '2020-05-01', + ResourceType.MGMT_COMPUTE: SDKProfile('2020-06-01', { 'resource_skus': '2019-04-01', - 'disks': '2019-11-01', + 'disks': '2020-05-01', + 'disk_encryption_sets': '2020-05-01', 'snapshots': '2019-07-01', 'galleries': '2019-12-01', 'gallery_images': '2019-12-01', 'gallery_image_versions': '2019-12-01', - 'virtual_machine_scale_sets': '2019-12-01' + 'virtual_machine_scale_sets': '2020-06-01' }), ResourceType.MGMT_RESOURCE_FEATURES: '2015-12-01', ResourceType.MGMT_RESOURCE_LINKS: '2016-09-01', ResourceType.MGMT_RESOURCE_LOCKS: '2016-09-01', ResourceType.MGMT_RESOURCE_POLICY: '2019-09-01', ResourceType.MGMT_RESOURCE_RESOURCES: '2019-07-01', - ResourceType.MGMT_RESOURCE_SUBSCRIPTIONS: '2019-06-01', + ResourceType.MGMT_RESOURCE_SUBSCRIPTIONS: '2019-11-01', ResourceType.MGMT_RESOURCE_DEPLOYMENTSCRIPTS: '2019-10-01-preview', ResourceType.MGMT_NETWORK_DNS: '2018-05-01', ResourceType.MGMT_KEYVAULT: '2019-09-01', @@ -163,7 +164,7 @@ ResourceType.DATA_STORAGE_FILESHARE: '2019-07-07', ResourceType.DATA_STORAGE_QUEUE: '2018-03-28', ResourceType.DATA_COSMOS_TABLE: '2017-04-17', - ResourceType.MGMT_EVENTHUB: '2017-04-01', + ResourceType.MGMT_EVENTHUB: '2018-01-01-preview', ResourceType.MGMT_MONITOR: SDKProfile('2019-06-01', { 'activity_log_alerts': '2017-04-01', 'activity_logs': '2015-04-01', @@ -193,7 +194,8 @@ 'private_link_scoped_resources': '2019-10-17-preview', 'private_link_scope_operation_status': '2019-10-17-preview', 'private_link_scopes': '2019-10-17-preview', - 'private_endpoint_connections': '2019-10-17-preview' + 'private_endpoint_connections': '2019-10-17-preview', + 'subscription_diagnostic_settings': '2017-05-01-preview' }), ResourceType.MGMT_APPSERVICE: '2019-08-01', ResourceType.MGMT_IOTHUB: '2020-03-01', @@ -230,7 +232,7 @@ # to have commands show up in the hybrid profile which happens to have the latest # API versions ResourceType.MGMT_APPSERVICE: '2018-02-01', - ResourceType.MGMT_EVENTHUB: '2017-04-01', + ResourceType.MGMT_EVENTHUB: '2018-01-01-preview', ResourceType.MGMT_IOTHUB: '2019-03-22' }, '2018-03-01-hybrid': { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-core-2.7.0/azure/cli/core/telemetry.py new/azure-cli-core-2.9.1/azure/cli/core/telemetry.py --- old/azure-cli-core-2.7.0/azure/cli/core/telemetry.py 2020-05-29 10:06:31.000000000 +0200 +++ new/azure-cli-core-2.9.1/azure/cli/core/telemetry.py 2020-07-16 10:09:55.000000000 +0200 @@ -437,6 +437,7 @@ def _get_shell_type(): + # This method is not accurate and needs improvement, for instance all shells on Windows return 'cmd'. if 'ZSH_VERSION' in os.environ: return 'zsh' if 'BASH_VERSION' in os.environ: @@ -445,6 +446,9 @@ return 'ksh' if 'WINDIR' in os.environ: return 'cmd' + from azure.cli.core.util import in_cloud_console + if in_cloud_console(): + return 'cloud-shell' return _remove_cmd_chars(_remove_symbols(os.environ.get('SHELL'))) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-core-2.7.0/azure/cli/core/util.py new/azure-cli-core-2.9.1/azure/cli/core/util.py --- old/azure-cli-core-2.7.0/azure/cli/core/util.py 2020-05-29 10:06:31.000000000 +0200 +++ new/azure-cli-core-2.9.1/azure/cli/core/util.py 2020-07-16 10:09:55.000000000 +0200 @@ -17,12 +17,8 @@ import logging from six.moves.urllib.request import urlopen # pylint: disable=import-error - -from azure.common import AzureException -from azure.core.exceptions import AzureError from knack.log import get_logger from knack.util import CLIError, to_snake_case -from inspect import getfullargspec as get_arg_spec logger = get_logger(__name__) @@ -61,6 +57,8 @@ from msrestazure.azure_exceptions import CloudError from msrest.exceptions import HttpOperationError, ValidationError, ClientRequestError from azure.cli.core.azlogging import CommandLoggerContext + from azure.common import AzureException + from azure.core.exceptions import AzureError with CommandLoggerContext(logger): if isinstance(ex, JMESPathTypeError): @@ -477,6 +475,7 @@ """ IS this client a autorestv3/track2 one?. Could be refined later if necessary. """ + from inspect import getfullargspec as get_arg_spec args = get_arg_spec(client_class.__init__).args return "credential" in args @@ -726,34 +725,27 @@ result[key] = value uri_parameters = result or None + endpoints = cli_ctx.cloud.endpoints # If url is an ARM resource ID, like /subscriptions/xxx/resourcegroups/xxx?api-version=2019-07-01, # default to Azure Resource Manager. - # https://management.azure.com/ + subscriptions/xxx/resourcegroups/xxx?api-version=2019-07-01 + # https://management.azure.com + /subscriptions/xxx/resourcegroups/xxx?api-version=2019-07-01 if '://' not in url: - url = cli_ctx.cloud.endpoints.resource_manager + url.lstrip('/') + url = endpoints.resource_manager.rstrip('/') + url # Replace common tokens with real values. It is for smooth experience if users copy and paste the url from # Azure Rest API doc from azure.cli.core._profile import Profile - profile = Profile() + profile = Profile(cli_ctx=cli_ctx) if '{subscriptionId}' in url: url = url.replace('{subscriptionId}', cli_ctx.data['subscription_id'] or profile.get_subscription_id()) - token_subscription = None - _subscription_regexes = [re.compile('https://management.azure.com/subscriptions/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})'), - re.compile('https://graph.windows.net/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})')] - for regex in _subscription_regexes: - match = regex.match(url) - if match: - token_subscription = match.groups()[0] - logger.debug('Retrieve token from subscription %s', token_subscription) - + # Prepare the Bearer token for `Authorization` header if not skip_authorization_header and url.lower().startswith('https://'): + # Prepare `resource` for `get_raw_token` if not resource: - endpoints = cli_ctx.cloud.endpoints - # If url starts with ARM endpoint, like https://management.azure.com/, - # use active_directory_resource_id for resource. - # This follows the same behavior as azure.cli.core.commands.client_factory._get_mgmt_service_client + # If url starts with ARM endpoint, like `https://management.azure.com/`, + # use `active_directory_resource_id` for resource, like `https://management.core.windows.net/`. + # This follows the same behavior as `azure.cli.core.commands.client_factory._get_mgmt_service_client` if url.lower().startswith(endpoints.resource_manager.rstrip('/')): resource = endpoints.active_directory_resource_id else: @@ -767,8 +759,20 @@ resource = value break if resource: - token_info, _, _ = profile.get_raw_token(resource, subscription=token_subscription) - logger.debug('Retrievd AAD token for resource: %s', resource or 'ARM') + # Prepare `subscription` for `get_raw_token` + # If this is an ARM request, try to extract subscription ID from the URL. + # But there are APIs which don't require subscription ID, like /subscriptions, /tenants + # TODO: In the future when multi-tenant subscription is supported, we won't be able to uniquely identify + # the token from subscription anymore. + token_subscription = None + if url.lower().startswith(endpoints.resource_manager.rstrip('/')): + token_subscription = _extract_subscription_id(url) + if token_subscription: + logger.debug('Retrieving token for resource %s, subscription %s', resource, token_subscription) + token_info, _, _ = profile.get_raw_token(resource, subscription=token_subscription) + else: + logger.debug('Retrieving token for resource %s', resource) + token_info, _, _ = profile.get_raw_token(resource) token_type, token, _ = token_info headers = headers or {} headers['Authorization'] = '{} {}'.format(token_type, token) @@ -801,6 +805,18 @@ return r +def _extract_subscription_id(url): + """Extract the subscription ID from an ARM request URL.""" + subscription_regex = '/subscriptions/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})' + match = re.search(subscription_regex, url, re.IGNORECASE) + if match: + subscription_id = match.groups()[0] + logger.debug('Found subscription ID %s in the URL %s', subscription_id, url) + return subscription_id + logger.debug('No subscription ID specified in the URL %s', url) + return None + + def _log_request(request): """Log a client request. Copied from msrest https://github.com/Azure/msrest-for-python/blob/3653d29fc44da408898b07c710290a83d196b777/msrest/http_logger.py#L39 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-core-2.7.0/azure_cli_core.egg-info/PKG-INFO new/azure-cli-core-2.9.1/azure_cli_core.egg-info/PKG-INFO --- old/azure-cli-core-2.7.0/azure_cli_core.egg-info/PKG-INFO 2020-05-29 10:06:40.000000000 +0200 +++ new/azure-cli-core-2.9.1/azure_cli_core.egg-info/PKG-INFO 2020-07-16 10:10:07.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: azure-cli-core -Version: 2.7.0 +Version: 2.9.1 Summary: Microsoft Azure Command-Line Tools Core Module Home-page: https://github.com/Azure/azure-cli Author: Microsoft Corporation @@ -15,6 +15,18 @@ Release History =============== + 2.9.1 + ++++++ + * Minor fixes + + 2.9.0 + ++++++ + * Fix get_token() issue in msi login and `expiresIn` key error in cloud shell login credentials for track 2 SDK related commands (#14187) + + 2.8.0 + ++++++ + * Add get_command_loader() entry to support to load customized CommandLoader (#13763) + 2.7.0 ++++++ * Enable local context for location (#13682) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-core-2.7.0/azure_cli_core.egg-info/requires.txt new/azure-cli-core-2.9.1/azure_cli_core.egg-info/requires.txt --- old/azure-cli-core-2.7.0/azure_cli_core.egg-info/requires.txt 2020-05-29 10:06:40.000000000 +0200 +++ new/azure-cli-core-2.9.1/azure_cli_core.egg-info/requires.txt 2020-07-16 10:10:07.000000000 +0200 @@ -15,7 +15,7 @@ requests~=2.22 six~=1.12 pkginfo>=1.5.0.1 -azure-mgmt-resource==9.0.0 +azure-mgmt-resource==10.0.0 azure-mgmt-core==1.0.0 [:python_version<"3.0"] diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/azure-cli-core-2.7.0/setup.py new/azure-cli-core-2.9.1/setup.py --- old/azure-cli-core-2.7.0/setup.py 2020-05-29 10:06:31.000000000 +0200 +++ new/azure-cli-core-2.9.1/setup.py 2020-07-16 10:09:55.000000000 +0200 @@ -17,7 +17,7 @@ logger.warn("Wheel is not available, disabling bdist_wheel hook") cmdclass = {} -VERSION = "2.7.0" +VERSION = "2.9.1" # If we have source, validate that our version numbers match # This should prevent uploading releases with mismatched versions. try: @@ -67,7 +67,7 @@ 'requests~=2.22', 'six~=1.12', 'pkginfo>=1.5.0.1', - 'azure-mgmt-resource==9.0.0', + 'azure-mgmt-resource==10.0.0', 'azure-mgmt-core==1.0.0' ]
