AMBARI-21734 - Upgrade Associated Repository is Wrong for Downgrades and Patches (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ba15d80e Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ba15d80e Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ba15d80e Branch: refs/heads/branch-2.6 Commit: ba15d80e26b05468ec241470cf38b8530146460d Parents: 53fb9af Author: Jonathan Hurley <[email protected]> Authored: Wed Aug 16 15:04:55 2017 -0400 Committer: Jonathan Hurley <[email protected]> Committed: Thu Aug 17 11:51:07 2017 -0400 ---------------------------------------------------------------------- .../libraries/functions/constants.py | 1 - .../libraries/functions/upgrade_summary.py | 25 +++++++++- .../internal/UpgradeResourceProvider.java | 6 +-- .../orm/entities/UpgradeHistoryEntity.java | 2 +- .../ambari/server/state/UpgradeContext.java | 22 +++------ .../package/scripts/hive_metastore.py | 12 ++--- .../package/scripts/hive_server_upgrade.py | 8 ++-- .../0.12.0.2.0/package/scripts/params_linux.py | 4 ++ .../KAFKA/0.8.1/package/scripts/kafka.py | 29 ------------ .../KAFKA/0.8.1/package/scripts/kafka_broker.py | 19 ++++---- .../KAFKA/0.8.1/package/scripts/params.py | 5 ++ .../KAFKA/0.8.1/package/scripts/upgrade.py | 2 +- .../4.0/properties/stack_features.json | 6 --- .../HIVE/package/scripts/hive_metastore.py | 8 +++- .../HIVE/package/scripts/hive_server_upgrade.py | 7 ++- .../4.0/services/HIVE/package/scripts/params.py | 4 ++ .../KAFKA/package/scripts/kafka_broker.py | 19 +++++--- .../services/KAFKA/package/scripts/params.py | 4 ++ .../services/KAFKA/package/scripts/upgrade.py | 2 +- .../HIVE/package/scripts/hive_metastore.py | 11 +++-- .../HIVE/package/scripts/hive_server_upgrade.py | 12 +++-- .../HIVE/package/scripts/params_linux.py | 4 ++ .../KAFKA/package/scripts/kafka_broker.py | 20 ++++---- .../HDP/2.0.6/properties/stack_features.json | 6 --- .../internal/UpgradeResourceProviderTest.java | 4 +- .../ambari/server/state/UpgradeContextTest.java | 2 +- .../src/test/python/TestUpgradeSummary.py | 50 ++++++++++++++++++++ .../stacks/2.1/HIVE/test_hive_metastore.py | 30 ++++++------ .../HIVE/package/scripts/hive_metastore.py | 7 ++- .../HIVE/package/scripts/hive_server_upgrade.py | 13 ++++- .../HIVE/package/scripts/params_linux.py | 4 ++ 31 files changed, 217 insertions(+), 131 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-common/src/main/python/resource_management/libraries/functions/constants.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py index 6ae71ef..f46b7cf 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py @@ -65,7 +65,6 @@ class StackFeature: SPARK_LIVY2 = "spark_livy2" STORM_KERBEROS = "storm_kerberos" STORM_AMS = "storm_ams" - CREATE_KAFKA_BROKER_ID = "create_kafka_broker_id" KAFKA_LISTENERS = "kafka_listeners" KAFKA_KERBEROS = "kafka_kerberos" PIG_ON_TEZ = "pig_on_tez" http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-common/src/main/python/resource_management/libraries/functions/upgrade_summary.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/upgrade_summary.py b/ambari-common/src/main/python/resource_management/libraries/functions/upgrade_summary.py index f2f5e42..507bc42 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/upgrade_summary.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/upgrade_summary.py @@ -20,6 +20,7 @@ limitations under the License. from collections import namedtuple from resource_management.libraries.script.script import Script +from resource_management.libraries.functions.constants import Direction UpgradeSummary = namedtuple("UpgradeSummary", "type direction orchestration is_revert services") UpgradeServiceSummary = namedtuple("UpgradeServiceSummary", "service_name source_stack source_version target_stack target_version") @@ -85,6 +86,28 @@ def get_upgrade_summary(): orchestration=upgrade_summary["orchestration"], is_revert = upgrade_summary["isRevert"], services = service_summary_dict) + +def get_downgrade_from_version(service_name = None): + """ + Gets the downgrade-from-version for the specificed service. If there is no downgrade or + the service isn't participating in the downgrade, then this will return None + :param service_name: the service, or optionally onmitted to infer it from the command. + :return: the downgrade-from-version or None + """ + upgrade_summary = get_upgrade_summary() + if upgrade_summary is None: + return None + + if Direction.DOWNGRADE.lower() != upgrade_summary.direction.lower(): + return None + + service_summary = _get_service_summary(service_name) + if service_summary is None: + return None + + return service_summary.source_version + + def _get_service_summary(service_name): """ Gets the service summary for the upgrade/downgrade for the given service, or None if @@ -104,4 +127,4 @@ def _get_service_summary(service_name): if service_name not in service_summary: return None - return service_summary[service_name] + return service_summary[service_name] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java index 4a28f56..1ac2864 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java @@ -570,7 +570,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider continue; } - serviceVersions = new RepositoryVersions(history.getFromReposistoryVersion(), + serviceVersions = new RepositoryVersions(history.getSourceRepositoryVersion(), history.getTargetRepositoryVersion()); repositoryVersions.put(history.getServiceName(), serviceVersions); @@ -960,7 +960,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider @Experimental(feature = ExperimentalFeature.PATCH_UPGRADES, comment = "Shouldn't be getting the overall downgrade-to version.") UpgradeHistoryEntity lastHistory = lastUpgrade.getHistory().iterator().next(); - params.put(KeyNames.VERSION, lastHistory.getFromReposistoryVersion().getVersion()); + params.put(KeyNames.VERSION, lastHistory.getSourceRepositoryVersion().getVersion()); } } @@ -1429,7 +1429,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider // depending on whether this is an upgrade or a downgrade, the history // will be different - if (upgradeContext.getDirection() == Direction.UPGRADE || upgradeContext.isPatchRevert()) { + if (upgradeContext.getDirection() == Direction.UPGRADE) { history.setFromRepositoryVersion(component.getDesiredRepositoryVersion()); history.setTargetRepositoryVersion(upgradeContext.getRepositoryVersion()); } else { http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/UpgradeHistoryEntity.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/UpgradeHistoryEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/UpgradeHistoryEntity.java index 0f7ac72..aa19608 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/UpgradeHistoryEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/UpgradeHistoryEntity.java @@ -134,7 +134,7 @@ public class UpgradeHistoryEntity { * * @return the repository that the upgrade is coming from (not {@code null}). */ - public RepositoryVersionEntity getFromReposistoryVersion() { + public RepositoryVersionEntity getSourceRepositoryVersion() { return fromRepositoryVersion; } http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java index 3d7f533..60f44bb 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java @@ -302,27 +302,20 @@ public class UpgradeContext { throw new AmbariException("Can only revert successful upgrades, not downgrades."); } - Set<RepositoryVersionEntity> priors = new HashSet<>(); for (UpgradeHistoryEntity history : revertUpgrade.getHistory()) { - priors.add(history.getFromReposistoryVersion()); - // !!! build all service-specific m_services.add(history.getServiceName()); m_sourceRepositoryMap.put(history.getServiceName(), history.getTargetRepositoryVersion()); - m_targetRepositoryMap.put(history.getServiceName(), history.getFromReposistoryVersion()); - } - - if (priors.size() != 1) { - String message = String.format("Upgrade from %s could not be reverted as there is no single " - + " repository across services.", revertUpgrade.getRepositoryVersion().getVersion()); - - throw new AmbariException(message); + m_targetRepositoryMap.put(history.getServiceName(), history.getSourceRepositoryVersion()); } - m_repositoryVersion = priors.iterator().next(); + // downgrades (which is what a revert really is) have the same associated + // repo, just like regular downgrades + m_repositoryVersion = revertUpgrade.getRepositoryVersion(); // !!! the version is used later in validators upgradeRequestMap.put(UPGRADE_REPO_VERSION_ID, m_repositoryVersion.getId().toString()); + // !!! use the same upgrade pack that was used in the upgrade being reverted upgradeRequestMap.put(UPGRADE_PACK, revertUpgrade.getUpgradePackage()); @@ -368,7 +361,7 @@ public class UpgradeContext { for (UpgradeHistoryEntity history : upgrade.getHistory()) { m_services.add(history.getServiceName()); m_sourceRepositoryMap.put(history.getServiceName(), m_repositoryVersion); - m_targetRepositoryMap.put(history.getServiceName(), history.getFromReposistoryVersion()); + m_targetRepositoryMap.put(history.getServiceName(), history.getSourceRepositoryVersion()); } break; @@ -379,7 +372,6 @@ public class UpgradeContext { } } - /** * For the unit tests tests, there are multiple upgrade packs for the same * type, so allow picking one of them. In prod, this is empty. @@ -454,7 +446,7 @@ public class UpgradeContext { List<UpgradeHistoryEntity> allHistory = upgradeEntity.getHistory(); for (UpgradeHistoryEntity history : allHistory) { String serviceName = history.getServiceName(); - RepositoryVersionEntity sourceRepositoryVersion = history.getFromReposistoryVersion(); + RepositoryVersionEntity sourceRepositoryVersion = history.getSourceRepositoryVersion(); RepositoryVersionEntity targetRepositoryVersion = history.getTargetRepositoryVersion(); m_sourceRepositoryMap.put(serviceName, sourceRepositoryVersion); m_targetRepositoryMap.put(serviceName, targetRepositoryVersion); http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py index b75ff13..dc8efa3 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py @@ -25,14 +25,10 @@ from resource_management.libraries.script import Script from resource_management.libraries.functions import stack_select from resource_management.libraries.functions.constants import Direction from resource_management.libraries.functions.format import format -from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions import upgrade_summary from resource_management.libraries.functions.stack_features import check_stack_feature -from resource_management.libraries.functions.security_commons import build_expectations from resource_management.libraries.functions.security_commons import cached_kinit_executor -from resource_management.libraries.functions.security_commons import get_params_from_filesystem -from resource_management.libraries.functions.security_commons import validate_security_config_properties -from resource_management.libraries.functions.security_commons import FILE_TYPE_XML from resource_management.core.resources.system import File from setup_ranger_hive import setup_ranger_hive_metastore_service @@ -180,7 +176,11 @@ class HiveMetastoreDefault(HiveMetastore): # since the configurations have not been written out yet during an upgrade # we need to choose the original legacy location schematool_hive_server_conf_dir = params.hive_server_conf_dir - if not(check_stack_feature(StackFeature.CONFIG_VERSIONING, params.version_for_stack_feature_checks)): + + upgrade_from_version = upgrade_summary.get_source_version("HIVE", + default_version = params.version_for_stack_feature_checks) + + if not (check_stack_feature(StackFeature.CONFIG_VERSIONING, upgrade_from_version)): schematool_hive_server_conf_dir = LEGACY_HIVE_SERVER_CONF env_dict = { http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_upgrade.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_upgrade.py index 9121923..12c9e1c 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_upgrade.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_upgrade.py @@ -68,9 +68,8 @@ def deregister(): # If upgrading, the upgrade-target hive binary should be used to call the --deregister command. # If downgrading, the downgrade-source hive binary should be used to call the --deregister command. # By now <stack-selector-tool> has been called to set 'current' to target-stack - if "downgrade" == params.upgrade_direction: - # hive_bin - hive_execute_path = _get_hive_execute_path(params.version_for_stack_feature_checks) + if params.downgrade_from_version is not None: + hive_execute_path = _get_hive_execute_path(params.downgrade_from_version) command = format('hive --config {hive_server_conf_dir} --service hiveserver2 --deregister ' + current_hiveserver_version) Execute(command, user=params.hive_user, path=hive_execute_path, tries=1 ) @@ -112,6 +111,9 @@ def _get_current_hiveserver_version(): try: # When downgrading the source version should be the version we are downgrading from source_version = params.version_for_stack_feature_checks + if params.downgrade_from_version is not None: + source_version = params.downgrade_from_version + hive_execute_path = _get_hive_execute_path(source_version) version_hive_bin = params.hive_bin formatted_source_version = format_stack_version(source_version) http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index 886156a..647beac 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -89,6 +89,10 @@ major_stack_version = get_major_version(stack_version_formatted_major) # It cannot be used during the initial Cluser Install because the version is not yet known. version = default("/commandParams/version", None) +# When downgrading the 'version' is pointing to the downgrade-target version +# downgrade_from_version provides the source-version the downgrade is happening from +downgrade_from_version = upgrade_summary.get_downgrade_from_version("HIVE") + # get the correct version to use for checking stack features version_for_stack_feature_checks = get_stack_feature_version(config) http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py index 101cd40..85c3347 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py @@ -45,35 +45,6 @@ def kafka(upgrade_type=None): effective_version = params.stack_version_formatted if upgrade_type is None else format_stack_version(params.version) Logger.info(format("Effective stack version: {effective_version}")) - # In HDP-2.2 (Apache Kafka 0.8.1.1) we used to generate broker.ids based on hosts and add them to - # kafka's server.properties. In future version brokers can generate their own ids based on zookeeper seq - # We need to preserve the broker.id when user is upgrading from HDP-2.2 to any higher version. - # Once its preserved it will be written to kafka.log.dirs/meta.properties and it will be used from there on - # similarly we need preserve port as well during the upgrade - - if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE and \ - check_stack_feature(StackFeature.CREATE_KAFKA_BROKER_ID, params.version_for_stack_feature_checks) and \ - check_stack_feature(StackFeature.KAFKA_LISTENERS, params.version_for_stack_feature_checks): - if len(params.kafka_hosts) > 0 and params.hostname in params.kafka_hosts: - brokerid = str(sorted(params.kafka_hosts).index(params.hostname)) - kafka_server_config['broker.id'] = brokerid - Logger.info(format("Calculating broker.id as {brokerid}")) - if 'port' in kafka_server_config: - port = kafka_server_config['port'] - Logger.info(format("Port config from previous verson: {port}")) - listeners = kafka_server_config['listeners'] - kafka_server_config['listeners'] = listeners.replace("6667", port) - Logger.info(format("Kafka listeners after the port update: {listeners}")) - del kafka_server_config['port'] - - - if effective_version is not None and effective_version != "" and \ - check_stack_feature(StackFeature.CREATE_KAFKA_BROKER_ID, effective_version): - if len(params.kafka_hosts) > 0 and params.hostname in params.kafka_hosts: - brokerid = str(sorted(params.kafka_hosts).index(params.hostname)) - kafka_server_config['broker.id'] = brokerid - Logger.info(format("Calculating broker.id as {brokerid}")) - # listeners and advertised.listeners are only added in 2.3.0.0 onwards. if effective_version is not None and effective_version != "" and \ check_stack_feature(StackFeature.KAFKA_LISTENERS, effective_version): http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka_broker.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka_broker.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka_broker.py index 769e6ec..72ef5e9 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka_broker.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka_broker.py @@ -18,10 +18,10 @@ limitations under the License. """ from resource_management import Script from resource_management.core.logger import Logger -from resource_management.core.resources.system import Execute, File, Directory +from resource_management.core.resources.system import Execute, File from resource_management.libraries.functions import stack_select from resource_management.libraries.functions import Direction -from resource_management.libraries.functions.version import format_stack_version +from resource_management.libraries.functions import upgrade_summary from resource_management.libraries.functions.format import format from resource_management.libraries.functions.check_process_status import check_process_status from resource_management.libraries.functions import StackFeature @@ -47,16 +47,19 @@ class KafkaBroker(Script): import params env.set_params(params) - # grab the current version of the component - pre_upgrade_version = stack_select.get_role_component_current_stack_version() - if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): stack_select.select_packages(params.version) # This is extremely important since it should only be called if crossing the HDP 2.3.4.0 boundary. - if pre_upgrade_version and params.version_for_stack_feature_checks and params.upgrade_direction: - src_version = format_stack_version(pre_upgrade_version) - dst_version = format_stack_version(params.version_for_stack_feature_checks) + if params.version and params.upgrade_direction: + src_version = dst_version = None + if params.upgrade_direction == Direction.UPGRADE: + src_version = upgrade_summary.get_source_version("KAFKA", default_version = params.version) + dst_version = upgrade_summary.get_target_version("KAFKA", default_version = params.version) + else: + # These represent the original values during the UPGRADE direction + src_version = upgrade_summary.get_target_version("KAFKA", default_version = params.version) + dst_version = upgrade_summary.get_source_version("KAFKA", default_version = params.version) if not check_stack_feature(StackFeature.KAFKA_ACL_MIGRATION_SUPPORT, src_version) and check_stack_feature(StackFeature.KAFKA_ACL_MIGRATION_SUPPORT, dst_version): # Calling the acl migration script requires the configs to be present. http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py index 349ff5c..539b469 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py @@ -33,6 +33,7 @@ from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.functions import stack_select from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import get_kinit_path +from resource_management.libraries.functions import upgrade_summary from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs, generate_ranger_service_config @@ -57,6 +58,10 @@ stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBERO stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks) stack_supports_core_site_for_ranger_plugin = check_stack_feature(StackFeature.CORE_SITE_FOR_RANGER_PLUGINS_SUPPORT, version_for_stack_feature_checks) +# When downgrading the 'version' is pointing to the downgrade-target version +# downgrade_from_version provides the source-version the downgrade is happening from +downgrade_from_version = upgrade_summary.get_downgrade_from_version("KAFKA") + hostname = config['hostname'] # default kafka parameters http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/upgrade.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/upgrade.py index e79a8ad..e119211 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/upgrade.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/upgrade.py @@ -50,7 +50,7 @@ def run_migration(env, upgrade_type): Logger.info("Upgrade type: {0}, direction: {1}".format(str(upgrade_type), params.upgrade_direction)) # If the schema upgrade script exists in the version upgrading to, then attempt to upgrade/downgrade it while still using the present bits. - kafka_acls_script = format("{stack_root}/{version_for_stack_feature_checks}/kafka/bin/kafka-acls.sh") + kafka_acls_script = format("{stack_root}/{downgrade_from_version}/kafka/bin/kafka-acls.sh") command_suffix = "" if params.upgrade_direction == Direction.UPGRADE: command_suffix = "--upgradeAcls" http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-server/src/main/resources/stacks/BigInsights/4.0/properties/stack_features.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/properties/stack_features.json b/ambari-server/src/main/resources/stacks/BigInsights/4.0/properties/stack_features.json index bf00f13..d29efb2 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/properties/stack_features.json +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/properties/stack_features.json @@ -77,12 +77,6 @@ "min_version": "4.2.0.0" }, { - "name": "create_kafka_broker_id", - "description": "Ambari should create Kafka Broker Id (AMBARI-12678)", - "min_version": "4.0.0.0", - "max_version": "4.2.0.0" - }, - { "name": "kafka_listeners", "description": "Kafka listeners (AMBARI-10984)", "min_version": "4.2.0.0" http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HIVE/package/scripts/hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HIVE/package/scripts/hive_metastore.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HIVE/package/scripts/hive_metastore.py index 7728ac6..6d8f763 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HIVE/package/scripts/hive_metastore.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HIVE/package/scripts/hive_metastore.py @@ -23,6 +23,7 @@ from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute from resource_management.libraries.script import Script from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import upgrade_summary from resource_management.libraries.functions.format import format from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions.version import compare_versions @@ -178,9 +179,12 @@ class HiveMetastoreDefault(HiveMetastore): # since the configurations have not been written out yet during an upgrade # we need to choose the original legacy location schematool_hive_server_conf_dir = params.hive_server_conf_dir + + upgrade_from_version = upgrade_summary.get_source_version("HIVE", + default_version = params.version_for_stack_feature_checks) + if params.version_for_stack_feature_checks is not None: - version_for_stack_feature_checks = format_stack_version(params.version_for_stack_feature_checks) - if compare_versions(version_for_stack_feature_checks, "4.1.0.0") < 0: + if compare_versions(upgrade_from_version, "4.1.0.0") < 0: schematool_hive_server_conf_dir = LEGACY_HIVE_SERVER_CONF env_dict = { http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HIVE/package/scripts/hive_server_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HIVE/package/scripts/hive_server_upgrade.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HIVE/package/scripts/hive_server_upgrade.py index 0105a7d..ae122cc 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HIVE/package/scripts/hive_server_upgrade.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HIVE/package/scripts/hive_server_upgrade.py @@ -139,10 +139,9 @@ def post_upgrade_deregister(): hive_execute_path = params.execute_path # If upgrading, the upgrade-target hive binary should be used to call the --deregister command. # If downgrading, the downgrade-source hive binary should be used to call the --deregister command. - # By now hdp-select has been called to set 'current' to target-stack - if "downgrade" == params.upgrade_direction: - # hive_bin - hive_execute_path = _get_hive_execute_path(params.version_for_stack_feature_checks) + # By now <stack-selector-tool> has been called to set 'current' to target-stack + if params.downgrade_from_version is not None: + hive_execute_path = _get_hive_execute_path(params.downgrade_from_version) command = format('hive --config {hive_server_conf_dir} --service hiveserver2 --deregister ' + current_hiveserver_version) Execute(command, user=params.hive_user, path=hive_execute_path, tries=1 ) http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HIVE/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HIVE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HIVE/package/scripts/params.py index e9d62b1..8f2132a 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HIVE/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HIVE/package/scripts/params.py @@ -61,6 +61,10 @@ version = default("/commandParams/version", None) # Upgrade direction upgrade_direction = default("/commandParams/upgrade_direction", None) +# When downgrading the 'version' is pointing to the downgrade-target version +# downgrade_from_version provides the source-version the downgrade is happening from +downgrade_from_version = upgrade_summary.get_downgrade_from_version("HIVE") + # get the correct version to use for checking stack features version_for_stack_feature_checks = get_stack_feature_version(config) http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/KAFKA/package/scripts/kafka_broker.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/KAFKA/package/scripts/kafka_broker.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/KAFKA/package/scripts/kafka_broker.py index b8d44f0..4515a84 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/KAFKA/package/scripts/kafka_broker.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/KAFKA/package/scripts/kafka_broker.py @@ -20,10 +20,11 @@ limitations under the License. from resource_management import * from resource_management.libraries.functions import stack_select from resource_management.libraries.functions import Direction +from resource_management.libraries.functions import upgrade_summary from resource_management.libraries.functions.version import compare_versions, format_stack_version from resource_management import Script from resource_management.core.logger import Logger -from resource_management.core.resources.system import Execute, File, Directory +from resource_management.core.resources.system import Execute, File from resource_management.libraries.functions.format import format from resource_management.libraries.functions.check_process_status import check_process_status from kafka import ensure_base_directories @@ -52,16 +53,20 @@ class KafkaBroker(Script): import params env.set_params(params) - # grab the current version of the component - pre_upgrade_version = stack_select.get_role_component_current_stack_version() - if params.version and compare_versions(format_stack_version(params.version), '4.1.0.0') >= 0: stack_select.select_packages(params.version) # This is extremely important since it should only be called if crossing the IOP 4.2 boundary. - if pre_upgrade_version and params.version_for_stack_feature_checks and params.upgrade_direction: - src_version = format_stack_version(pre_upgrade_version) - dst_version = format_stack_version(params.version_for_stack_feature_checks) + # This is extremely important since it should only be called if crossing the HDP 2.3.4.0 boundary. + if params.version and params.upgrade_direction: + src_version = dst_version = None + if params.upgrade_direction == Direction.UPGRADE: + src_version = upgrade_summary.get_source_version("KAFKA", default_version = params.version) + dst_version = upgrade_summary.get_target_version("KAFKA", default_version = params.version) + else: + # These represent the original values during the UPGRADE direction + src_version = upgrade_summary.get_target_version("KAFKA", default_version = params.version) + dst_version = upgrade_summary.get_source_version("KAFKA", default_version = params.version) if compare_versions(src_version, '4.2.0.0') < 0 and compare_versions(dst_version, '4.2.0.0') >= 0: # Upgrade from IOP 4.1 to 4.2, Calling the acl migration script requires the configs to be present. http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/KAFKA/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/KAFKA/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/KAFKA/package/scripts/params.py index 799866a..8bc29b2 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/KAFKA/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/KAFKA/package/scripts/params.py @@ -19,6 +19,7 @@ limitations under the License. """ from resource_management.libraries.functions import format from resource_management.libraries.script.script import Script +from resource_management.libraries.functions import upgrade_summary from resource_management.libraries.functions.version import format_stack_version, compare_versions from resource_management.libraries.functions.default import default from resource_management.libraries.functions.stack_features import get_stack_feature_version @@ -49,6 +50,9 @@ stack_version_unformatted = str(config['hostLevelParams']['stack_version']) iop_stack_version = format_stack_version(stack_version_unformatted) upgrade_direction = default("/commandParams/upgrade_direction", None) +# When downgrading the 'version' is pointing to the downgrade-target version +# downgrade_from_version provides the source-version the downgrade is happening from +downgrade_from_version = upgrade_summary.get_downgrade_from_version("KAFKA") # default kafka parameters kafka_home = '/usr/iop/current/kafka-broker' http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/KAFKA/package/scripts/upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/KAFKA/package/scripts/upgrade.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/KAFKA/package/scripts/upgrade.py index e405cb5..992e68c 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/KAFKA/package/scripts/upgrade.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/KAFKA/package/scripts/upgrade.py @@ -51,7 +51,7 @@ def run_migration(env, upgrade_type): Logger.info("Upgrade type: {0}, direction: {1}".format(str(upgrade_type), params.upgrade_direction)) # If the schema upgrade script exists in the version upgrading to, then attempt to upgrade/downgrade it while still using the present bits. - kafka_acls_script = format("/usr/hdp/{version_for_stack_feature_checks}/kafka/bin/kafka-acls.sh") + kafka_acls_script = format("/usr/hdp/{downgrade_from_version}/kafka/bin/kafka-acls.sh") command_suffix = "" if params.upgrade_direction == Direction.UPGRADE: command_suffix = "--upgradeAcls" http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive_metastore.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive_metastore.py index 2705e5a..f4b78b1 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive_metastore.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive_metastore.py @@ -23,6 +23,7 @@ from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute, Directory from resource_management.libraries.script import Script from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import upgrade_summary from resource_management.libraries.functions.constants import Direction from resource_management.libraries.functions.format import format from resource_management.libraries.functions.version import format_stack_version @@ -213,10 +214,12 @@ class HiveMetastoreDefault(HiveMetastore): # since the configurations have not been written out yet during an upgrade # we need to choose the original legacy location schematool_hive_server_conf_dir = params.hive_server_conf_dir - if params.version_for_stack_feature_checks: - version = format_stack_version(params.version_for_stack_feature_checks) - if compare_versions(version, "4.1.0.0") < 0: - schematool_hive_server_conf_dir = LEGACY_HIVE_SERVER_CONF + + upgrade_from_version = upgrade_summary.get_source_version("HIVE", + default_version = params.version_for_stack_feature_checks) + + if compare_versions(upgrade_from_version, "4.1.0.0") < 0: + schematool_hive_server_conf_dir = LEGACY_HIVE_SERVER_CONF env_dict = { 'HIVE_CONF_DIR': schematool_hive_server_conf_dir http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive_server_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive_server_upgrade.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive_server_upgrade.py index 71263d4..ccbbfe9 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive_server_upgrade.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive_server_upgrade.py @@ -62,9 +62,9 @@ def post_upgrade_deregister(): hive_execute_path = params.execute_path # If upgrading, the upgrade-target hive binary should be used to call the --deregister command. # If downgrading, the downgrade-source hive binary should be used to call the --deregister command. - # By now hdp-select has been called to set 'current' to target-stack - if "downgrade" == params.upgrade_direction: - hive_execute_path = _get_hive_execute_path(params.version_for_stack_feature_checks) + # By now <stack-selector-tool> has been called to set 'current' to target-stack + if params.downgrade_from_version is not None: + hive_execute_path = _get_hive_execute_path(params.downgrade_from_version) command = format('hive --config {hive_server_conf_dir} --service hiveserver2 --deregister ' + current_hiveserver_version) Execute(command, user=params.hive_user, path=hive_execute_path, tries=1 ) @@ -103,7 +103,11 @@ def _get_current_hiveserver_version(): try: # When downgrading the source version should be the version we are downgrading from - hive_execute_path = _get_hive_execute_path(params.version_for_stack_feature_checks) + source_version = params.version_for_stack_feature_checks + if params.downgrade_from_version is not None: + source_version = params.downgrade_from_version + + hive_execute_path = _get_hive_execute_path(source_version) version_hive_bin = params.hive_bin formatted_source_version = format_stack_version(params.version_for_stack_feature_checks) if formatted_source_version and compare_versions(formatted_source_version, "4.1") >= 0: http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/params_linux.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/params_linux.py index 770d3c5..528760a 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/params_linux.py @@ -64,6 +64,10 @@ iop_stack_version = functions.get_stack_version('hive-server2') # It cannot be used during the initial Cluser Install because the version is not yet known. version = default("/commandParams/version", None) +# When downgrading the 'version' is pointing to the downgrade-target version +# downgrade_from_version provides the source-version the downgrade is happening from +downgrade_from_version = upgrade_summary.get_downgrade_from_version("HIVE") + # get the correct version to use for checking stack features version_for_stack_feature_checks = get_stack_feature_version(config) http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/KAFKA/package/scripts/kafka_broker.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/KAFKA/package/scripts/kafka_broker.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/KAFKA/package/scripts/kafka_broker.py index 16e7039..511906d 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/KAFKA/package/scripts/kafka_broker.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/KAFKA/package/scripts/kafka_broker.py @@ -18,10 +18,11 @@ limitations under the License. """ from resource_management.libraries.functions import stack_select from resource_management.libraries.functions import Direction +from resource_management.libraries.functions import upgrade_summary from resource_management.libraries.functions.version import compare_versions, format_stack_version from resource_management import Script from resource_management.core.logger import Logger -from resource_management.core.resources.system import Execute, File, Directory +from resource_management.core.resources.system import Execute, File from resource_management.libraries.functions.format import format from resource_management.libraries.functions.check_process_status import check_process_status from kafka import ensure_base_directories @@ -44,16 +45,19 @@ class KafkaBroker(Script): import params env.set_params(params) - # grab the current version of the component - pre_upgrade_version = stack_select.get_role_component_current_stack_version() - if params.version and compare_versions(format_stack_version(params.version), '4.1.0.0') >= 0: stack_select.select_packages(params.version) - # This is extremely important since it should only be called if crossing the IOP 4.2 boundary. - if pre_upgrade_version and params.version_for_stack_feature_checks and params.upgrade_direction: - src_version = format_stack_version(pre_upgrade_version) - dst_version = format_stack_version(params.version_for_stack_feature_checks) + # This is extremely important since it should only be called if crossing the HDP 2.3.4.0 boundary. + if params.version and params.upgrade_direction: + src_version = dst_version = None + if params.upgrade_direction == Direction.UPGRADE: + src_version = upgrade_summary.get_source_version("KAFKA", default_version = params.version) + dst_version = upgrade_summary.get_target_version("KAFKA", default_version = params.version) + else: + # These represent the original values during the UPGRADE direction + src_version = upgrade_summary.get_target_version("KAFKA", default_version = params.version) + dst_version = upgrade_summary.get_source_version("KAFKA", default_version = params.version) if compare_versions(src_version, '4.2.0.0') < 0 and compare_versions(dst_version, '4.2.0.0') >= 0: # Upgrade from IOP 4.1 to 4.2, Calling the acl migration script requires the configs to be present. http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json index 973d9dd..86de20d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json @@ -104,12 +104,6 @@ "min_version": "2.2.0.0" }, { - "name": "create_kafka_broker_id", - "description": "Ambari should create Kafka Broker Id (AMBARI-12678)", - "min_version": "2.2.0.0", - "max_version": "2.3.0.0" - }, - { "name": "kafka_listeners", "description": "Kafka listeners (AMBARI-10984)", "min_version": "2.3.0.0" http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java index aa216f7..52f41ca 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java @@ -1559,7 +1559,7 @@ public class UpgradeResourceProviderTest extends EasyMockSupport { for( UpgradeHistoryEntity history : histories){ assertEquals( "ZOOKEEPER", history.getServiceName() ); - assertEquals(repoVersionEntity2110, history.getFromReposistoryVersion()); + assertEquals(repoVersionEntity2110, history.getSourceRepositoryVersion()); assertEquals(repoVersionEntity2200, history.getTargetRepositoryVersion()); } @@ -1585,7 +1585,7 @@ public class UpgradeResourceProviderTest extends EasyMockSupport { for (UpgradeHistoryEntity history : histories) { assertEquals("ZOOKEEPER", history.getServiceName()); - assertEquals(repoVersionEntity2200, history.getFromReposistoryVersion()); + assertEquals(repoVersionEntity2200, history.getSourceRepositoryVersion()); assertEquals(repoVersionEntity2110, history.getTargetRepositoryVersion()); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeContextTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeContextTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeContextTest.java index 629ea9b..500d229 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeContextTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeContextTest.java @@ -125,7 +125,7 @@ public class UpgradeContextTest extends EasyMockSupport { UpgradeHistoryEntity upgradeHistoryEntity = createNiceMock(UpgradeHistoryEntity.class); expect(upgradeHistoryEntity.getServiceName()).andReturn(HDFS_SERVICE_NAME).anyTimes(); - expect(upgradeHistoryEntity.getFromReposistoryVersion()).andReturn(m_sourceRepositoryVersion).anyTimes(); + expect(upgradeHistoryEntity.getSourceRepositoryVersion()).andReturn(m_sourceRepositoryVersion).anyTimes(); expect(upgradeHistoryEntity.getTargetRepositoryVersion()).andReturn(m_targetRepositoryVersion).anyTimes(); List<UpgradeHistoryEntity> upgradeHistory = Lists.newArrayList(upgradeHistoryEntity); http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-server/src/test/python/TestUpgradeSummary.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestUpgradeSummary.py b/ambari-server/src/test/python/TestUpgradeSummary.py index 614dcd2..7606867 100644 --- a/ambari-server/src/test/python/TestUpgradeSummary.py +++ b/ambari-server/src/test/python/TestUpgradeSummary.py @@ -49,6 +49,20 @@ class TestUpgradeSummary(TestCase): self.assertEqual("2.4.0.0-1234", upgrade_summary.get_source_version("HDFS")) self.assertEqual("2.5.9.9-9999", upgrade_summary.get_target_version("HDFS")) + self.assertIsNone(upgrade_summary.get_downgrade_from_version("HDFS")) + + + def test_get_downgrade_from_version(self): + """ + Tests that simple downgrade returns the correct version + :return: + """ + command_json = TestUpgradeSummary._get_cluster_simple_downgrade_json() + Script.config = command_json + + self.assertIsNone(upgrade_summary.get_downgrade_from_version("FOO")) + self.assertEqual("2.5.9.9-9999", upgrade_summary.get_downgrade_from_version("HDFS")) + @staticmethod def _get_cluster_simple_upgrade_json(): @@ -84,4 +98,40 @@ class TestUpgradeSummary(TestCase): "isRevert":False, "orchestration":"STANDARD" } + } + + @staticmethod + def _get_cluster_simple_downgrade_json(): + """ + A restart command during a downgrade. + :return: + """ + return { + "roleCommand":"ACTIONEXECUTE", + "hostLevelParams": { + "stack_name": "HDP", + "stack_version": "2.4", + }, + "commandParams": { + "source_stack": "2.5", + "target_stack": "2.4", + "upgrade_direction": "downgrade", + "version": "2.4.0.0-1234" + }, + "upgradeSummary": { + "services":{ + "HDFS":{ + "sourceRepositoryId":2, + "sourceStackId":"HDP-2.5", + "sourceVersion":"2.5.9.9-9999", + "targetRepositoryId":1, + "targetStackId":"HDP-2.4", + "targetVersion":"2.4.0.0-1234" + } + }, + "direction":"DOWNGRADE", + "type":"rolling_upgrade", + "isRevert":False, + "orchestration":"STANDARD" + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py index ac5d6ae..82ade7c 100644 --- a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py +++ b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py @@ -513,10 +513,10 @@ class TestHiveMetastore(RMFTestCase): @patch("resource_management.core.shell.call") @patch("resource_management.libraries.functions.get_stack_version") def test_upgrade_metastore_schema(self, get_stack_version_mock, call_mock, os_path_exists_mock): - get_stack_version_mock.return_value = '2.3.0.0-1234' + get_stack_version_mock.return_value = '2.4.0.0-1234' def side_effect(path): - if path == "/usr/hdp/2.2.7.0-1234/hive-server2/lib/mysql-connector-java.jar": + if path == "/usr/hdp/2.3.7.0-1234/hive-server2/lib/mysql-connector-java.jar": return True if ".j2" in path: return True @@ -529,19 +529,19 @@ class TestHiveMetastore(RMFTestCase): with open(config_file, "r") as f: json_content = json.load(f) - # must be HDP 2.3+ - version = '2.3.0.0-1234' + # must be HDP 2.4+ + version = '2.4.0.0-1234' json_content['commandParams']['version'] = version json_content['commandParams']['upgrade_direction'] = Direction.UPGRADE - json_content['hostLevelParams']['stack_version'] = "2.3" + json_content['hostLevelParams']['stack_version'] = "2.4" json_content["upgradeSummary"] = { "services":{ "HIVE":{ "sourceRepositoryId":1, - "sourceStackId":"HDP-2.2", - "sourceVersion":"2.2.7.0-1234", + "sourceStackId":"HDP-2.3", + "sourceVersion":"2.3.7.0-1234", "targetRepositoryId":2, - "targetStackId":"HDP-2.3", + "targetStackId":"HDP-2.4", "targetVersion":version } }, @@ -569,7 +569,7 @@ class TestHiveMetastore(RMFTestCase): '/usr/bin/hdp-select', 'set', 'hive-metastore', - '2.3.0.0-1234'), + '2.4.0.0-1234'), sudo = True) # we don't care about configure here - the strings are different anyway because this @@ -587,23 +587,23 @@ class TestHiveMetastore(RMFTestCase): self.assertResourceCalled('Execute', ('cp', '--remove-destination', '/tmp/mysql-connector-java.jar', - '/usr/hdp/2.3.0.0-1234/hive/lib/mysql-connector-java.jar'), + '/usr/hdp/2.4.0.0-1234/hive/lib/mysql-connector-java.jar'), path = ['/bin', '/usr/bin/'], sudo = True) - self.assertResourceCalled('File', '/usr/hdp/2.3.0.0-1234/hive/lib/mysql-connector-java.jar', + self.assertResourceCalled('File', '/usr/hdp/2.4.0.0-1234/hive/lib/mysql-connector-java.jar', mode = 0644) self.assertResourceCalled('Execute', ('cp', - '/usr/hdp/2.2.7.0-1234/hive/lib/mysql-connector-java.jar', - '/usr/hdp/2.3.0.0-1234/hive/lib'), + '/usr/hdp/2.3.7.0-1234/hive/lib/mysql-connector-java.jar', + '/usr/hdp/2.4.0.0-1234/hive/lib'), path = ['/bin', '/usr/bin/'], sudo = True) - self.assertResourceCalled('File', '/usr/hdp/2.3.0.0-1234/hive/lib/mysql-connector-java.jar', + self.assertResourceCalled('File', '/usr/hdp/2.4.0.0-1234/hive/lib/mysql-connector-java.jar', mode = 0644) - self.assertResourceCalled('Execute', '/usr/hdp/2.3.0.0-1234/hive/bin/schematool -dbType mysql -upgradeSchema', + self.assertResourceCalled('Execute', '/usr/hdp/2.4.0.0-1234/hive/bin/schematool -dbType mysql -upgradeSchema', logoutput = True, environment = {'HIVE_CONF_DIR': '/usr/hdp/current/hive-metastore/conf/conf.server'}, tries = 1, http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/package/scripts/hive_metastore.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/package/scripts/hive_metastore.py b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/package/scripts/hive_metastore.py index a49bbd9..b88f385 100755 --- a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/package/scripts/hive_metastore.py +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/package/scripts/hive_metastore.py @@ -27,6 +27,7 @@ from resource_management.libraries.functions import stack_select from resource_management.libraries.functions.constants import Direction from resource_management.libraries.functions.format import format from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions import upgrade_summary from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions.security_commons import build_expectations from resource_management.libraries.functions.security_commons import cached_kinit_executor @@ -233,7 +234,11 @@ class HiveMetastoreDefault(HiveMetastore): # since the configurations have not been written out yet during an upgrade # we need to choose the original legacy location schematool_hive_server_conf_dir = params.hive_server_conf_dir - if not(check_stack_feature(StackFeature.CONFIG_VERSIONING, params.version_for_stack_feature_checks)): + + upgrade_from_version = upgrade_summary.get_source_version("HIVE", + default_version = params.version_for_stack_feature_checks) + + if not (check_stack_feature(StackFeature.CONFIG_VERSIONING, upgrade_from_version)): schematool_hive_server_conf_dir = LEGACY_HIVE_SERVER_CONF env_dict = { http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/package/scripts/hive_server_upgrade.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/package/scripts/hive_server_upgrade.py b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/package/scripts/hive_server_upgrade.py index 9aa3d96..892f65b 100755 --- a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/package/scripts/hive_server_upgrade.py +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/package/scripts/hive_server_upgrade.py @@ -60,8 +60,13 @@ def post_upgrade_deregister(): hive_server_conf_dir = "/etc/hive/conf.server" # deregister - source_version = params.version_for_stack_feature_checks - hive_execute_path = _get_hive_execute_path(source_version) + hive_execute_path = params.execute_path + # If upgrading, the upgrade-target hive binary should be used to call the --deregister command. + # If downgrading, the downgrade-source hive binary should be used to call the --deregister command. + # By now <stack-selector-tool> has been called to set 'current' to target-stack + if params.downgrade_from_version is not None: + hive_execute_path = _get_hive_execute_path(params.downgrade_from_version) + command = format('hive --config {hive_server_conf_dir} --service hiveserver2 --deregister ' + current_hiveserver_version) Execute(command, user=params.hive_user, path=hive_execute_path, tries=1 ) @@ -100,7 +105,11 @@ def _get_current_hiveserver_version(): import params try: + # When downgrading the source version should be the version we are downgrading from source_version = params.version_for_stack_feature_checks + if params.downgrade_from_version is not None: + source_version = params.downgrade_from_version + hive_execute_path = _get_hive_execute_path(source_version) version_hive_bin = params.hive_bin formatted_source_version = format_stack_version(source_version) http://git-wip-us.apache.org/repos/asf/ambari/blob/ba15d80e/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/package/scripts/params_linux.py b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/package/scripts/params_linux.py index 1e4487d..e767bd0 100755 --- a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/package/scripts/params_linux.py +++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/package/scripts/params_linux.py @@ -75,6 +75,10 @@ stack_version_formatted = functions.get_stack_version('hive-server2') # It cannot be used during the initial Cluser Install because the version is not yet known. version = default("/commandParams/version", None) +# When downgrading the 'version' is pointing to the downgrade-target version +# downgrade_from_version provides the source-version the downgrade is happening from +downgrade_from_version = upgrade_summary.get_downgrade_from_version("HIVE") + # get the correct version to use for checking stack features version_for_stack_feature_checks = get_stack_feature_version(config)
