AMBARI-18591 - Hive Rolling Upgrade Is No Longer Supported In Ambari (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9b0f6317 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9b0f6317 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9b0f6317 Branch: refs/heads/branch-feature-AMBARI-18456 Commit: 9b0f631765f1e1c5855b95943c5cfb8c9cfff879 Parents: 78f4350 Author: Jonathan Hurley <jhur...@hortonworks.com> Authored: Thu Oct 13 13:13:39 2016 -0400 Committer: Jonathan Hurley <jhur...@hortonworks.com> Committed: Fri Oct 14 09:30:26 2016 -0400 ---------------------------------------------------------------------- .../ambari/server/checks/CheckDescription.java | 6 +- .../server/checks/HiveNotRollingWarning.java | 74 +++++++++++++++ .../checks/HiveRollingPortChangeWarning.java | 69 -------------- .../0.12.0.2.0/package/scripts/hive_server.py | 27 +++--- .../package/scripts/hive_server_upgrade.py | 10 +- .../stacks/HDP/2.2/upgrades/config-upgrade.xml | 52 ----------- .../stacks/HDP/2.2/upgrades/upgrade-2.2.xml | 24 ----- .../stacks/HDP/2.2/upgrades/upgrade-2.3.xml | 47 +++------- .../stacks/HDP/2.2/upgrades/upgrade-2.4.xml | 37 ++------ .../stacks/HDP/2.3/upgrades/config-upgrade.xml | 77 ---------------- .../stacks/HDP/2.3/upgrades/upgrade-2.3.xml | 24 ----- .../stacks/HDP/2.3/upgrades/upgrade-2.4.xml | 21 +---- .../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 21 +---- .../stacks/HDP/2.4/upgrades/config-upgrade.xml | 26 ------ .../stacks/HDP/2.4/upgrades/upgrade-2.4.xml | 24 ----- .../stacks/HDP/2.4/upgrades/upgrade-2.5.xml | 21 +---- .../stacks/HDP/2.5/upgrades/config-upgrade.xml | 31 ------- .../stacks/HDP/2.5/upgrades/upgrade-2.5.xml | 24 ----- .../checks/HiveNotRollingWarningTest.java | 96 ++++++++++++++++++++ .../HiveRollingPortChangeWarningTest.java | 96 -------------------- .../stacks/2.0.6/HIVE/test_hive_server.py | 27 +++++- 21 files changed, 242 insertions(+), 592 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9b0f6317/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java index 7fdd0ce..b5cb6ce 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java @@ -290,12 +290,12 @@ public class CheckDescription { .put(AbstractCheckDescriptor.DEFAULT, "Kafka is currently not Kerberized, but your cluster is. After upgrading, Kafka will automatically be Kerberized for you.").build()); - public static CheckDescription SERVICES_HIVE_ROLLING_PORT_WARNING = new CheckDescription("SERVICES_HIVE_ROLLING_PORT_WARNING", + public static CheckDescription SERVICES_HIVE_ROLLING_WARNING = new CheckDescription("SERVICES_HIVE_ROLLING_WARNING", PrereqCheckType.SERVICE, - "Hive Server Port Change", + "Hive Server Downtime", new ImmutableMap.Builder<String, String>() .put(AbstractCheckDescriptor.DEFAULT, - "In order to support rolling upgrades, the Hive server is required to change its port. Applications and users which use a URL that includes the port will no longer be able to connect after Hive has upgraded. If this behavior is not desired, then the port can be restored to its original value after the upgrade has been finalized.").build()); + "Hive does not currently support rolling upgrades. Hive will be upgraded, however existing queries which have been submitted will fail and need to be resubmitted after Hive has been upgraded.").build()); public static CheckDescription SERVICES_STORM_ROLLING_WARNING = new CheckDescription("SERVICES_STORM_ROLLING_WARNING", PrereqCheckType.SERVICE, http://git-wip-us.apache.org/repos/asf/ambari/blob/9b0f6317/ambari-server/src/main/java/org/apache/ambari/server/checks/HiveNotRollingWarning.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/HiveNotRollingWarning.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/HiveNotRollingWarning.java new file mode 100644 index 0000000..5e44c7c --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/HiveNotRollingWarning.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.checks; + +import java.util.Arrays; + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.controller.PrereqCheckRequest; +import org.apache.ambari.server.state.stack.PrereqCheckStatus; +import org.apache.ambari.server.state.stack.PrerequisiteCheck; +import org.apache.ambari.server.state.stack.upgrade.UpgradeType; + +import com.google.inject.Singleton; + +/** + * The {@link HiveNotRollingWarning} to see if Hive is installed and if the + * upgrade type is {@link UpgradeType#ROLLING}. If so, then a + * {@link PrereqCheckStatus#WARNING} is produced which will let the operator + * know that Hive does not support rolling upgrades. + * <p/> + * In actuality, it does, however in order to support no downtime, a new Hive + * server is spun up on a new port while the old server drains. If clients are + * not using the ZK discovery service for Hive and connecting via a URL + * directly, they will cease to function. For this reason, it's been determined + * that at this point in time, Hive will not be upgraded in a rolling fashion. + */ +@Singleton +@UpgradeCheck(group = UpgradeCheckGroup.INFORMATIONAL_WARNING, required = true) +public class HiveNotRollingWarning extends AbstractCheckDescriptor { + + /** + * Constructor. + */ + public HiveNotRollingWarning() { + super(CheckDescription.SERVICES_HIVE_ROLLING_WARNING); + } + + /** + * {@inheritDoc} + * <p/> + * This check is only applicable if Hive is installed and the upgrade type is + * {@link UpgradeType#ROLLING}. + */ + @Override + public boolean isApplicable(PrereqCheckRequest request) throws AmbariException { + boolean isApplicable = super.isApplicable(request, Arrays.asList("HIVE"), true); + return isApplicable && request.getUpgradeType() == UpgradeType.ROLLING; + } + + /** + * {@inheritDoc} + */ + @Override + public void perform(PrerequisiteCheck prerequisiteCheck, PrereqCheckRequest request) throws AmbariException { + prerequisiteCheck.getFailedOn().add("HIVE"); + prerequisiteCheck.setStatus(PrereqCheckStatus.WARNING); + prerequisiteCheck.setFailReason(getFailReason(prerequisiteCheck, request)); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/9b0f6317/ambari-server/src/main/java/org/apache/ambari/server/checks/HiveRollingPortChangeWarning.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/HiveRollingPortChangeWarning.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/HiveRollingPortChangeWarning.java deleted file mode 100644 index 3db31ea..0000000 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/HiveRollingPortChangeWarning.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.server.checks; - -import java.util.Arrays; - -import org.apache.ambari.server.AmbariException; -import org.apache.ambari.server.controller.PrereqCheckRequest; -import org.apache.ambari.server.state.stack.PrereqCheckStatus; -import org.apache.ambari.server.state.stack.PrerequisiteCheck; -import org.apache.ambari.server.state.stack.upgrade.UpgradeType; - -import com.google.inject.Singleton; - -/** - * The {@link HiveRollingPortChangeWarning} to see if Hive is installed and if - * the upgrade type is {@link UpgradeType#ROLLING}. If so, then a - * {@link PrereqCheckStatus#WARNING} is produced which will let the operator - * know that the port for Hive must change in order to preserve the uptime of - * the service. - */ -@Singleton -@UpgradeCheck(group = UpgradeCheckGroup.INFORMATIONAL_WARNING, required = true) -public class HiveRollingPortChangeWarning extends AbstractCheckDescriptor { - - /** - * Constructor. - */ - public HiveRollingPortChangeWarning() { - super(CheckDescription.SERVICES_HIVE_ROLLING_PORT_WARNING); - } - - /** - * {@inheritDoc} - * <p/> - * This check is only applicable if Hive is installed and the upgrade type is - * {@link UpgradeType#ROLLING}. - */ - @Override - public boolean isApplicable(PrereqCheckRequest request) throws AmbariException { - boolean isApplicable = super.isApplicable(request, Arrays.asList("HIVE"), true); - return isApplicable && request.getUpgradeType() == UpgradeType.ROLLING; - } - - /** - * {@inheritDoc} - */ - @Override - public void perform(PrerequisiteCheck prerequisiteCheck, PrereqCheckRequest request) throws AmbariException { - prerequisiteCheck.getFailedOn().add("HIVE"); - prerequisiteCheck.setStatus(PrereqCheckStatus.WARNING); - prerequisiteCheck.setFailReason(getFailReason(prerequisiteCheck, request)); - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/9b0f6317/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py index 614b2a9..12743ec 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py @@ -20,12 +20,9 @@ limitations under the License. from resource_management.libraries.script.script import Script -from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select -from resource_management.libraries.functions import format from resource_management.libraries.functions.copy_tarball import copy_to_hdfs -from resource_management.libraries.functions.get_stack_version import get_stack_version from resource_management.libraries.functions.check_process_status import check_process_status from resource_management.libraries.functions import StackFeature from resource_management.libraries.functions.stack_features import check_stack_feature @@ -37,7 +34,6 @@ if OSCheck.is_windows_family(): from resource_management.libraries.functions.windows_service_utils import check_windows_service_status from setup_ranger_hive import setup_ranger_hive from ambari_commons.os_family_impl import OsFamilyImpl -from ambari_commons.constants import UPGRADE_TYPE_ROLLING from resource_management.core.logger import Logger import hive_server_upgrade @@ -47,7 +43,6 @@ from hive_service import hive_service class HiveServer(Script): def install(self, env): - import params self.install_packages(env) def configure(self, env): @@ -87,21 +82,23 @@ class HiveServerDefault(HiveServer): setup_ranger_hive(upgrade_type=upgrade_type) hive_service('hiveserver2', action = 'start', upgrade_type=upgrade_type) - # only perform this if upgrading and rolling; a non-rolling upgrade doesn't need - # to do this since hive is already down - if upgrade_type == UPGRADE_TYPE_ROLLING: - hive_server_upgrade.post_upgrade_deregister() - def stop(self, env, upgrade_type=None): import params env.set_params(params) - # During rolling upgrade, HiveServer2 should not be stopped before new server is available. - # Once new server is started, old one is stopped by the --deregister command which is - # invoked by the 'hive_server_upgrade.post_upgrade_deregister()' method - if upgrade_type != UPGRADE_TYPE_ROLLING: - hive_service( 'hiveserver2', action = 'stop' ) + # always de-register the old hive instance so that ZK can route clients + # to the newly created hive server + try: + if upgrade_type is not None: + hive_server_upgrade.deregister() + except Exception as exception: + Logger.exception(str(exception)) + + # even during rolling upgrades, Hive Server will be stopped - this is because Ambari will + # not support the "port-change/deregister" workflow as it would impact Hive clients + # which do not use ZK discovery. + hive_service( 'hiveserver2', action = 'stop' ) def status(self, env): http://git-wip-us.apache.org/repos/asf/ambari/blob/9b0f6317/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_upgrade.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_upgrade.py index 478c240..17db489 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_upgrade.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_upgrade.py @@ -30,14 +30,18 @@ from resource_management.libraries.functions.stack_features import check_stack_f from resource_management.libraries.functions.version import format_stack_version -def post_upgrade_deregister(): +def deregister(): """ Runs the "hive --service hiveserver2 --deregister <version>" command to de-provision the server in preparation for an upgrade. This will contact ZooKeeper to remove the server so that clients that attempt to connect will be directed to other servers automatically. Once all - clients have drained, the server will shutdown automatically; this process - could take a very long time. + clients have drained, the server will shutdown automatically. + + However, since Ambari does not support Hive Server rolling upgrades due to the port change + affecting Hive Clients not using the ZK discovery service, the daemon might be forcefully + killed before it has been deregistered and drained. + This function will obtain the Kerberos ticket if security is enabled. :return: """ http://git-wip-us.apache.org/repos/asf/ambari/blob/9b0f6317/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml index 5957da4..2d336ba 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml @@ -243,51 +243,12 @@ <service name="HIVE"> <component name="HIVE_SERVER"> <changes> - <definition xsi:type="configure" id="hdp_2_2_0_0_hive_server_set_transport_mode"> - <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> - <type>hive-site</type> - <key>hive.server2.thrift.port</key> - <value>10010</value> - </condition> - <condition type="hive-site" key="hive.server2.transport.mode" value="http"> - <type>hive-site</type> - <key>hive.server2.http.port</key> - <value>10011</value> - </condition> - </definition> - - <definition xsi:type="configure" id="hdp_2_2_0_0_hive_server_restore_transport_mode_on_downgrade"> - <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> - <type>hive-site</type> - <key>hive.server2.thrift.port</key> - <value>10000</value> - </condition> - <condition type="hive-site" key="hive.server2.transport.mode" value="http"> - <type>hive-site</type> - <key>hive.server2.http.port</key> - <value>10001</value> - </condition> - </definition> - <definition xsi:type="configure" id="hdp_2_2_0_0_hive_server_delegation_token_store_class"> <type>hive-site</type> <set key="hive.cluster.delegation.token.store.class" value="org.apache.hadoop.hive.thrift.ZooKeeperTokenStore"/> </definition> - <definition xsi:type="configure" id="hdp_2_3_0_0_hive_server_set_transport_mode"> - <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> - <type>hive-site</type> - <key>hive.server2.thrift.port</key> - <value>10010</value> - </condition> - <condition type="hive-site" key="hive.server2.transport.mode" value="http"> - <type>hive-site</type> - <key>hive.server2.http.port</key> - <value>10011</value> - </condition> - </definition> - <definition xsi:type="configure" id="hdp_2_3_0_0_hive_server_replace_auth_manager" summary="Update Hive Authentication Manager"> <type>hiveserver2-site</type> @@ -440,19 +401,6 @@ <transfer operation="delete" delete-key="XAAUDIT.DB.HOSTNAME"/> </definition> - <definition xsi:type="configure" id="hdp_2_3_0_0_hive_server_restore_transport_mode_on_downgrade"> - <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> - <type>hive-site</type> - <key>hive.server2.thrift.port</key> - <value>10000</value> - </condition> - <condition type="hive-site" key="hive.server2.transport.mode" value="http"> - <type>hive-site</type> - <key>hive.server2.http.port</key> - <value>10001</value> - </condition> - </definition> - <definition xsi:type="configure" id="hdp_2_3_0_0_hive_server_remove_datastore_classname"> <type>hive-site</type> <transfer operation="delete" delete-key="datanucleus.rdbms.datastoreAdapterClassName"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/9b0f6317/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml index 6920b2c..586d04a 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml @@ -361,12 +361,6 @@ </task> </execute-stage> - <execute-stage service="HIVE" component="HIVE_SERVER" title="Hive Server Port Change"> - <task xsi:type="manual"> - <message>If there are users or applications using URLS that specify the Hive Server port number, they will now fail to connect. You can restore the port back to its original value after the upgrade has been finalized.</message> - </task> - </execute-stage> - <execute-stage service="HDFS" component="NAMENODE" title="Execute HDFS Finalize"> <task xsi:type="execute" hosts="master"> <script>scripts/namenode.py</script> @@ -545,24 +539,6 @@ </component> <component name="HIVE_SERVER"> - <pre-upgrade> - <task xsi:type="manual"> - <summary>HiveServer Port Availability</summary> - <message>Please note that the HiveServer port will now change to 10010 if Hive is using a binary transfer mode or 10011 if Hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> - </task> - - <task xsi:type="configure" id="hdp_2_2_0_0_hive_server_set_transport_mode"/> - </pre-upgrade> - - <pre-downgrade> - <task xsi:type="manual"> - <summary>HiveServer Port Availability</summary> - <message>Please note that the HiveServer port will now change to 10000 if Hive is using a binary transfer mode or 10001 if Hive is using an http transport mode. You can use "netstat -anp | grep 1000[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> - </task> - - <task xsi:type="configure" id="hdp_2_2_0_0_hive_server_restore_transport_mode_on_downgrade"/> - </pre-downgrade> - <upgrade> <task xsi:type="restart-task"/> </upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/9b0f6317/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml index 1b04237..d1ba1c2 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml @@ -390,7 +390,7 @@ <group xsi:type="cluster" name="FINALIZE_PRE_CHECK" title="Finalize {{direction.text.proper}} Pre-Check"> <direction>UPGRADE</direction> - + <execute-stage title="Check Component Versions"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.ComponentVersionCheckAction" /> </execute-stage> @@ -407,12 +407,6 @@ </task> </execute-stage> - <execute-stage service="HIVE" component="HIVE_SERVER" title="Hive Server Port Change"> - <task xsi:type="manual"> - <message>If there are users or applications using URLS that specify the Hive Server port number, they will now fail to connect. You can restore the port back to its original value after the upgrade has been finalized.</message> - </task> - </execute-stage> - <execute-stage service="HDFS" component="NAMENODE" title="Execute HDFS Finalize"> <task xsi:type="execute" hosts="master"> <script>scripts/namenode.py</script> @@ -459,9 +453,9 @@ <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.RangerConfigCalculation"> <summary>Calculating Ranger Properties</summary> </task> - + <task xsi:type="configure" id="hdp_2_3_0_0_update_ranger_usersync"/> - + <task xsi:type="configure" id="hdp_2_3_0_0_update_ranger_site"/> <task xsi:type="configure" id="hdp_2_3_0_0_update_ranger_usersync_sync_source"/> @@ -522,9 +516,9 @@ <task xsi:type="configure" id="hdp_2_3_0_0_hdfs_transition_ranger_hdfs_policy"/> <task xsi:type="configure" id="hdp_2_3_0_0_hdfs_transition_ranger_hdfs_audit"/> - + <task xsi:type="configure" id="hdp_2_3_0_0_hdfs_transition_ranger_hdfs_security"/> - + <task xsi:type="configure" id="hdp_2_3_0_0_hdfs_ranger_hdfs_delete_old_properties"/> </pre-upgrade> @@ -592,9 +586,9 @@ <task xsi:type="configure" id="hdp_2_3_0_0_yarn_ats_enable_recovery"/> <task xsi:type="configure" id="hdp_2_3_0_0_yarn_keep_ats_v1"/> </pre-upgrade> - + <pre-downgrade/> <!-- no-op to prevent config changes on downgrade --> - + <upgrade> <task xsi:type="restart-task"/> </upgrade> @@ -611,9 +605,9 @@ <task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_check_cs_root_max_capacity"/> </pre-upgrade> - + <pre-downgrade/> <!-- no-op to prevent config changes on downgrade --> - + <upgrade> <task xsi:type="restart-task"/> </upgrade> @@ -660,7 +654,7 @@ <!-- These HBASE configs changed in HDP 2.3.4.0, but Ambari can't distinguish HDP 2.3.2.0 vs HDP 2.3.4.0, so easier to always do them. --> <task xsi:type="configure" id="hdp_2_3_4_0_hbase_remove_local_indexing"/> </pre-upgrade> - + <pre-downgrade/> <!-- no-op to prevent config changes on downgrade --> <upgrade> @@ -723,12 +717,6 @@ <component name="HIVE_SERVER"> <pre-upgrade> - <task xsi:type="manual"> - <summary>HiveServer Port Availability</summary> - <message>Please note that the HiveServer port will now change to 10010 if Hive is using a binary transfer mode or 10011 if Hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> - </task> - - <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_set_transport_mode"/> <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_replace_auth_manager"/> <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_authentication"/> <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_ranger_policy"/> @@ -738,14 +726,7 @@ <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_remove_datastore_classname"/> </pre-upgrade> - <pre-downgrade> - <task xsi:type="manual"> - <summary>HiveServer Port Availability</summary> - <message>Please note that the HiveServer port will now change to 10000 if Hive is using a binary transfer mode or 10001 if Hive is using an http transport mode. You can use "netstat -anp | grep 1000[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> - </task> - - <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_restore_transport_mode_on_downgrade"/> - </pre-downgrade> + <pre-downgrade/> <upgrade> <task xsi:type="restart-task"/> @@ -889,7 +870,7 @@ <task xsi:type="configure" id="hdp_2_3_0_0_knox_remove_deprecated_ranger_properties"/> </pre-upgrade> - + <pre-downgrade/> <!-- no-op to prevent config changes on downgrade --> <upgrade> @@ -900,7 +881,7 @@ <service name="STORM"> <component name="NIMBUS"> - + <pre-upgrade> <task xsi:type="manual"> <message>Before continuing, please deactivate and kill any currently running topologies.</message> @@ -921,7 +902,7 @@ <task xsi:type="configure" id="hdp_2_3_0_0_nimbus_monitor_freq_adjustment"/> <task xsi:type="configure" id="hdp_2_3_0_0_nimbus_convert_nimbus_host_to_seeds"/> - + <task xsi:type="configure" id="hdp_2_3_0_0_update_storm_env"/> <task xsi:type="configure" id="hdp_2_3_0_0_nimbus_update_env_vars"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/9b0f6317/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml index 8c5f9aa..09fa146 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml @@ -395,7 +395,7 @@ <group xsi:type="cluster" name="FINALIZE_PRE_CHECK" title="Finalize {{direction.text.proper}} Pre-Check"> <direction>UPGRADE</direction> - + <execute-stage title="Check Component Versions"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.ComponentVersionCheckAction" /> </execute-stage> @@ -412,12 +412,6 @@ </task> </execute-stage> - <execute-stage service="HIVE" component="HIVE_SERVER" title="Hive Server Port Change"> - <task xsi:type="manual"> - <message>If there are users or applications using URLS that specify the Hive Server port number, they will now fail to connect. You can restore the port back to its original value after the upgrade has been finalized.</message> - </task> - </execute-stage> - <execute-stage service="HDFS" component="NAMENODE" title="Execute HDFS Finalize"> <task xsi:type="execute" hosts="master"> <script>scripts/namenode.py</script> @@ -463,9 +457,9 @@ <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.RangerConfigCalculation"> <summary>Calculating Ranger Properties</summary> </task> - + <task xsi:type="configure" id="hdp_2_3_0_0_update_ranger_usersync"/> - + <task xsi:type="configure" id="hdp_2_3_0_0_update_ranger_site"/> <task xsi:type="configure" id="hdp_2_3_0_0_update_ranger_usersync_sync_source"/> @@ -611,9 +605,9 @@ <summary>Calculating Yarn Properties for Spark Shuffle</summary> </task> </pre-upgrade> - + <pre-downgrade/> <!-- no-op to prevent config changes on downgrade --> - + <upgrade> <task xsi:type="restart-task"/> </upgrade> @@ -697,7 +691,7 @@ <summary>Verifying LZO codec path for Tez</summary> </task> </pre-upgrade> - + <upgrade> <task xsi:type="restart-task"/> </upgrade> @@ -729,12 +723,6 @@ <component name="HIVE_SERVER"> <pre-upgrade> - <task xsi:type="manual"> - <summary>HiveServer Port Availability</summary> - <message>Please note that the HiveServer port will now change to 10010 if Hive is using a binary transfer mode or 10011 if Hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> - </task> - - <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_set_transport_mode"/> <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_replace_auth_manager"/> <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_authentication"/> <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_ranger_policy"/> @@ -744,14 +732,7 @@ <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_remove_datastore_classname"/> </pre-upgrade> - <pre-downgrade> - <task xsi:type="manual"> - <summary>HiveServer Port Availability</summary> - <message>Please note that the HiveServer port will now change to 10000 if Hive is using a binary transfer mode or 10001 if Hive is using an http transport mode. You can use "netstat -anp | grep 1000[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> - </task> - - <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_restore_transport_mode_on_downgrade"/> - </pre-downgrade> + <pre-downgrade/> <upgrade> <task xsi:type="restart-task"/> @@ -908,7 +889,7 @@ <service name="STORM"> <component name="NIMBUS"> - + <pre-upgrade> <task xsi:type="manual"> <message>Before continuing, please deactivate and kill any currently running topologies.</message> @@ -929,7 +910,7 @@ <task xsi:type="configure" id="hdp_2_3_0_0_nimbus_monitor_freq_adjustment"/> <task xsi:type="configure" id="hdp_2_3_0_0_nimbus_convert_nimbus_host_to_seeds"/> - + <task xsi:type="configure" id="hdp_2_3_0_0_update_storm_env"/> <task xsi:type="configure" id="hdp_2_3_0_0_nimbus_update_env_vars"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/9b0f6317/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml index 6105665..e8a2e2f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml @@ -118,57 +118,6 @@ <service name="HIVE"> <component name="HIVE_SERVER"> <changes> - <definition xsi:type="configure" id="hdp_2_3_0_0_hive_server_set_transport_mode"> - <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> - <type>hive-site</type> - <key>hive.server2.thrift.port</key> - <value>10010</value> - </condition> - <condition type="hive-site" key="hive.server2.transport.mode" value="http"> - <type>hive-site</type> - <key>hive.server2.http.port</key> - <value>10011</value> - </condition> - </definition> - <definition xsi:type="configure" id="hdp_2_3_0_0_hive_server_restore_transport_mode_on_downgrade"> - <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> - <type>hive-site</type> - <key>hive.server2.thrift.port</key> - <value>10000</value> - </condition> - <condition type="hive-site" key="hive.server2.transport.mode" value="http"> - <type>hive-site</type> - <key>hive.server2.http.port</key> - <value>10001</value> - </condition> - </definition> - - <definition xsi:type="configure" id="hdp_2_4_0_0_hive_server_set_transport_mode"> - <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> - <type>hive-site</type> - <key>hive.server2.thrift.port</key> - <value>10010</value> - </condition> - <condition type="hive-site" key="hive.server2.transport.mode" value="http"> - <type>hive-site</type> - <key>hive.server2.http.port</key> - <value>10011</value> - </condition> - </definition> - - <definition xsi:type="configure" id="hdp_2_4_0_0_hive_server_restore_transport_mode_on_downgrade"> - <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> - <type>hive-site</type> - <key>hive.server2.thrift.port</key> - <value>10000</value> - </condition> - <condition type="hive-site" key="hive.server2.transport.mode" value="http"> - <type>hive-site</type> - <key>hive.server2.http.port</key> - <value>10001</value> - </condition> - </definition> - <definition xsi:type="configure" id="hdp_2_4_0_0_hive_server_configure_authentication" summary="Removing unused properties for current hive authentication type"> <type>hive-site</type> <transfer operation="delete" delete-key="hive.server2.authentication.ldap.url" if-key="hive.server2.authentication" if-type="hive-site" if-value="NONE"/> @@ -220,32 +169,6 @@ <transfer operation="delete" delete-key="atlas.hook.hive.maxThreads" /> </definition> - <definition xsi:type="configure" id="hdp_2_5_0_0_hive_server_set_transport_mode"> - <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> - <type>hive-site</type> - <key>hive.server2.thrift.port</key> - <value>10010</value> - </condition> - <condition type="hive-site" key="hive.server2.transport.mode" value="http"> - <type>hive-site</type> - <key>hive.server2.http.port</key> - <value>10011</value> - </condition> - </definition> - - <definition xsi:type="configure" id="hdp_2_5_0_0_hive_server_restore_transport_mode_on_downgrade"> - <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> - <type>hive-site</type> - <key>hive.server2.thrift.port</key> - <value>10000</value> - </condition> - <condition type="hive-site" key="hive.server2.transport.mode" value="http"> - <type>hive-site</type> - <key>hive.server2.http.port</key> - <value>10001</value> - </condition> - </definition> - </changes> </component> <component name="WEBHCAT_SERVER"> http://git-wip-us.apache.org/repos/asf/ambari/blob/9b0f6317/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml index 9b9848e..0b326a7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml @@ -409,12 +409,6 @@ </task> </execute-stage> - <execute-stage service="HIVE" component="HIVE_SERVER" title="Hive Server Port Change"> - <task xsi:type="manual"> - <message>If there are users or applications using URLS that specify the Hive Server port number, they will now fail to connect. You can restore the port back to its original value after the upgrade has been finalized.</message> - </task> - </execute-stage> - <execute-stage service="HDFS" component="NAMENODE" title="Execute HDFS Finalize"> <task xsi:type="execute" hosts="master"> <script>scripts/namenode.py</script> @@ -706,24 +700,6 @@ </component> <component name="HIVE_SERVER"> - <pre-upgrade> - <task xsi:type="manual"> - <summary>HiveServer Port Availability</summary> - <message>Please note that the HiveServer port will now change to 10010 if Hive is using a binary transfer mode or 10011 if Hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> - </task> - - <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_set_transport_mode"/> - </pre-upgrade> - - <pre-downgrade> - <task xsi:type="manual"> - <summary>HiveServer Port Availability</summary> - <message>Please note that the HiveServer port will now change to 10000 if Hive is using a binary transfer mode or 10001 if Hive is using an http transport mode. You can use "netstat -anp | grep 1000[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> - </task> - - <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_restore_transport_mode_on_downgrade"/> - </pre-downgrade> - <upgrade> <task xsi:type="restart-task"/> </upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/9b0f6317/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml index 51fbcbf..a5c243f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml @@ -453,12 +453,6 @@ </task> </execute-stage> - <execute-stage service="HIVE" component="HIVE_SERVER" title="Hive Server Port Change"> - <task xsi:type="manual"> - <message>If there are users or applications using URLS that specify the Hive Server port number, they will now fail to connect. You can restore the port back to its original value after the upgrade has been finalized.</message> - </task> - </execute-stage> - <execute-stage service="HDFS" component="NAMENODE" title="Execute HDFS Finalize"> <task xsi:type="execute" hosts="master"> <script>scripts/namenode.py</script> @@ -730,23 +724,10 @@ <component name="HIVE_SERVER"> <pre-upgrade> - <task xsi:type="manual"> - <summary>HiveServer Port Availability</summary> - <message>Please note that the HiveServer port will now change to 10010 if Hive is using a binary transfer mode or 10011 if Hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> - </task> - - <task xsi:type="configure" id="hdp_2_4_0_0_hive_server_set_transport_mode"/> <task xsi:type="configure" id="hdp_2_4_0_0_hive_server_configure_authentication"/> </pre-upgrade> - <pre-downgrade> - <task xsi:type="manual"> - <summary>HiveServer Port Availability</summary> - <message>Please note that the HiveServer port will now change to 10000 if Hive is using a binary transfer mode or 10001 if Hive is using an http transport mode. You can use "netstat -anp | grep 1000[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> - </task> - - <task xsi:type="configure" id="hdp_2_4_0_0_hive_server_restore_transport_mode_on_downgrade"/> - </pre-downgrade> + <pre-downgrade/> <upgrade> <task xsi:type="restart-task" /> http://git-wip-us.apache.org/repos/asf/ambari/blob/9b0f6317/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml index cca4b14..e89d62b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml @@ -508,12 +508,6 @@ </task> </execute-stage> - <execute-stage service="HIVE" component="HIVE_SERVER" title="Hive Server Port Change"> - <task xsi:type="manual"> - <message>If there are users or applications using URLS that specify the Hive Server port number, they will now fail to connect. You can restore the port back to its original value after the upgrade has been finalized.</message> - </task> - </execute-stage> - <execute-stage service="RANGER" component="RANGER_ADMIN" title="Ranger Kerberos Keytab Check"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.KerberosKeytabsAction"> <summary>Ranger Kerberos Keytab Check</summary> @@ -830,12 +824,6 @@ <component name="HIVE_SERVER"> <pre-upgrade> - <task xsi:type="manual"> - <summary>HiveServer Port Availability</summary> - <message>Please note that the HiveServer port will now change to 10010 if Hive is using a binary transfer mode or 10011 if Hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> - </task> - - <task xsi:type="configure" id="hdp_2_5_0_0_hive_server_set_transport_mode"/> <task xsi:type="configure" id="hdp_2_4_0_0_hive_server_configure_authentication"/> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hive_audit_db" /> @@ -847,14 +835,7 @@ </task> </pre-upgrade> - <pre-downgrade> - <task xsi:type="manual"> - <summary>HiveServer Port Availability</summary> - <message>Please note that the HiveServer port will now change to 10000 if Hive is using a binary transfer mode or 10001 if Hive is using an http transport mode. You can use "netstat -anp | grep 1000[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> - </task> - - <task xsi:type="configure" id="hdp_2_5_0_0_hive_server_restore_transport_mode_on_downgrade"/> - </pre-downgrade> + <pre-downgrade/> <upgrade> <task xsi:type="restart-task" /> http://git-wip-us.apache.org/repos/asf/ambari/blob/9b0f6317/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml index db59f8c..e127dff 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml @@ -66,32 +66,6 @@ <service name="HIVE"> <component name="HIVE_SERVER"> <changes> - <definition xsi:type="configure" id="hdp_2_4_0_0_hive_server_set_transport_mode"> - <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> - <type>hive-site</type> - <key>hive.server2.thrift.port</key> - <value>10010</value> - </condition> - <condition type="hive-site" key="hive.server2.transport.mode" value="http"> - <type>hive-site</type> - <key>hive.server2.http.port</key> - <value>10011</value> - </condition> - </definition> - - <definition xsi:type="configure" id="hdp_2_4_0_0_hive_server_restore_transport_mode_on_downgrade"> - <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> - <type>hive-site</type> - <key>hive.server2.thrift.port</key> - <value>10000</value> - </condition> - <condition type="hive-site" key="hive.server2.transport.mode" value="http"> - <type>hive-site</type> - <key>hive.server2.http.port</key> - <value>10001</value> - </condition> - </definition> - <definition xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hive_audit_db"> <type>ranger-hive-audit</type> <transfer operation="delete" delete-key="xasecure.audit.destination.db" /> http://git-wip-us.apache.org/repos/asf/ambari/blob/9b0f6317/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml index 17d7d9c..5b53c86 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml @@ -406,12 +406,6 @@ </task> </execute-stage> - <execute-stage service="HIVE" component="HIVE_SERVER" title="Hive Server Port Change"> - <task xsi:type="manual"> - <message>If there are users or applications using URLS that specify the Hive Server port number, they will now fail to connect. You can restore the port back to its original value after the upgrade has been finalized.</message> - </task> - </execute-stage> - <execute-stage service="HDFS" component="NAMENODE" title="Execute HDFS Finalize"> <task xsi:type="execute" hosts="master"> <script>scripts/namenode.py</script> @@ -680,24 +674,6 @@ </component> <component name="HIVE_SERVER"> - <pre-upgrade> - <task xsi:type="manual"> - <summary>HiveServer Port Availability</summary> - <message>Please note that the HiveServer port will now change to 10010 if Hive is using a binary transfer mode or 10011 if Hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> - </task> - - <task xsi:type="configure" id="hdp_2_4_0_0_hive_server_set_transport_mode"/> - </pre-upgrade> - - <pre-downgrade> - <task xsi:type="manual"> - <summary>HiveServer Port Availability</summary> - <message>Please note that the HiveServer port will now change to 10000 if Hive is using a binary transfer mode or 10001 if Hive is using an http transport mode. You can use "netstat -anp | grep 1000[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> - </task> - - <task xsi:type="configure" id="hdp_2_4_0_0_hive_server_restore_transport_mode_on_downgrade" /> - </pre-downgrade> - <upgrade> <task xsi:type="restart-task" /> </upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/9b0f6317/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml index 921fc9c..fa1a555 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml @@ -502,12 +502,6 @@ </task> </execute-stage> - <execute-stage service="HIVE" component="HIVE_SERVER" title="Hive Server Port Change"> - <task xsi:type="manual"> - <message>If there are users or applications using URLS that specify the Hive Server port number, they will now fail to connect. You can restore the port back to its original value after the upgrade has been finalized.</message> - </task> - </execute-stage> - <execute-stage service="HDFS" component="NAMENODE" title="Execute HDFS Finalize"> <task xsi:type="execute" hosts="master"> <script>scripts/namenode.py</script> @@ -832,22 +826,9 @@ <summary>Update hive-env content</summary> </task> - <task xsi:type="manual"> - <summary>HiveServer Port Availability</summary> - <message>Please note that the HiveServer port will now change to 10010 if Hive is using a binary transfer mode or 10011 if Hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> - </task> - - <task xsi:type="configure" id="hdp_2_4_0_0_hive_server_set_transport_mode"/> </pre-upgrade> - <pre-downgrade> - <task xsi:type="manual"> - <summary>HiveServer Port Availability</summary> - <message>Please note that the HiveServer port will now change to 10000 if Hive is using a binary transfer mode or 10001 if Hive is using an http transport mode. You can use "netstat -anp | grep 1000[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> - </task> - - <task xsi:type="configure" id="hdp_2_4_0_0_hive_server_restore_transport_mode_on_downgrade" /> - </pre-downgrade> + <pre-downgrade/> <upgrade> <task xsi:type="restart-task" /> http://git-wip-us.apache.org/repos/asf/ambari/blob/9b0f6317/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml index 962d92e..065f30f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml @@ -19,37 +19,6 @@ <upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> <services> - <service name="HIVE"> - <component name="HIVE_SERVER"> - <changes> - <definition xsi:type="configure" id="hdp_2_5_0_0_hive_server_set_transport_mode"> - <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> - <type>hive-site</type> - <key>hive.server2.thrift.port</key> - <value>10010</value> - </condition> - <condition type="hive-site" key="hive.server2.transport.mode" value="http"> - <type>hive-site</type> - <key>hive.server2.http.port</key> - <value>10011</value> - </condition> - </definition> - - <definition xsi:type="configure" id="hdp_2_5_0_0_hive_server_restore_transport_mode_on_downgrade"> - <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> - <type>hive-site</type> - <key>hive.server2.thrift.port</key> - <value>10000</value> - </condition> - <condition type="hive-site" key="hive.server2.transport.mode" value="http"> - <type>hive-site</type> - <key>hive.server2.http.port</key> - <value>10001</value> - </condition> - </definition> - </changes> - </component> - </service> <service name="STORM"> <component name="NIMBUS"> <changes> http://git-wip-us.apache.org/repos/asf/ambari/blob/9b0f6317/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml index 28b34a2..2932b6c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml @@ -453,12 +453,6 @@ </task> </execute-stage> - <execute-stage service="HIVE" component="HIVE_SERVER" title="Hive Server Port Change"> - <task xsi:type="manual"> - <message>If there are users or applications using URLS that specify the Hive Server port number, they will now fail to connect. You can restore the port back to its original value after the upgrade has been finalized.</message> - </task> - </execute-stage> - <execute-stage service="HDFS" component="NAMENODE" title="Execute HDFS Finalize"> <task xsi:type="execute" hosts="master"> <script>scripts/namenode.py</script> @@ -731,24 +725,6 @@ </component> <component name="HIVE_SERVER"> - <pre-upgrade> - <task xsi:type="manual"> - <summary>HiveServer Port Availability</summary> - <message>Please note that the HiveServer port will now change to 10010 if Hive is using a binary transfer mode or 10011 if Hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> - </task> - - <task xsi:type="configure" id="hdp_2_5_0_0_hive_server_set_transport_mode"/> - </pre-upgrade> - - <pre-downgrade> - <task xsi:type="manual"> - <summary>HiveServer Port Availability</summary> - <message>Please note that the HiveServer port will now change to 10000 if Hive is using a binary transfer mode or 10001 if Hive is using an http transport mode. You can use "netstat -anp | grep 1000[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> - </task> - - <task xsi:type="configure" id="hdp_2_5_0_0_hive_server_restore_transport_mode_on_downgrade" /> - </pre-downgrade> - <upgrade> <task xsi:type="restart-task" /> </upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/9b0f6317/ambari-server/src/test/java/org/apache/ambari/server/checks/HiveNotRollingWarningTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/checks/HiveNotRollingWarningTest.java b/ambari-server/src/test/java/org/apache/ambari/server/checks/HiveNotRollingWarningTest.java new file mode 100644 index 0000000..665d6ec --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/checks/HiveNotRollingWarningTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.checks; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.ambari.server.controller.PrereqCheckRequest; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.Service; +import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.state.stack.PrereqCheckStatus; +import org.apache.ambari.server.state.stack.PrerequisiteCheck; +import org.apache.ambari.server.state.stack.upgrade.UpgradeType; +import org.easymock.EasyMock; +import org.easymock.EasyMockSupport; +import org.junit.Assert; +import org.junit.Test; + +import com.google.inject.Provider; + +/** + * Tests {@link HiveNotRollingWarning}. + */ +public class HiveNotRollingWarningTest extends EasyMockSupport { + + private final String m_clusterName = "c1"; + private final Clusters m_clusters = niceMock(Clusters.class); + + /** + * @throws Exception + */ + @Test + public void testIsApplicable() throws Exception { + final HiveNotRollingWarning hiveWarningCheck = new HiveNotRollingWarning(); + hiveWarningCheck.clustersProvider = new Provider<Clusters>() { + + @Override + public Clusters get() { + return m_clusters; + } + }; + + final Cluster cluster = niceMock(Cluster.class); + final Service hive = niceMock(Service.class); + + final Map<String, Service> services = new HashMap<>(); + services.put("HIVE", hive); + + EasyMock.expect(cluster.getClusterId()).andReturn(1L).anyTimes(); + + EasyMock.expect(cluster.getCurrentStackVersion()).andReturn(new StackId("HDP", "2.3")).anyTimes(); + EasyMock.expect(cluster.getServices()).andReturn(services).atLeastOnce(); + EasyMock.expect(m_clusters.getCluster(m_clusterName)).andReturn(cluster).atLeastOnce(); + + PrereqCheckRequest request = niceMock(PrereqCheckRequest.class); + EasyMock.expect(request.getClusterName()).andReturn(m_clusterName); + EasyMock.expect(request.getUpgradeType()).andReturn(UpgradeType.ROLLING); + + replayAll(); + + Assert.assertTrue(hiveWarningCheck.isApplicable(request)); + + verifyAll(); + } + + /** + * @throws Exception + */ + @Test + public void testPerform() throws Exception { + final HiveNotRollingWarning hiveWarningCheck = new HiveNotRollingWarning(); + + PrereqCheckRequest request = new PrereqCheckRequest(m_clusterName); + PrerequisiteCheck check = new PrerequisiteCheck(null, null); + + hiveWarningCheck.perform(check, request); + Assert.assertEquals(PrereqCheckStatus.WARNING, check.getStatus()); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/9b0f6317/ambari-server/src/test/java/org/apache/ambari/server/checks/HiveRollingPortChangeWarningTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/checks/HiveRollingPortChangeWarningTest.java b/ambari-server/src/test/java/org/apache/ambari/server/checks/HiveRollingPortChangeWarningTest.java deleted file mode 100644 index cf76d12..0000000 --- a/ambari-server/src/test/java/org/apache/ambari/server/checks/HiveRollingPortChangeWarningTest.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.server.checks; - -import java.util.HashMap; -import java.util.Map; - -import org.apache.ambari.server.controller.PrereqCheckRequest; -import org.apache.ambari.server.state.Cluster; -import org.apache.ambari.server.state.Clusters; -import org.apache.ambari.server.state.Service; -import org.apache.ambari.server.state.StackId; -import org.apache.ambari.server.state.stack.PrereqCheckStatus; -import org.apache.ambari.server.state.stack.PrerequisiteCheck; -import org.apache.ambari.server.state.stack.upgrade.UpgradeType; -import org.easymock.EasyMock; -import org.easymock.EasyMockSupport; -import org.junit.Assert; -import org.junit.Test; - -import com.google.inject.Provider; - -/** - * Tests {@link HiveRollingPortChangeWarning}. - */ -public class HiveRollingPortChangeWarningTest extends EasyMockSupport { - - private final String m_clusterName = "c1"; - private final Clusters m_clusters = niceMock(Clusters.class); - - /** - * @throws Exception - */ - @Test - public void testIsApplicable() throws Exception { - final HiveRollingPortChangeWarning portChangeWarning = new HiveRollingPortChangeWarning(); - portChangeWarning.clustersProvider = new Provider<Clusters>() { - - @Override - public Clusters get() { - return m_clusters; - } - }; - - final Cluster cluster = niceMock(Cluster.class); - final Service hive = niceMock(Service.class); - - final Map<String, Service> services = new HashMap<>(); - services.put("HIVE", hive); - - EasyMock.expect(cluster.getClusterId()).andReturn(1L).anyTimes(); - - EasyMock.expect(cluster.getCurrentStackVersion()).andReturn(new StackId("HDP", "2.3")).anyTimes(); - EasyMock.expect(cluster.getServices()).andReturn(services).atLeastOnce(); - EasyMock.expect(m_clusters.getCluster(m_clusterName)).andReturn(cluster).atLeastOnce(); - - PrereqCheckRequest request = niceMock(PrereqCheckRequest.class); - EasyMock.expect(request.getClusterName()).andReturn(m_clusterName); - EasyMock.expect(request.getUpgradeType()).andReturn(UpgradeType.ROLLING); - - replayAll(); - - Assert.assertTrue(portChangeWarning.isApplicable(request)); - - verifyAll(); - } - - /** - * @throws Exception - */ - @Test - public void testPerform() throws Exception { - final HiveRollingPortChangeWarning portChangeWarning = new HiveRollingPortChangeWarning(); - - PrereqCheckRequest request = new PrereqCheckRequest(m_clusterName); - PrerequisiteCheck check = new PrerequisiteCheck(null, null); - - portChangeWarning.perform(check, request); - Assert.assertEquals(PrereqCheckStatus.WARNING, check.getStatus()); - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/9b0f6317/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py index b3e2494..35d2144 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py @@ -773,11 +773,20 @@ From source with checksum 150f554beae04f76f814f59549dead8b""" call_mocks = call_side_effects ) - self.assertResourceCalled('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'hive-server2', '2.2.1.0-2065'), sudo=True,) + # ensure deregister is called self.assertResourceCalledIgnoreEarlier('Execute', 'hive --config /etc/hive/conf.server --service hiveserver2 --deregister 1.2.1.2.3.0.0-2434', path=['/bin:/usr/hdp/current/hive-server2/bin:/usr/hdp/current/hadoop-client/bin'], tries=1, user='hive') + # ensure stop is called + self.assertResourceCalled('Execute', "ambari-sudo.sh kill 123", + not_if = "! (ls /var/run/hive/hive-server.pid >/dev/null 2>&1 && ps -p 123 >/dev/null 2>&1)") + + # skip any other stop stuff since it's covered by other tests and verify hdp-select + self.assertResourceCalledIgnoreEarlier('Execute', + ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'hive-server2', '2.2.1.0-2065'), + sudo=True) + @patch("resource_management.libraries.script.Script.post_start") @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs") @@ -797,11 +806,17 @@ From source with checksum 150f554beae04f76f814f59549dead8b""" call_mocks = call_side_effects ) - self.assertResourceCalled('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'hive-server2', '2.2.1.0-2065'), sudo=True,) + # ensure that deregister is called self.assertResourceCalledIgnoreEarlier( 'Execute', 'hive --config /etc/hive/conf.server --service hiveserver2 --deregister 1.2.1.2.3.0.0-2434', path=['/bin:/usr/hdp/current/hive-server2/bin:/usr/hdp/current/hadoop-client/bin'], tries=1, user='hive') + # ensure hdp-select is called + self.assertResourceCalledIgnoreEarlier('Execute', + ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'hive-server2', '2.2.1.0-2065'), + sudo = True, ) + + def test_stop_during_upgrade_bad_hive_version(self): try: self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py", @@ -813,7 +828,13 @@ From source with checksum 150f554beae04f76f814f59549dead8b""" except: pass - self.assertResourceCalled('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'hive-server2', '2.2.1.0-2065'), sudo=True,) + self.assertResourceCalled('Execute', "ambari-sudo.sh kill 123", + not_if = "! (ls /var/run/hive/hive-server.pid >/dev/null 2>&1 && ps -p 123 >/dev/null 2>&1)") + + self.assertResourceCalledIgnoreEarlier('Execute', + ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'hive-server2', '2.2.1.0-2065'), + sudo=True) + self.assertNoMoreResources() @patch("resource_management.libraries.functions.security_commons.build_expectations")
