AMBARI-20225. Take care of hive-interactive-site's 'hive.tez.container.size' config during HDP upgrade from 2.5 to 2.6 and update default value for tez-interactive-site's config 'tez.am.resource.memory.mb'.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/baefbcca Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/baefbcca Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/baefbcca Branch: refs/heads/branch-feature-AMBARI-12556 Commit: baefbcca0b60caac801f3addf0d015df23a5e96d Parents: 2072511 Author: Swapan Shridhar <sshrid...@hortonworks.com> Authored: Mon Feb 27 22:56:54 2017 -0800 Committer: Swapan Shridhar <sshrid...@hortonworks.com> Committed: Wed Mar 1 12:24:06 2017 -0800 ---------------------------------------------------------------------- .../YARN/3.0.0.3.0/service_advisor.py | 51 ++++++++++++++---- .../configuration/hive-interactive-site.xml | 4 +- .../HIVE/configuration/tez-interactive-site.xml | 2 +- .../stacks/HDP/2.5/services/stack_advisor.py | 56 +++++++++++++++----- .../stacks/HDP/2.5/upgrades/config-upgrade.xml | 5 ++ .../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 4 ++ .../stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 1 + .../stacks/2.5/common/test_stack_advisor.py | 44 +++++++-------- 8 files changed, 117 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/baefbcca/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py index e1e03c1..18938a3 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py @@ -48,6 +48,8 @@ class YARNServiceAdvisor(service_advisor.ServiceAdvisor): self.as_super = super(YARNServiceAdvisor, self) self.as_super.__init__(*args, **kwargs) + self.CLUSTER_CREATE_OPERATION = "ClusterCreate" + # Always call these methods self.modifyMastersWithMultipleInstances() self.modifyCardinalitiesDict() @@ -488,6 +490,14 @@ class YARNRecommender(service_advisor.ServiceAdvisor): Note: All memory calculations are in MB, unless specified otherwise. """ Logger.info("DBG: Entered updateLlapConfigs") + + # Determine if we entered here during cluster creation. + operation = getUserOperationContext(services, "operation") + is_cluster_create_opr = False + if operation == self.CLUSTER_CREATE_OPERATION: + is_cluster_create_opr = True + Logger.info("Is cluster create operation ? = {0}".format(is_cluster_create_opr)) + putHiveInteractiveSiteProperty = self.putProperty(configurations, YARNRecommender.HIVE_INTERACTIVE_SITE, services) putHiveInteractiveSitePropertyAttribute = self.putPropertyAttribute(configurations, YARNRecommender.HIVE_INTERACTIVE_SITE) putHiveInteractiveEnvProperty = self.putProperty(configurations, "hive-interactive-env", services) @@ -605,7 +615,8 @@ class YARNRecommender(service_advisor.ServiceAdvisor): Logger.info("DBG: Calculated total_cluster_capacity : {0}, using following : node_manager_cnt : {1}, " "yarn_nm_mem_in_mb : {2}".format(total_cluster_capacity, node_manager_cnt, yarn_nm_mem_in_mb)) yarn_min_container_size = float(self.get_yarn_min_container_size(services, configurations)) - tez_am_container_size = self.calculate_tez_am_container_size(services, long(total_cluster_capacity)) + tez_am_container_size = self.calculate_tez_am_container_size(services, long(total_cluster_capacity), is_cluster_create_opr, + changed_configs_has_enable_hive_int) normalized_tez_am_container_size = self._normalizeUp(tez_am_container_size, yarn_min_container_size) if yarn_site and "yarn.nodemanager.resource.cpu-vcores" in yarn_site: @@ -619,7 +630,8 @@ class YARNRecommender(service_advisor.ServiceAdvisor): # Calculate the available memory for LLAP app yarn_nm_mem_in_mb_normalized = self._normalizeDown(yarn_nm_mem_in_mb, yarn_min_container_size) - mem_per_thread_for_llap = self.calculate_mem_per_thread_for_llap(services, yarn_nm_mem_in_mb_normalized, cpu_per_nm_host) + mem_per_thread_for_llap = float(self.calculate_mem_per_thread_for_llap(services, yarn_nm_mem_in_mb_normalized, cpu_per_nm_host, + is_cluster_create_opr, changed_configs_has_enable_hive_int)) Logger.info("DBG: Calculated mem_per_thread_for_llap : {0}, using following: yarn_nm_mem_in_mb_normalized : {1}, " "cpu_per_nm_host : {2}".format(mem_per_thread_for_llap, yarn_nm_mem_in_mb_normalized, cpu_per_nm_host)) @@ -856,8 +868,10 @@ class YARNRecommender(service_advisor.ServiceAdvisor): # Done with calculations, updating calculated configs. Logger.info("DBG: Applying the calculated values....") - normalized_tez_am_container_size = long(normalized_tez_am_container_size) - putTezInteractiveSiteProperty('tez.am.resource.memory.mb', normalized_tez_am_container_size) + if is_cluster_create_opr or changed_configs_has_enable_hive_int: + normalized_tez_am_container_size = long(normalized_tez_am_container_size) + putTezInteractiveSiteProperty('tez.am.resource.memory.mb', normalized_tez_am_container_size) + Logger.info("DBG: Setting 'tez.am.resource.memory.mb' config value as : {0}".format(normalized_tez_am_container_size)) if not llap_concurrency_in_changed_configs: min_llap_concurrency = 1 @@ -872,18 +886,19 @@ class YARNRecommender(service_advisor.ServiceAdvisor): putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "maximum", node_manager_cnt) #TODO A single value is not being set for numNodes in case of a custom queue. Also the attribute is set to non-visible, so the UI likely ends up using an old cached value if (num_llap_nodes != num_llap_nodes_requested): - Logger.info("User requested num_llap_nodes : {0}, but used/adjusted value for calculations is : {1}".format(num_llap_nodes_requested, num_llap_nodes)) + Logger.info("DBG: User requested num_llap_nodes : {0}, but used/adjusted value for calculations is : {1}".format(num_llap_nodes_requested, num_llap_nodes)) else: - Logger.info("Used num_llap_nodes for calculations : {0}".format(num_llap_nodes_requested)) + Logger.info("DBG: Used num_llap_nodes for calculations : {0}".format(num_llap_nodes_requested)) llap_container_size = long(llap_daemon_mem_per_node) putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', llap_container_size) # Set 'hive.tez.container.size' only if it is read as "SET_ON_FIRST_INVOCATION", implying initialization. # Else, we don't (1). Override the previous calculated value or (2). User provided value. - if self.get_hive_tez_container_size(services) == YARNRecommender.CONFIG_VALUE_UINITIALIZED: + if is_cluster_create_opr or changed_configs_has_enable_hive_int: mem_per_thread_for_llap = long(mem_per_thread_for_llap) putHiveInteractiveSiteProperty('hive.tez.container.size', mem_per_thread_for_llap) + Logger.info("DBG: Setting 'hive.tez.container.size' config value as : {0}".format(mem_per_thread_for_llap)) putTezInteractiveSiteProperty('tez.runtime.io.sort.mb', tez_runtime_io_sort_mb) if "tez-site" in services["configurations"] and "tez.runtime.sorter.class" in services["configurations"]["tez-site"]["properties"]: @@ -970,13 +985,14 @@ class YARNRecommender(service_advisor.ServiceAdvisor): # TODO: This potentially takes up the entire node leaving no space for AMs. return min(floor(nm_mem_per_node_normalized / mem_per_thread), nm_cpus_per_node) - def calculate_mem_per_thread_for_llap(self, services, nm_mem_per_node_normalized, cpu_per_nm_host): + def calculate_mem_per_thread_for_llap(self, services, nm_mem_per_node_normalized, cpu_per_nm_host, is_cluster_create_opr=False, + enable_hive_interactive_1st_invocation=False): """ Calculates 'mem_per_thread_for_llap' for 1st time initialization. Else returns 'hive.tez.container.size' read value. """ hive_tez_container_size = self.get_hive_tez_container_size(services) - if hive_tez_container_size == self.CONFIG_VALUE_UINITIALIZED: + if is_cluster_create_opr or enable_hive_interactive_1st_invocation: if nm_mem_per_node_normalized <= 1024: calculated_hive_tez_container_size = min(512, nm_mem_per_node_normalized) elif nm_mem_per_node_normalized <= 4096: @@ -1003,6 +1019,18 @@ class YARNRecommender(service_advisor.ServiceAdvisor): if hsi_site and 'hive.tez.container.size' in hsi_site: hive_container_size = hsi_site['hive.tez.container.size'] + if not hive_container_size: + # This can happen (1). If config is missing in hive-interactive-site or (2). its an + # upgrade scenario from Ambari 2.4 to Ambari 2.5 with HDP 2.5 installed. Read it + # from hive-site. + # + # If Ambari 2.5 after upgrade from 2.4 is managing HDP 2.6 here, this config would have + # already been added in hive-interactive-site as part of HDP upgrade from 2.5 to 2.6, + # and we wont end up in this block to look up in hive-site. + hive_site = self.getServicesSiteProperties(services, "hive-site") + if hive_site and 'hive.tez.container.size' in hive_site: + hive_container_size = hive_site['hive.tez.container.size'] + return hive_container_size def get_llap_headroom_space(self, services, configurations): @@ -1383,14 +1411,15 @@ yarn.scheduler.capacity.root.{0}.maximum-am-resource-percent=1""".format(llap_qu return yarn_nm_mem_in_mb - def calculate_tez_am_container_size(self, services, total_cluster_capacity): + def calculate_tez_am_container_size(self, services, total_cluster_capacity, is_cluster_create_opr=False, + enable_hive_interactive_1st_invocation=False): """ Calculates Tez App Master container size (tez.am.resource.memory.mb) for tez_hive2/tez-site on initialization if values read is 0. Else returns the read value. """ tez_am_resource_memory_mb = self.get_tez_am_resource_memory_mb(services) calculated_tez_am_resource_memory_mb = None - if tez_am_resource_memory_mb == YARNRecommender.CONFIG_VALUE_UINITIALIZED: + if is_cluster_create_opr or enable_hive_interactive_1st_invocation: if total_cluster_capacity <= 4096: calculated_tez_am_resource_memory_mb = 256 elif total_cluster_capacity > 4096 and total_cluster_capacity <= 73728: http://git-wip-us.apache.org/repos/asf/ambari/blob/baefbcca/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml index 8008bef..e11bb18 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml @@ -647,7 +647,7 @@ limitations under the License. </property> <property> <name>hive.tez.container.size</name> - <value>SET_ON_FIRST_INVOCATION</value> + <value>682</value> <depends-on> <property> <type>hive-interactive-env</type> @@ -670,7 +670,7 @@ limitations under the License. <name>hive.server2.tez.sessions.per.default.queue</name> </property> </depends-on> - <on-ambari-upgrade add="true"/> + <on-ambari-upgrade add="false"/> </property> <property> <name>hive.auto.convert.join.noconditionaltask.size</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/baefbcca/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml index 1c5117e..e914d69 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml @@ -83,7 +83,7 @@ </property> <property> <name>tez.am.resource.memory.mb</name> - <value>SET_ON_FIRST_INVOCATION</value> + <value>1536</value> <description>The amount of memory to be used by the AppMaster</description> <depends-on> <property> http://git-wip-us.apache.org/repos/asf/ambari/blob/baefbcca/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index 6f3dfa7..a450d4d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -34,6 +34,8 @@ class HDP25StackAdvisor(HDP24StackAdvisor): self.YARN_ROOT_DEFAULT_QUEUE_NAME = 'default' self.AMBARI_MANAGED_LLAP_QUEUE_NAME = 'llap' self.CONFIG_VALUE_UINITIALIZED = 'SET_ON_FIRST_INVOCATION' + self.CLUSTER_CREATE_OPERATION = "ClusterCreate" + def recommendOozieConfigurations(self, configurations, clusterData, services, hosts): super(HDP25StackAdvisor,self).recommendOozieConfigurations(configurations, clusterData, services, hosts) @@ -780,6 +782,14 @@ class HDP25StackAdvisor(HDP24StackAdvisor): Note: All memory calculations are in MB, unless specified otherwise. """ Logger.info("DBG: Entered updateLlapConfigs"); + + # Determine if we entered here during cluster creation. + operation = getUserOperationContext(services, "operation") + is_cluster_create_opr = False + if operation == self.CLUSTER_CREATE_OPERATION: + is_cluster_create_opr = True + Logger.info("Is cluster create operation ? = {0}".format(is_cluster_create_opr)) + putHiveInteractiveSiteProperty = self.putProperty(configurations, self.HIVE_INTERACTIVE_SITE, services) putHiveInteractiveSitePropertyAttribute = self.putPropertyAttribute(configurations, self.HIVE_INTERACTIVE_SITE) putHiveInteractiveEnvProperty = self.putProperty(configurations, "hive-interactive-env", services) @@ -901,7 +911,8 @@ class HDP25StackAdvisor(HDP24StackAdvisor): Logger.info("DBG: Calculated total_cluster_capacity : {0}, using following : node_manager_cnt : {1}, " "yarn_nm_mem_in_mb : {2}".format(total_cluster_capacity, node_manager_cnt, yarn_nm_mem_in_mb)) yarn_min_container_size = float(self.get_yarn_min_container_size(services, configurations)) - tez_am_container_size = self.calculate_tez_am_container_size(services, long(total_cluster_capacity)) + tez_am_container_size = self.calculate_tez_am_container_size(services, long(total_cluster_capacity), is_cluster_create_opr, + changed_configs_has_enable_hive_int) normalized_tez_am_container_size = self._normalizeUp(tez_am_container_size, yarn_min_container_size) if yarn_site and "yarn.nodemanager.resource.cpu-vcores" in yarn_site: @@ -915,7 +926,8 @@ class HDP25StackAdvisor(HDP24StackAdvisor): # Calculate the available memory for LLAP app yarn_nm_mem_in_mb_normalized = self._normalizeDown(yarn_nm_mem_in_mb, yarn_min_container_size) - mem_per_thread_for_llap = float(self.calculate_mem_per_thread_for_llap(services, yarn_nm_mem_in_mb_normalized, cpu_per_nm_host)) + mem_per_thread_for_llap = float(self.calculate_mem_per_thread_for_llap(services, yarn_nm_mem_in_mb_normalized, cpu_per_nm_host, + is_cluster_create_opr, changed_configs_has_enable_hive_int)) Logger.info("DBG: Calculated mem_per_thread_for_llap : {0}, using following: yarn_nm_mem_in_mb_normalized : {1}, " "cpu_per_nm_host : {2}".format(mem_per_thread_for_llap, yarn_nm_mem_in_mb_normalized, cpu_per_nm_host)) @@ -939,7 +951,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor): Logger.info("DBG: Calculated 'min_node_required': {0}, using following : min_memory_required : {1}, yarn_nm_mem_in_mb_normalized " ": {2}".format(min_nodes_required, min_memory_required, yarn_nm_mem_in_mb_normalized)) if min_nodes_required > node_manager_cnt: - Logger.warn("ERROR: Not enough memory/nodes to run LLAP"); + Logger.warning("ERROR: Not enough memory/nodes to run LLAP"); self.recommendDefaultLlapConfiguration(configurations, services, hosts) return @@ -1176,8 +1188,10 @@ class HDP25StackAdvisor(HDP24StackAdvisor): # Done with calculations, updating calculated configs. Logger.info("DBG: Applying the calculated values....") - normalized_tez_am_container_size = long(normalized_tez_am_container_size) - putTezInteractiveSiteProperty('tez.am.resource.memory.mb', normalized_tez_am_container_size) + if is_cluster_create_opr or changed_configs_has_enable_hive_int: + normalized_tez_am_container_size = long(normalized_tez_am_container_size) + putTezInteractiveSiteProperty('tez.am.resource.memory.mb', normalized_tez_am_container_size) + Logger.info("DBG: Setting 'tez.am.resource.memory.mb' config value as : {0}".format(normalized_tez_am_container_size)) if not llap_concurrency_in_changed_configs: min_llap_concurrency = 1 @@ -1193,20 +1207,22 @@ class HDP25StackAdvisor(HDP24StackAdvisor): putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "maximum", node_manager_cnt) #TODO A single value is not being set for numNodes in case of a custom queue. Also the attribute is set to non-visible, so the UI likely ends up using an old cached value if (num_llap_nodes != num_llap_nodes_requested): - Logger.info("User requested num_llap_nodes : {0}, but used/adjusted value for calculations is : {1}".format(num_llap_nodes_requested, num_llap_nodes)) + Logger.info("DBG: User requested num_llap_nodes : {0}, but used/adjusted value for calculations is : {1}".format(num_llap_nodes_requested, num_llap_nodes)) else: - Logger.info("Used num_llap_nodes for calculations : {0}".format(num_llap_nodes_requested)) + Logger.info("DBG: Used num_llap_nodes for calculations : {0}".format(num_llap_nodes_requested)) putHiveInteractiveEnvProperty('num_llap_nodes_for_llap_daemons', num_llap_nodes) - Logger.info("Setting config 'num_llap_nodes_for_llap_daemons' as : {0}".format(num_llap_nodes)) + Logger.info("DBG: Setting config 'num_llap_nodes_for_llap_daemons' as : {0}".format(num_llap_nodes)) llap_container_size = long(llap_daemon_mem_per_node) putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', llap_container_size) # Set 'hive.tez.container.size' only if it is read as "SET_ON_FIRST_INVOCATION", implying initialization. # Else, we don't (1). Override the previous calculated value or (2). User provided value. - if self.get_hive_tez_container_size(services) == self.CONFIG_VALUE_UINITIALIZED: + if is_cluster_create_opr or changed_configs_has_enable_hive_int: mem_per_thread_for_llap = long(mem_per_thread_for_llap) putHiveInteractiveSiteProperty('hive.tez.container.size', mem_per_thread_for_llap) + Logger.info("DBG: Setting 'hive.tez.container.size' config value as : {0}".format(mem_per_thread_for_llap)) + putTezInteractiveSiteProperty('tez.runtime.io.sort.mb', tez_runtime_io_sort_mb) if "tez-site" in services["configurations"] and "tez.runtime.sorter.class" in services["configurations"]["tez-site"]["properties"]: @@ -1296,13 +1312,13 @@ class HDP25StackAdvisor(HDP24StackAdvisor): # TODO: This potentially takes up the entire node leaving no space for AMs. return min(math.floor(nm_mem_per_node_normalized / mem_per_thread), nm_cpus_per_node) - def calculate_mem_per_thread_for_llap(self, services, nm_mem_per_node_normalized, cpu_per_nm_host): + def calculate_mem_per_thread_for_llap(self, services, nm_mem_per_node_normalized, cpu_per_nm_host, is_cluster_create_opr=False, + enable_hive_interactive_1st_invocation=False): """ Calculates 'mem_per_thread_for_llap' for 1st time initialization. Else returns 'hive.tez.container.size' read value. """ hive_tez_container_size = self.get_hive_tez_container_size(services) - - if hive_tez_container_size == self.CONFIG_VALUE_UINITIALIZED: + if is_cluster_create_opr or enable_hive_interactive_1st_invocation: if nm_mem_per_node_normalized <= 1024: calculated_hive_tez_container_size = min(512, nm_mem_per_node_normalized) elif nm_mem_per_node_normalized <= 4096: @@ -1328,6 +1344,18 @@ class HDP25StackAdvisor(HDP24StackAdvisor): hsi_site = self.getServicesSiteProperties(services, self.HIVE_INTERACTIVE_SITE) if hsi_site and 'hive.tez.container.size' in hsi_site: hive_container_size = hsi_site['hive.tez.container.size'] + + if not hive_container_size: + # This can happen (1). If config is missing in hive-interactive-site or (2). its an + # upgrade scenario from Ambari 2.4 to Ambari 2.5 with HDP 2.5 installed. Read it + # from hive-site. + # + # If Ambari 2.5 after upgrade from 2.4 is managing HDP 2.6 here, this config would have + # already been added in hive-interactive-site as part of HDP upgrade from 2.5 to 2.6, + # and we wont end up in this block to look up in hive-site. + hive_site = self.getServicesSiteProperties(services, "hive-site") + if hive_site and 'hive.tez.container.size' in hive_site: + hive_container_size = hive_site['hive.tez.container.size'] return hive_container_size def get_llap_headroom_space(self, services, configurations): @@ -1401,14 +1429,14 @@ class HDP25StackAdvisor(HDP24StackAdvisor): if yarn_min_container_size < 256: return 256 - def calculate_tez_am_container_size(self, services, total_cluster_capacity): + def calculate_tez_am_container_size(self, services, total_cluster_capacity, is_cluster_create_opr=False, enable_hive_interactive_1st_invocation=False): """ Calculates Tez App Master container size (tez.am.resource.memory.mb) for tez_hive2/tez-site on initialization if values read is 0. Else returns the read value. """ tez_am_resource_memory_mb = self.get_tez_am_resource_memory_mb(services) calculated_tez_am_resource_memory_mb = None - if tez_am_resource_memory_mb == self.CONFIG_VALUE_UINITIALIZED: + if is_cluster_create_opr or enable_hive_interactive_1st_invocation: if total_cluster_capacity <= 4096: calculated_tez_am_resource_memory_mb = 256 elif total_cluster_capacity > 4096 and total_cluster_capacity <= 73728: http://git-wip-us.apache.org/repos/asf/ambari/blob/baefbcca/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml index da334be..e5f07ba 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml @@ -444,6 +444,11 @@ <set key="llap_headroom_space" value="12288"/> </definition> + <definition xsi:type="configure" id="hdp_2_6_0_0_copy_hive_tez_container_size_to_hiveInteractive"> + <type>hive-interactive-site</type> + <transfer operation="copy" from-type="hive-site" from-key="hive.tez.container.size" to-key="hive.tez.container.size" default-value="682" if-type="hive-interactive-site" if-key="hive.tez.container.size" if-key-state="absent"/> + </definition> + </changes> </component> http://git-wip-us.apache.org/repos/asf/ambari/blob/baefbcca/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml index 8e98ba8..0d2f1bc 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml @@ -504,6 +504,10 @@ <task xsi:type="configure" id="hdp_2_6_0_0_tez_append_heap_dump_options_for_tez_am"/> </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Copying hive.tez.container.size to HIVE SERVER INTERACTIVE's hive-site"> + <task xsi:type="configure" id="hdp_2_6_0_0_copy_hive_tez_container_size_to_hiveInteractive"/> + </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Set hive.enforce.bucketing property"> <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property"> <summary>Updating hive.enforce.bucketing property to true value</summary> http://git-wip-us.apache.org/repos/asf/ambari/blob/baefbcca/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml index c1e8f8e..f0a4f05 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml @@ -798,6 +798,7 @@ <task xsi:type="configure" id="hive_log4j_parameterize" /> <task xsi:type="configure" id="hive_llap_log4j_parameterize" /> <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property" /> + <task xsi:type="configure" id="hdp_2_6_0_0_copy_hive_tez_container_size_to_hiveInteractive" /> </pre-upgrade> <pre-downgrade /> http://git-wip-us.apache.org/repos/asf/ambari/blob/baefbcca/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index cf1c0ee..2d7bcd3 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -1214,7 +1214,7 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'llap') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'], '286261248') - self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '512') + self.assertTrue('tez.am.resource.memory.mb' not in configurations['tez-interactive-site']['properties']) self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], {'entries': [{'value': 'default', 'label': 'default'}, {'value': 'llap', 'label': 'llap'}]}) @@ -1414,7 +1414,7 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_mb'], '341') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'], '286261248') - self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '341') + self.assertTrue('tez.am.resource.memory.mb' not in configurations['tez-interactive-site']['properties']) self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], {'entries': [{'value': 'default', 'label': 'default'}, {'value': 'llap', 'label': 'llap'}]}) @@ -1612,7 +1612,7 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'llap') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'], '572522496') - self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '1024') + self.assertTrue('tez.am.resource.memory.mb' not in configurations['tez-interactive-site']['properties']) self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], {'entries': [{'value': 'default', 'label': 'default'}, {'value': 'llap', 'label': 'llap'}]}) @@ -1817,7 +1817,7 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'llap') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'], '572522496') - self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '2048') + self.assertTrue('tez.am.resource.memory.mb' not in configurations['tez-interactive-site']['properties']) self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], {'entries': [{'value': 'default', 'label': 'default'}, {'value': 'llap', 'label': 'llap'}]}) @@ -2002,23 +2002,23 @@ class TestHDP25StackAdvisor(TestCase): self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties']) self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3) - self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '10571') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '11253') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '3') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.threadpool.size'], '3') - self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'], '4427') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'], '2037') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.enabled'], 'true') - self.assertEqual(configurations['hive-interactive-env']['properties']['llap_heap_size'], '4915') + self.assertEqual(configurations['hive-interactive-env']['properties']['llap_heap_size'], '7372') self.assertEqual(configurations['hive-interactive-env']['properties']['hive_heapsize'], '2048') self.assertEqual(configurations['hive-interactive-env']['property_attributes']['num_llap_nodes'], {'maximum': '3', 'minimum': '1', 'read_only': 'false'}) self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_mb'], '341') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'llap') - self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'], '572522496') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'], '858783744') - self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '1364') + self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '682') self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], {'entries': [{'value': 'default', 'label': 'default'}, {'value': 'llap', 'label': 'llap'}]}) @@ -2417,7 +2417,7 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'llap') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'], '572522496') - self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '2048') + self.assertTrue('tez.am.resource.memory.mb' not in configurations['tez-interactive-site']['properties']) self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], {'entries': [{'value': 'default', 'label': 'default'}, {'value': 'llap', 'label': 'llap'}]}) @@ -2624,7 +2624,7 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'llap') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'], '572522496') - self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '3072') + self.assertTrue('tez.am.resource.memory.mb' not in configurations['tez-interactive-site']['properties']) self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], {'entries': [{'value': 'default', 'label': 'default'}, {'value': 'llap', 'label': 'llap'}]}) @@ -2809,23 +2809,23 @@ class TestHDP25StackAdvisor(TestCase): self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3) self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties']) - self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '203918') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '202554') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '10') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.threadpool.size'], '10') - self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'], '183438') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'], '161594') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.enabled'], 'true') - self.assertEqual(configurations['hive-interactive-env']['properties']['llap_heap_size'], '16384') + self.assertEqual(configurations['hive-interactive-env']['properties']['llap_heap_size'], '34816') self.assertEqual(configurations['hive-interactive-env']['properties']['hive_heapsize'], '2048') self.assertEqual(configurations['hive-interactive-env']['property_attributes']['num_llap_nodes'], {'maximum': '5', 'minimum': '1', 'read_only': 'false'}) self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_mb'], '341') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'llap') - self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'], '572522496') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'], '1145044992') - self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '341') + self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '1705') self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], {'entries': [{'value': 'default', 'label': 'default'}, {'value': 'llap', 'label': 'llap'}]}) @@ -3022,7 +3022,7 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'llap') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'], '572522496') - self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '2048') + self.assertTrue('tez.am.resource.memory.mb' not in configurations['tez-interactive-site']['properties']) self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], {'entries': [{'value': 'default', 'label': 'default'}, {'value': 'llap', 'label': 'llap'}]}) @@ -3214,7 +3214,7 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_mb'], '2048') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'], '572522496') - self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '2048') + self.assertTrue('tez.am.resource.memory.mb' not in configurations['tez-interactive-site']['properties']) self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], {'entries': [{'value': 'default', 'label': 'default'}]}) @@ -3438,7 +3438,7 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_mb'], '512') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'], '572522496') - self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '1024') + self.assertTrue('tez.am.resource.memory.mb' not in configurations['tez-interactive-site']['properties']) self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], {'entries': [{'value': 'a1', 'label': 'a1'}, {'value': 'b', 'label': 'b'}, {'value': 'llap', 'label': 'llap'}]}) @@ -3845,8 +3845,8 @@ class TestHDP25StackAdvisor(TestCase): {'maximum': '1', 'minimum': '1', 'read_only': 'false'}) self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.threadpool.size'], '3') - self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'], '197632') - self.assertEqual(configurations['hive-interactive-env']['properties']['llap_heap_size'], '2457') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'], '188416') + self.assertEqual(configurations['hive-interactive-env']['properties']['llap_heap_size'], '9830') @@ -4043,7 +4043,7 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'default') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'], '572522496') - self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '2048') + self.assertTrue('tez.am.resource.memory.mb' not in configurations['tez-interactive-site']['properties']) self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], {'entries': [{'value': 'default', 'label': 'default'}]})
