Repository: ambari Updated Branches: refs/heads/trunk ef355ae0d -> cce9489c2
AMBARI-20109. HDP stack - enable shuffle connection keep-alive. (Siddharth Seth via Swapan Shridhar). Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cce9489c Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cce9489c Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cce9489c Branch: refs/heads/trunk Commit: cce9489c2ed551940c70e9003fa421d4c9880e38 Parents: ef355ae Author: Swapan Shridhar <[email protected]> Authored: Thu Feb 23 16:46:20 2017 -0800 Committer: Swapan Shridhar <[email protected]> Committed: Thu Feb 23 16:46:20 2017 -0800 ---------------------------------------------------------------------- .../stacks/HDP/2.5/upgrades/config-upgrade.xml | 24 +++++++++++++++++ .../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 20 +++++++++++++++ .../stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 6 +++++ .../HIVE/configuration/tez-interactive-site.xml | 6 +++++ .../2.6/services/TEZ/configuration/tez-site.xml | 7 +++++ .../YARN/configuration-mapred/mapred-site.xml | 27 ++++++++++++++++++++ 6 files changed, 90 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/cce9489c/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml index 3069b34..bca641c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml @@ -177,6 +177,14 @@ </definition> </changes> </component> + <component name="NODEMANAGER"> + <changes> + <definition xsi:type="configure" id="hdp_2_6_0_0_enable_mapreduce_shuffle_connection_keepalive"> + <type>mapred-site</type> + <set key="mapreduce.shuffle.connection-keep-alive.enable" value="true"/> + </definition> + </changes> + </component> </service> <service name="MAPREDUCE2"> @@ -354,6 +362,17 @@ </component> </service> + <service name="TEZ"> + <component name="TEZ_CLIENT"> + <changes> + <definition xsi:type="configure" id="hdp_2_6_0_0_tez_client_enable_connection_keepalive"> + <type>tez-site</type> + <set key="tez.runtime.shuffle.keep-alive.enabled" value="true"/> + </definition> + </changes> + </component> + </service> + <service name="PIG"> <component name="PIG"> <changes> @@ -433,6 +452,11 @@ <set key="llap_headroom_space" value="12288"/> </definition> + <definition xsi:type="configure" id="hdp_2_6_0_0_tez_interactive_enable_connection_keepalive"> + <type>tez-interactive-site</type> + <set key="tez.runtime.shuffle.keep-alive.enabled" value="true"/> + </definition> + </changes> </component> http://git-wip-us.apache.org/repos/asf/ambari/blob/cce9489c/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml index 9516b6a..8641146 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml @@ -287,6 +287,13 @@ </task> </execute-stage> + <!--YARN--> + <execute-stage service="YARN" component="NODEMANAGER" title="Enable ShuffleHandler keep-alive"> + <task xsi:type="configure" id="hdp_2_6_0_0_enable_mapreduce_shuffle_connection_keepalive"> + <summary>Enabling connection keep alive for the MapReduce Shuffle Handler</summary> + </task> + </execute-stage> + <!--Yarn--> <execute-stage service="YARN" component="RESOURCEMANAGER" title="Parameterizing Yarn Log4J Properties Resource Manager"> <task xsi:type="configure" id="yarn_log4j_parameterize"> @@ -334,6 +341,13 @@ </task> </execute-stage> + <!--TEZ--> + <execute-stage service="TEZ" component="TEZ_CLIENT" title="Enable Tez Client Shuffle Connectio Keep-Alive"> + <task xsi:type="configure" id="hdp_2_6_0_0_tez_client_enable_connection_keepalive"> + <summary>Enabling connection keep alive for Tez shuffle client</summary> + </task> + </execute-stage> + <!--OOZIE--> <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie Server"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation"> @@ -527,6 +541,12 @@ <summary>Updating the Hive Log4J2 properties to include parameterizations</summary> </task> </execute-stage> + + <execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Enable Tez-HiveInteractive Client Shuffle Connectio Keep-Alive"> + <task xsi:type="configure" id="hdp_2_6_0_0_tez_interactive_enable_connection_keepalive"> + <summary>Enabling connection keep alive for Tez-HiveInteractive shuffle client</summary> + </task> + </execute-stage> <!-- SPARK --> <execute-stage service="SPARK" component="LIVY_SERVER" title="Apply config changes for Livy Server"> http://git-wip-us.apache.org/repos/asf/ambari/blob/cce9489c/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml index 04d603b..689c36b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml @@ -683,6 +683,10 @@ </component> <component name="NODEMANAGER"> + <pre-upgrade> + <task xsi:type="configure" id="hdp_2_6_0_0_enable_mapreduce_shuffle_connection_keepalive"/> + </pre-upgrade> + <pre-downgrade /> <upgrade> <task xsi:type="restart-task" /> </upgrade> @@ -802,6 +806,7 @@ <task xsi:type="configure" id="hive_log4j2_parameterize"/> <task xsi:type="configure" id="llap_cli_log4j2_parameterize"/> <task xsi:type="configure" id="llap_update_headroom"/> + <task xsi:type="configure" id="hdp_2_6_0_0_tez_interactive_enable_connection_keepalive"/> </pre-upgrade> <pre-downgrade /> @@ -898,6 +903,7 @@ <script>scripts/oozie_server_upgrade.py</script> <function>upgrade_oozie_database_and_sharelib</function> </task> + <task xsi:type="configure" id="hdp_2_6_0_0_tez_client_enable_connection_keepalive"/> </pre-upgrade> <pre-downgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/cce9489c/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml index 3ce5fed..3bcb49a 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml @@ -74,5 +74,11 @@ </description> <on-ambari-upgrade add="true"/> </property> + <property> + <name>tez.runtime.shuffle.keep-alive.enabled</name> + <value>true</value> + <description>Enables/Disables connection keep from the client side</description> + <on-ambari-upgrade add="false"/> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/cce9489c/ambari-server/src/main/resources/stacks/HDP/2.6/services/TEZ/configuration/tez-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/TEZ/configuration/tez-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/TEZ/configuration/tez-site.xml index 4bd1a19..463a2ed 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/TEZ/configuration/tez-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/TEZ/configuration/tez-site.xml @@ -25,4 +25,11 @@ </description> <on-ambari-upgrade add="false"/> </property> + <property> + <name>tez.runtime.shuffle.keep-alive.enabled</name> + <value>true</value> + <description>Enables/Disables connection keep from the client side</description> + <on-ambari-upgrade add="false"/> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/cce9489c/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration-mapred/mapred-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration-mapred/mapred-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration-mapred/mapred-site.xml new file mode 100644 index 0000000..1654c1c --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration-mapred/mapred-site.xml @@ -0,0 +1,27 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<!-- Put site-specific property overrides in this file. --> +<configuration xmlns:xi="http://www.w3.org/2001/XInclude"; supports_final="true"> + <property> + <name>mapreduce.shuffle.connection-keep-alive.enable</name> + <value>true</value> + <description>Enable/Disable connection keep-alive for the MapReduce shuffle handler</description> + <on-ambari-upgrade add="false"/> + </property> +</configuration>
