AMBARI-21770 - Spark1 Shuffle Property Is Removed Incorrectly on a Stack Upgrade (jonathanhurley)
(cherry picked from commit 24debeed14654249055077ad5483dc45d127a074) Change-Id: Ic475ed2895a14740b576aeaf801f320320017bfd Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a0b3f4fc Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a0b3f4fc Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a0b3f4fc Branch: refs/heads/2.5-maint Commit: a0b3f4fc738aa28e2ac39a175da4e53e31208b3f Parents: 2054332 Author: Jonathan Hurley <[email protected]> Authored: Mon Aug 21 16:09:57 2017 -0400 Committer: Sumit Mohanty <[email protected]> Committed: Mon Aug 21 19:19:29 2017 -0700 ---------------------------------------------------------------------- .../stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml | 7 ++++++- .../stacks/BigInsights/4.2/upgrades/config-upgrade.xml | 6 +++++- 2 files changed, 11 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/a0b3f4fc/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml index f55f9fb..f90a20e 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml @@ -76,7 +76,12 @@ <changes> <definition xsi:type="configure" id="biginsights_4_2_yarn_config_update" summary="Update Yarn configurations"> <type>yarn-site</type> - <replace key="yarn.nodemanager.aux-services" find=",spark_shuffle" replace-with=""/> + <!-- adjust the spark shuffle values for spark1 and spark2 --> + <set key="yarn.nodemanager.aux-services" value="mapreduce_shuffle,spark_shuffle,spark2_shuffle"/> + <set key="yarn.nodemanager.aux-services.spark_shuffle.class" value="org.apache.spark.network.yarn.YarnShuffleService"/> + <set key="yarn.nodemanager.aux-services.spark_shuffle.classpath" value="{{stack_root}}/${hdp.version}/spark/aux/*"/> + <set key="yarn.nodemanager.aux-services.spark2_shuffle.class" value="org.apache.spark.network.yarn.YarnShuffleService"/> + <set key="yarn.nodemanager.aux-services.spark2_shuffle.classpath" value="{{stack_root}}/${hdp.version}/spark2/aux/*"/> </definition> <definition xsi:type="configure" id="yarn_env_security_opts" summary="Adding YARN Security ACLs"> <type>yarn-env</type> http://git-wip-us.apache.org/repos/asf/ambari/blob/a0b3f4fc/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml index 02d3a01..d85c7ee 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml @@ -164,7 +164,11 @@ <changes> <definition xsi:type="configure" id="biginsights_4_2_yarn_config_update" summary="Update Yarn configurations"> <type>yarn-site</type> - <replace key="yarn.nodemanager.aux-services" find=",spark_shuffle" replace-with=""/> + <!-- adjust the spark shuffle values for spark1 based on whether spark is installed --> + <set key="yarn.nodemanager.aux-services" value="mapreduce_shuffle,spark_shuffle" if-type="spark-env" if-key="content" if-key-state="present"/> + <set key="yarn.nodemanager.aux-services.spark_shuffle.class" value="org.apache.spark.network.yarn.YarnShuffleService" if-type="spark-env" if-key="content" if-key-state="present"/> + <set key="yarn.nodemanager.aux-services.spark_shuffle.classpath" value="{{stack_root}}/${hdp.version}/spark/aux/*" if-type="spark-env" if-key="content" if-key-state="present"/> + <insert key="yarn.application.classpath" insert-type="append" newline-after="false" newline-before="false" value=",/usr/hdp/current/ext/hadoop/*"/> </definition> <definition xsi:type="configure" id="yarn_env_security_opts" summary="Adding YARN Security ACLs">
