AMBARI-22386 - Patch Upgrades Broken For Clients Due To Versioned LD Library (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/feccebcb Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/feccebcb Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/feccebcb Branch: refs/heads/branch-feature-AMBARI-20859 Commit: feccebcb9b4cf56d8ea0cc021d5250a711f37596 Parents: c693de3 Author: Jonathan Hurley <jhur...@hortonworks.com> Authored: Wed Nov 8 16:20:30 2017 -0500 Committer: Jonathan Hurley <jhur...@hortonworks.com> Committed: Thu Nov 9 16:36:49 2017 -0500 ---------------------------------------------------------------------- .../TEZ/0.9.0.3.0/configuration/tez-site.xml | 2 +- .../configuration-mapred/mapred-site.xml | 2 +- .../configuration-mapred/mapred-site.xml | 2 +- .../2.2/services/TEZ/configuration/tez-site.xml | 4 ++-- .../YARN/configuration-mapred/mapred-site.xml | 2 +- .../resources/stacks/HDP/2.6/repos/repoinfo.xml | 2 +- .../stacks/HDP/2.6/upgrades/config-upgrade.xml | 23 ++++++++++++++++++++ .../HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml | 9 ++++++++ .../stacks/HDP/2.6/upgrades/upgrade-2.6.xml | 4 ++++ .../YARN/configuration-mapred/mapred-site.xml | 2 +- 10 files changed, 44 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/feccebcb/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/configuration/tez-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/configuration/tez-site.xml b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/configuration/tez-site.xml index 5c17044..58558af 100644 --- a/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/configuration/tez-site.xml +++ b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/configuration/tez-site.xml @@ -78,7 +78,7 @@ </property> <property> <name>tez.am.launch.env</name> - <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value> + <value>LD_LIBRARY_PATH=./tezlib/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value> <description> Additional execution environment entries for tez. This is not an additive property. You must preserve the original value if you want to have access to native libraries. http://git-wip-us.apache.org/repos/asf/ambari/blob/feccebcb/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml index a7d8cd6..3438c45 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml @@ -438,7 +438,7 @@ </property> <property> <name>mapreduce.admin.user.env</name> - <value>LD_LIBRARY_PATH={{hadoop_lib_home}}/native/Linux-{{architecture}}-64</value> + <value>LD_LIBRARY_PATH=./mr-framework/hadoop/lib/native:{{hadoop_lib_home}}/native/Linux-{{architecture}}-64</value> <description> Additional execution environment entries for map and reduce task processes. This is not an additive property. You must preserve the original value if http://git-wip-us.apache.org/repos/asf/ambari/blob/feccebcb/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration-mapred/mapred-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration-mapred/mapred-site.xml b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration-mapred/mapred-site.xml index 705763f..882cf83 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration-mapred/mapred-site.xml +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration-mapred/mapred-site.xml @@ -438,7 +438,7 @@ </property> <property> <name>mapreduce.admin.user.env</name> - <value>LD_LIBRARY_PATH=/usr/lib/hadoop/lib/native:/usr/lib/hadoop/lib/native/Linux-amd64-64</value> + <value>LD_LIBRARY_PATH=./mr-framework/hadoop/lib/native:/usr/lib/hadoop/lib/native:/usr/lib/hadoop/lib/native/Linux-amd64-64</value> <description> Additional execution environment entries for map and reduce task processes. This is not an additive property. You must preserve the original value if http://git-wip-us.apache.org/repos/asf/ambari/blob/feccebcb/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml index 1427a6f..4ffb7a4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml @@ -78,7 +78,7 @@ </property> <property> <name>tez.am.launch.env</name> - <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value> + <value>LD_LIBRARY_PATH=./tezlib/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value> <description> Additional execution environment entries for tez. This is not an additive property. You must preserve the original value if you want to have access to native libraries. @@ -124,7 +124,7 @@ </property> <property> <name>tez.task.launch.env</name> - <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value> + <value>LD_LIBRARY_PATH=./tezlib/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value> <description> Additional execution environment entries for tez. This is not an additive property. You must preserve the original value if you want to have access to native libraries. http://git-wip-us.apache.org/repos/asf/ambari/blob/feccebcb/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-site.xml index 4ad08ce..084e912 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-site.xml @@ -20,7 +20,7 @@ <configuration xmlns:xi="http://www.w3.org/2001/XInclude" supports_final="true"> <property> <name>mapreduce.admin.user.env</name> - <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value> + <value>LD_LIBRARY_PATH=./mr-framework/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value> <description> Additional execution environment entries for map and reduce task processes. This is not an additive property. You must preserve the original value if http://git-wip-us.apache.org/repos/asf/ambari/blob/feccebcb/ambari-server/src/main/resources/stacks/HDP/2.6/repos/repoinfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/repos/repoinfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/repos/repoinfo.xml index ff132aa..eb0b0ef 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/repos/repoinfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/repos/repoinfo.xml @@ -33,7 +33,7 @@ </os> <os family="redhat-ppc7"> <repo> - <baseurl>http://public-repo-1.hortonworks.com/HDP/centos7/2.x/updates/2.6.0.3</baseurl> + <baseurl>http://public-repo-1.hortonworks.com/HDP/centos7-ppc/2.x/updates/2.6.0.3</baseurl> <repoid>HDP-2.6</repoid> <reponame>HDP</reponame> <unique>true</unique> http://git-wip-us.apache.org/repos/asf/ambari/blob/feccebcb/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml index 63624d6..45b1707 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml @@ -258,5 +258,28 @@ </component> </service> + <service name="TEZ"> + <component name="TEZ_CLIENT"> + <changes> + <definition xsi:type="configure" id="hdp_2_6_tez_tarball_ld_library"> + <type>tez-site</type> + <set key="tez.am.launch.env" value="LD_LIBRARY_PATH=./tezlib/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64"/> + <set key="tez.task.launch.env" value="LD_LIBRARY_PATH=./tezlib/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64"/> + </definition> + </changes> + </component> + </service> + + <service name="MAPREDUCE2"> + <component name="MAPREDUCE2_CLIENT"> + <changes> + <definition xsi:type="configure" id="hdp_2_6_mapreduce_tarball_ld_library"> + <type>mapred-site</type> + <set key="mapreduce.admin.user.env" value="LD_LIBRARY_PATH=./mr-framework/hadoop/lib/native:{{hadoop_lib_home}}/native/Linux-{{architecture}}-64"/> + </definition> + </changes> + </component> + </service> + </services> </upgrade-config-changes> http://git-wip-us.apache.org/repos/asf/ambari/blob/feccebcb/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml index c9e90a9..ae2a855 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml @@ -409,6 +409,15 @@ </task> </execute-stage> + <!-- TEZ --> + <execute-stage service="TEZ" component="TEZ_CLIENT" title="Updating LD Library Classpath for Tarball" > + <task xsi:type="configure" id="hdp_2_6_tez_tarball_ld_library" supports-patch="true"/> + </execute-stage> + + <!-- MapR --> + <execute-stage service="MAPREDUCE2" component="MAPREDUCE2_CLIENT" title="Updating LD Library Classpath for Tarball"> + <task xsi:type="configure" id="hdp_2_6_mapreduce_tarball_ld_library" supports-patch="true"/> + </execute-stage> </group> http://git-wip-us.apache.org/repos/asf/ambari/blob/feccebcb/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml index 176143c..37847a2 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml @@ -675,6 +675,8 @@ <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"> <summary>Verifying LZO codec path for mapreduce</summary> </task> + + <task xsi:type="configure" id="hdp_2_6_mapreduce_tarball_ld_library" supports-patch="true"/> </pre-upgrade> <pre-downgrade copy-upgrade="true" /> @@ -757,6 +759,8 @@ <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"> <summary>Verifying LZO codec path for Tez</summary> </task> + + <task xsi:type="configure" id="hdp_2_6_tez_tarball_ld_library" supports-patch="true"/> </pre-upgrade> <pre-downgrade copy-upgrade="true" /> http://git-wip-us.apache.org/repos/asf/ambari/blob/feccebcb/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration-mapred/mapred-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration-mapred/mapred-site.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration-mapred/mapred-site.xml index 6ce4d72..489754f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration-mapred/mapred-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration-mapred/mapred-site.xml @@ -31,7 +31,7 @@ <!-- These configs were inherited from HDP 2.2 --> <property> <name>mapreduce.admin.user.env</name> - <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value> + <value>LD_LIBRARY_PATH=./mr-framework/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value> <description> Additional execution environment entries for map and reduce task processes. This is not an additive property. You must preserve the original value if