Repository: ambari Updated Branches: refs/heads/branch-2.1 cfa8023d5 -> d9f600e23 refs/heads/trunk 544f96e85 -> 1c9df2d3f
HADOOP-11764. [ HADOOP-11764] NodeManager should use directory other than tmp for extracting and loading leveldbjni (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1c9df2d3 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1c9df2d3 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1c9df2d3 Branch: refs/heads/trunk Commit: 1c9df2d3f9fbfb98eeb7be651cd511cbe6727358 Parents: 544f96e Author: Andrew Onishuk <[email protected]> Authored: Fri Sep 11 17:16:23 2015 +0300 Committer: Andrew Onishuk <[email protected]> Committed: Fri Sep 11 17:16:23 2015 +0300 ---------------------------------------------------------------------- .../common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml | 2 ++ .../YARN/2.1.0.2.0/package/scripts/params_linux.py | 1 + .../stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py | 2 ++ .../2.0.6/hooks/before-ANY/scripts/shared_initialization.py | 7 +++++++ .../stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml | 2 ++ .../HDP/2.2/services/YARN/configuration-mapred/mapred-env.xml | 2 ++ .../stacks/HDP/2.3/services/YARN/configuration/yarn-env.xml | 2 ++ .../python/stacks/2.0.6/hooks/before-ANY/test_before_any.py | 5 +++++ 8 files changed, 23 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/1c9df2d3/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml index a1dfa57..6eb1a1d 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml @@ -85,6 +85,7 @@ export YARN_LOG_DIR={{yarn_log_dir_prefix}}/$USER export YARN_PID_DIR={{yarn_pid_dir_prefix}}/$USER export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}} export JAVA_HOME={{java64_home}} +export JAVA_LIBRARY_PATH="${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}" # User for YARN daemons export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn} @@ -190,6 +191,7 @@ if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" fi YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE" +YARN_OPTS="$YARN_OPTS -Djava.io.tmpdir={{hadoop_java_io_tmpdir}}" </value> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/1c9df2d3/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index 70ae4b8..9b242ea 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -74,6 +74,7 @@ hadoop_mapred2_jar_location = "/usr/lib/hadoop-mapreduce" mapred_bin = "/usr/lib/hadoop-mapreduce/sbin" yarn_bin = "/usr/lib/hadoop-yarn/sbin" yarn_container_bin = "/usr/lib/hadoop-yarn/bin" +hadoop_java_io_tmpdir = os.path.join(tmp_dir, "hadoop_java_io_tmpdir") # hadoop parameters for 2.2+ if Script.is_hdp_stack_greater_or_equal("2.2"): http://git-wip-us.apache.org/repos/asf/ambari/blob/1c9df2d3/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py index cca0fd1..ea3e866 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py @@ -19,6 +19,7 @@ limitations under the License. import collections import re +import os import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. @@ -96,6 +97,7 @@ hadoop_conf_empty_dir = "/etc/hadoop/conf.empty" hadoop_secure_dn_user = hdfs_user hadoop_dir = "/etc/hadoop" versioned_hdp_root = '/usr/hdp/current' +hadoop_java_io_tmpdir = os.path.join(tmp_dir, "hadoop_java_io_tmpdir") # HDP 2.2+ params if Script.is_hdp_stack_greater_or_equal("2.2"): http://git-wip-us.apache.org/repos/asf/ambari/blob/1c9df2d3/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py index f288f95..420c364 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py @@ -156,6 +156,13 @@ def setup_hadoop_env(): group=params.user_group, content=InlineTemplate(params.hadoop_env_sh_template)) + # Create tmp dir for java.io.tmpdir + # Handle a situation when /tmp is set to noexec + Directory(params.hadoop_java_io_tmpdir, + owner=params.hdfs_user, + group=params.user_group, + mode=0777 + ) def setup_java(): """ http://git-wip-us.apache.org/repos/asf/ambari/blob/1c9df2d3/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml index 4c99823..9ad95ac 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml @@ -37,6 +37,7 @@ export YARN_LOG_DIR={{yarn_log_dir_prefix}}/$USER export YARN_PID_DIR={{yarn_pid_dir_prefix}}/$USER export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}} export JAVA_HOME={{java64_home}} +export JAVA_LIBRARY_PATH="${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}" # User for YARN daemons export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn} @@ -142,6 +143,7 @@ if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" fi YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE" +YARN_OPTS="$YARN_OPTS -Djava.io.tmpdir={{hadoop_java_io_tmpdir}}" </value> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/1c9df2d3/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-env.xml index b5280ce..0046eb9 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-env.xml @@ -41,6 +41,8 @@ export HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA #export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default #export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0. export HADOOP_OPTS="-Dhdp.version=$HDP_VERSION $HADOOP_OPTS" +export HADOOP_OPTS="-Djava.io.tmpdir={{hadoop_java_io_tmpdir}} $HADOOP_OPTS" +export JAVA_LIBRARY_PATH="${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}" </value> </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/1c9df2d3/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/yarn-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/yarn-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/yarn-env.xml index 44f3dc2..699463b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/yarn-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/yarn-env.xml @@ -35,6 +35,7 @@ export YARN_PID_DIR={{yarn_pid_dir_prefix}}/$USER export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}} export JAVA_HOME={{java64_home}} + export JAVA_LIBRARY_PATH="${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}" # We need to add the EWMA appender for the yarn daemons only; # however, YARN_ROOT_LOGGER is shared by the yarn client and the @@ -148,6 +149,7 @@ YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" fi YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE" + YARN_OPTS="$YARN_OPTS -Djava.io.tmpdir={{hadoop_java_io_tmpdir}}" </value> </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/1c9df2d3/ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py index b553a29..a7172dd 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py +++ b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py @@ -166,6 +166,11 @@ class TestHookBeforeInstall(RMFTestCase): owner = 'hdfs', group = 'hadoop' ) + self.assertResourceCalled('Directory', '/tmp/hadoop_java_io_tmpdir', + owner = 'hdfs', + group = 'hadoop', + mode = 0777 + ) self.assertResourceCalled('Directory', '/tmp/AMBARI-artifacts/', recursive = True,
