HADOOP-11764. [ HADOOP-11764] NodeManager should use directory other than tmp for extracting and loading leveldbjni (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d9f600e2 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d9f600e2 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d9f600e2 Branch: refs/heads/branch-2.1 Commit: d9f600e23d42544a3d8f67e2da75dc3db98bc555 Parents: cfa8023 Author: Andrew Onishuk <[email protected]> Authored: Fri Sep 11 17:16:26 2015 +0300 Committer: Andrew Onishuk <[email protected]> Committed: Fri Sep 11 17:16:26 2015 +0300 ---------------------------------------------------------------------- .../common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml | 2 ++ .../YARN/2.1.0.2.0/package/scripts/params_linux.py | 1 + .../stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py | 2 ++ .../2.0.6/hooks/before-ANY/scripts/shared_initialization.py | 7 +++++++ .../stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml | 2 ++ .../HDP/2.2/services/YARN/configuration-mapred/mapred-env.xml | 2 ++ .../stacks/HDP/2.3/services/YARN/configuration/yarn-env.xml | 2 ++ .../python/stacks/2.0.6/hooks/before-ANY/test_before_any.py | 5 +++++ 8 files changed, 23 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/d9f600e2/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml index a1dfa57..6eb1a1d 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml @@ -85,6 +85,7 @@ export YARN_LOG_DIR={{yarn_log_dir_prefix}}/$USER export YARN_PID_DIR={{yarn_pid_dir_prefix}}/$USER export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}} export JAVA_HOME={{java64_home}} +export JAVA_LIBRARY_PATH="${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}" # User for YARN daemons export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn} @@ -190,6 +191,7 @@ if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" fi YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE" +YARN_OPTS="$YARN_OPTS -Djava.io.tmpdir={{hadoop_java_io_tmpdir}}" </value> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/d9f600e2/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index 70ae4b8..9b242ea 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -74,6 +74,7 @@ hadoop_mapred2_jar_location = "/usr/lib/hadoop-mapreduce" mapred_bin = "/usr/lib/hadoop-mapreduce/sbin" yarn_bin = "/usr/lib/hadoop-yarn/sbin" yarn_container_bin = "/usr/lib/hadoop-yarn/bin" +hadoop_java_io_tmpdir = os.path.join(tmp_dir, "hadoop_java_io_tmpdir") # hadoop parameters for 2.2+ if Script.is_hdp_stack_greater_or_equal("2.2"): http://git-wip-us.apache.org/repos/asf/ambari/blob/d9f600e2/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py index 8abd72b..d871751 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py @@ -19,6 +19,7 @@ limitations under the License. import collections import re +import os import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. @@ -94,6 +95,7 @@ hadoop_conf_empty_dir = "/etc/hadoop/conf.empty" hadoop_secure_dn_user = hdfs_user hadoop_dir = "/etc/hadoop" versioned_hdp_root = '/usr/hdp/current' +hadoop_java_io_tmpdir = os.path.join(tmp_dir, "hadoop_java_io_tmpdir") # HDP 2.2+ params if Script.is_hdp_stack_greater_or_equal("2.2"): http://git-wip-us.apache.org/repos/asf/ambari/blob/d9f600e2/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py index bd44caf..c2cbe3c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py @@ -155,6 +155,13 @@ def setup_hadoop_env(): group=params.user_group, content=InlineTemplate(params.hadoop_env_sh_template)) + # Create tmp dir for java.io.tmpdir + # Handle a situation when /tmp is set to noexec + Directory(params.hadoop_java_io_tmpdir, + owner=params.hdfs_user, + group=params.user_group, + mode=0777 + ) def setup_java(): """ http://git-wip-us.apache.org/repos/asf/ambari/blob/d9f600e2/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml index 4c99823..9ad95ac 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml @@ -37,6 +37,7 @@ export YARN_LOG_DIR={{yarn_log_dir_prefix}}/$USER export YARN_PID_DIR={{yarn_pid_dir_prefix}}/$USER export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}} export JAVA_HOME={{java64_home}} +export JAVA_LIBRARY_PATH="${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}" # User for YARN daemons export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn} @@ -142,6 +143,7 @@ if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" fi YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE" +YARN_OPTS="$YARN_OPTS -Djava.io.tmpdir={{hadoop_java_io_tmpdir}}" </value> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/d9f600e2/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-env.xml index b5280ce..0046eb9 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-env.xml @@ -41,6 +41,8 @@ export HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA #export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default #export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0. export HADOOP_OPTS="-Dhdp.version=$HDP_VERSION $HADOOP_OPTS" +export HADOOP_OPTS="-Djava.io.tmpdir={{hadoop_java_io_tmpdir}} $HADOOP_OPTS" +export JAVA_LIBRARY_PATH="${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}" </value> </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/d9f600e2/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/yarn-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/yarn-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/yarn-env.xml index 44f3dc2..699463b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/yarn-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/yarn-env.xml @@ -35,6 +35,7 @@ export YARN_PID_DIR={{yarn_pid_dir_prefix}}/$USER export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}} export JAVA_HOME={{java64_home}} + export JAVA_LIBRARY_PATH="${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}" # We need to add the EWMA appender for the yarn daemons only; # however, YARN_ROOT_LOGGER is shared by the yarn client and the @@ -148,6 +149,7 @@ YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" fi YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE" + YARN_OPTS="$YARN_OPTS -Djava.io.tmpdir={{hadoop_java_io_tmpdir}}" </value> </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/d9f600e2/ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py index 39fd3d5..242dc44 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py +++ b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py @@ -163,6 +163,11 @@ class TestHookBeforeInstall(RMFTestCase): owner = 'hdfs', group = 'hadoop' ) + self.assertResourceCalled('Directory', '/tmp/hadoop_java_io_tmpdir', + owner = 'hdfs', + group = 'hadoop', + mode = 0777 + ) self.assertResourceCalled('Directory', '/tmp/AMBARI-artifacts/', recursive = True,
