Updated Branches: refs/heads/trunk 9c3940bd0 -> 32399749c
AMBARI-2907. yarn-env.sh is missing few critical attributes. (smohanty) Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/32399749 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/32399749 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/32399749 Branch: refs/heads/trunk Commit: 32399749c9394cef0088328583f192c6e548165c Parents: 9c3940b Author: Sumit Mohanty <[email protected]> Authored: Wed Aug 14 22:50:58 2013 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Wed Aug 14 22:50:58 2013 -0700 ---------------------------------------------------------------------- .../puppet/modules/hdp-yarn/manifests/params.pp | 9 +- .../modules/hdp-yarn/templates/yarn-env.sh.erb | 172 +++++++++++-------- .../services/YARN/configuration/global.xml | 15 ++ .../services/YARN/configuration/global.xml | 15 ++ 4 files changed, 130 insertions(+), 81 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/32399749/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp b/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp index d09483b..8916196 100644 --- a/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp +++ b/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp @@ -32,12 +32,11 @@ class hdp-yarn::params( ## yarn-env $hadoop_libexec_dir = hdp_default("yarn/yarn-env/hadoop_libexec_dir","/usr/lib/hadoop/libexec") - - $hadoop_common_home = hdp_default("yarn/yarn-env/hadoop_common_home","/usr/lib/hadoop") - $hadoop_hdfs_home = hdp_default("yarn/yarn-env/hadoop_hdfs_home","/usr/lib/hadoop-hdfs") - $hadoop_mapred_home = hdp_default("yarn/yarn-env/hadoop_mapred_home","/usr/lib/hadoop-mapreduce") $hadoop_yarn_home = hdp_default("yarn/yarn-env/hadoop_yarn_home","/usr/lib/hadoop-yarn") - + $yarn_heapsize = hdp_default("yarn/yarn-env/yarn_heapsize","1024") + $resourcemanager_heapsize = hdp_default("yarn/yarn-env/resourcemanager_heapsize","1024") + $nodemanager_heapsize = hdp_default("yarn/yarn-env/nodemanager_heapsize","1024") + $yarn_log_dir_prefix = hdp_default("hadoop/yarn-env/yarn_log_dir_prefix","/var/log/hadoop-yarn") $yarn_pid_dir_prefix = hdp_default("hadoop/yarn-env/yarn_pid_dir_prefix","/var/run/hadoop-yarn") http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/32399749/ambari-agent/src/main/puppet/modules/hdp-yarn/templates/yarn-env.sh.erb ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/puppet/modules/hdp-yarn/templates/yarn-env.sh.erb b/ambari-agent/src/main/puppet/modules/hdp-yarn/templates/yarn-env.sh.erb index d1bd482..4bf1779 100644 --- a/ambari-agent/src/main/puppet/modules/hdp-yarn/templates/yarn-env.sh.erb +++ b/ambari-agent/src/main/puppet/modules/hdp-yarn/templates/yarn-env.sh.erb @@ -16,84 +16,104 @@ # * limitations under the License. # */ -# Set Hadoop-specific environment variables here. - -# The only required environment variable is JAVA_HOME. All others are -# optional. When running a distributed configuration it is best to -# set JAVA_HOME in this file, so that it is correctly defined on -# remote nodes. - -export HADOOP_COMMON_HOME=<%=scope.function_hdp_template_var("hadoop_common_home")%> -export HADOOP_HDFS_HOME=<%=scope.function_hdp_template_var("hadoop_hdfs_home")%> -export HADOOP_MAPRED_HOME=<%=scope.function_hdp_template_var("hadoop_mapred_home")%> export HADOOP_YARN_HOME=<%=scope.function_hdp_template_var("hadoop_yarn_home")%> - export YARN_LOG_DIR=<%=scope.function_hdp_template_var("yarn_log_dir_prefix")%>/$USER export YARN_PID_DIR=<%=scope.function_hdp_template_var("yarn_pid_dir_prefix")%>/$USER - -export HADOOP_LIBEXEC_DIR=/usr/lib/hadoop/libexec - -# The java implementation to use. Required. +export HADOOP_LIBEXEC_DIR=<%=scope.function_hdp_template_var("hadoop_libexec_dir")%> export JAVA_HOME=<%=scope.function_hdp_java_home()%> -export HADOOP_HOME_WARN_SUPPRESS=1 - -# Hadoop Configuration Directory -#TODO: if env var set that can cause problems -export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop/conf} - -# The maximum amount of heap to use, in MB. Default is 1000. -export HADOOP_HEAPSIZE="1024" - -export HADOOP_NAMENODE_INIT_HEAPSIZE="-Xms1024m" - -# Extra Java runtime options. Empty by default. -export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}" - -# Command specific options appended to HADOOP_OPTS when specified -export HADOOP_NAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=640m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms1024m -Xmx1024m -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_NAMENODE_OPTS}" -HADOOP_JOBTRACKER_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx1024m -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}" - -HADOOP_TASKTRACKER_OPTS="-server -Xmx1024m -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}" -HADOOP_DATANODE_OPTS="-Xmx1024m -Dhadoop.security.logger=ERROR,DRFAS ${HADOOP_DATANODE_OPTS}" -HADOOP_BALANCER_OPTS="-server -Xmx1024m ${HADOOP_BALANCER_OPTS}" - -export HADOOP_SECONDARYNAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=640m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps ${HADOOP_NAMENODE_INIT_HEAPSIZE} -Xmx1024m -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_SECONDARYNAMENODE_OPTS}" - -# The following applies to multiple commands (fs, dfs, fsck, distcp etc) -export HADOOP_CLIENT_OPTS="-Xmx128m ${HADOOP_CLIENT_OPTS}" -#HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData ${HADOOP_JAVA_PLATFORM_OPTS}" - -# On secure datanodes, user to run the datanode as after dropping privileges -export HADOOP_SECURE_DN_USER=hdfs - -# Extra ssh options. Empty by default. -export HADOOP_SSH_OPTS="-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR" - -# Where log files are stored. $HADOOP_HOME/logs by default. -export HADOOP_LOG_DIR=/var/log/hadoop/$USER - - -# Where log files are stored in the secure data environment. -export HADOOP_SECURE_DN_LOG_DIR=/var/log/hadoop/$HADOOP_SECURE_DN_USER - -# File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default. -# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves - -# host:path where hadoop code should be rsync'd from. Unset by default. -# export HADOOP_MASTER=master:/home/$USER/src/hadoop - -# Seconds to sleep between slave commands. Unset by default. This -# can be useful in large clusters, where, e.g., slave rsyncs can -# otherwise arrive faster than the master can service them. -# export HADOOP_SLAVE_SLEEP=0.1 - -# The directory where pid files are stored. /tmp by default. -export HADOOP_PID_DIR=/var/run/hadoop/$USER -export HADOOP_SECURE_DN_PID_DIR=/var/run/hadoop/$HADOOP_SECURE_DN_USER - -# A string representing this instance of hadoop. $USER by default. -export HADOOP_IDENT_STRING=$USER - -# The scheduling priority for daemon processes. See 'man nice'. -# export HADOOP_NICENESS=10 +# User for YARN daemons +export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn} + +# resolve links - $0 may be a softlink +export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}" + +# some Java parameters +# export JAVA_HOME=/home/y/libexec/jdk1.6.0/ +if [ "$JAVA_HOME" != "" ]; then + #echo "run java in $JAVA_HOME" + JAVA_HOME=$JAVA_HOME +fi + +if [ "$JAVA_HOME" = "" ]; then + echo "Error: JAVA_HOME is not set." + exit 1 +fi + +JAVA=$JAVA_HOME/bin/java +JAVA_HEAP_MAX=-Xmx1000m + +# For setting YARN specific HEAP sizes please use this +# Parameter and set appropriately +YARN_HEAPSIZE=<%=scope.function_hdp_template_var("yarn_heapsize")%> + +# check envvars which might override default args +if [ "$YARN_HEAPSIZE" != "" ]; then + JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m" +fi + +# Resource Manager specific parameters + +# Specify the max Heapsize for the ResourceManager using a numerical value +# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set +# the value to 1000. +# This value will be overridden by an Xmx setting specified in either YARN_OPTS +# and/or YARN_RESOURCEMANAGER_OPTS. +# If not specified, the default value will be picked from either YARN_HEAPMAX +# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. +export YARN_RESOURCEMANAGER_HEAPSIZE=<%=scope.function_hdp_template_var("resourcemanager_heapsize")%> + +# Specify the JVM options to be used when starting the ResourceManager. +# These options will be appended to the options specified as YARN_OPTS +# and therefore may override any similar flags set in YARN_OPTS +#export YARN_RESOURCEMANAGER_OPTS= + +# Node Manager specific parameters + +# Specify the max Heapsize for the NodeManager using a numerical value +# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set +# the value to 1000. +# This value will be overridden by an Xmx setting specified in either YARN_OPTS +# and/or YARN_NODEMANAGER_OPTS. +# If not specified, the default value will be picked from either YARN_HEAPMAX +# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. +export YARN_NODEMANAGER_HEAPSIZE=<%=scope.function_hdp_template_var("nodemanager_heapsize")%> + +# Specify the JVM options to be used when starting the NodeManager. +# These options will be appended to the options specified as YARN_OPTS +# and therefore may override any similar flags set in YARN_OPTS +#export YARN_NODEMANAGER_OPTS= + +# so that filenames w/ spaces are handled correctly in loops below +IFS= + + +# default log directory & file +if [ "$YARN_LOG_DIR" = "" ]; then + YARN_LOG_DIR="$HADOOP_YARN_HOME/logs" +fi +if [ "$YARN_LOGFILE" = "" ]; then + YARN_LOGFILE='yarn.log' +fi + +# default policy file for service-level authorization +if [ "$YARN_POLICYFILE" = "" ]; then + YARN_POLICYFILE="hadoop-policy.xml" +fi + +# restore ordinary behaviour +unset IFS + + +YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR" +YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR" +YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE" +YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE" +YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME" +YARN_OPTS="$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING" +YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}" +YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}" +if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then + YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" +fi +YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE" http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/32399749/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/YARN/configuration/global.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/YARN/configuration/global.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/YARN/configuration/global.xml index 7ddd6e3..edd1636 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/YARN/configuration/global.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/YARN/configuration/global.xml @@ -46,4 +46,19 @@ <value>yarn</value> <description>YARN User</description> </property> + <property> + <name>yarn_heapsize</name> + <value>1024</value> + <description>Max heapsize for all YARN components using a numerical value in the scale of MB</description> + </property> + <property> + <name>resourcemanager_heapsize</name> + <value>1024</value> + <description>Max heapsize for ResourceManager using a numerical value in the scale of MB</description> + </property> + <property> + <name>nodemanager_heapsize</name> + <value>1024</value> + <description>Max heapsize for NodeManager using a numerical value in the scale of MB</description> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/32399749/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/YARN/configuration/global.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/YARN/configuration/global.xml b/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/YARN/configuration/global.xml index 7ddd6e3..7d43a97 100644 --- a/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/YARN/configuration/global.xml +++ b/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/YARN/configuration/global.xml @@ -46,4 +46,19 @@ <value>yarn</value> <description>YARN User</description> </property> + <property> + <name>yarn_heapsize</name> + <value>1000</value> + <description>Max heapsize for all YARN components using a numerical value in the scale of MB</description> + </property> + <property> + <name>resourcemanager_heapsize</name> + <value>1000</value> + <description>Max heapsize for ResourceManager using a numerical value in the scale of MB</description> + </property> + <property> + <name>nodemanager_heapsize</name> + <value>1000</value> + <description>Max heapsize for NodeManager using a numerical value in the scale of MB</description> + </property> </configuration>
