This is an automated email from the ASF dual-hosted git repository.
wuzhiguo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/bigtop-manager.git
The following commit(s) were added to refs/heads/main by this push:
new 388c265 BIGTOP-4124: Add YARN component on Bigtop-3.3.0 stack (#16)
388c265 is described below
commit 388c265d558fcb3b7e25538332293aa88992fa11
Author: timyuer <[email protected]>
AuthorDate: Sat Jul 13 14:31:27 2024 +0800
BIGTOP-4124: Add YARN component on Bigtop-3.3.0 stack (#16)
---
.../services/hdfs/configuration/core-site.xml | 6 +-
.../stacks/bigtop/3.3.0/services/hdfs/metainfo.xml | 2 +-
.../bigtop/3.3.0/services/kafka/metainfo.xml | 2 +-
.../services/yarn/configuration/exclude-nodes.xml | 54 +++
.../3.3.0/services/yarn/configuration/yarn-env.xml | 215 ++++++++++
.../services/yarn/configuration/yarn-log4j.xml | 101 +++++
.../services/yarn/configuration/yarn-site.xml | 476 +++++++++++++++++++++
.../services/yarn/configuration/yarn.conf.xml | 51 +++
.../3.3.0/services/{hdfs => yarn}/metainfo.xml | 56 +--
.../stacks/bigtop/3.3.0/services/yarn/order.json | 20 +
.../bigtop/3.3.0/services/zookeeper/metainfo.xml | 2 +-
.../stack/bigtop/v3_3_0/hdfs/HdfsParams.java | 33 +-
.../bigtop/v3_3_0/yarn/NodeManagerScript.java | 76 ++++
.../bigtop/v3_3_0/yarn/ResourceManagerScript.java | 76 ++++
.../stack/bigtop/v3_3_0/yarn/YarnClientScript.java | 43 ++
.../stack/bigtop/v3_3_0/yarn/YarnParams.java | 141 ++++++
.../stack/bigtop/v3_3_0/yarn/YarnSetup.java | 119 ++++++
.../stack/common/utils/template/BaseTemplate.java | 3 +-
18 files changed, 1426 insertions(+), 50 deletions(-)
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/core-site.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/core-site.xml
index 23ce6ba..1d072df 100644
---
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/core-site.xml
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/core-site.xml
@@ -167,11 +167,11 @@
</property>
<property>
<name>fs.azure.user.agent.prefix</name>
- <value>User-Agent: APN/1.0 Hortonworks/1.0 HDP/{{version}}</value>
+ <value/>
</property>
<property>
<name>fs.gs.application.name.suffix</name>
- <value>(GPN:Hortonworks; version 1.0) HDP/{{version}}</value>
+ <value/>
</property>
<property>
<name>fs.gs.path.encoding</name>
@@ -184,6 +184,6 @@
</property>
<property>
<name>fs.s3a.user.agent.prefix</name>
- <value>User-Agent: APN/1.0 Hortonworks/1.0 HDP/{{version}}</value>
+ <value/>
</property>
</configuration>
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml
index 070040f..2d4f9f0 100644
---
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml
@@ -24,7 +24,7 @@
<name>hdfs</name>
<display-name>HDFS</display-name>
<desc>Apache Hadoop Distributed File System.</desc>
- <version>3.3.5</version>
+ <version>3.3.6-1</version>
<user>hdfs</user>
<group>hdfs</group>
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml
index febc9b3..5e379f3 100644
---
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml
@@ -28,7 +28,7 @@
high-performance data pipelines, streaming analytics, data
integration, and mission-critical
applications.
</desc>
- <version>2.8.2</version>
+ <version>2.8.2-1</version>
<user>kafka</user>
<group>kafka</group>
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/exclude-nodes.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/exclude-nodes.xml
new file mode 100644
index 0000000..3f63b79
--- /dev/null
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/exclude-nodes.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+-->
+
+<configuration>
+ <property>
+ <name>content</name>
+ <display-name>hdfs slaves template</display-name>
+ <description>This is the freemarker template for hdfs
file</description>
+ <value><![CDATA[
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+<#if exclude_hosts?? >
+<#list exclude_hosts as host>
+${host}
+</#list>
+</#if>
+
+]]>
+ </value>
+ </property>
+</configuration>
\ No newline at end of file
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-env.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-env.xml
new file mode 100644
index 0000000..ccf4fba
--- /dev/null
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-env.xml
@@ -0,0 +1,215 @@
+<?xml version="1.0"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+-->
+<configuration>
+ <property>
+ <name>yarn_log_dir_prefix</name>
+ <value>/var/log/hadoop-yarn</value>
+ <display-name>YARN Log Dir Prefix</display-name>
+ <description>YARN Log Dir Prefix</description>
+ </property>
+ <property>
+ <name>yarn_pid_dir_prefix</name>
+ <value>/var/run/hadoop-yarn</value>
+ <display-name>YARN PID Dir Prefix</display-name>
+ <description>YARN PID Dir Prefix</description>
+ </property>
+ <property>
+ <name>yarn_heapsize</name>
+ <value>1024</value>
+ <display-name>YARN Java heap size</display-name>
+ <description>Max heapsize for all YARN components using a numerical
value in the scale of MB</description>
+ </property>
+ <property>
+ <name>resourcemanager_heapsize</name>
+ <value>1024</value>
+ <display-name>ResourceManager Java heap size</display-name>
+ <description>Max heapsize for ResourceManager using a numerical value
in the scale of MB</description>
+ </property>
+ <property>
+ <name>nodemanager_heapsize</name>
+ <value>1024</value>
+ <display-name>NodeManager Java heap size</display-name>
+ <description>Max heapsize for NodeManager using a numerical value in
the scale of MB</description>
+ </property>
+ <property>
+ <name>min_user_id</name>
+ <value>1000</value>
+ <display-name>Minimum user ID for submitting job</display-name>
+ <description>Set to 0 to disallow root from submitting jobs. Set to
1000 to disallow all superusers from submitting jobs</description>
+ </property>
+ <property>
+ <name>is_supported_yarn_ranger</name>
+ <value>false</value>
+ <description>Set to false by default, needs to be set to true in
stacks that use Ranger Yarn Plugin</description>
+ </property>
+ <property>
+ <name>yarn_user_nofile_limit</name>
+ <value>32768</value>
+ <description>Max open files limit setting for YARN user.</description>
+ </property>
+ <property>
+ <name>yarn_user_nproc_limit</name>
+ <value>65536</value>
+ <description>Max number of processes limit setting for YARN
user.</description>
+ </property>
+ <property>
+ <name>hadoop_java_io_tmpdir</name>
+ <value>/tmp/hadoop-yarn</value>
+ <description>Max number of processes limit setting for YARN
user.</description>
+ </property>
+ <!-- yarn-env.sh -->
+ <property>
+ <name>content</name>
+ <display-name>yarn-env template</display-name>
+ <description>This is the freemarker template for yarn-env.sh
file</description>
+ <value><![CDATA[
+export HADOOP_HOME=${hadoop_home}
+export HADOOP_YARN_HOME=${hadoop_yarn_home}
+export HADOOP_MAPRED_HOME=${hadoop_mapred_home}
+USER="$(whoami)"
+export HADOOP_LOG_DIR=${yarn_log_dir_prefix}/$USER
+export HADOOP_PID_DIR=${yarn_pid_dir_prefix}/$USER
+export HADOOP_LIBEXEC_DIR=${hadoop_libexec_dir}
+export JAVA_HOME=${java_home}
+export
JAVA_LIBRARY_PATH="<#noparse>${JAVA_LIBRARY_PATH}</#noparse>:${hadoop_java_io_tmpdir}"
+<#noparse>
+# User for YARN daemons
+export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
+
+
+# resolve links - $0 may be a softlink
+export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_YARN_HOME/etc/hadoop}"
+
+# some Java parameters
+# export JAVA_HOME=/home/y/libexec/jdk1.6.0/
+if [ "$JAVA_HOME" != "" ]; then
+#echo "run java in $JAVA_HOME"
+JAVA_HOME=$JAVA_HOME
+fi
+
+if [ "$JAVA_HOME" = "" ]; then
+echo "Error: JAVA_HOME is not set."
+exit 1
+fi
+
+JAVA=$JAVA_HOME/bin/java
+JAVA_HEAP_MAX=-Xmx1000m
+</#noparse>
+# For setting YARN specific HEAP sizes please use this
+# Parameter and set appropriately
+YARN_HEAPSIZE=${yarn_heapsize}
+
+<#noparse>
+# check envvars which might override default args
+if [ "$YARN_HEAPSIZE" != "" ]; then
+JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m"
+fi
+</#noparse>
+
+# Resource Manager specific parameters
+
+# Specify the max Heapsize for the ResourceManager using a numerical value
+# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+# the value to 1000.
+# This value will be overridden by an Xmx setting specified in either
HADOOP_OPTS
+# and/or YARN_RESOURCEMANAGER_OPTS.
+# If not specified, the default value will be picked from either YARN_HEAPMAX
+# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+export YARN_RESOURCEMANAGER_HEAPSIZE=${resourcemanager_heapsize}
+
+# Specify the JVM options to be used when starting the ResourceManager.
+# These options will be appended to the options specified as HADOOP_OPTS
+# and therefore may override any similar flags set in HADOOP_OPTS
+#export YARN_RESOURCEMANAGER_OPTS=
+
+# Node Manager specific parameters
+
+# Specify the max Heapsize for the NodeManager using a numerical value
+# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+# the value to 1000.
+# This value will be overridden by an Xmx setting specified in either
HADOOP_OPTS
+# and/or YARN_NODEMANAGER_OPTS.
+# If not specified, the default value will be picked from either YARN_HEAPMAX
+# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+export YARN_NODEMANAGER_HEAPSIZE=${nodemanager_heapsize}
+
+# Specify the max Heapsize for the HistoryManager using a numerical value
+# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+# the value to 1024.
+# This value will be overridden by an Xmx setting specified in either
HADOOP_OPTS
+# and/or YARN_HISTORYSERVER_OPTS.
+# If not specified, the default value will be picked from either YARN_HEAPMAX
+# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+export YARN_HISTORYSERVER_HEAPSIZE=${apptimelineserver_heapsize}
+
+# Specify the JVM options to be used when starting the NodeManager.
+# These options will be appended to the options specified as HADOOP_OPTS
+# and therefore may override any similar flags set in HADOOP_OPTS
+#export YARN_NODEMANAGER_OPTS=
+
+# so that filenames w/ spaces are handled correctly in loops below
+IFS=
+
+<#noparse>
+# default log directory and file
+if [ "$HADOOP_LOG_DIR" = "" ]; then
+HADOOP_LOG_DIR="$HADOOP_YARN_HOME/logs"
+fi
+if [ "$HADOOP_LOGFILE" = "" ]; then
+HADOOP_LOGFILE='yarn.log'
+fi
+
+# default policy file for service-level authorization
+if [ "$YARN_POLICYFILE" = "" ]; then
+YARN_POLICYFILE="hadoop-policy.xml"
+fi
+
+# restore ordinary behaviour
+unset IFS
+
+
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR"
+HADOOP_OPTS="$HADOOP_OPTS -Dyarn.log.dir=$HADOOP_LOG_DIR"
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_LOGFILE"
+HADOOP_OPTS="$HADOOP_OPTS -Dyarn.log.file=$HADOOP_LOGFILE"
+HADOOP_OPTS="$HADOOP_OPTS -Dyarn.home.dir=$HADOOP_YARN_HOME"
+HADOOP_OPTS="$HADOOP_OPTS -Dyarn.id.str=$HADOOP_IDENT_STRING"
+HADOOP_OPTS="$HADOOP_OPTS
-Dhadoop.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}"
+HADOOP_OPTS="$HADOOP_OPTS
-Dyarn.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}"
+export YARN_NODEMANAGER_OPTS="$YARN_NODEMANAGER_OPTS
-Dnm.audit.logger=INFO,NMAUDIT"
+export YARN_RESOURCEMANAGER_OPTS="$YARN_RESOURCEMANAGER_OPTS
-Drm.audit.logger=INFO,RMAUDIT"
+if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
+HADOOP_OPTS="$HADOOP_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
+fi
+HADOOP_OPTS="$HADOOP_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"
+</#noparse>
+HADOOP_OPTS="$HADOOP_OPTS -Djava.io.tmpdir=${hadoop_java_io_tmpdir}"
+HADOOP_OPTS="$HADOOP_OPTS --add-opens java.base/java.lang=ALL-UNNAMED"
+]]>
+ </value>
+ </property>
+ <property>
+ <name>service_check.queue.name</name>
+ <value>default</value>
+ <description>
+ The queue that used by service check.
+ </description>
+ </property>
+</configuration>
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-log4j.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-log4j.xml
new file mode 100644
index 0000000..55f1f2c
--- /dev/null
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-log4j.xml
@@ -0,0 +1,101 @@
+<?xml version="1.0"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+-->
+<configuration>
+ <property>
+ <name>yarn_rm_summary_log_max_backup_size</name>
+ <value>256</value>
+ <description>The maximum size of backup file before the log is
rotated</description>
+ <display-name>YARN Log: backup file size</display-name>
+ </property>
+ <property>
+ <name>yarn_rm_summary_log_number_of_backup_files</name>
+ <value>20</value>
+ <description>The number of backup files</description>
+ <display-name>YARN Log: # of backup files</display-name>
+ </property>
+ <property>
+ <name>content</name>
+ <display-name>yarn-log4j template</display-name>
+ <description>Custom log4j.properties</description>
+ <value><![CDATA[
+#Relative to Yarn Log Dir Prefix
+yarn.log.dir=.
+#
+# Job Summary Appender
+#
+# Use following logger to send summary to separate file defined by
+# hadoop.mapreduce.jobsummary.log.file rolled daily:
+# hadoop.mapreduce.jobsummary.logger=INFO,JSA
+#
+<#noparse>hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger}</#noparse>
+hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log
+log4j.appender.JSA=org.apache.log4j.DailyRollingFileAppender
+# Set the ResourceManager summary log filename
+yarn.server.resourcemanager.appsummary.log.file=hadoop-mapreduce.jobsummary.log
+# Set the ResourceManager summary log level and appender
+<#noparse>yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger}</#noparse>
+#yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY
+
+# To enable AppSummaryLogging for the RM,
+# set yarn.server.resourcemanager.appsummary.logger to
+# LEVEL,RMSUMMARY in hadoop-env.sh
+
+# Appender for ResourceManager Application Summary Log
+# Requires the following properties to be set
+# - hadoop.log.dir (Hadoop Log directory)
+# - yarn.server.resourcemanager.appsummary.log.file (resource manager app
summary log filename)
+# - yarn.server.resourcemanager.appsummary.logger (resource manager app
summary log level and appender)
+log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender
+<#noparse>log4j.appender.RMSUMMARY.File=${yarn.log.dir}/${yarn.server.resourcemanager.appsummary.log.file}</#noparse>
+log4j.appender.RMSUMMARY.MaxFileSize=${yarn_rm_summary_log_max_backup_size}MB
+log4j.appender.RMSUMMARY.MaxBackupIndex=${yarn_rm_summary_log_number_of_backup_files}
+log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout
+log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
+log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
+log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}:
%m%n
+log4j.appender.JSA.DatePattern=.yyyy-MM-dd
+log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
+<#noparse>log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger}</#noparse>
+<#noparse>log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false</#noparse>
+
+# Audit logging for ResourceManager
+<#noparse>rm.audit.logger=${hadoop.root.logger}</#noparse>
+<#noparse>log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger=${rm.audit.logger}</#noparse>
+log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger=false
+log4j.appender.RMAUDIT=org.apache.log4j.DailyRollingFileAppender
+<#noparse>log4j.appender.RMAUDIT.File=${yarn.log.dir}/rm-audit.log</#noparse>
+log4j.appender.RMAUDIT.layout=org.apache.log4j.PatternLayout
+log4j.appender.RMAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
+log4j.appender.RMAUDIT.DatePattern=.yyyy-MM-dd
+
+# Audit logging for NodeManager
+<#noparse>nm.audit.logger=${hadoop.root.logger}</#noparse>
+<#noparse>log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger=${nm.audit.logger}</#noparse>
+log4j.additivity.org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger=false
+log4j.appender.NMAUDIT=org.apache.log4j.DailyRollingFileAppender
+<#noparse>log4j.appender.NMAUDIT.File=${yarn.log.dir}/nm-audit.log</#noparse>
+log4j.appender.NMAUDIT.layout=org.apache.log4j.PatternLayout
+log4j.appender.NMAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
+log4j.appender.NMAUDIT.DatePattern=.yyyy-MM-dd
+ ]]>
+ </value>
+ </property>
+</configuration>
+
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml
new file mode 100644
index 0000000..f7d5dd3
--- /dev/null
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml
@@ -0,0 +1,476 @@
+<?xml version="1.0"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+-->
+<configuration>
+ <!-- ResourceManager -->
+ <property>
+ <name>yarn.resourcemanager.hostname</name>
+ <value>localhost</value>
+ <description>The hostname of the RM.</description>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.resource-tracker.address</name>
+ <value>localhost:8025</value>
+ <description> The address of ResourceManager. </description>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.scheduler.address</name>
+ <value>localhost:8030</value>
+ <description>The address of the scheduler interface.</description>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value>localhost:8050</value>
+ <description>
+ The address of the applications manager interface in the
+ RM.
+ </description>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.admin.address</name>
+ <value>localhost:8141</value>
+ <description>The address of the RM admin interface.</description>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.scheduler.class</name>
+
<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
+ <description>The class to use as the resource scheduler.</description>
+ </property>
+ <property>
+ <name>yarn.scheduler.minimum-allocation-mb</name>
+ <value>512</value>
+ <description>
+ The minimum allocation for every container request at the RM,
+ in MBs. Memory requests lower than this won't take effect,
+ and the specified value will get allocated at minimum.
+ </description>
+ <display-name>Minimum Container Size (Memory)</display-name>
+ </property>
+ <property>
+ <name>yarn.scheduler.maximum-allocation-mb</name>
+ <value>5120</value>
+ <description>
+ The maximum allocation for every container request at the RM,
+ in MBs. Memory requests higher than this won't take effect,
+ and will get capped to this value.
+ </description>
+ <display-name>Maximum Container Size (Memory)</display-name>
+ </property>
+ <property>
+ <name>yarn.acl.enable</name>
+ <value>false</value>
+ <description> Are acls enabled. </description>
+ </property>
+ <property>
+ <name>yarn.admin.acl</name>
+ <value />
+ <description> ACL of who can be admin of the YARN cluster.
</description>
+ </property>
+ <!-- NodeManager -->
+ <property>
+ <name>yarn.nodemanager.address</name>
+ <value>0.0.0.0:45454</value>
+ <description>The address of the container manager in the
NM.</description>
+ </property>
+ <property>
+ <name>yarn.nodemanager.resource.memory-mb</name>
+ <value>5120</value>
+ <description>Amount of physical memory, in MB, that can be allocated
+ for containers.</description>
+ <display-name>Memory allocated for all YARN containers on a
node</display-name>
+ </property>
+ <property>
+ <name>yarn.application.classpath</name>
+ <value>
+
{{hadoop_conf_dir}},{{hadoop_home}}/*,{{hadoop_home}}/lib/*,{{hadoop_hdfs_home}}/*,{{hadoop_hdfs_home}}/lib/*,{{hadoop_yarn_home}}/*,{{hadoop_yarn_home}}/lib/*</value>
+ <description>Classpath for typical applications.</description>
+ </property>
+ <property>
+ <name>yarn.nodemanager.vmem-pmem-ratio</name>
+ <value>2.1</value>
+ <description>Ratio between virtual memory to physical memory when
+ setting memory limits for containers. Container allocations are
+ expressed in terms of physical memory, and virtual memory usage
+ is allowed to exceed this allocation by this ratio.
+ </description>
+ <display-name>Virtual Memory Ratio</display-name>
+ </property>
+ <property>
+ <name>yarn.nodemanager.container-executor.class</name>
+
<value>org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor</value>
+ <description>ContainerExecutor for launching containers</description>
+ </property>
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.group</name>
+ <value>hadoop</value>
+ <description>Unix group of the NodeManager</description>
+ </property>
+ <property>
+ <name>yarn.nodemanager.aux-services</name>
+ <value>mapreduce_shuffle</value>
+ <description>Auxilliary services of NodeManager. A valid service name
should only contain
+ a-zA-Z0-9_ and can
+ not start with numbers</description>
+ </property>
+ <property>
+ <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
+ <value>org.apache.hadoop.mapred.ShuffleHandler</value>
+ <description>The auxiliary service class to use </description>
+ </property>
+ <property>
+ <name>yarn.nodemanager.log-dirs</name>
+ <display-name>YARN NodeManager Log directories</display-name>
+ <value>/hadoop/yarn/log</value>
+ <description>
+ Where to store container logs. An application's localized log
directory
+ will be found in ${yarn.nodemanager.log-dirs}/application_${appid}.
+ Individual containers' log directories will be below this, in
directories
+ named container_{$contid}. Each container directory will contain
the files
+ stderr, stdin, and syslog generated by that container.
+ </description>
+ </property>
+ <property>
+ <name>yarn.nodemanager.local-dirs</name>
+ <display-name>YARN NodeManager Local directories</display-name>
+ <value>/hadoop/yarn/local</value>
+ <description>
+ List of directories to store localized files in. An
+ application's localized file directory will be found in:
+
${yarn.nodemanager.local-dirs}/usercache/${user}/appcache/application_${appid}.
+ Individual containers' work directories, called
container_${contid}, will
+ be subdirectories of this.
+ </description>
+ </property>
+ <property>
+ <name>yarn.nodemanager.container-monitor.interval-ms</name>
+ <value>3000</value>
+ <description>
+ The interval, in milliseconds, for which the node manager
+ waits between two cycles of monitoring its containers' memory
usage.
+ </description>
+ </property>
+ <!--
+ <property>
+ <name>yarn.nodemanager.health-checker.script.path</name>
+ <value>/etc/hadoop/conf/health_check_nodemanager</value>
+ <description>The health check script to run.</description>
+ </property>
+ -->
+ <property>
+ <name>yarn.nodemanager.health-checker.interval-ms</name>
+ <value>135000</value>
+ <description>Frequency of running node health script.</description>
+ </property>
+ <property>
+ <name>yarn.nodemanager.health-checker.script.timeout-ms</name>
+ <value>60000</value>
+ <description>Script time out period.</description>
+ </property>
+ <property>
+ <name>yarn.nodemanager.log.retain-seconds</name>
+ <value>604800</value>
+ <description>
+ Time in seconds to retain user logs. Only applicable if
+ log aggregation is disabled.
+ </description>
+ </property>
+ <property>
+ <name>yarn.log-aggregation-enable</name>
+ <value>true</value>
+ <description>Whether to enable log aggregation. </description>
+ <display-name>Enable Log Aggregation</display-name>
+ </property>
+ <property>
+ <name>yarn.nodemanager.remote-app-log-dir</name>
+ <display-name>YARN NodeManager Remote App Log directory</display-name>
+ <value>/app-logs</value>
+ <description>Location to aggregate logs to. </description>
+ </property>
+ <property>
+ <name>yarn.nodemanager.remote-app-log-dir-suffix</name>
+ <value>logs</value>
+ <description>
+ The remote log dir will be created at
+ {yarn.nodemanager.remote-app-log-dir}/${user}/{thisParam}.
+ </description>
+ </property>
+ <property>
+ <name>yarn.nodemanager.log-aggregation.compression-type</name>
+ <value>gz</value>
+ <description>
+ T-file compression types used to compress aggregated logs.
+ </description>
+ </property>
+ <property>
+ <name>yarn.nodemanager.delete.debug-delay-sec</name>
+ <value>0</value>
+ <description>
+ Number of seconds after an application finishes before the
nodemanager's
+ DeletionService will delete the application's localized file
directory
+ and log directory.
+
+ To diagnose Yarn application problems, set this property's value
large
+ enough (for example, to 600 = 10 minutes) to permit examination of
these
+ directories. After changing the property's value, you must restart
the
+ nodemanager in order for it to have an effect.
+
+ The roots of Yarn applications' work directories is configurable
with
+ the yarn.nodemanager.local-dirs property (see below), and the roots
+ of the Yarn applications' log directories is configurable with the
+ yarn.nodemanager.log-dirs property (see also below).
+ </description>
+ </property>
+ <property>
+ <name>yarn.log-aggregation.retain-seconds</name>
+ <value>2592000</value>
+ <description>
+ How long to keep aggregation logs before deleting them. -1
disables.
+ Be careful set this too small and you will spam the name node.
+ </description>
+ </property>
+ <property>
+ <name>yarn.nodemanager.admin-env</name>
+ <value>MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX</value>
+ <description>
+ Environment variables that should be forwarded from the
NodeManager's
+ environment to the container's.
+ </description>
+ </property>
+ <property>
+ <name>yarn.nodemanager.disk-health-checker.min-healthy-disks</name>
+ <value>0.25</value>
+ <description>
+ The minimum fraction of number of disks to be healthy for the
nodemanager
+ to launch new containers. This correspond to both
+ yarn-nodemanager.local-dirs and yarn.nodemanager.log-dirs. i.e.
+ If there are less number of healthy local-dirs (or log-dirs)
available,
+ then new containers will not be launched on this node.
+ </description>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.am.max-attempts</name>
+ <value>2</value>
+ <description>
+ The maximum number of application attempts. It's a global
+ setting for all application masters. Each application master can
specify
+ its individual maximum number of application attempts via the API,
but the
+ individual number cannot be more than the global upper bound. If
it is,
+ the resourcemanager will override it. The default number is set to
2, to
+ allow at least one retry for AM.
+ </description>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.webapp.address</name>
+ <value>localhost:8088</value>
+ <description>
+ The address of the RM web application.
+ </description>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.webapp.https.address</name>
+ <value>localhost:8090</value>
+ <description>
+ The https address of the RM web application.
+ </description>
+ </property>
+ <property>
+ <name>yarn.nodemanager.vmem-check-enabled</name>
+ <value>false</value>
+ <description>
+ Whether virtual memory limits will be enforced for containers.
+ </description>
+ </property>
+ <property>
+ <name>yarn.log.server.url</name>
+ <value>http://localhost:19888/jobhistory/logs</value>
+ <description>
+ URI for the HistoryServer's log resource
+ </description>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.nodes.exclude-path</name>
+ <value>/etc/hadoop/conf/yarn.exclude</value>
+ <description>
+ Names a file that contains a list of hosts that are
+ not permitted to connect to the resource manager. The full
pathname of the
+ file must be specified. If the value is empty, no hosts are
+ excluded.
+ </description>
+ </property>
+ <property>
+ <name>manage.include.files</name>
+ <value>false</value>
+ <description>If true Ambari will manage include file if
+ yarn.resourcemanager.nodes.include-path is
configured.</description>
+ </property>
+ <property>
+ <name>yarn.http.policy</name>
+ <value>HTTP_ONLY</value>
+ <description>
+ This configures the HTTP endpoint for Yarn Daemons.The following
values are supported: -
+ HTTP_ONLY : Service is provided only on http - HTTPS_ONLY :
Service is provided only on
+ https
+ </description>
+ </property>
+ <property>
+ <name>yarn.timeline-service.enabled</name>
+ <value>true</value>
+ <description>Indicate to clients whether timeline service is enabled
or not.
+ If enabled, clients will put entities and events to the timeline
server.
+ </description>
+ </property>
+ <property>
+
<name>yarn.timeline-service.generic-application-history.store-class</name>
+
<value>org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore</value>
+ <description>
+ Store class name for history store, defaulting to file system store
+ </description>
+ </property>
+ <property>
+ <name>yarn.timeline-service.leveldb-timeline-store.path</name>
+ <value>/var/log/hadoop-yarn/timeline</value>
+ <description>
+ Store file name for leveldb timeline store
+ </description>
+ </property>
+ <property>
+ <name>yarn.timeline-service.webapp.address</name>
+ <value>localhost:8188</value>
+ <description>
+ The http address of the timeline service web application.
+ </description>
+ </property>
+ <property>
+ <name>yarn.timeline-service.webapp.https.address</name>
+ <value>localhost:8190</value>
+ <description>
+ The http address of the timeline service web application.
+ </description>
+ </property>
+ <property>
+ <name>yarn.timeline-service.address</name>
+ <value>localhost:10200</value>
+ <description>
+ This is default address for the timeline server to start
+ the RPC server.
+ </description>
+ </property>
+ <property>
+ <description>Enable age off of timeline store data.</description>
+ <name>yarn.timeline-service.ttl-enable</name>
+ <value>true</value>
+ </property>
+ <property>
+ <description>Time to live for timeline store data in
milliseconds.</description>
+ <name>yarn.timeline-service.ttl-ms</name>
+ <value>2678400000</value>
+ </property>
+ <property>
+ <description>Length of time to wait between deletion cycles of leveldb
timeline store in
+ milliseconds.</description>
+
<name>yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms</name>
+ <value>300000</value>
+ </property>
+ <property>
+ <name>yarn.timeline-service.recovery.enabled</name>
+ <description>
+ Enable timeline server to recover state after starting. If
+ true, then yarn.timeline-service.state-store-class must be
specified.
+ </description>
+ <value>true</value>
+ </property>
+ <property>
+ <name>yarn.acl.enable</name>
+ <value>false</value>
+ <description> Are acls enabled. </description>
+ </property>
+ <property>
+ <name>yarn.authorization-provider</name>
+ <description> Yarn authorization provider class. </description>
+ </property>
+ <property>
+ <name>yarn.admin.acl</name>
+ <value>yarn</value>
+ <description> ACL of who can be admin of the YARN cluster.
</description>
+ </property>
+ <!--ats
+ v1.5 properties-->
+ <property>
+ <name>yarn.timeline-service.store-class</name>
+
<value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>
+ <description>Main storage class for YARN timeline server.</description>
+ </property>
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.active-dir</name>
+ <value>/ats/active/</value>
+ <description>DFS path to store active application’s timeline
data</description>
+ </property>
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.done-dir</name>
+ <value>/ats/done/</value>
+ <description>DFS path to store done application’s timeline
data</description>
+ </property>
+ <property>
+
<name>yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes</name>
+ <value />
+ <description>Plugins that can translate a timeline entity read request
into a list of
+ timeline cache ids, separated by commas. </description>
+ </property>
+ <!-- advanced ats v1.5 properties-->
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.summary-store</name>
+ <description>Summary storage for ATS v1.5</description>
+ <!-- Use rolling leveldb, advanced -->
+
<value>org.apache.hadoop.yarn.server.timeline.RollingLevelDBTimelineStore</value>
+ </property>
+ <property>
+
<name>yarn.timeline-service.entity-group-fs-store.scan-interval-seconds</name>
+ <description>
+ Scan interval for ATS v1.5 entity group file system storage
reader.This
+ value controls how frequent the reader will scan the HDFS active
directory
+ for application status.
+ </description>
+ <!-- Default is 60 seconds, advanced -->
+ <value>60</value>
+ </property>
+ <property>
+
<name>yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds</name>
+ <description>
+ Scan interval for ATS v1.5 entity group file system storage
cleaner.This
+ value controls how frequent the reader will scan the HDFS done
directory
+ for stale application data.
+ </description>
+ <!-- 3600 is default, advanced -->
+ <value>3600</value>
+ </property>
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.retain-seconds</name>
+ <description>
+ How long the ATS v1.5 entity group file system storage will keep an
+ application's data in the done directory.
+ </description>
+ <!-- 7 days is default, advanced -->
+ <value>604800</value>
+ </property>
+ <property>
+ <name>yarn.log.server.web-service.url</name>
+ <value>http://localhost:8188/ws/v1/applicationhistory</value>
+ <description>Log Server Web Service URL.</description>
+ </property>
+</configuration>
\ No newline at end of file
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn.conf.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn.conf.xml
new file mode 100644
index 0000000..84f2d3f
--- /dev/null
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn.conf.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+-->
+
+<configuration>
+ <property>
+ <name>content</name>
+ <display-name>yarn.conf template</display-name>
+ <description>This is the freemarker template for yarn
file</description>
+ <value><![CDATA[
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+${yarn_user} - nofile ${yarn_user_nofile_limit}
+${yarn_group} - nproc ${yarn_user_nproc_limit}
+
+]]>
+ </value>
+ </property>
+</configuration>
\ No newline at end of file
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml
similarity index 68%
copy from
bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml
copy to
bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml
index 070040f..d3fc224 100644
---
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml
@@ -21,37 +21,37 @@
<metainfo>
<schema-version>2.0</schema-version>
<service>
- <name>hdfs</name>
- <display-name>HDFS</display-name>
- <desc>Apache Hadoop Distributed File System.</desc>
- <version>3.3.5</version>
- <user>hdfs</user>
- <group>hdfs</group>
+ <name>yarn</name>
+ <display-name>YARN</display-name>
+ <desc>Apache Hadoop NextGen MapReduce (YARN)</desc>
+ <version>3.3.6-1</version>
+ <user>yarn</user>
+ <group>yarn</group>
<components>
<component>
- <name>namenode</name>
- <display-name>NameNode</display-name>
+ <name>resourcemanager</name>
+ <display-name>ResourceManager</display-name>
<category>master</category>
<cardinality>1-2</cardinality>
<command-script>
-
<script-id>org.apache.bigtop.manager.stack.bigtop.v3_3_0.hdfs.NameNodeScript</script-id>
+
<script-id>org.apache.bigtop.manager.stack.bigtop.v3_3_0.yarn.ResourceManagerScript</script-id>
<script-type>java</script-type>
<timeout>1200</timeout>
</command-script>
<custom-commands>
<custom-command>
- <name>rebalance_hdfs</name>
+ <name>decommission</name>
<command-script>
-
<script-id>org.apache.bigtop.manager.stack.bigtop.v3_3_0.hdfs.NameNodeScript</script-id>
+
<script-id>org.apache.bigtop.manager.stack.bigtop.v3_3_0.yarn.ResourceManagerScript</script-id>
<script-type>java</script-type>
<timeout>600</timeout>
</command-script>
</custom-command>
<custom-command>
- <name>print_topology</name>
+ <name>refreshqueues</name>
<command-script>
-
<script-id>org.apache.bigtop.manager.stack.bigtop.v3_3_0.hdfs.NameNodeScript</script-id>
+
<script-id>org.apache.bigtop.manager.stack.bigtop.v3_3_0.yarn.ResourceManagerScript</script-id>
<script-type>java</script-type>
<timeout>600</timeout>
</command-script>
@@ -59,34 +59,23 @@
</custom-commands>
</component>
<component>
- <name>datanode</name>
- <display-name>DataNode</display-name>
+ <name>nodemanager</name>
+ <display-name>NodeManager</display-name>
<category>slave</category>
<cardinality>1+</cardinality>
<command-script>
-
<script-id>org.apache.bigtop.manager.stack.bigtop.v3_3_0.hdfs.DataNodeScript</script-id>
+
<script-id>org.apache.bigtop.manager.stack.bigtop.v3_3_0.yarn.NodeManagerScript</script-id>
<script-type>java</script-type>
<timeout>1200</timeout>
</command-script>
</component>
<component>
- <name>secondary_namenode</name>
- <display-name>SNameNode</display-name>
- <category>master</category>
- <cardinality>1</cardinality>
- <command-script>
-
<script-id>org.apache.bigtop.manager.stack.bigtop.v3_3_0.hdfs.SNameNodeScript</script-id>
- <script-type>java</script-type>
- <timeout>1200</timeout>
- </command-script>
- </component>
- <component>
- <name>hdfs_client</name>
- <display-name>HDFS Client</display-name>
+ <name>yarn_client</name>
+ <display-name>YARN Client</display-name>
<category>client</category>
<cardinality>1+</cardinality>
<command-script>
-
<script-id>org.apache.bigtop.manager.stack.bigtop.v3_3_0.hdfs.HdfsClientScript</script-id>
+
<script-id>org.apache.bigtop.manager.stack.bigtop.v3_3_0.yarn.YarnClientScript</script-id>
<script-type>java</script-type>
<timeout>1200</timeout>
</command-script>
@@ -103,15 +92,16 @@
<arch>x86_64</arch>
</architectures>
<packages>
- <package>hadoop_3_3_0</package>
- <package>hadoop_3_3_0-client</package>
- <package>hadoop_3_3_0-libhdfs</package>
+ <package>hadoop_3_3_0-yarn</package>
+ <package>hadoop_3_3_0-hdfs</package>
+ <package>hadoop_3_3_0-mapreduce</package>
</packages>
</os-specific>
</os-specifics>
<required-services>
<service>zookeeper</service>
+ <service>hdfs</service>
</required-services>
</service>
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/order.json
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/order.json
new file mode 100644
index 0000000..9c9addb
--- /dev/null
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/order.json
@@ -0,0 +1,20 @@
+{
+ "RESOURCEMANAGER-START": [
+ "ZOOKEEPER_SERVER-START"
+ ],
+ "NODEMANAGER-START": [
+ "NAMENODE-START",
+ "DATANODE-START",
+ "RESOURCEMANAGER-START"
+ ],
+ "RESOURCEMANAGER-START": [
+ "NAMENODE-START",
+ "DATANODE-START"
+ ],
+ "RESOURCEMANAGER-RESTART": [
+ "NAMENODE-RESTART"
+ ],
+ "NODEMANAGER-RESTART": [
+ "NAMENODE-RESTART"
+ ]
+}
\ No newline at end of file
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml
index 8f53e43..64f10b1 100644
---
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml
@@ -27,7 +27,7 @@
Apache ZooKeeper is an effort to develop and maintain an
open-source server which enables highly
reliable distributed coordination.
</desc>
- <version>3.6.4</version>
+ <version>3.7.2-1</version>
<user>zookeeper</user>
<group>zookeeper</group>
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsParams.java
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsParams.java
index bc782f7..114e17d 100644
---
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsParams.java
+++
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsParams.java
@@ -62,10 +62,6 @@ public class HdfsParams extends BaseParams {
globalParamsMap.put("hadoop_hdfs_home", hdfsHome());
globalParamsMap.put("hadoop_conf_dir", confDir());
globalParamsMap.put("hadoop_libexec_dir", serviceHome() + "/libexec");
- List<String> namenodeList = LocalSettings.hosts("namenode");
- if (!namenodeList.isEmpty()) {
- coreSite().put("fs.defaultFS",
MessageFormat.format("hdfs://{0}:8020", namenodeList.get(0)));
- }
}
public String hdfsLimits() {
@@ -85,7 +81,12 @@ public class HdfsParams extends BaseParams {
@GlobalParams
public Map<String, Object> coreSite() {
- return LocalSettings.configurations(serviceName(), "core-site");
+ Map<String, Object> coreSite =
LocalSettings.configurations(serviceName(), "core-site");
+ List<String> namenodeList = LocalSettings.hosts("namenode");
+ if (!namenodeList.isEmpty()) {
+ coreSite.put("fs.defaultFS",
MessageFormat.format("hdfs://{0}:8020", namenodeList.get(0)));
+ }
+ return coreSite;
}
@GlobalParams
@@ -96,6 +97,18 @@ public class HdfsParams extends BaseParams {
@GlobalParams
public Map<String, Object> hdfsSite() {
Map<String, Object> hdfsSite =
LocalSettings.configurations(serviceName(), "hdfs-site");
+ List<String> namenodeList = LocalSettings.hosts("namenode");
+ if (!namenodeList.isEmpty()) {
+ hdfsSite.put("dfs.namenode.rpc-address",
MessageFormat.format("{0}:8020", namenodeList.get(0)));
+ hdfsSite.put("dfs.namenode.http-address",
MessageFormat.format("{0}:50070", namenodeList.get(0)));
+ hdfsSite.put("dfs.namenode.https-address",
MessageFormat.format("{0}:50470", namenodeList.get(0)));
+ }
+ List<String> snamenodeList = LocalSettings.hosts("secondary_namenode");
+ if (!snamenodeList.isEmpty()) {
+ hdfsSite.put(
+ "dfs.namenode.secondary.http-address",
MessageFormat.format("{0}:50090", snamenodeList.get(0)));
+ }
+
dfsDataDir = (String) hdfsSite.get("dfs.datanode.data.dir");
dfsNameNodeDir = (String) hdfsSite.get("dfs.namenode.name.dir");
nameNodeFormattedDirs = Arrays.stream(dfsNameNodeDir.split(","))
@@ -116,11 +129,11 @@ public class HdfsParams extends BaseParams {
hadoopLogDir = (String) hadoopEnv.get("hadoop_log_dir_prefix");
hadoopPidDir = (String) hadoopEnv.get("hadoop_pid_dir_prefix");
- nameNodePidFile = MessageFormat.format("{0}/hadoop-{1}-namenode.pid",
hadoopPidDir, user());
- dataNodePidFile = MessageFormat.format("{0}/hadoop-{1}-datanode.pid",
hadoopPidDir, user());
- sNameNodePidFile =
MessageFormat.format("{0}/hadoop-{1}-secondarynamenode.pid", hadoopPidDir,
user());
- journalNodePidFile =
MessageFormat.format("{0}/hadoop-{1}-journalnode.pid", hadoopPidDir, user());
- zkfcPidFile = MessageFormat.format("{0}/hadoop-{1}-zkfc.pid",
hadoopPidDir, user());
+ nameNodePidFile =
MessageFormat.format("{0}/{1}/hadoop-{1}-namenode.pid", hadoopPidDir, user());
+ dataNodePidFile =
MessageFormat.format("{0}/{1}/hadoop-{1}-datanode.pid", hadoopPidDir, user());
+ sNameNodePidFile =
MessageFormat.format("{0}/{1}/hadoop-{1}-secondarynamenode.pid", hadoopPidDir,
user());
+ journalNodePidFile =
MessageFormat.format("{0}/{1}/hadoop-{1}-journalnode.pid", hadoopPidDir,
user());
+ zkfcPidFile = MessageFormat.format("{0}/{1}/hadoop-{1}-zkfc.pid",
hadoopPidDir, user());
return hadoopEnv;
}
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/NodeManagerScript.java
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/NodeManagerScript.java
new file mode 100644
index 0000000..3348c1b
--- /dev/null
+++
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/NodeManagerScript.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bigtop.manager.stack.bigtop.v3_3_0.yarn;
+
+import org.apache.bigtop.manager.common.shell.ShellResult;
+import org.apache.bigtop.manager.spi.stack.Params;
+import org.apache.bigtop.manager.spi.stack.Script;
+import org.apache.bigtop.manager.stack.common.exception.StackException;
+import org.apache.bigtop.manager.stack.common.utils.PackageUtils;
+import org.apache.bigtop.manager.stack.common.utils.linux.LinuxOSUtils;
+
+import com.google.auto.service.AutoService;
+import lombok.extern.slf4j.Slf4j;
+
+import java.text.MessageFormat;
+
+@Slf4j
+@AutoService(Script.class)
+public class NodeManagerScript implements Script {
+
+ @Override
+ public ShellResult install(Params params) {
+ return PackageUtils.install(params.getPackageList());
+ }
+
+ @Override
+ public ShellResult configure(Params params) {
+ return YarnSetup.config(params, "nodemanager");
+ }
+
+ @Override
+ public ShellResult start(Params params) {
+ configure(params);
+ YarnParams yarnParams = (YarnParams) params;
+
+ String cmd = MessageFormat.format("{0} --daemon start nodemanager",
yarnParams.yarnExec());
+ try {
+ return LinuxOSUtils.sudoExecCmd(cmd, yarnParams.user());
+ } catch (Exception e) {
+ throw new StackException(e);
+ }
+ }
+
+ @Override
+ public ShellResult stop(Params params) {
+ YarnParams yarnParams = (YarnParams) params;
+ String cmd = MessageFormat.format("{0} --daemon stop nodemanager",
yarnParams.yarnExec());
+ try {
+ return LinuxOSUtils.sudoExecCmd(cmd, yarnParams.user());
+ } catch (Exception e) {
+ throw new StackException(e);
+ }
+ }
+
+ @Override
+ public ShellResult status(Params params) {
+ YarnParams yarnParams = (YarnParams) params;
+ return LinuxOSUtils.checkProcess(yarnParams.getNodeManagerPidFile());
+ }
+}
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/ResourceManagerScript.java
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/ResourceManagerScript.java
new file mode 100644
index 0000000..36930af
--- /dev/null
+++
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/ResourceManagerScript.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bigtop.manager.stack.bigtop.v3_3_0.yarn;
+
+import org.apache.bigtop.manager.common.shell.ShellResult;
+import org.apache.bigtop.manager.spi.stack.Params;
+import org.apache.bigtop.manager.spi.stack.Script;
+import org.apache.bigtop.manager.stack.common.exception.StackException;
+import org.apache.bigtop.manager.stack.common.utils.PackageUtils;
+import org.apache.bigtop.manager.stack.common.utils.linux.LinuxOSUtils;
+
+import com.google.auto.service.AutoService;
+import lombok.extern.slf4j.Slf4j;
+
+import java.text.MessageFormat;
+
+@Slf4j
+@AutoService(Script.class)
+public class ResourceManagerScript implements Script {
+
+ @Override
+ public ShellResult install(Params params) {
+ return PackageUtils.install(params.getPackageList());
+ }
+
+ @Override
+ public ShellResult configure(Params params) {
+ return YarnSetup.config(params, "resourcemanager");
+ }
+
+ @Override
+ public ShellResult start(Params params) {
+ configure(params);
+ YarnParams yarnParams = (YarnParams) params;
+
+ String cmd = MessageFormat.format("{0} --daemon start
resourcemanager", yarnParams.yarnExec());
+ try {
+ return LinuxOSUtils.sudoExecCmd(cmd, yarnParams.user());
+ } catch (Exception e) {
+ throw new StackException(e);
+ }
+ }
+
+ @Override
+ public ShellResult stop(Params params) {
+ YarnParams yarnParams = (YarnParams) params;
+ String cmd = MessageFormat.format("{0} --daemon stop resourcemanager",
yarnParams.yarnExec());
+ try {
+ return LinuxOSUtils.sudoExecCmd(cmd, yarnParams.user());
+ } catch (Exception e) {
+ throw new StackException(e);
+ }
+ }
+
+ @Override
+ public ShellResult status(Params params) {
+ YarnParams yarnParams = (YarnParams) params;
+ return
LinuxOSUtils.checkProcess(yarnParams.getResourceManagerPidFile());
+ }
+}
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnClientScript.java
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnClientScript.java
new file mode 100644
index 0000000..31c3854
--- /dev/null
+++
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnClientScript.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bigtop.manager.stack.bigtop.v3_3_0.yarn;
+
+import org.apache.bigtop.manager.common.shell.ShellResult;
+import org.apache.bigtop.manager.spi.stack.ClientScript;
+import org.apache.bigtop.manager.spi.stack.Params;
+import org.apache.bigtop.manager.spi.stack.Script;
+import org.apache.bigtop.manager.stack.common.utils.PackageUtils;
+
+import com.google.auto.service.AutoService;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@AutoService(Script.class)
+public class YarnClientScript implements ClientScript {
+
+ @Override
+ public ShellResult install(Params params) {
+ return PackageUtils.install(params.getPackageList());
+ }
+
+ @Override
+ public ShellResult configure(Params params) {
+ return YarnSetup.config(params);
+ }
+}
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnParams.java
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnParams.java
new file mode 100644
index 0000000..cc9e0eb
--- /dev/null
+++
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnParams.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bigtop.manager.stack.bigtop.v3_3_0.yarn;
+
+import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload;
+import org.apache.bigtop.manager.stack.common.annotations.GlobalParams;
+import org.apache.bigtop.manager.stack.common.utils.BaseParams;
+import org.apache.bigtop.manager.stack.common.utils.LocalSettings;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
+import java.text.MessageFormat;
+import java.util.List;
+import java.util.Map;
+
+@Getter
+@Slf4j
+public class YarnParams extends BaseParams {
+
+ private String yarnLogDir = "/var/log/hadoop-yarn";
+ private String yarnPidDir = "/var/run/hadoop-yarn";
+ private String rmNodesExcludeDir = "/etc/hadoop/conf/yarn.exclude";
+ private String tmpDir = "/tmp/hadoop-yarn";
+ private String nodemanagerLogDir = "/hadoop/yarn/log";
+ private String nodemanagerLocalDir = "/hadoop/yarn/local";
+ /* pid file */
+ private String resourceManagerPidFile = yarnPidDir +
"/yarn/hadoop-yarn-resourcemanager.pid";
+ private String nodeManagerPidFile = yarnPidDir +
"/yarn/hadoop-yarn-nodemanager.pid";
+ /* pid file */
+ private List<String> excludeHosts = List.of();
+
+ public YarnParams(CommandPayload commandPayload) {
+ super(commandPayload);
+ globalParamsMap.put("yarn_user", user());
+ globalParamsMap.put("yarn_group", group());
+ globalParamsMap.put("java_home", "/usr/local/java");
+ globalParamsMap.put("hadoop_home", serviceHome());
+ globalParamsMap.put("hadoop_hdfs_home", hdfsHome());
+ globalParamsMap.put("hadoop_yarn_home", yarnHome());
+ globalParamsMap.put("hadoop_mapred_home", mapredHome());
+ globalParamsMap.put("hadoop_conf_dir", confDir());
+ globalParamsMap.put("hadoop_libexec_dir", serviceHome() + "/libexec");
+ globalParamsMap.put("exclude_hosts", excludeHosts);
+ }
+
+ public String yarnLimits() {
+ Map<String, Object> yarnConf =
LocalSettings.configurations(serviceName(), "yarn.conf");
+ return (String) yarnConf.get("content");
+ }
+
+ public String excludeNodesContent() {
+ Map<String, Object> excludeNodesContent =
LocalSettings.configurations(serviceName(), "exclude-nodes");
+ return (String) excludeNodesContent.get("content");
+ }
+
+ @GlobalParams
+ public Map<String, Object> yarnLog4j() {
+ return LocalSettings.configurations(serviceName(), "yarn-log4j");
+ }
+
+ @GlobalParams
+ public Map<String, Object> yarnSite() {
+ Map<String, Object> yarnSite =
LocalSettings.configurations(serviceName(), "yarn-site");
+ List<String> resourcemanagerList =
LocalSettings.hosts("resourcemanager");
+ if (!resourcemanagerList.isEmpty()) {
+ yarnSite.put("yarn.resourcemanager.hostname",
MessageFormat.format("{0}", resourcemanagerList.get(0)));
+ yarnSite.put(
+ "yarn.resourcemanager.resource-tracker.address",
+ MessageFormat.format("{0}:8025",
resourcemanagerList.get(0)));
+ yarnSite.put("yarn.resourcemanager.address",
MessageFormat.format("{0}:8050", resourcemanagerList.get(0)));
+ yarnSite.put(
+ "yarn.resourcemanager.admin.address",
MessageFormat.format("{0}:8141", resourcemanagerList.get(0)));
+ yarnSite.put(
+ "yarn.resourcemanager.webapp.address",
+ MessageFormat.format("{0}:8088",
resourcemanagerList.get(0)));
+ yarnSite.put(
+ "yarn.resourcemanager.webapp.https.address",
+ MessageFormat.format("{0}:8090",
resourcemanagerList.get(0)));
+ }
+
+ rmNodesExcludeDir = (String)
yarnSite.get("yarn.resourcemanager.nodes.exclude-path");
+ nodemanagerLogDir = (String) yarnSite.get("yarn.nodemanager.log-dirs");
+ nodemanagerLocalDir = (String)
yarnSite.get("yarn.nodemanager.local-dirs");
+ return yarnSite;
+ }
+
+ @GlobalParams
+ public Map<String, Object> yarnEnv() {
+ Map<String, Object> yarnEnv =
LocalSettings.configurations(serviceName(), "yarn-env");
+
+ tmpDir = (String) yarnEnv.get("hadoop_java_io_tmpdir");
+ yarnLogDir = (String) yarnEnv.get("yarn_log_dir_prefix");
+ yarnPidDir = (String) yarnEnv.get("yarn_pid_dir_prefix");
+ resourceManagerPidFile =
MessageFormat.format("{0}/{1}/hadoop-{1}-resourcemanager.pid", yarnPidDir,
user());
+ nodeManagerPidFile =
MessageFormat.format("{0}/{1}/hadoop-{1}-nodemanager.pid", yarnPidDir, user());
+ return yarnEnv;
+ }
+
+ @Override
+ public String confDir() {
+ return "/etc/hadoop/conf";
+ }
+
+ @Override
+ public String serviceHome() {
+ return stackLibDir() + "/hadoop";
+ }
+
+ public String hdfsHome() {
+ return stackLibDir() + "/hadoop-hdfs";
+ }
+
+ public String yarnExec() {
+ return stackBinDir() + "/yarn";
+ }
+
+ public String yarnHome() {
+ return stackLibDir() + "/hadoop-yarn";
+ }
+
+ public String mapredHome() {
+ return stackLibDir() + "/hadoop-mapreduce";
+ }
+}
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnSetup.java
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnSetup.java
new file mode 100644
index 0000000..236dda9
--- /dev/null
+++
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnSetup.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bigtop.manager.stack.bigtop.v3_3_0.yarn;
+
+import org.apache.bigtop.manager.common.constants.Constants;
+import org.apache.bigtop.manager.common.shell.ShellResult;
+import org.apache.bigtop.manager.spi.stack.Params;
+import org.apache.bigtop.manager.stack.common.enums.ConfigType;
+import org.apache.bigtop.manager.stack.common.utils.BaseParams;
+import org.apache.bigtop.manager.stack.common.utils.linux.LinuxFileUtils;
+
+import org.apache.commons.lang3.StringUtils;
+
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import java.text.MessageFormat;
+import java.util.Map;
+
+@Slf4j
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class YarnSetup {
+
+ public static ShellResult config(Params params) {
+ return config(params, null);
+ }
+
+ public static ShellResult config(Params params, String componentName) {
+ log.info("starting YARN config");
+ YarnParams yarnParams = (YarnParams) params;
+
+ String confDir = yarnParams.confDir();
+ String yarnUser = yarnParams.user();
+ String yarnGroup = yarnParams.group();
+ Map<String, Object> yarnEnv = yarnParams.yarnEnv();
+
+ if (StringUtils.isNotBlank(componentName)) {
+ switch (componentName) {
+ case "resourcemanager": {
+ LinuxFileUtils.toFileByTemplate(
+ yarnParams.excludeNodesContent(),
+ yarnParams.getRmNodesExcludeDir(),
+ yarnUser,
+ yarnGroup,
+ Constants.PERMISSION_644,
+ yarnParams.getGlobalParamsMap());
+ }
+ case "nodemanager": {
+ LinuxFileUtils.createDirectories(
+ yarnParams.getNodemanagerLogDir(), yarnUser,
yarnGroup, Constants.PERMISSION_755, true);
+ LinuxFileUtils.createDirectories(
+ yarnParams.getNodemanagerLocalDir(), yarnUser,
yarnGroup, Constants.PERMISSION_755, true);
+ }
+ }
+ }
+
+ // mkdir directories
+ LinuxFileUtils.createDirectories(
+ yarnParams.getYarnLogDir(), yarnUser, yarnGroup,
Constants.PERMISSION_755, true);
+ LinuxFileUtils.createDirectories(
+ yarnParams.getYarnPidDir(), yarnUser, yarnGroup,
Constants.PERMISSION_755, true);
+ LinuxFileUtils.createDirectories(yarnParams.getTmpDir(), yarnUser,
yarnGroup, Constants.PERMISSION_755, true);
+
+ // hdfs.limits
+ LinuxFileUtils.toFileByTemplate(
+ yarnParams.yarnLimits(),
+ MessageFormat.format("{0}/yarn.conf",
BaseParams.LIMITS_CONF_DIR),
+ Constants.ROOT_USER,
+ Constants.ROOT_USER,
+ Constants.PERMISSION_644,
+ yarnParams.getGlobalParamsMap());
+
+ // hadoop-env.sh
+ LinuxFileUtils.toFileByTemplate(
+ yarnEnv.get("content").toString(),
+ MessageFormat.format("{0}/yarn-env.sh", confDir),
+ yarnUser,
+ yarnGroup,
+ Constants.PERMISSION_644,
+ yarnParams.getGlobalParamsMap());
+
+ // hdfs-site.xml
+ LinuxFileUtils.toFile(
+ ConfigType.XML,
+ MessageFormat.format("{0}/yarn-site.xml", confDir),
+ yarnUser,
+ yarnGroup,
+ Constants.PERMISSION_644,
+ yarnParams.yarnSite());
+
+ // log4j
+ LinuxFileUtils.toFileByTemplate(
+ yarnParams.yarnLog4j().get("content").toString(),
+ MessageFormat.format("{0}/yarnservice-log4j.properties",
confDir),
+ yarnUser,
+ yarnGroup,
+ Constants.PERMISSION_644,
+ yarnParams.getGlobalParamsMap());
+
+ return ShellResult.success("YARN Configure success!");
+ }
+}
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-common/src/main/java/org/apache/bigtop/manager/stack/common/utils/template/BaseTemplate.java
b/bigtop-manager-stack/bigtop-manager-stack-common/src/main/java/org/apache/bigtop/manager/stack/common/utils/template/BaseTemplate.java
index 309ff8c..c97c3c0 100644
---
a/bigtop-manager-stack/bigtop-manager-stack-common/src/main/java/org/apache/bigtop/manager/stack/common/utils/template/BaseTemplate.java
+++
b/bigtop-manager-stack/bigtop-manager-stack-common/src/main/java/org/apache/bigtop/manager/stack/common/utils/template/BaseTemplate.java
@@ -42,13 +42,14 @@ public class BaseTemplate {
CONFIGURATION.setClassForTemplateLoading(BaseTemplate.class,
"/templates");
CONFIGURATION.setDefaultEncoding("UTF-8");
CONFIGURATION.setOutputFormat(UndefinedOutputFormat.INSTANCE);
-
CONFIGURATION.setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER);
CONFIGURATION.setLogTemplateExceptions(false);
CONFIGURATION.setWrapUncheckedExceptions(true);
CONFIGURATION.setFallbackOnNullLoopVariable(false);
CONFIGURATION.setSQLDateAndTimeTimeZone(TimeZone.getDefault());
/* fix number_format display error */
CONFIGURATION.setNumberFormat("0.##");
+ // Set the behavior when the attribute does not exist, IGNORE_HANDLER
will ignore the error
+
CONFIGURATION.setTemplateExceptionHandler(TemplateExceptionHandler.IGNORE_HANDLER);
}
public static void writeTemplate(String path, Object dataModel, String
type) {
