This is an automated email from the ASF dual-hosted git repository.
wuzhiguo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/bigtop-manager.git
The following commit(s) were added to refs/heads/main by this push:
new 2c90eaa BIGTOP-4130: Add MapReduce2 component on Bigtop-3.3.0 stack
(#45)
2c90eaa is described below
commit 2c90eaa9762caff0d8243048c02b8568f25d6808
Author: Zhiguo Wu <[email protected]>
AuthorDate: Fri Aug 16 10:32:40 2024 +0800
BIGTOP-4130: Add MapReduce2 component on Bigtop-3.3.0 stack (#45)
---
.../service/ComponentStatusServiceGrpcImpl.java | 1 +
.../main/resources/proto/component_status.proto | 11 +-
.../server/command/job/AbstractServiceJob.java | 11 +-
.../server/scheduler/ComponentStatusScheduler.java | 1 +
.../services/mapred/configuration/mapred-env.xml | 76 +++++
.../services/mapred/configuration/mapred-site.xml | 325 +++++++++++++++++++++
.../configuration/mapreduce.conf.xml} | 8 +-
.../3.3.0/services/{yarn => mapred}/metainfo.xml | 65 ++---
.../stacks/bigtop/3.3.0/services/mapred/order.json | 4 +
.../services/yarn/configuration/yarn-site.xml | 12 +-
.../services/yarn/configuration/yarn.conf.xml | 4 +-
.../stacks/bigtop/3.3.0/services/yarn/metainfo.xml | 2 +-
.../bigtop/v3_3_0/mapred/HistoryServerScript.java | 74 +++++
.../bigtop/v3_3_0/mapred/MapredClientScript.java | 40 +--
.../stack/bigtop/v3_3_0/mapred/MapredSetup.java | 94 ++++++
.../bigtop/v3_3_0/mapred/Mapreduce2Params.java | 99 +++++++
.../manager/stack/bigtop/v3_3_0/tez/TezSetup.java | 4 +-
.../stack/bigtop/v3_3_0/yarn/YarnSetup.java | 6 +-
18 files changed, 745 insertions(+), 92 deletions(-)
diff --git
a/bigtop-manager-agent/src/main/java/org/apache/bigtop/manager/agent/service/ComponentStatusServiceGrpcImpl.java
b/bigtop-manager-agent/src/main/java/org/apache/bigtop/manager/agent/service/ComponentStatusServiceGrpcImpl.java
index f2cba36..0f43995 100644
---
a/bigtop-manager-agent/src/main/java/org/apache/bigtop/manager/agent/service/ComponentStatusServiceGrpcImpl.java
+++
b/bigtop-manager-agent/src/main/java/org/apache/bigtop/manager/agent/service/ComponentStatusServiceGrpcImpl.java
@@ -45,6 +45,7 @@ public class ComponentStatusServiceGrpcImpl extends
ComponentStatusServiceGrpc.C
CommandPayload commandPayload = new CommandPayload();
commandPayload.setCommand(Command.STATUS);
commandPayload.setServiceName(request.getServiceName());
+ commandPayload.setServiceUser(request.getServiceUser());
commandPayload.setComponentName(request.getComponentName());
commandPayload.setCommandScript(JsonUtils.readFromString(request.getCommandScript(),
ScriptInfo.class));
commandPayload.setRoot(request.getRoot());
diff --git
a/bigtop-manager-grpc/src/main/resources/proto/component_status.proto
b/bigtop-manager-grpc/src/main/resources/proto/component_status.proto
index 06799da..7431b63 100644
--- a/bigtop-manager-grpc/src/main/resources/proto/component_status.proto
+++ b/bigtop-manager-grpc/src/main/resources/proto/component_status.proto
@@ -28,12 +28,13 @@ service ComponentStatusService {
message ComponentStatusRequest {
string service_name = 1;
- string component_name = 2;
- string command_script = 3;
+ string service_user = 2;
+ string component_name = 3;
+ string command_script = 4;
// TODO Unnecessary, should be removed in the future
- string root = 4;
- string stack_name = 5;
- string stack_version = 6;
+ string root = 5;
+ string stack_name = 6;
+ string stack_version = 7;
}
message ComponentStatusReply {
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java
index 4a51ba4..76bf6f5 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java
@@ -142,6 +142,11 @@ public abstract class AbstractServiceJob extends
AbstractJob {
return
componentDTO.getCategory().equalsIgnoreCase(ComponentCategories.SLAVE);
}
+ protected Boolean isClientComponent(String componentName) {
+ ComponentDTO componentDTO = StackUtils.getComponentDTO(stackName,
stackVersion, componentName);
+ return
componentDTO.getCategory().equalsIgnoreCase(ComponentCategories.CLIENT);
+ }
+
protected List<String> findHostnamesByComponentName(String componentName) {
List<HostComponentPO> hostComponentPOList =
hostComponentRepository.findAllByComponentPOClusterPOIdAndComponentPOComponentName(
@@ -205,7 +210,7 @@ public abstract class AbstractServiceJob extends
AbstractJob {
String componentName = split[0];
String serviceName = findServiceNameByComponentName(componentName);
- if (!(isMasterComponent(componentName) ||
isSlaveComponent(componentName))) {
+ if (isClientComponent(componentName)) {
continue;
}
@@ -227,7 +232,7 @@ public abstract class AbstractServiceJob extends
AbstractJob {
String componentName = split[0];
String serviceName = findServiceNameByComponentName(componentName);
- if (!(isMasterComponent(componentName) ||
isSlaveComponent(componentName))) {
+ if (isClientComponent(componentName)) {
continue;
}
@@ -249,7 +254,7 @@ public abstract class AbstractServiceJob extends
AbstractJob {
String componentName = split[0];
String serviceName = findServiceNameByComponentName(componentName);
- if (!isMasterComponent(componentName)) {
+ if (isClientComponent(componentName)) {
continue;
}
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/scheduler/ComponentStatusScheduler.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/scheduler/ComponentStatusScheduler.java
index 9ef5683..43882ab 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/scheduler/ComponentStatusScheduler.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/scheduler/ComponentStatusScheduler.java
@@ -66,6 +66,7 @@ public class ComponentStatusScheduler {
ComponentStatusRequest request =
ComponentStatusRequest.newBuilder()
.setServiceName(servicePO.getServiceName())
+ .setServiceUser(servicePO.getServiceUser())
.setComponentName(componentPO.getComponentName())
.setCommandScript(componentPO.getCommandScript())
.setRoot(clusterPO.getRoot())
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapred-env.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapred-env.xml
new file mode 100644
index 0000000..ad177ca
--- /dev/null
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapred-env.xml
@@ -0,0 +1,76 @@
+<?xml version="1.0"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+-->
+
+<configuration>
+ <property>
+ <name>mapred_log_dir_prefix</name>
+ <value>/var/log/hadoop-mapreduce</value>
+ <display-name>Mapreduce Log Dir Prefix</display-name>
+ <description>Mapreduce Log Dir Prefix</description>
+ </property>
+ <property>
+ <name>mapred_pid_dir_prefix</name>
+ <value>/var/run/hadoop-mapreduce</value>
+ <display-name>Mapreduce PID Dir Prefix</display-name>
+ <description>Mapreduce PID Dir Prefix</description>
+ </property>
+ <property>
+ <name>jobhistory_heapsize</name>
+ <display-name>History Server heap size</display-name>
+ <value>1024</value>
+ <description>Value for JobHistoryServer heap_size variable in
hadoop-env.sh</description>
+ </property>
+ <property>
+ <name>mapred_user_nofile_limit</name>
+ <value>32768</value>
+ <description>Max open files limit setting for MAPREDUCE
user.</description>
+ </property>
+ <property>
+ <name>mapred_user_nproc_limit</name>
+ <value>65536</value>
+ <description>Max number of processes limit setting for MAPREDUCE
user.</description>
+ </property>
+ <property>
+ <name>content</name>
+ <display-name>mapred-env template</display-name>
+ <description>This is the freemarker template for mapred-env.sh
file</description>
+ <value><![CDATA[
+export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=${jobhistory_heapsize!}
+
+export HADOOP_ROOT_LOGGER=INFO,RFA
+
+export HADOOP_HOME=${hadoop_home}
+export HADOOP_YARN_HOME=${hadoop_yarn_home}
+export HADOOP_MAPRED_HOME=${hadoop_mapred_home}
+
+# Could be enabled from deployment option if necessary
+export HADOOP_LOG_DIR=${mapred_log_dir_prefix!}/$USER
+export HADOOP_PID_DIR=${mapred_pid_dir_prefix!}/$USER
+export MAPRED_HISTORYSERVER_OPTS="$MAPRED_HISTORYSERVER_OPTS
-Dhadoop.log.dir=$HADOOP_LOG_DIR"
+export HADOOP_LIBEXEC_DIR=${hadoop_libexec_dir}
+export HADOOP_OPTS="$HADOOP_OPTS --add-opens java.base/java.lang=ALL-UNNAMED"
+export JAVA_HOME=${java_home}
+]]>
+ </value>
+ <attrs>
+ <type>longtext</type>
+ </attrs>
+ </property>
+</configuration>
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapred-site.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapred-site.xml
new file mode 100644
index 0000000..2cd6f6d
--- /dev/null
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapred-site.xml
@@ -0,0 +1,325 @@
+<?xml version="1.0"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+-->
+
+<configuration>
+ <!-- i/o properties -->
+ <property>
+ <name>mapreduce.task.io.sort.mb</name>
+ <value>512</value>
+ <display-name>Sort Allocation Memory</display-name>
+ <description>
+ The total amount of buffer memory to use while sorting files, in
megabytes.
+ By default, gives each merge stream 1MB, which should minimize
seeks.
+ </description>
+ </property>
+ <property>
+ <name>mapreduce.map.sort.spill.percent</name>
+ <value>0.7</value>
+ <description>
+ The soft limit in the serialization buffer. Once reached, a thread
will
+ begin to spill the contents to disk in the background. Note that
+ collection will not block if this threshold is exceeded while a
spill
+ is already in progress, so spills may be larger than this
threshold when
+ it is set to less than .5
+ </description>
+ </property>
+ <property>
+ <name>mapreduce.task.io.sort.factor</name>
+ <value>100</value>
+ <description>
+ The number of streams to merge at once while sorting files.
+ This determines the number of open file handles.
+ </description>
+ </property>
+ <!-- map/reduce properties -->
+ <property>
+ <name>mapreduce.cluster.administrators</name>
+ <value>hadoop</value>
+ <description>Administrators for MapReduce applications.</description>
+ </property>
+ <property>
+ <name>mapreduce.reduce.shuffle.parallelcopies</name>
+ <value>30</value>
+ <description>
+ The default number of parallel transfers run by reduce during the
copy(shuffle) phase.
+ </description>
+ </property>
+ <property>
+ <name>mapreduce.map.speculative</name>
+ <value>false</value>
+ <description>
+ If true, then multiple instances of some map tasks may be executed
in parallel.
+ </description>
+ </property>
+ <property>
+ <name>mapreduce.reduce.speculative</name>
+ <value>false</value>
+ <description>
+ If true, then multiple instances of some reduce tasks may be
+ executed in parallel.
+ </description>
+ </property>
+ <property>
+ <name>mapreduce.job.reduce.slowstart.completedmaps</name>
+ <value>0.05</value>
+ <description>
+ Fraction of the number of maps in the job which should be complete
before reduces are scheduled for the job.
+ </description>
+ </property>
+ <property>
+ <name>mapreduce.job.counters.max</name>
+ <value>130</value>
+ <description>Limit on the number of counters allowed per
job.</description>
+ </property>
+ <property>
+ <name>mapreduce.reduce.shuffle.merge.percent</name>
+ <value>0.66</value>
+ <description>
+ The usage threshold at which an in-memory merge will be
+ initiated, expressed as a percentage of the total memory allocated
to
+ storing in-memory map outputs, as defined by
+ mapreduce.reduce.shuffle.input.buffer.percent.
+ </description>
+ </property>
+ <property>
+ <name>mapreduce.reduce.shuffle.input.buffer.percent</name>
+ <value>0.7</value>
+ <description>
+ The percentage of memory to be allocated from the maximum heap
+ size to storing map outputs during the shuffle.
+ </description>
+ </property>
+ <property>
+ <name>mapreduce.output.fileoutputformat.compress.type</name>
+ <value>BLOCK</value>
+ <description>
+ If the job outputs are to compressed as SequenceFiles, how should
+ they be compressed? Should be one of NONE, RECORD or BLOCK.
+ </description>
+ </property>
+ <property>
+ <name>mapreduce.reduce.input.buffer.percent</name>
+ <value>0.0</value>
+ <description>
+ The percentage of memory- relative to the maximum heap size- to
+ retain map outputs during the reduce. When the shuffle is
concluded, any
+ remaining map outputs in memory must consume less than this
threshold before
+ the reduce can begin.
+ </description>
+ </property>
+ <!-- copied from kryptonite configuration -->
+ <property>
+ <name>mapreduce.map.output.compress</name>
+ <value>false</value>
+ <description>
+ Should the outputs of the maps be compressed before being sent
across the network. Uses SequenceFile compression.
+ </description>
+ </property>
+ <property>
+ <name>mapreduce.task.timeout</name>
+ <value>300000</value>
+ <description>
+ The number of milliseconds before a task will be
+ terminated if it neither reads an input, writes an output, nor
+ updates its status string.
+ </description>
+ </property>
+ <property>
+ <name>mapreduce.map.memory.mb</name>
+ <value>512</value>
+ <description>Virtual memory for single Map task</description>
+ <display-name>Map Memory</display-name>
+ </property>
+ <property>
+ <name>mapreduce.reduce.memory.mb</name>
+ <value>512</value>
+ <description>Virtual memory for single Reduce task</description>
+ <display-name>Reduce Memory</display-name>
+ </property>
+ <property>
+ <name>mapreduce.shuffle.port</name>
+ <value>13562</value>
+ <description>
+ Default port that the ShuffleHandler will run on.
+ ShuffleHandler is a service run at the NodeManager to facilitate
+ transfers of intermediate Map outputs to requesting Reducers.
+ </description>
+ </property>
+ <property>
+ <name>mapreduce.jobhistory.intermediate-done-dir</name>
+ <display-name>Mapreduce JobHistory Intermediate Done
directory</display-name>
+ <value>/apps/mapred/history/tmp</value>
+ <description>Directory where history files are written by MapReduce
jobs.</description>
+ </property>
+ <property>
+ <name>mapreduce.jobhistory.done-dir</name>
+ <display-name>Mapreduce JobHistory Done directory</display-name>
+ <value>/apps/mapred/history/done</value>
+ <description>Directory where history files are managed by the MR
JobHistory Server.</description>
+ </property>
+ <property>
+ <name>mapreduce.jobhistory.address</name>
+ <value>0.0.0.0:10020</value>
+ <description>Enter your JobHistoryServer hostname.</description>
+ </property>
+ <property>
+ <name>mapreduce.jobhistory.webapp.address</name>
+ <value>0.0.0.0:19888</value>
+ <description>Enter your JobHistoryServer hostname.</description>
+ </property>
+ <property>
+ <name>mapreduce.framework.name</name>
+ <value>yarn</value>
+ <description>
+ The runtime framework for executing MapReduce jobs. Can be one of
local, classic or yarn.
+ </description>
+ </property>
+ <property>
+ <name>yarn.app.mapreduce.am.staging-dir</name>
+ <display-name>YARN App Mapreduce AM Staging directory</display-name>
+ <value>/apps/mapred/staging</value>
+ <description>The staging dir used while submitting jobs.</description>
+ </property>
+ <property>
+ <name>yarn.app.mapreduce.am.resource.mb</name>
+ <display-name>AppMaster Memory</display-name>
+ <value>512</value>
+ <description>The amount of memory the MR AppMaster needs.</description>
+ </property>
+ <property>
+ <name>yarn.app.mapreduce.am.command-opts</name>
+ <value>-Xmx1024m</value>
+ <description>
+ Java opts for the MR App Master processes.
+ The following symbol, if present, will be interpolated: @taskid@
is replaced
+ by current TaskID. Any other occurrences of '@' will go unchanged.
+ For example, to enable verbose gc logging to a file named for the
taskid in
+ /tmp and to set the heap maximum to be a gigabyte, pass a 'value'
of:
+ -Xmx1024m -verbose:gc -Xloggc:/tmp/@[email protected]
+
+ Usage of -Djava.library.path can cause programs to no longer
function if
+ hadoop native libraries are used. These values should instead be
set as part
+ of LD_LIBRARY_PATH in the map / reduce JVM env using the
mapreduce.map.env and
+ mapreduce.reduce.env config settings.
+ </description>
+ <display-name>MR AppMaster Java Heap Size</display-name>
+ </property>
+ <property>
+ <name>yarn.app.mapreduce.am.admin-command-opts</name>
+ <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true
-Dhadoop.metrics.log.level=WARN</value>
+ <description>
+ Java opts for the MR App Master processes for admin purposes.
+ It will appears before the opts set by
yarn.app.mapreduce.am.command-opts and
+ thus its options can be overridden user.
+
+ Usage of -Djava.library.path can cause programs to no longer
function if
+ hadoop native libraries are used. These values should instead be
set as part
+ of LD_LIBRARY_PATH in the map / reduce JVM env using the
mapreduce.map.env and
+ mapreduce.reduce.env config settings.
+ </description>
+ <display-name>MR AppMaster Java Heap Size</display-name>
+ </property>
+ <property>
+ <name>yarn.app.mapreduce.am.log.level</name>
+ <value>INFO</value>
+ <description>MR App Master process log level.</description>
+ </property>
+ <property>
+ <name>mapreduce.admin.map.child.java.opts</name>
+ <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true
-Dhadoop.metrics.log.level=WARN</value>
+ <description>This property stores Java options for map
tasks.</description>
+ </property>
+ <property>
+ <name>mapreduce.admin.reduce.child.java.opts</name>
+ <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true
-Dhadoop.metrics.log.level=WARN</value>
+ <description>This property stores Java options for reduce
tasks.</description>
+ </property>
+ <property>
+ <name>mapreduce.application.classpath</name>
+
<value>${hadoop_conf_dir!},${hadoop_home!}/*,${hadoop_home!}/lib/*,${hadoop_hdfs_home!}/*,${hadoop_hdfs_home!}/lib/*,${hadoop_yarn_home!}/*,${hadoop_yarn_home!}/lib/*,${hadoop_mapred_home!}/*,${hadoop_mapred_home!}/lib/*</value>
+ <description>CLASSPATH for MR applications. A comma-separated list of
CLASSPATH entries.</description>
+ </property>
+ <property>
+ <name>mapreduce.am.max-attempts</name>
+ <value>2</value>
+ <description>
+ The maximum number of application attempts. It is a
+ application-specific setting. It should not be larger than the
global number
+ set by resourcemanager. Otherwise, it will be override. The
default number is
+ set to 2, to allow at least one retry for AM.
+ </description>
+ </property>
+ <property>
+ <name>mapreduce.map.java.opts</name>
+ <display-name>MR Map Java Heap Size</display-name>
+ <value>-Xmx1024m</value>
+ <description>Larger heap-size for child jvms of maps.</description>
+ </property>
+ <property>
+ <name>mapreduce.reduce.java.opts</name>
+ <display-name>MR Reduce Java Heap Size</display-name>
+ <value>-Xmx1024m</value>
+ <description>Larger heap-size for child jvms of reduces.</description>
+ </property>
+ <property>
+ <name>mapreduce.map.log.level</name>
+ <value>INFO</value>
+ <description>
+ The logging level for the map task. The allowed levels are:
+ OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.
+ </description>
+ </property>
+ <property>
+ <name>mapreduce.reduce.log.level</name>
+ <value>INFO</value>
+ <description>
+ The logging level for the reduce task. The allowed levels are:
+ OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.
+ </description>
+ </property>
+ <property>
+ <name>mapreduce.admin.user.env</name>
+ <value>LD_LIBRARY_PATH=${hadoop_home!}/lib/native</value>
+ <description>
+ Additional execution environment entries for map and reduce task
processes.
+ This is not an additive property. You must preserve the original
value if
+ you want your map and reduce tasks to have access to native
libraries (compression, etc)
+ </description>
+ </property>
+ <property>
+ <name>mapreduce.output.fileoutputformat.compress</name>
+ <value>false</value>
+ <description>Should the job outputs be compressed?</description>
+ </property>
+ <property>
+ <name>mapreduce.jobhistory.http.policy</name>
+ <value>HTTP_ONLY</value>
+ <description>
+ This configures the HTTP endpoint for JobHistoryServer web UI.
+ The following values are supported: - HTTP_ONLY : Service is
provided only
+ on http - HTTPS_ONLY : Service is provided only on https
+ </description>
+ </property>
+ <property>
+ <name>mapreduce.job.queuename</name>
+ <value>default</value>
+ <description>Queue to which a job is submitted.</description>
+ </property>
+</configuration>
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn.conf.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapreduce.conf.xml
similarity index 87%
copy from
bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn.conf.xml
copy to
bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapreduce.conf.xml
index 79438b7..5280789 100644
---
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn.conf.xml
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapreduce.conf.xml
@@ -21,8 +21,8 @@
<configuration>
<property>
<name>content</name>
- <display-name>yarn.conf template</display-name>
- <description>This is the freemarker template for yarn
file</description>
+ <display-name>mapreduce.conf template</display-name>
+ <description>This is the freemarker template for mapreduce
file</description>
<value><![CDATA[
#
# Licensed to the Apache Software Foundation (ASF) under one
@@ -42,8 +42,8 @@
# limitations under the License.
#
-${yarn_user} - nofile ${yarn_user_nofile_limit}
-${yarn_group} - nproc ${yarn_user_nproc_limit}
+${mapred_user} - nofile ${mapred_user_nofile_limit}
+${mapred_user} - nproc ${mapred_user_nproc_limit}
]]>
</value>
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/metainfo.xml
similarity index 50%
copy from
bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml
copy to
bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/metainfo.xml
index eeb5863..95d6894 100644
---
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/metainfo.xml
@@ -20,67 +20,38 @@
<metainfo>
<service>
- <name>yarn</name>
- <display-name>YARN</display-name>
- <desc>Apache Hadoop NextGen MapReduce (YARN)</desc>
+ <name>mapreduce2</name>
+ <display-name>MapReduce2</display-name>
+ <desc>Apache Hadoop NextGen MapReduce.</desc>
<version>3.3.6-1</version>
- <user>yarn</user>
+ <user>mapred</user>
<components>
<component>
- <name>resourcemanager</name>
- <display-name>ResourceManager</display-name>
+ <name>history_server</name>
+ <display-name>History Server</display-name>
<category>master</category>
- <cardinality>1-2</cardinality>
+ <cardinality>1</cardinality>
<command-script>
-
<script-id>org.apache.bigtop.manager.stack.bigtop.v3_3_0.yarn.ResourceManagerScript</script-id>
+
<script-id>org.apache.bigtop.manager.stack.bigtop.v3_3_0.mapred.HistoryServerScript</script-id>
<script-type>java</script-type>
<timeout>1200</timeout>
</command-script>
- <custom-commands>
- <custom-command>
- <name>decommission</name>
- <command-script>
-
<script-id>org.apache.bigtop.manager.stack.bigtop.v3_3_0.yarn.ResourceManagerScript</script-id>
- <script-type>java</script-type>
- <timeout>600</timeout>
- </command-script>
- </custom-command>
- <custom-command>
- <name>refreshqueues</name>
- <command-script>
-
<script-id>org.apache.bigtop.manager.stack.bigtop.v3_3_0.yarn.ResourceManagerScript</script-id>
- <script-type>java</script-type>
- <timeout>600</timeout>
- </command-script>
- </custom-command>
- </custom-commands>
<quick-link>
- <display-name>ResourceManager UI</display-name>
-
<http-port-property>yarn.resourcemanager.webapp.address</http-port-property>
- <http-port-default>8088</http-port-default>
-
<https-port-property>yarn.resourcemanager.webapp.https.address</https-port-property>
- <https-port-default>8090</https-port-default>
+ <display-name>JobHistory UI</display-name>
+
<http-port-property>mapreduce.jobhistory.webapp.address</http-port-property>
+ <http-port-default>19888</http-port-default>
+
<https-port-property>mapreduce.jobhistory.webapp.https.address</https-port-property>
+ <https-port-default>19890</https-port-default>
</quick-link>
</component>
<component>
- <name>nodemanager</name>
- <display-name>NodeManager</display-name>
- <category>slave</category>
- <cardinality>1+</cardinality>
- <command-script>
-
<script-id>org.apache.bigtop.manager.stack.bigtop.v3_3_0.yarn.NodeManagerScript</script-id>
- <script-type>java</script-type>
- <timeout>1200</timeout>
- </command-script>
- </component>
- <component>
- <name>yarn_client</name>
- <display-name>YARN Client</display-name>
+ <name>mapreduce2_client</name>
+ <display-name>MapReduce2 Client</display-name>
<category>client</category>
<cardinality>1+</cardinality>
<command-script>
-
<script-id>org.apache.bigtop.manager.stack.bigtop.v3_3_0.yarn.YarnClientScript</script-id>
+
<script-id>org.apache.bigtop.manager.stack.bigtop.v3_3_0.mapred.MapredClientScript</script-id>
<script-type>java</script-type>
<timeout>1200</timeout>
</command-script>
@@ -105,9 +76,7 @@
</os-specifics>
<required-services>
- <service>zookeeper</service>
- <service>hdfs</service>
+ <service>yarn</service>
</required-services>
-
</service>
</metainfo>
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/order.json
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/order.json
new file mode 100644
index 0000000..883fc62
--- /dev/null
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/order.json
@@ -0,0 +1,4 @@
+{
+ "HISTORY_SERVER-START": ["NAMENODE-START", "DATANODE-START"],
+ "HISTORY_SERVER-RESTART": ["NAMENODE-RESTART"]
+}
\ No newline at end of file
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml
index 15ba17e..653fd96 100644
---
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml
@@ -72,12 +72,12 @@
<property>
<name>yarn.acl.enable</name>
<value>false</value>
- <description> Are acls enabled. </description>
+ <description>Are acls enabled.</description>
</property>
<property>
<name>yarn.admin.acl</name>
<value />
- <description> ACL of who can be admin of the YARN cluster.
</description>
+ <description>ACL of who can be admin of the YARN cluster.</description>
</property>
<!-- NodeManager -->
<property>
@@ -99,7 +99,8 @@
<property>
<name>yarn.nodemanager.vmem-pmem-ratio</name>
<value>2.1</value>
- <description>Ratio between virtual memory to physical memory when
+ <description>
+ Ratio between virtual memory to physical memory when
setting memory limits for containers. Container allocations are
expressed in terms of physical memory, and virtual memory usage
is allowed to exceed this allocation by this ratio.
@@ -119,8 +120,9 @@
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
- <description>Auxilliary services of NodeManager. A valid service name
should only contain
- a-zA-Z0-9_ and can not start with numbers
+ <description>
+ Auxilliary services of NodeManager.
+ A valid service name should only contain a-zA-Z0-9_ and can not
start with numbers
</description>
</property>
<property>
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn.conf.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn.conf.xml
index 79438b7..facf11f 100644
---
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn.conf.xml
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn.conf.xml
@@ -42,8 +42,8 @@
# limitations under the License.
#
-${yarn_user} - nofile ${yarn_user_nofile_limit}
-${yarn_group} - nproc ${yarn_user_nproc_limit}
+${yarn_user} - nofile ${yarn_user_nofile_limit}
+${yarn_user} - nproc ${yarn_user_nproc_limit}
]]>
</value>
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml
index eeb5863..f66e0ae 100644
---
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml
@@ -22,7 +22,7 @@
<service>
<name>yarn</name>
<display-name>YARN</display-name>
- <desc>Apache Hadoop NextGen MapReduce (YARN)</desc>
+ <desc>Apache Hadoop NextGen YARN.</desc>
<version>3.3.6-1</version>
<user>yarn</user>
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/HistoryServerScript.java
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/HistoryServerScript.java
new file mode 100644
index 0000000..58e5b39
--- /dev/null
+++
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/HistoryServerScript.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bigtop.manager.stack.bigtop.v3_3_0.mapred;
+
+import org.apache.bigtop.manager.common.shell.ShellResult;
+import org.apache.bigtop.manager.stack.core.exception.StackException;
+import org.apache.bigtop.manager.stack.core.param.Params;
+import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript;
+import org.apache.bigtop.manager.stack.core.spi.script.Script;
+import org.apache.bigtop.manager.stack.core.utils.PackageUtils;
+import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils;
+
+import com.google.auto.service.AutoService;
+
+import java.text.MessageFormat;
+
+@AutoService(Script.class)
+public class HistoryServerScript extends AbstractServerScript {
+
+ @Override
+ public ShellResult install(Params params) {
+ return PackageUtils.install(params.getPackageList());
+ }
+
+ @Override
+ public ShellResult configure(Params params) {
+ return MapredSetup.config(params);
+ }
+
+ @Override
+ public ShellResult start(Params params) {
+ configure(params);
+ Mapreduce2Params mapreduce2Params = (Mapreduce2Params) params;
+ String cmd = MessageFormat.format("{0} --daemon start historyserver",
mapreduce2Params.mapredExec());
+ try {
+ return LinuxOSUtils.sudoExecCmd(cmd, mapreduce2Params.user());
+ } catch (Exception e) {
+ throw new StackException(e);
+ }
+ }
+
+ @Override
+ public ShellResult stop(Params params) {
+ Mapreduce2Params mapreduce2Params = (Mapreduce2Params) params;
+ String cmd = MessageFormat.format("{0} --daemon stop historyserver",
mapreduce2Params.mapredExec());
+ try {
+ return LinuxOSUtils.sudoExecCmd(cmd, mapreduce2Params.user());
+ } catch (Exception e) {
+ throw new StackException(e);
+ }
+ }
+
+ @Override
+ public ShellResult status(Params params) {
+ Mapreduce2Params mapreduce2Params = (Mapreduce2Params) params;
+ return
LinuxOSUtils.checkProcess(mapreduce2Params.getHistoryServerPidFile());
+ }
+}
diff --git
a/bigtop-manager-grpc/src/main/resources/proto/component_status.proto
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/MapredClientScript.java
similarity index 50%
copy from bigtop-manager-grpc/src/main/resources/proto/component_status.proto
copy to
bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/MapredClientScript.java
index 06799da..48429f5 100644
--- a/bigtop-manager-grpc/src/main/resources/proto/component_status.proto
+++
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/MapredClientScript.java
@@ -16,26 +16,28 @@
* specific language governing permissions and limitations
* under the License.
*/
-syntax = "proto3";
+package org.apache.bigtop.manager.stack.bigtop.v3_3_0.mapred;
-option java_multiple_files = true;
-option java_package = "org.apache.bigtop.manager.grpc.generated";
-option java_outer_classname = "ComponentStatusProto";
+import org.apache.bigtop.manager.common.shell.ShellResult;
+import org.apache.bigtop.manager.stack.core.param.Params;
+import org.apache.bigtop.manager.stack.core.spi.script.AbstractClientScript;
+import org.apache.bigtop.manager.stack.core.spi.script.Script;
+import org.apache.bigtop.manager.stack.core.utils.PackageUtils;
-service ComponentStatusService {
- rpc GetComponentStatus (ComponentStatusRequest) returns
(ComponentStatusReply) {}
-}
+import com.google.auto.service.AutoService;
+import lombok.extern.slf4j.Slf4j;
-message ComponentStatusRequest {
- string service_name = 1;
- string component_name = 2;
- string command_script = 3;
- // TODO Unnecessary, should be removed in the future
- string root = 4;
- string stack_name = 5;
- string stack_version = 6;
-}
+@Slf4j
+@AutoService(Script.class)
+public class MapredClientScript extends AbstractClientScript {
-message ComponentStatusReply {
- int32 status = 1;
-}
\ No newline at end of file
+ @Override
+ public ShellResult install(Params params) {
+ return PackageUtils.install(params.getPackageList());
+ }
+
+ @Override
+ public ShellResult configure(Params params) {
+ return MapredSetup.config(params);
+ }
+}
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/MapredSetup.java
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/MapredSetup.java
new file mode 100644
index 0000000..79ba53f
--- /dev/null
+++
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/MapredSetup.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bigtop.manager.stack.bigtop.v3_3_0.mapred;
+
+import org.apache.bigtop.manager.common.constants.Constants;
+import org.apache.bigtop.manager.common.shell.ShellResult;
+import org.apache.bigtop.manager.stack.bigtop.utils.HdfsUtil;
+import org.apache.bigtop.manager.stack.core.enums.ConfigType;
+import org.apache.bigtop.manager.stack.core.param.BaseParams;
+import org.apache.bigtop.manager.stack.core.param.Params;
+import org.apache.bigtop.manager.stack.core.utils.LocalSettings;
+import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils;
+
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import java.text.MessageFormat;
+
+@Slf4j
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class MapredSetup {
+
+ public static ShellResult config(Params params) {
+ Mapreduce2Params mapreduce2Params = (Mapreduce2Params) params;
+ String hdfsUser = LocalSettings.users().get("hdfs");
+ String mapredUser = params.user();
+ String mapredGroup = params.group();
+ String confDir = mapreduce2Params.confDir();
+
+ LinuxFileUtils.createDirectories(
+ mapreduce2Params.getMapredLogDir(), mapredUser, mapredGroup,
Constants.PERMISSION_755, true);
+ LinuxFileUtils.createDirectories(
+ mapreduce2Params.getMapredPidDir(), mapredUser, mapredGroup,
Constants.PERMISSION_755, true);
+
+ // mapreduce.conf
+ log.info("Generating [{}/mapreduce.conf] file",
BaseParams.LIMITS_CONF_DIR);
+ LinuxFileUtils.toFileByTemplate(
+ mapreduce2Params.mapredLimits(),
+ MessageFormat.format("{0}/mapreduce.conf",
BaseParams.LIMITS_CONF_DIR),
+ Constants.ROOT_USER,
+ Constants.ROOT_USER,
+ Constants.PERMISSION_644,
+ mapreduce2Params.getGlobalParamsMap());
+
+ // mapred-env.sh
+ log.info("Generating [{}/mapred-env.sh] file", confDir);
+ LinuxFileUtils.toFileByTemplate(
+ mapreduce2Params.mapredEnv().get("content").toString(),
+ MessageFormat.format("{0}/mapred-env.sh", confDir),
+ mapredUser,
+ mapredGroup,
+ Constants.PERMISSION_644,
+ mapreduce2Params.getGlobalParamsMap());
+
+ // mapred-site.xml
+ log.info("Generating [{}/mapred-site.xml] file", confDir);
+ LinuxFileUtils.toFile(
+ ConfigType.XML,
+ MessageFormat.format("{0}/mapred-site.xml", confDir),
+ mapredUser,
+ mapredGroup,
+ Constants.PERMISSION_644,
+ mapreduce2Params.mapredSite(),
+ mapreduce2Params.getGlobalParamsMap());
+
+ HdfsUtil.createDirectory(hdfsUser, "/apps");
+ HdfsUtil.createDirectory(hdfsUser, "/app-logs");
+ HdfsUtil.createDirectory(mapredUser, "/apps/mapred");
+ HdfsUtil.createDirectory(mapredUser, "/apps/mapred/staging");
+ HdfsUtil.createDirectory(mapredUser, "/apps/mapred/history");
+ HdfsUtil.createDirectory(mapredUser, "/apps/mapred/history/tmp");
+ HdfsUtil.createDirectory(mapredUser, "/apps/mapred/history/done");
+
+ log.info("Successfully configured MapReduce2");
+ return ShellResult.success("MapReduce2 Configure success!");
+ }
+}
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/Mapreduce2Params.java
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/Mapreduce2Params.java
new file mode 100644
index 0000000..f690b09
--- /dev/null
+++
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/Mapreduce2Params.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bigtop.manager.stack.bigtop.v3_3_0.mapred;
+
+import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload;
+import org.apache.bigtop.manager.stack.core.annotations.GlobalParams;
+import org.apache.bigtop.manager.stack.core.param.BaseParams;
+import org.apache.bigtop.manager.stack.core.utils.LocalSettings;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
+import java.text.MessageFormat;
+import java.util.Map;
+
+@Getter
+@Slf4j
+public class Mapreduce2Params extends BaseParams {
+
+ private String mapredEnvContent;
+ private String mapredLogDir;
+ private String mapredPidDir;
+ private String historyServerPidFile;
+
+ public Mapreduce2Params(CommandPayload commandPayload) {
+ super(commandPayload);
+ globalParamsMap.put("mapred_user", user());
+ globalParamsMap.put("mapred_group", group());
+ globalParamsMap.put("java_home", "/usr/local/java");
+ globalParamsMap.put("hadoop_home", serviceHome());
+ globalParamsMap.put("hadoop_conf_dir", confDir());
+ globalParamsMap.put("hadoop_hdfs_home", hdfsHome());
+ globalParamsMap.put("hadoop_yarn_home", yarnHome());
+ globalParamsMap.put("hadoop_mapred_home", mapredHome());
+ globalParamsMap.put("hadoop_libexec_dir", serviceHome() + "/libexec");
+ }
+
+ public String mapredLimits() {
+ Map<String, Object> yarnConf =
LocalSettings.configurations(serviceName(), "mapreduce.conf");
+ return (String) yarnConf.get("content");
+ }
+
+ @GlobalParams
+ public Map<String, Object> mapredSite() {
+ return LocalSettings.configurations(serviceName(), "mapred-site");
+ }
+
+ @GlobalParams
+ public Map<String, Object> mapredEnv() {
+ Map<String, Object> mapredEnv =
LocalSettings.configurations(serviceName(), "mapred-env");
+ mapredEnvContent = (String) mapredEnv.get("content");
+ mapredLogDir = (String) mapredEnv.get("mapred_log_dir_prefix");
+ mapredPidDir = (String) mapredEnv.get("mapred_pid_dir_prefix");
+ historyServerPidFile =
MessageFormat.format("{0}/{1}/hadoop-{1}-historyserver.pid", mapredPidDir,
user());
+ return mapredEnv;
+ }
+
+ @Override
+ public String confDir() {
+ return "/etc/hadoop/conf";
+ }
+
+ @Override
+ public String serviceHome() {
+ return stackLibDir() + "/hadoop";
+ }
+
+ public String hdfsHome() {
+ return stackLibDir() + "/hadoop-hdfs";
+ }
+
+ public String yarnHome() {
+ return stackLibDir() + "/hadoop-yarn";
+ }
+
+ public String mapredHome() {
+ return stackLibDir() + "/hadoop-mapreduce";
+ }
+
+ public String mapredExec() {
+ return stackBinDir() + "/mapred";
+ }
+}
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java
index aaf4ac9..739bd30 100644
---
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java
+++
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java
@@ -47,7 +47,7 @@ public class TezSetup {
String tezGroup = tezParams.group();
// tez-site
- log.info("Generating {}/tez-site.xml file", confDir);
+ log.info("Generating [{}/tez-site.xml] file", confDir);
LinuxFileUtils.toFile(
ConfigType.XML,
MessageFormat.format("{0}/tez-site.xml", confDir),
@@ -58,7 +58,7 @@ public class TezSetup {
tezParams.getGlobalParamsMap());
// tez-env
- log.info("Generating {}/tez-env.sh file", confDir);
+ log.info("Generating [{}/tez-env.sh] file", confDir);
LinuxFileUtils.toFileByTemplate(
tezParams.getTezEnvContent(),
MessageFormat.format("{0}/tez-env.sh", confDir),
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnSetup.java
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnSetup.java
index 980b0b0..ccce820 100644
---
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnSetup.java
+++
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnSetup.java
@@ -78,7 +78,7 @@ public class YarnSetup {
yarnParams.getYarnPidDir(), yarnUser, yarnGroup,
Constants.PERMISSION_755, true);
LinuxFileUtils.createDirectories(yarnParams.getTmpDir(), yarnUser,
yarnGroup, Constants.PERMISSION_755, true);
- // hdfs.limits
+ // yarn.conf
LinuxFileUtils.toFileByTemplate(
yarnParams.yarnLimits(),
MessageFormat.format("{0}/yarn.conf",
BaseParams.LIMITS_CONF_DIR),
@@ -87,7 +87,7 @@ public class YarnSetup {
Constants.PERMISSION_644,
yarnParams.getGlobalParamsMap());
- // hadoop-env.sh
+ // yarn-env.sh
LinuxFileUtils.toFileByTemplate(
yarnEnv.get("content").toString(),
MessageFormat.format("{0}/yarn-env.sh", confDir),
@@ -96,7 +96,7 @@ public class YarnSetup {
Constants.PERMISSION_644,
yarnParams.getGlobalParamsMap());
- // hdfs-site.xml
+ // yarn-site.xml
LinuxFileUtils.toFile(
ConfigType.XML,
MessageFormat.format("{0}/yarn-site.xml", confDir),
