This is an automated email from the ASF dual-hosted git repository.
wuzhiguo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/bigtop-manager.git
The following commit(s) were added to refs/heads/main by this push:
new b38a616 BIGTOP-4131: Add Tez component on Bigtop-3.3.0 stack (#37)
b38a616 is described below
commit b38a616a862bd0f5b555323545da3331f4141bc7
Author: Zhiguo Wu <[email protected]>
AuthorDate: Fri Aug 9 17:45:38 2024 +0800
BIGTOP-4131: Add Tez component on Bigtop-3.3.0 stack (#37)
---
.../src/main/resources/bin/start.sh | 29 ++
.../bigtop/manager/common/constants/Constants.java | 11 +-
.../server/command/job/AbstractServiceJob.java | 19 ++
.../server/command/job/ServiceConfigureJob.java | 8 +
.../server/command/job/ServiceInstallJob.java | 3 +
.../ComponentConfigureStage.java} | 17 +-
.../ComponentConfigureTask.java} | 16 +-
.../manager/server/stack/xml/ConfigurationXml.java | 3 -
.../server/stack/xml/ServiceMetainfoXml.java | 4 -
.../manager/server/stack/xml/StackMetainfoXml.java | 3 -
.../src/main/resources/bin/start.sh | 1 -
.../resources/stacks/bigtop/3.3.0/metainfo.xml | 1 -
.../services/hdfs/configuration/hadoop-env.xml | 11 -
.../stacks/bigtop/3.3.0/services/hdfs/metainfo.xml | 1 -
.../bigtop/3.3.0/services/kafka/metainfo.xml | 1 -
.../stacks/bigtop/3.3.0/services/kafka/order.json | 3 -
.../3.3.0/services/tez/configuration/tez-env.xml | 54 ++++
.../3.3.0/services/tez/configuration/tez-site.xml | 355 +++++++++++++++++++++
.../3.3.0/services/{zookeeper => tez}/metainfo.xml | 43 +--
.../services/yarn/configuration/yarn-site.xml | 13 +-
.../stacks/bigtop/3.3.0/services/yarn/metainfo.xml | 1 -
.../stacks/bigtop/3.3.0/services/yarn/order.json | 8 +-
.../bigtop/3.3.0/services/zookeeper/metainfo.xml | 1 -
.../bigtop/3.3.0/services/zookeeper/order.json | 4 +-
.../main/resources/stacks/nop/1.0.0/metainfo.xml | 1 -
.../stacks/nop/1.0.0/services/kafka/metainfo.xml | 1 -
.../nop/1.0.0/services/zookeeper/metainfo.xml | 1 -
.../bigtop-manager-stack-bigtop/pom.xml | 77 +++++
.../manager/stack/bigtop/utils/HdfsUtil.java | 110 +++++++
.../stack/bigtop/v3_3_0/hdfs/HdfsSetup.java | 6 +-
.../bigtop/v3_3_0/kafka/KafkaBrokerScript.java | 63 +---
.../{KafkaBrokerScript.java => KafkaSetup.java} | 61 +---
.../stack/bigtop/v3_3_0/tez/TezClientScript.java | 25 +-
.../manager/stack/bigtop/v3_3_0/tez/TezParams.java | 101 ++++++
.../manager/stack/bigtop/v3_3_0/tez/TezSetup.java | 74 +++++
.../stack/bigtop/v3_3_0/yarn/YarnSetup.java | 3 +-
.../stack/common/utils/linux/LinuxFileUtils.java | 39 ++-
.../manager/stack/core/hook/AbstractHook.java | 2 +-
bigtop-manager-stack/pom.xml | 1 +
dev-support/docker/rocky8/build-containers.sh | 4 +-
40 files changed, 955 insertions(+), 224 deletions(-)
diff --git a/bigtop-manager-agent/src/main/resources/bin/start.sh
b/bigtop-manager-agent/src/main/resources/bin/start.sh
index 1a6f9cc..638b5b1 100755
--- a/bigtop-manager-agent/src/main/resources/bin/start.sh
+++ b/bigtop-manager-agent/src/main/resources/bin/start.sh
@@ -23,12 +23,41 @@ BIGTOP_MANAGER_HOME=${BIGTOP_MANAGER_HOME:-$(cd
$BIN_DIR/..; pwd)}
source "${BIGTOP_MANAGER_HOME}/bin/env.sh"
+usage() {
+ echo "usage: $PROG [--debug]"
+ echo " --debug - enable debug mode"
+ echo " -h, --help"
+ exit 1
+}
+
+DEBUG="false"
+DOCKER="false"
+
+while [ $# -gt 0 ]; do
+ case "$1" in
+ --debug)
+ echo "enable debug mode."
+ DEBUG="true"
+ shift;;
+ -h|--help)
+ usage
+ shift;;
+ *)
+ echo "Unknown argument: '$1'" 1>&2
+ usage;;
+ esac
+done
+
JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE}
-Xms4g -Xmx4g -Xmn2g -XX:+IgnoreUnrecognizedVMOptions -XX:+PrintGCDateStamps
-XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=dump.hprof"}
if [[ "$DOCKER" == "true" ]]; then
JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport"
fi
+if [[ "$DEBUG" == "true" ]]; then
+ JAVA_OPTS="${JAVA_OPTS}
-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5006"
+fi
+
cd $BIGTOP_MANAGER_HOME
$JAVA_HOME/bin/java $JAVA_OPTS \
diff --git
a/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/constants/Constants.java
b/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/constants/Constants.java
index 66c12b9..14fecb1 100644
---
a/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/constants/Constants.java
+++
b/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/constants/Constants.java
@@ -42,14 +42,21 @@ public final class Constants {
*/
public static final String ALL_HOST_KEY = "all";
+ /**
+ * permission 644
+ */
+ public static final String PERMISSION_644 = "rw-r--r--";
+
/**
* permission 755
*/
public static final String PERMISSION_755 = "rwxr-xr-x";
+
/**
- * permission 644
+ * permission 775
*/
- public static final String PERMISSION_644 = "rw-r--r--";
+ public static final String PERMISSION_775 = "rwxrwxr-x";
+
/**
* permission 777
*/
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java
index 1315077..4a51ba4 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java
@@ -28,6 +28,7 @@ import
org.apache.bigtop.manager.dao.repository.ComponentRepository;
import org.apache.bigtop.manager.dao.repository.HostComponentRepository;
import org.apache.bigtop.manager.server.command.stage.CacheFileUpdateStage;
import org.apache.bigtop.manager.server.command.stage.ComponentCheckStage;
+import org.apache.bigtop.manager.server.command.stage.ComponentConfigureStage;
import org.apache.bigtop.manager.server.command.stage.ComponentInstallStage;
import org.apache.bigtop.manager.server.command.stage.ComponentStartStage;
import org.apache.bigtop.manager.server.command.stage.ComponentStopStage;
@@ -35,6 +36,7 @@ import
org.apache.bigtop.manager.server.command.stage.StageContext;
import org.apache.bigtop.manager.server.exception.ServerException;
import org.apache.bigtop.manager.server.holder.SpringContextHolder;
import org.apache.bigtop.manager.server.model.dto.ComponentDTO;
+import org.apache.bigtop.manager.server.model.dto.ComponentHostDTO;
import org.apache.bigtop.manager.server.model.dto.ServiceDTO;
import org.apache.bigtop.manager.server.model.dto.command.ServiceCommandDTO;
import org.apache.bigtop.manager.server.stack.dag.ComponentCommandWrapper;
@@ -178,6 +180,23 @@ public abstract class AbstractServiceJob extends
AbstractJob {
}
}
+ protected void createConfigureStages() {
+ for (ServiceCommandDTO serviceCommand :
jobContext.getCommandDTO().getServiceCommands()) {
+ if (serviceCommand.getInstalled()) {
+ continue;
+ }
+
+ for (ComponentHostDTO componentHost :
serviceCommand.getComponentHosts()) {
+ String serviceName = serviceCommand.getServiceName();
+ String componentName = componentHost.getComponentName();
+ List<String> hostnames = componentHost.getHostnames();
+
+ StageContext stageContext = createStageContext(serviceName,
componentName, hostnames);
+ stages.add(new ComponentConfigureStage(stageContext));
+ }
+ }
+ }
+
protected void createStartStages() {
List<String> todoList = getTodoListForCommand(Command.START);
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceConfigureJob.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceConfigureJob.java
index 9dad58a..6f60fc6 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceConfigureJob.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceConfigureJob.java
@@ -26,7 +26,15 @@ public class ServiceConfigureJob extends AbstractServiceJob {
@Override
protected void createStages() {
+ // Update cache files
super.createCacheStage();
+
+ // Configure services
+ super.createConfigureStages();
+
+ // Restart services
+ super.createStopStages();
+ super.createStartStages();
}
@Override
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceInstallJob.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceInstallJob.java
index d97b209..dd5c727 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceInstallJob.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceInstallJob.java
@@ -67,6 +67,9 @@ public class ServiceInstallJob extends AbstractServiceJob {
// Update cache files after installed
super.createCacheStage();
+ // Configure services
+ super.createConfigureStages();
+
// Start all master components
super.createStartStages();
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceConfigureJob.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentConfigureStage.java
similarity index 61%
copy from
bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceConfigureJob.java
copy to
bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentConfigureStage.java
index 9dad58a..08eea2a 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceConfigureJob.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentConfigureStage.java
@@ -16,21 +16,24 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.bigtop.manager.server.command.job;
+package org.apache.bigtop.manager.server.command.stage;
-public class ServiceConfigureJob extends AbstractServiceJob {
+import org.apache.bigtop.manager.server.command.task.ComponentConfigureTask;
+import org.apache.bigtop.manager.server.command.task.Task;
- public ServiceConfigureJob(JobContext jobContext) {
- super(jobContext);
+public class ComponentConfigureStage extends AbstractComponentStage {
+
+ public ComponentConfigureStage(StageContext stageContext) {
+ super(stageContext);
}
@Override
- protected void createStages() {
- super.createCacheStage();
+ protected Task createTask(String hostname) {
+ return new ComponentConfigureTask(createTaskContext(hostname));
}
@Override
public String getName() {
- return "Configure services";
+ return "Configure " + stageContext.getComponentDTO().getDisplayName();
}
}
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceConfigureJob.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentConfigureTask.java
similarity index 66%
copy from
bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceConfigureJob.java
copy to
bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentConfigureTask.java
index 9dad58a..553d35a 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceConfigureJob.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentConfigureTask.java
@@ -16,21 +16,23 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.bigtop.manager.server.command.job;
+package org.apache.bigtop.manager.server.command.task;
-public class ServiceConfigureJob extends AbstractServiceJob {
+import org.apache.bigtop.manager.common.enums.Command;
- public ServiceConfigureJob(JobContext jobContext) {
- super(jobContext);
+public class ComponentConfigureTask extends AbstractComponentTask {
+
+ public ComponentConfigureTask(TaskContext taskContext) {
+ super(taskContext);
}
@Override
- protected void createStages() {
- super.createCacheStage();
+ protected Command getCommand() {
+ return Command.CONFIGURE;
}
@Override
public String getName() {
- return "Configure services";
+ return "Configure " + taskContext.getComponentDisplayName() + " on " +
taskContext.getHostname();
}
}
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/ConfigurationXml.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/ConfigurationXml.java
index 07f6a10..7981699 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/ConfigurationXml.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/ConfigurationXml.java
@@ -33,9 +33,6 @@ import java.util.List;
@XmlAccessorType(XmlAccessType.FIELD)
public class ConfigurationXml {
- @XmlElement(name = "schema-version")
- private String schemaVersion;
-
@XmlElement(name = "property")
private List<PropertyModel> propertyModels;
}
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/ServiceMetainfoXml.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/ServiceMetainfoXml.java
index cd8fb56..b8c352e 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/ServiceMetainfoXml.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/ServiceMetainfoXml.java
@@ -24,7 +24,6 @@ import lombok.Data;
import jakarta.xml.bind.annotation.XmlAccessType;
import jakarta.xml.bind.annotation.XmlAccessorType;
-import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlRootElement;
@Data
@@ -32,8 +31,5 @@ import jakarta.xml.bind.annotation.XmlRootElement;
@XmlAccessorType(XmlAccessType.FIELD)
public class ServiceMetainfoXml {
- @XmlElement(name = "schema-version")
- private String schemaVersion;
-
private ServiceModel service;
}
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/StackMetainfoXml.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/StackMetainfoXml.java
index 341589b..65ea700 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/StackMetainfoXml.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/StackMetainfoXml.java
@@ -32,9 +32,6 @@ import jakarta.xml.bind.annotation.XmlRootElement;
@XmlAccessorType(XmlAccessType.FIELD)
public class StackMetainfoXml {
- @XmlElement(name = "schema-version")
- private String schemaVersion;
-
@XmlElement(name = "stack")
private StackModel stack;
}
diff --git a/bigtop-manager-server/src/main/resources/bin/start.sh
b/bigtop-manager-server/src/main/resources/bin/start.sh
index 855eea8..7769d50 100755
--- a/bigtop-manager-server/src/main/resources/bin/start.sh
+++ b/bigtop-manager-server/src/main/resources/bin/start.sh
@@ -58,7 +58,6 @@ if [[ "$DEBUG" == "true" ]]; then
JAVA_OPTS="${JAVA_OPTS}
-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005"
fi
-
cd $BIGTOP_MANAGER_HOME
$JAVA_HOME/bin/java $JAVA_OPTS \
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/metainfo.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/metainfo.xml
index 78099df..ea14359 100644
--- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/metainfo.xml
+++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/metainfo.xml
@@ -19,7 +19,6 @@
-->
<metainfo>
- <schema-version>2.0</schema-version>
<stack>
<stack-name>bigtop</stack-name>
<stack-version>3.3.0</stack-version>
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hadoop-env.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hadoop-env.xml
index b085204..01ec608 100644
---
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hadoop-env.xml
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hadoop-env.xml
@@ -86,21 +86,13 @@
USER="$(whoami)"
export JAVA_HOME=${java_home!}
-
export HADOOP_HOME=${hadoop_home!}
-
export HADOOP_COMMON_HOME=${hadoop_home!}
-
export HADOOP_HDFS_HOME=${hadoop_hdfs_home!}
-
export HADOOP_CONF_DIR=${hadoop_conf_dir!}
-
export HADOOP_HEAPSIZE_MAX=${hadoop_heapsize_max}
-
export HADOOP_HEAPSIZE_MIN=${hadoop_heapsize_min}
-export HADOOP_JAAS_DEBUG=true
-
# export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true
-Dsun.security.krb5.debug=true -Dsun.security.spnego.debug"
<#noparse>
@@ -108,11 +100,8 @@ export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)}
</#noparse>
export HADOOP_LOG_DIR=${hadoop_log_dir_prefix}/$USER
-
export HADOOP_PID_DIR=${hadoop_pid_dir_prefix}/$USER
-
export HADOOP_ROOT_LOGGER=${hadoop_root_logger}
-
export HADOOP_LIBEXEC_DIR=${hadoop_libexec_dir}
]]>
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml
index c2fc518..26f77bd 100644
---
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml
@@ -19,7 +19,6 @@
-->
<metainfo>
- <schema-version>2.0</schema-version>
<service>
<name>hdfs</name>
<display-name>HDFS</display-name>
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml
index 5e379f3..844ba3e 100644
---
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml
@@ -19,7 +19,6 @@
-->
<metainfo>
- <schema-version>2.0</schema-version>
<service>
<name>kafka</name>
<display-name>Kafka</display-name>
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/order.json
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/order.json
index 78adc63..09585eb 100644
---
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/order.json
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/order.json
@@ -7,8 +7,5 @@
],
"KAFKA_BROKER-RESTART": [
"ZOOKEEPER_SERVER-RESTART"
- ],
- "ZOOKEEPER_SERVER-STOP": [
- "KAFKA_BROKER-STOP"
]
}
\ No newline at end of file
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-env.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-env.xml
new file mode 100644
index 0000000..0f9c9ff
--- /dev/null
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-env.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+-->
+
+<configuration>
+ <!-- tez-env.sh -->
+ <property>
+ <name>content</name>
+ <display-name>tez-env template</display-name>
+ <description>This is the freemarker template for tez-env.sh
file</description>
+ <value><![CDATA[
+# Tez specific configuration
+export TEZ_CONF_DIR=${tez_conf_dir!}
+
+# Set HADOOP_HOME to point to a specific hadoop install directory
+export HADOOP_HOME=${hadoop_home!}
+
+# The java implementation to use.
+export JAVA_HOME=${java_home!}
+]]>
+ </value>
+ <attrs>
+ <type>longtext</type>
+ </attrs>
+ </property>
+ <property>
+ <name>enable_heap_dump</name>
+ <value>false</value>
+ <description>Enable or disable taking Heap Dump.
(true/false)</description>
+ <display-name>Enable heap dump</display-name>
+ </property>
+ <property>
+ <name>heap_dump_location</name>
+ <value>/tmp</value>
+ <description>Location for heap dump file</description>
+ <display-name>Heap dump location</display-name>
+ </property>
+</configuration>
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-site.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-site.xml
new file mode 100644
index 0000000..98d5f29
--- /dev/null
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-site.xml
@@ -0,0 +1,355 @@
+<?xml version="1.0"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+-->
+
+<configuration>
+ <property>
+ <name>tez.lib.uris.classpath</name>
+ <display-name>TEZ Lib URIs Classpath</display-name>
+
<value>${hadoop_conf_dir!},${hadoop_home!}/*,${hadoop_home!}/lib/*,${hadoop_hdfs_home!}/*,${hadoop_hdfs_home!}/lib/*,${hadoop_yarn_home!}/*,${hadoop_yarn_home!}/lib/*,${tez_home!}/*,${tez_home!}/lib/*,${tez_conf_dir!}</value>
+ <description>
+ Comma-delimited list of the location of the Tez libraries
Classpath which will be localized for DAGs.
+ </description>
+ </property>
+ <property>
+ <name>tez.lib.uris</name>
+ <display-name>TEZ Lib URIs</display-name>
+ <value>${tez_lib_uris!}</value>
+ <description>Comma-delimited list of the location of the Tez libraries
which will be localized for DAGs.
+ Specifying a single .tar.gz or .tgz assumes that a compressed
version of the tez libs is being used. This is
+ uncompressed into a tezlibs directory when running containers, and
tezlibs/;tezlibs/lib/ are added to the
+ classpath (after . and .*).
+ If multiple files are specified - files are localized as regular
files, contents of directories are
+ localized as regular files (non-recursive).
+ </description>
+ </property>
+ <property>
+ <name>tez.cluster.additional.classpath.prefix</name>
+ <value>/etc/hadoop/conf/secure</value>
+ <description/>
+ </property>
+ <property>
+ <name>tez.am.log.level</name>
+ <value>INFO</value>
+ <description>Root Logging level passed to the Tez app
master</description>
+ </property>
+ <property>
+ <name>tez.generate.debug.artifacts</name>
+ <value>false</value>
+ <description>Generate debug artifacts such as a text representation of
the submitted DAG plan</description>
+ </property>
+ <property>
+ <name>tez.staging-dir</name>
+ <display-name>TEZ Staging directory</display-name>
+ <value>/tmp/${user.name}/staging</value>
+ <description>The staging dir used while submitting DAGs</description>
+ </property>
+ <property>
+ <name>tez.am.resource.memory.mb</name>
+ <value>2048</value>
+ <description>The amount of memory to be used by the AppMaster.
+ Used only if the value is not specified explicitly by the DAG
definition.
+ </description>
+ </property>
+ <property>
+ <name>tez.am.launch.cmd-opts</name>
+ <value>-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps
-XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB ${heap_dump_opts!}</value>
+ <description>Java options for the Tez AppMaster process. The Xmx value
is derived based on
+ tez.am.resource.memory.mb and is 80% of the value by default.
+ Used only if the value is not specified explicitly by the DAG
definition.
+ </description>
+ </property>
+ <property>
+ <name>tez.am.launch.cluster-default.cmd-opts</name>
+ <value>-server -Djava.net.preferIPv4Stack=true</value>
+ <description>Cluster default Java options for the Tez AppMaster
process. These will be prepended to the
+ properties specified via tez.am.launch.cmd-opts
+ </description>
+ </property>
+ <property>
+ <name>tez.am.launch.env</name>
+ <value>LD_LIBRARY_PATH=${hadoop_home!}/lib/native</value>
+ <description>
+ Additional execution environment entries for tez. This is not an
additive property. You must preserve the
+ original value if
+ you want to have access to native libraries.
+ Used only if the value is not specified explicitly by the DAG
definition.
+ </description>
+ </property>
+ <property>
+ <name>tez.task.resource.memory.mb</name>
+ <value>1536</value>
+ <description>The amount of memory to be used by launched tasks.
+ Used only if the value is not specified explicitly by the DAG
definition.
+ </description>
+ </property>
+ <property>
+ <name>tez.task.launch.cmd-opts</name>
+ <value>-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps
-XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB ${heap_dump_opts!}</value>
+ <description>Java options for tasks. The Xmx value is derived based on
tez.task.resource.memory.mb and is 80% of
+ this value by default.
+ Used only if the value is not specified explicitly by the DAG
definition.
+ </description>
+ </property>
+ <property>
+ <name>tez.task.launch.cluster-default.cmd-opts</name>
+ <value>-server -Djava.net.preferIPv4Stack=true</value>
+ <description>Cluster default Java options for tasks. These will be
prepended to the properties specified via
+ tez.task.launch.cmd-opts
+ </description>
+ </property>
+ <property>
+ <name>tez.task.launch.env</name>
+ <value>LD_LIBRARY_PATH=${hadoop_home!}/lib/native</value>
+ <description>
+ Additional execution environment entries for tez. This is not an
additive property. You must preserve the
+ original value if
+ you want to have access to native libraries.
+ Used only if the value is not specified explicitly by the DAG
definition.
+ </description>
+ </property>
+ <property>
+ <name>tez.shuffle-vertex-manager.min-src-fraction</name>
+ <value>0.2</value>
+ <description>In case of a ScatterGather connection, the fraction of
source tasks which should
+ complete before tasks for the current vertex are schedule
+ </description>
+ </property>
+ <property>
+ <name>tez.shuffle-vertex-manager.max-src-fraction</name>
+ <value>0.4</value>
+ <description>In case of a ScatterGather connection, once this fraction
of source tasks have
+ completed, all tasks on the current vertex can be scheduled.
Number of tasks ready for
+ scheduling on the current vertex scales linearly between
min-fraction and max-fraction
+ </description>
+ </property>
+ <property>
+ <name>tez.am.am-rm.heartbeat.interval-ms.max</name>
+ <value>250</value>
+ <description>The maximum heartbeat interval between the AM and RM in
milliseconds</description>
+ </property>
+ <property>
+ <name>tez.grouping.split-waves</name>
+ <value>1.7</value>
+ <description>The multiplier for available queue capacity when
determining number of tasks for
+ a Vertex. 1.7 with 100% queue available implies generating a
number of tasks roughly equal
+ to 170% of the available containers on the queue
+ </description>
+ </property>
+ <property>
+ <name>tez.grouping.min-size</name>
+ <value>16777216</value>
+ <description>Lower bound on the size (in bytes) of a grouped split, to
avoid generating
+ too many splits
+ </description>
+ </property>
+ <property>
+ <name>tez.grouping.max-size</name>
+ <value>1073741824</value>
+ <description>Upper bound on the size (in bytes) of a grouped split, to
avoid generating
+ excessively large split
+ </description>
+ </property>
+ <property>
+ <name>tez.am.container.reuse.enabled</name>
+ <value>true</value>
+ <description>Configuration to specify whether container should be
reused</description>
+ </property>
+ <property>
+ <name>tez.am.container.reuse.rack-fallback.enabled</name>
+ <value>true</value>
+ <description>Whether to reuse containers for rack local tasks. Active
only if reuse is enabled
+ </description>
+ </property>
+ <property>
+ <name>tez.am.container.reuse.non-local-fallback.enabled</name>
+ <value>false</value>
+ <description>Whether to reuse containers for non-local tasks. Active
only if reuse is enabled
+ </description>
+ </property>
+ <property>
+ <name>tez.am.container.idle.release-timeout-min.millis</name>
+ <value>10000</value>
+ <description>The minimum amount of time to hold on to a container that
is idle. Only active when reuse is
+ enabled.
+ </description>
+ </property>
+ <property>
+ <name>tez.am.container.idle.release-timeout-max.millis</name>
+ <value>20000</value>
+ <description>The maximum amount of time to hold on to a container if
no task can be assigned to it immediately.
+ Only active when reuse is enabled.
+ </description>
+ </property>
+ <property>
+ <name>tez.am.container.reuse.locality.delay-allocation-millis</name>
+ <value>250</value>
+ <description>The amount of time to wait before assigning a container
to the next level of
+ locality. NODE -> RACK -> NON_LOCAL
+ </description>
+ </property>
+ <property>
+ <name>tez.am.max.app.attempts</name>
+ <value>2</value>
+ <description>Specifies the total number of time the app master will
run in case recovery is triggered</description>
+ </property>
+ <property>
+ <name>tez.am.maxtaskfailures.per.node</name>
+ <value>10</value>
+ <description>The maximum number of allowed task attempt failures on a
node before it gets marked as blacklisted</description>
+ </property>
+ <property>
+ <name>tez.task.am.heartbeat.counter.interval-ms.max</name>
+ <value>4000</value>
+ <description>Time interval at which task counters are sent to the
AM</description>
+ </property>
+ <property>
+ <name>tez.task.get-task.sleep.interval-ms.max</name>
+ <value>200</value>
+ <description>The maximum amount of time, in seconds, to wait before a
task asks an AM for another task</description>
+ </property>
+ <property>
+ <name>tez.task.max-events-per-heartbeat</name>
+ <value>500</value>
+ <description>Maximum number of of events to fetch from the AM by the
tasks in a single heartbeat.</description>
+ </property>
+ <property>
+ <name>tez.session.client.timeout.secs</name>
+ <value>-1</value>
+ <description>Time (in seconds) to wait for AM to come up when trying
to submit a DAG from the client</description>
+ </property>
+ <property>
+ <name>tez.session.am.dag.submit.timeout.secs</name>
+ <value>300</value>
+ <description>
+ Time (in seconds) for which the Tez AM should wait for a DAG to be
submitted before shutting down
+ </description>
+ </property>
+ <property>
+ <name>tez.runtime.compress</name>
+ <value>true</value>
+ <description>Whether intermediate data should be compressed or
not</description>
+ </property>
+ <property>
+ <name>tez.runtime.compress.codec</name>
+ <value>org.apache.hadoop.io.compress.SnappyCodec</value>
+ <description>
+ The coded to be used if compressing intermediate data. Only
applicable if tez.runtime.compress is enabled
+ </description>
+ </property>
+ <property>
+ <name>tez.runtime.unordered.output.buffer.size-mb</name>
+ <value>100</value>
+ <description>The size of the buffer when output does not require to be
sorted</description>
+ </property>
+ <property>
+ <name>tez.runtime.convert.user-payload.to.history-text</name>
+ <value>false</value>
+ <description>Whether to publish configuration information to History
logger</description>
+ </property>
+ <property>
+ <name>tez.use.cluster.hadoop-libs</name>
+ <value>false</value>
+ <description>
+ This being true implies that the deployment is relying on hadoop
jars being available on the cluster on all nodes.
+ </description>
+ </property>
+ <property>
+ <name>tez.am.tez-ui.history-url.template</name>
+
<value>__HISTORY_URL_BASE__?viewPath=%2F%23%2Ftez-app%2F__APPLICATION_ID__</value>
+ <description>Template to generate the History URL for a particular Tez
Application.
+ Template replaces __APPLICATION_ID__ with the actual applicationId
and
+ __HISTORY_URL_BASE__ with the value from the
tez.tez-ui.history-url.base config property
+ </description>
+ </property>
+ <property>
+ <name>tez.tez-ui.history-url.base</name>
+ <value/>
+ <description/>
+ </property>
+ <property>
+ <name>tez.am.view-acls</name>
+ <value>*</value>
+ <description>AM view ACLs. This allows the specified users/groups to
view the status of the AM and all DAGs that
+ run within this AM.
+ Value format: Comma separated list of users, followed by
whitespace, followed by a comma separated list of
+ groups.
+ </description>
+ </property>
+ <property>
+ <name>tez.runtime.optimize.local.fetch</name>
+ <value>true</value>
+ <description>If the shuffle input is on the local host bypass the http
fetch and access the files directly.
+ </description>
+ </property>
+ <property>
+ <name>tez.task.generate.counters.per.io</name>
+ <value>true</value>
+ <description>Whether to generate counters on a per-edge basis for a
Tez DAG. Helpful for in-depth analysis.
+ </description>
+ </property>
+ <property>
+ <name>tez.runtime.sorter.class</name>
+ <value>PIPELINED</value>
+ <description>Which sorter implementation to use</description>
+ </property>
+ <property>
+ <name>tez.runtime.pipelined.sorter.sort.threads</name>
+ <value>2</value>
+ <description>Tez runtime pipelined sorter sort threads</description>
+ </property>
+ <property>
+ <name>tez.runtime.io.sort.mb</name>
+ <value>272</value>
+ <description>The size of the sort buffer when output needs to be
sorted</description>
+ </property>
+ <property>
+ <name>tez.counters.max</name>
+ <value>10000</value>
+ <description>The number of allowed counters for the executing
DAG</description>
+ </property>
+ <property>
+ <name>tez.counters.max.groups</name>
+ <value>3000</value>
+ <description>The number of allowed counter groups for the executing
DAG</description>
+ </property>
+ <property>
+ <name>tez.runtime.shuffle.fetch.buffer.percent</name>
+ <value>0.6</value>
+ <description>Fraction (0-1) of the available memory which can be used
to retain shuffled data</description>
+ </property>
+ <property>
+ <name>tez.runtime.shuffle.memory.limit.percent</name>
+ <value>0.25</value>
+ <description>This property determines the maximum size of a shuffle
segment
+ which can be fetched to memory. Fraction (0-1) of shuffle memory
+ (after applying tez.runtime.shuffle.fetch.buffer.percent)
+ </description>
+ </property>
+ <property>
+
<name>tez.history.logging.timeline-cache-plugin.old-num-dags-per-group</name>
+ <value>5</value>
+ <description>DAGs per group</description>
+ </property>
+ <property>
+ <name>yarn.timeline-service.enabled</name>
+ <value>false</value>
+ <description>Timeline service version we’re currently
using.</description>
+ </property>
+</configuration>
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml
similarity index 60%
copy from
bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml
copy to
bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml
index 64f10b1..42ef5e2 100644
---
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml
@@ -19,39 +19,24 @@
-->
<metainfo>
- <schema-version>2.0</schema-version>
<service>
- <name>zookeeper</name>
- <display-name>ZooKeeper</display-name>
- <desc>
- Apache ZooKeeper is an effort to develop and maintain an
open-source server which enables highly
- reliable distributed coordination.
- </desc>
- <version>3.7.2-1</version>
- <user>zookeeper</user>
- <group>zookeeper</group>
+ <name>tez</name>
+ <display-name>Tez</display-name>
+ <desc>Tez is the next generation Hadoop Query Processing framework
written on top of YARN.</desc>
+ <version>0.10.2-1</version>
+ <user>tez</user>
+ <group>tez</group>
<components>
<component>
- <name>zookeeper_server</name>
- <display-name>ZooKeeper Server</display-name>
- <category>master</category>
- <cardinality>1+</cardinality>
- <command-script>
-
<script-id>org.apache.bigtop.manager.stack.bigtop.v3_3_0.zookeeper.ZookeeperServerScript</script-id>
- <script-type>java</script-type>
- <timeout>1200</timeout>
- </command-script>
- </component>
-
- <component>
- <name>zookeeper_client</name>
- <display-name>ZooKeeper Client</display-name>
+ <name>tez_client</name>
+ <display-name>Tez Client</display-name>
<category>client</category>
<cardinality>1+</cardinality>
<command-script>
-
<script-id>org.apache.bigtop.manager.stack.bigtop.v3_3_0.zookeeper.ZookeeperClientScript</script-id>
+
<script-id>org.apache.bigtop.manager.stack.bigtop.v3_3_0.tez.TezClientScript</script-id>
<script-type>java</script-type>
+ <timeout>1200</timeout>
</command-script>
</component>
</components>
@@ -66,9 +51,13 @@
<arch>x86_64</arch>
</architectures>
<packages>
- <package>zookeeper_3_3_0</package>
+ <package>tez_3_3_0</package>
</packages>
</os-specific>
</os-specifics>
+
+ <required-services>
+ <service>hdfs</service>
+ </required-services>
</service>
-</metainfo>
\ No newline at end of file
+</metainfo>
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml
index 996cb2e..a718fae 100644
---
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml
@@ -38,8 +38,7 @@
<name>yarn.resourcemanager.address</name>
<value>0.0.0.0:8032</value>
<description>
- The address of the applications manager interface in the
- RM.
+ The address of the applications manager interface in the RM.
</description>
</property>
<property>
@@ -91,14 +90,12 @@
<property>
<name>yarn.nodemanager.resource.memory-mb</name>
<value>5120</value>
- <description>Amount of physical memory, in MB, that can be allocated
- for containers.</description>
+ <description>Amount of physical memory, in MB, that can be allocated
for containers.</description>
<display-name>Memory allocated for all YARN containers on a
node</display-name>
</property>
<property>
<name>yarn.application.classpath</name>
- <value>
-
{{hadoop_conf_dir}},{{hadoop_home}}/*,{{hadoop_home}}/lib/*,{{hadoop_hdfs_home}}/*,{{hadoop_hdfs_home}}/lib/*,{{hadoop_yarn_home}}/*,{{hadoop_yarn_home}}/lib/*</value>
+
<value>${hadoop_conf_dir!},${hadoop_home!}/*,${hadoop_home!}/lib/*,${hadoop_hdfs_home!}/*,${hadoop_hdfs_home!}/lib/*,${hadoop_yarn_home!}/*,${hadoop_yarn_home!}/lib/*</value>
<description>Classpath for typical applications.</description>
</property>
<property>
@@ -402,12 +399,12 @@
</property>
<property>
<name>yarn.authorization-provider</name>
- <description> Yarn authorization provider class. </description>
+ <description> Yarn authorization provider class.</description>
</property>
<property>
<name>yarn.admin.acl</name>
<value>yarn</value>
- <description> ACL of who can be admin of the YARN cluster.
</description>
+ <description>ACL of who can be admin of the YARN cluster.</description>
</property>
<!--ats
v1.5 properties-->
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml
index 44ea8ff..9e0d03d 100644
---
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml
@@ -19,7 +19,6 @@
-->
<metainfo>
- <schema-version>2.0</schema-version>
<service>
<name>yarn</name>
<display-name>YARN</display-name>
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/order.json
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/order.json
index 9c9addb..ea235ec 100644
---
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/order.json
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/order.json
@@ -1,16 +1,14 @@
{
"RESOURCEMANAGER-START": [
- "ZOOKEEPER_SERVER-START"
+ "ZOOKEEPER_SERVER-START",
+ "NAMENODE-START",
+ "DATANODE-START"
],
"NODEMANAGER-START": [
"NAMENODE-START",
"DATANODE-START",
"RESOURCEMANAGER-START"
],
- "RESOURCEMANAGER-START": [
- "NAMENODE-START",
- "DATANODE-START"
- ],
"RESOURCEMANAGER-RESTART": [
"NAMENODE-RESTART"
],
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml
index 64f10b1..043b67e 100644
---
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml
@@ -19,7 +19,6 @@
-->
<metainfo>
- <schema-version>2.0</schema-version>
<service>
<name>zookeeper</name>
<display-name>ZooKeeper</display-name>
diff --git
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/order.json
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/order.json
index a0d75ea..0da4717 100644
---
a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/order.json
+++
b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/order.json
@@ -1,5 +1,5 @@
{
- "KAFKA_BROKER-INSTALL": [
- "ZOOKEEPER_SERVER-INSTALL"
+ "ZOOKEEPER_SERVER-STOP": [
+ "KAFKA_BROKER-STOP"
]
}
\ No newline at end of file
diff --git
a/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/metainfo.xml
b/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/metainfo.xml
index 1c956f0..c87d7b7 100644
--- a/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/metainfo.xml
+++ b/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/metainfo.xml
@@ -19,7 +19,6 @@
-->
<metainfo>
- <schema-version>2.0</schema-version>
<stack>
<stack-name>nop</stack-name>
<stack-version>1.0.0</stack-version>
diff --git
a/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/services/kafka/metainfo.xml
b/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/services/kafka/metainfo.xml
index cd3d23e..dd3d0c2 100644
---
a/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/services/kafka/metainfo.xml
+++
b/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/services/kafka/metainfo.xml
@@ -19,7 +19,6 @@
-->
<metainfo>
- <schema-version>2.0</schema-version>
<service>
<name>kafka</name>
<display-name>Kafka</display-name>
diff --git
a/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/services/zookeeper/metainfo.xml
b/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/services/zookeeper/metainfo.xml
index 55db55e..1ebb84c 100644
---
a/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/services/zookeeper/metainfo.xml
+++
b/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/services/zookeeper/metainfo.xml
@@ -19,7 +19,6 @@
-->
<metainfo>
- <schema-version>2.0</schema-version>
<service>
<name>zookeeper</name>
<display-name>ZooKeeper</display-name>
diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/pom.xml
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/pom.xml
index ad85d1a..2e2bbe6 100644
--- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/pom.xml
+++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/pom.xml
@@ -29,6 +29,11 @@
<name>${project.artifactId}</name>
<description>Bigtop Manager Stack Bigtop</description>
+ <properties>
+ <!-- Import component's API, for bigtop stack only -->
+ <hadoop.version>3.3.6</hadoop.version>
+ </properties>
+
<dependencies>
<dependency>
<groupId>org.apache.bigtop</groupId>
@@ -39,6 +44,78 @@
<groupId>com.google.auto.service</groupId>
<artifactId>auto-service</artifactId>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
</project>
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/utils/HdfsUtil.java
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/utils/HdfsUtil.java
new file mode 100644
index 0000000..fe65590
--- /dev/null
+++
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/utils/HdfsUtil.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bigtop.manager.stack.bigtop.utils;
+
+import org.apache.bigtop.manager.stack.common.exception.StackException;
+import org.apache.bigtop.manager.stack.common.utils.LocalSettings;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
+
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+
+import java.net.URI;
+import java.security.PrivilegedAction;
+import java.text.MessageFormat;
+import java.util.List;
+
+@Data
+@Slf4j
+public class HdfsUtil {
+ public static void createDirectory(String user, String directory) {
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
+ try {
+ ugi.doAs((PrivilegedAction<Void>) () -> {
+ try (FileSystem fs = getFileSystem()) {
+ // Create dest dir if not exist
+ Path destDirPath = new Path(directory);
+ if (!fs.exists(destDirPath)) {
+ fs.mkdirs(destDirPath);
+ }
+ } catch (Exception e) {
+ log.error("Error while creating directory on hdfs", e);
+ throw new StackException(e);
+ }
+
+ return null;
+ });
+ } catch (Exception e) {
+ log.error("Error while creating directory on hdfs", e);
+ throw new StackException(e);
+ }
+ }
+
+ public static void uploadFile(String user, String localFilePath, String
destDir) {
+ uploadFile(user, localFilePath, destDir, null);
+ }
+
+ public static void uploadFile(String user, String localFilePath, String
destDir, String destFilename) {
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
+ try {
+ ugi.doAs((PrivilegedAction<Void>) () -> {
+ try (FileSystem fs = getFileSystem()) {
+ // Create dest dir if not exist
+ Path destDirPath = new Path(destDir);
+ if (!fs.exists(destDirPath)) {
+ fs.mkdirs(destDirPath);
+ }
+
+ // upload file
+ Path destFilePath = destFilename == null ? new
Path(destDir) : new Path(destDir, destFilename);
+ fs.copyFromLocalFile(new Path(localFilePath),
destFilePath);
+ } catch (Exception e) {
+ log.error("Error while uploading file to hdfs", e);
+ throw new StackException(e);
+ }
+
+ return null;
+ });
+ } catch (Exception e) {
+ log.error("Error while uploading file to hdfs", e);
+ throw new StackException(e);
+ }
+ }
+
+ private static FileSystem getFileSystem() throws Exception {
+ Configuration conf = new Configuration();
+ conf.addResource(new Path("/etc/hadoop/conf/core-site.xml"));
+ conf.addResource(new Path("/etc/hadoop/conf/hdfs-site.xml"));
+
+ List<String> namenodeList = LocalSettings.hosts("namenode");
+ if (CollectionUtils.isEmpty(namenodeList)) {
+ String msg = "No namenode found in the cluster";
+ log.error(msg);
+ throw new StackException(msg);
+ }
+
+ String hdfsUri = MessageFormat.format("hdfs://{0}:8020",
namenodeList.get(0));
+ return FileSystem.get(new URI(hdfsUri), conf);
+ }
+}
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsSetup.java
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsSetup.java
index 274eac1..3d37bdf 100644
---
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsSetup.java
+++
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsSetup.java
@@ -83,7 +83,7 @@ public class HdfsSetup {
LinuxFileUtils.createDirectories(
hdfsParams.getDfsDataDir(), hdfsUser, hdfsGroup,
Constants.PERMISSION_755, true);
LinuxFileUtils.createDirectories(
- hdfsParams.getHadoopLogDir(), hdfsUser, hdfsGroup,
Constants.PERMISSION_755, true);
+ hdfsParams.getHadoopLogDir(), hdfsUser, hdfsGroup,
Constants.PERMISSION_775, true);
LinuxFileUtils.createDirectories(
hdfsParams.getHadoopPidDir(), hdfsUser, hdfsGroup,
Constants.PERMISSION_755, true);
@@ -150,6 +150,10 @@ public class HdfsSetup {
Constants.PERMISSION_644,
hdfsParams.getGlobalParamsMap());
+ // log.info("Creating /apps on hdfs");
+ // HdfsUtil.createDirectory(hdfsUser, "/apps");
+
+ log.info("Successfully configured HDFS");
return ShellResult.success("HDFS Configure success!");
}
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java
index e72467d..c44c85c 100644
---
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java
+++
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java
@@ -21,11 +21,8 @@ package org.apache.bigtop.manager.stack.bigtop.v3_3_0.kafka;
import org.apache.bigtop.manager.common.shell.ShellResult;
import org.apache.bigtop.manager.spi.stack.Params;
import org.apache.bigtop.manager.spi.stack.Script;
-import org.apache.bigtop.manager.stack.common.enums.ConfigType;
import org.apache.bigtop.manager.stack.common.exception.StackException;
-import org.apache.bigtop.manager.stack.common.utils.LocalSettings;
import org.apache.bigtop.manager.stack.common.utils.PackageUtils;
-import org.apache.bigtop.manager.stack.common.utils.linux.LinuxFileUtils;
import org.apache.bigtop.manager.stack.common.utils.linux.LinuxOSUtils;
import com.google.auto.service.AutoService;
@@ -33,13 +30,6 @@ import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
import java.text.MessageFormat;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static
org.apache.bigtop.manager.common.constants.Constants.PERMISSION_644;
-import static
org.apache.bigtop.manager.common.constants.Constants.PERMISSION_755;
-import static org.apache.bigtop.manager.common.constants.Constants.ROOT_USER;
@Slf4j
@AutoService(Script.class)
@@ -52,58 +42,7 @@ public class KafkaBrokerScript implements Script {
@Override
public ShellResult configure(Params params) {
- KafkaParams kafkaParams = (KafkaParams) params;
-
- String confDir = kafkaParams.confDir();
- String kafkaUser = kafkaParams.user();
- String kafkaGroup = kafkaParams.group();
-
- LinuxFileUtils.createDirectories(kafkaParams.getKafkaDataDir(),
kafkaUser, kafkaGroup, PERMISSION_755, true);
- LinuxFileUtils.createDirectories(kafkaParams.getKafkaLogDir(),
kafkaUser, kafkaGroup, PERMISSION_755, true);
- LinuxFileUtils.createDirectories(kafkaParams.getKafkaPidDir(),
kafkaUser, kafkaGroup, PERMISSION_755, true);
-
- // server.properties
- List<String> zookeeperServerHosts =
LocalSettings.hosts("zookeeper_server");
- Map<String, Object> paramMap = new HashMap<>();
- paramMap.put("zk_server_list", zookeeperServerHosts);
- paramMap.put("host", kafkaParams.hostname());
- LinuxFileUtils.toFile(
- ConfigType.PROPERTIES,
- MessageFormat.format("{0}/server.properties", confDir),
- kafkaUser,
- kafkaGroup,
- PERMISSION_644,
- kafkaParams.kafkaBroker(),
- paramMap);
-
- // kafka-env
- LinuxFileUtils.toFileByTemplate(
- kafkaParams.getKafkaEnvContent(),
- MessageFormat.format("{0}/kafka-env.sh", confDir),
- kafkaUser,
- kafkaGroup,
- PERMISSION_644,
- kafkaParams.getGlobalParamsMap());
-
- // log4j
- LinuxFileUtils.toFileByTemplate(
- kafkaParams.getKafkaLog4jContent(),
- MessageFormat.format("{0}/log4j.properties", confDir),
- kafkaUser,
- kafkaGroup,
- PERMISSION_644,
- kafkaParams.getGlobalParamsMap());
-
- // kafka.limits
- LinuxFileUtils.toFileByTemplate(
- kafkaParams.kafkaLimits(),
- MessageFormat.format("{0}/kafka.conf",
KafkaParams.LIMITS_CONF_DIR),
- ROOT_USER,
- ROOT_USER,
- PERMISSION_644,
- kafkaParams.getGlobalParamsMap());
-
- return ShellResult.success("Kafka Server Configure success!");
+ return KafkaSetup.config(params);
}
@Override
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaSetup.java
similarity index 64%
copy from
bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java
copy to
bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaSetup.java
index e72467d..ca5c236 100644
---
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java
+++
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaSetup.java
@@ -20,18 +20,14 @@ package org.apache.bigtop.manager.stack.bigtop.v3_3_0.kafka;
import org.apache.bigtop.manager.common.shell.ShellResult;
import org.apache.bigtop.manager.spi.stack.Params;
-import org.apache.bigtop.manager.spi.stack.Script;
import org.apache.bigtop.manager.stack.common.enums.ConfigType;
-import org.apache.bigtop.manager.stack.common.exception.StackException;
import org.apache.bigtop.manager.stack.common.utils.LocalSettings;
-import org.apache.bigtop.manager.stack.common.utils.PackageUtils;
import org.apache.bigtop.manager.stack.common.utils.linux.LinuxFileUtils;
-import org.apache.bigtop.manager.stack.common.utils.linux.LinuxOSUtils;
-import com.google.auto.service.AutoService;
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
-import java.io.IOException;
import java.text.MessageFormat;
import java.util.HashMap;
import java.util.List;
@@ -42,16 +38,10 @@ import static
org.apache.bigtop.manager.common.constants.Constants.PERMISSION_75
import static org.apache.bigtop.manager.common.constants.Constants.ROOT_USER;
@Slf4j
-@AutoService(Script.class)
-public class KafkaBrokerScript implements Script {
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class KafkaSetup {
- @Override
- public ShellResult install(Params params) {
- return PackageUtils.install(params.getPackageList());
- }
-
- @Override
- public ShellResult configure(Params params) {
+ public static ShellResult config(Params params) {
KafkaParams kafkaParams = (KafkaParams) params;
String confDir = kafkaParams.confDir();
@@ -105,45 +95,4 @@ public class KafkaBrokerScript implements Script {
return ShellResult.success("Kafka Server Configure success!");
}
-
- @Override
- public ShellResult start(Params params) {
- configure(params);
- KafkaParams kafkaParams = (KafkaParams) params;
-
- String cmd = MessageFormat.format(
- "sh {0}/bin/kafka-server-start.sh {0}/config/server.properties
> /dev/null 2>&1 & echo -n $!>{1}",
- kafkaParams.serviceHome(), kafkaParams.getKafkaPidFile());
- try {
- return LinuxOSUtils.sudoExecCmd(cmd, kafkaParams.user());
- } catch (IOException e) {
- throw new StackException(e);
- }
- }
-
- @Override
- public ShellResult stop(Params params) {
- KafkaParams kafkaParams = (KafkaParams) params;
- String cmd = MessageFormat.format("sh {0}/bin/kafka-server-stop.sh",
kafkaParams.serviceHome());
- try {
- return LinuxOSUtils.sudoExecCmd(cmd, kafkaParams.user());
- } catch (IOException e) {
- throw new StackException(e);
- }
- }
-
- @Override
- public ShellResult status(Params params) {
- KafkaParams kafkaParams = (KafkaParams) params;
- return LinuxOSUtils.checkProcess(kafkaParams.getKafkaPidFile());
- }
-
- public ShellResult test(Params params) {
- KafkaParams kafkaParams = (KafkaParams) params;
- try {
- return LinuxOSUtils.sudoExecCmd("date", kafkaParams.user());
- } catch (IOException e) {
- throw new StackException(e);
- }
- }
}
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceConfigureJob.java
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java
similarity index 54%
copy from
bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceConfigureJob.java
copy to
bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java
index 9dad58a..648680c 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceConfigureJob.java
+++
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java
@@ -16,21 +16,28 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.bigtop.manager.server.command.job;
+package org.apache.bigtop.manager.stack.bigtop.v3_3_0.tez;
-public class ServiceConfigureJob extends AbstractServiceJob {
+import org.apache.bigtop.manager.common.shell.ShellResult;
+import org.apache.bigtop.manager.spi.stack.ClientScript;
+import org.apache.bigtop.manager.spi.stack.Params;
+import org.apache.bigtop.manager.spi.stack.Script;
+import org.apache.bigtop.manager.stack.common.utils.PackageUtils;
- public ServiceConfigureJob(JobContext jobContext) {
- super(jobContext);
- }
+import com.google.auto.service.AutoService;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@AutoService(Script.class)
+public class TezClientScript implements ClientScript {
@Override
- protected void createStages() {
- super.createCacheStage();
+ public ShellResult install(Params params) {
+ return PackageUtils.install(params.getPackageList());
}
@Override
- public String getName() {
- return "Configure services";
+ public ShellResult configure(Params params) {
+ return TezSetup.config(params);
}
}
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java
new file mode 100644
index 0000000..f5df822
--- /dev/null
+++
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bigtop.manager.stack.bigtop.v3_3_0.tez;
+
+import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload;
+import org.apache.bigtop.manager.stack.common.annotations.GlobalParams;
+import org.apache.bigtop.manager.stack.common.utils.BaseParams;
+import org.apache.bigtop.manager.stack.common.utils.LocalSettings;
+
+import org.apache.commons.lang3.StringUtils;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
+import java.text.MessageFormat;
+import java.util.Map;
+
+@Getter
+@Slf4j
+public class TezParams extends BaseParams {
+
+ private String headDumpOpts;
+ private String tezEnvContent;
+
+ public TezParams(CommandPayload commandPayload) {
+ super(commandPayload);
+ globalParamsMap.put("tez_user", user());
+ globalParamsMap.put("tez_group", group());
+ globalParamsMap.put("java_home", "/usr/local/java");
+ globalParamsMap.put("hadoop_home", hadoopHome());
+ globalParamsMap.put("hadoop_conf_dir", hadoopConfDir());
+ globalParamsMap.put("hadoop_hdfs_home", hdfsHome());
+ globalParamsMap.put("hadoop_yarn_home", yarnHome());
+ globalParamsMap.put("tez_home", serviceHome());
+ globalParamsMap.put("tez_conf_dir", confDir());
+ globalParamsMap.put("head_dump_opts", headDumpOpts);
+ }
+
+ @GlobalParams
+ public Map<String, Object> tezSite() {
+ return LocalSettings.configurations(serviceName(), "tez-site");
+ }
+
+ @GlobalParams
+ public Map<String, Object> tezEnv() {
+ Map<String, Object> tezEnv =
LocalSettings.configurations(serviceName(), "tez-env");
+
+ String heapDumpEnabled = (String) tezEnv.get("enable_heap_dump");
+ if (StringUtils.isNotBlank(heapDumpEnabled) &&
Boolean.parseBoolean(heapDumpEnabled)) {
+ String heapDumpLocation = StringUtils.defaultIfBlank((String)
tezEnv.get("heap_dump_location"), "/tmp");
+ headDumpOpts =
+ MessageFormat.format("-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath={0}", heapDumpLocation);
+ }
+
+ tezEnvContent = (String) tezEnv.get("content");
+
+ return tezEnv;
+ }
+
+ @Override
+ public String confDir() {
+ return "/etc/tez/conf";
+ }
+
+ @Override
+ public String serviceHome() {
+ return stackLibDir() + "/tez";
+ }
+
+ public String hadoopConfDir() {
+ return "/etc/hadoop/conf";
+ }
+
+ public String hadoopHome() {
+ return stackLibDir() + "/hadoop";
+ }
+
+ public String hdfsHome() {
+ return stackLibDir() + "/hadoop-hdfs";
+ }
+
+ public String yarnHome() {
+ return stackLibDir() + "/hadoop-yarn";
+ }
+}
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java
new file mode 100644
index 0000000..68cc1fa
--- /dev/null
+++
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bigtop.manager.stack.bigtop.v3_3_0.tez;
+
+import org.apache.bigtop.manager.common.constants.Constants;
+import org.apache.bigtop.manager.common.shell.ShellResult;
+import org.apache.bigtop.manager.spi.stack.Params;
+import org.apache.bigtop.manager.stack.common.enums.ConfigType;
+import org.apache.bigtop.manager.stack.common.utils.linux.LinuxFileUtils;
+
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import java.text.MessageFormat;
+
+import static
org.apache.bigtop.manager.common.constants.Constants.PERMISSION_755;
+
+@Slf4j
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class TezSetup {
+
+ public static ShellResult config(Params params) {
+ TezParams tezParams = (TezParams) params;
+
+ String confDir = tezParams.confDir();
+ String tezUser = tezParams.user();
+ String tezGroup = tezParams.group();
+
+ // tez-site
+ log.info("Generating {}/tez-site.xml file", confDir);
+ LinuxFileUtils.toFile(
+ ConfigType.XML,
+ MessageFormat.format("{0}/tez-site.xml", confDir),
+ tezUser,
+ tezGroup,
+ Constants.PERMISSION_644,
+ tezParams.tezSite(),
+ tezParams.getGlobalParamsMap());
+
+ // tez-env
+ log.info("Generating {}/tez-env.sh file", confDir);
+ LinuxFileUtils.toFileByTemplate(
+ tezParams.getTezEnvContent(),
+ MessageFormat.format("{0}/tez-env.sh", confDir),
+ tezUser,
+ tezGroup,
+ PERMISSION_755,
+ tezParams.getGlobalParamsMap());
+
+ // maybe we should upload tez.tar.gz to HDFS here?
+ // log.info("Uploading tez.tar.gz to HDFS");
+ // HdfsUtil.uploadFile(tezUser, tezParams.serviceHome() +
"/tez.tar.gz", "/apps/tez");
+
+ log.info("Successfully configured Tez");
+ return ShellResult.success("Tez Configure success!");
+ }
+}
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnSetup.java
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnSetup.java
index f11c908..24e35b2 100644
---
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnSetup.java
+++
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnSetup.java
@@ -103,7 +103,8 @@ public class YarnSetup {
yarnUser,
yarnGroup,
Constants.PERMISSION_644,
- yarnParams.yarnSite());
+ yarnParams.yarnSite(),
+ yarnParams.getGlobalParamsMap());
// log4j
LinuxFileUtils.toFileByTemplate(
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-common/src/main/java/org/apache/bigtop/manager/stack/common/utils/linux/LinuxFileUtils.java
b/bigtop-manager-stack/bigtop-manager-stack-common/src/main/java/org/apache/bigtop/manager/stack/common/utils/linux/LinuxFileUtils.java
index a84e8c6..dff623c 100644
---
a/bigtop-manager-stack/bigtop-manager-stack-common/src/main/java/org/apache/bigtop/manager/stack/common/utils/linux/LinuxFileUtils.java
+++
b/bigtop-manager-stack/bigtop-manager-stack-common/src/main/java/org/apache/bigtop/manager/stack/common/utils/linux/LinuxFileUtils.java
@@ -22,6 +22,7 @@ import org.apache.bigtop.manager.common.constants.Constants;
import org.apache.bigtop.manager.common.utils.JsonUtils;
import org.apache.bigtop.manager.common.utils.YamlUtils;
import org.apache.bigtop.manager.stack.common.enums.ConfigType;
+import org.apache.bigtop.manager.stack.common.exception.StackException;
import org.apache.bigtop.manager.stack.common.utils.template.TemplateUtils;
import org.apache.commons.lang3.StringUtils;
@@ -226,7 +227,7 @@ public class LinuxFileUtils {
Path path = Paths.get(dirPath);
if (Files.isSymbolicLink(path)) {
- log.warn("unable to create symbolic link: [{}]", dirPath);
+ log.error("Directory is symbolic link: [{}]", dirPath);
return;
}
@@ -240,4 +241,40 @@ public class LinuxFileUtils {
updateOwner(dirPath, owner, group, recursive);
updatePermissions(dirPath, permissions, recursive);
}
+
+ /**
+ * create symbolic link
+ *
+ * @param target target file
+ * @param source source file
+ */
+ public static void createSymbolicLink(String target, String source) {
+ if (StringUtils.isBlank(target) || StringUtils.isBlank(source)) {
+ log.error("target, source must not be null");
+ return;
+ }
+
+ Path targetPath = Paths.get(target);
+ Path sourcePath = Paths.get(source);
+
+ try {
+ log.info("Creating symbolic link: [{}] -> [{}]", target, source);
+ if (Files.exists(targetPath)) {
+ if (Files.isSymbolicLink(targetPath)) {
+ Path existingSourcePath =
Files.readSymbolicLink(targetPath);
+ if (!existingSourcePath.equals(sourcePath)) {
+ throw new IOException(
+ "Symbolic link already exists and points to a
different source: " + existingSourcePath);
+ }
+ } else {
+ throw new IOException("Target path exists and is not a
symbolic link: " + target);
+ }
+ }
+
+ Files.createSymbolicLink(targetPath, sourcePath);
+ } catch (IOException e) {
+ log.error("Error when create symbolic link", e);
+ throw new StackException(e);
+ }
+ }
}
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/hook/AbstractHook.java
b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/hook/AbstractHook.java
index c07b5ec..a6b2b67 100644
---
a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/hook/AbstractHook.java
+++
b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/hook/AbstractHook.java
@@ -58,7 +58,7 @@ public abstract class AbstractHook implements Hook {
}
log.info(
- "Adding user: {} to group: {} and groups: [{}]",
+ "Adding user: {} to primary group: {} and supplementary
groups: [{}]",
user.getKey(),
userGroup,
String.join(",", groups));
diff --git a/bigtop-manager-stack/pom.xml b/bigtop-manager-stack/pom.xml
index 98556bb..c9bf863 100644
--- a/bigtop-manager-stack/pom.xml
+++ b/bigtop-manager-stack/pom.xml
@@ -52,6 +52,7 @@
<type>pom</type>
<scope>import</scope>
</dependency>
+
<dependency>
<groupId>com.google.auto.service</groupId>
<artifactId>auto-service</artifactId>
diff --git a/dev-support/docker/rocky8/build-containers.sh
b/dev-support/docker/rocky8/build-containers.sh
index fda727a..effd34f 100755
--- a/dev-support/docker/rocky8/build-containers.sh
+++ b/dev-support/docker/rocky8/build-containers.sh
@@ -50,7 +50,7 @@ echo -e "\033[32mCreating network bigtop-manager\033[0m"
docker network create --driver bridge bigtop-manager
echo -e "\033[32mCreating container bigtop-manager-server\033[0m"
-docker run -it -d -p 13306:3306 -p 15005:5005 -p 18080:8080 --name
bigtop-manager-server --hostname bigtop-manager-server --network bigtop-manager
--cap-add=SYS_TIME bigtop-manager/develop:trunk-rocky-8
+docker run -it -d -p 13306:3306 -p 15005:5005 -p 15006:5006 -p 18080:8080
--name bigtop-manager-server --hostname bigtop-manager-server --network
bigtop-manager --cap-add=SYS_TIME bigtop-manager/develop:trunk-rocky-8
docker cp ../../../bigtop-manager-server/target/bigtop-manager-server
bigtop-manager-server:/opt/
docker cp ../../../bigtop-manager-agent/target/bigtop-manager-agent
bigtop-manager-server:/opt/
SERVER_PUB_KEY=`docker exec bigtop-manager-server /bin/cat
/root/.ssh/id_rsa.pub`
@@ -116,7 +116,7 @@ docker exec bigtop-manager-agent-01 bash -c "sed -i
's/host: localhost/host: $BI
docker exec bigtop-manager-agent-02 bash -c "sed -i 's/host: localhost/host:
$BIGTOP_MANAGER_SERVER_IP/' /opt/bigtop-manager-agent/conf/application.yml"
docker exec bigtop-manager-server bash -c "nohup /bin/bash
/opt/bigtop-manager-server/bin/start.sh --debug > /dev/null 2>&1 &"
-docker exec bigtop-manager-server bash -c "nohup /bin/bash
/opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &"
+docker exec bigtop-manager-server bash -c "nohup /bin/bash
/opt/bigtop-manager-agent/bin/start.sh --debug > /dev/null 2>&1 &"
docker exec bigtop-manager-agent-01 bash -c "nohup /bin/bash
/opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &"
docker exec bigtop-manager-agent-02 bash -c "nohup /bin/bash
/opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &"
