This is an automated email from the ASF dual-hosted git repository.
wuzhiguo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/bigtop-manager.git
The following commit(s) were added to refs/heads/main by this push:
new f64b03e BIGTOP-4187: Upload tez.tar.tz to hdfs after installed (#42)
f64b03e is described below
commit f64b03e5d5a22f6ab50264975fb81a3876870e68
Author: Zhiguo Wu <[email protected]>
AuthorDate: Wed Aug 14 14:32:40 2024 +0800
BIGTOP-4187: Upload tez.tar.tz to hdfs after installed (#42)
---
.../manager/stack/bigtop/utils/HdfsUtil.java | 41 ++++++++++++++++++++--
.../stack/bigtop/v3_3_0/hdfs/HdfsSetup.java | 3 --
.../manager/stack/bigtop/v3_3_0/tez/TezSetup.java | 8 +++--
.../manager/stack/core/executor/StackExecutor.java | 9 +++++
pom.xml | 4 +--
5 files changed, 55 insertions(+), 10 deletions(-)
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/utils/HdfsUtil.java
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/utils/HdfsUtil.java
index 74929f0..d16b54b 100644
---
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/utils/HdfsUtil.java
+++
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/utils/HdfsUtil.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.security.UserGroupInformation;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
+import java.io.File;
import java.net.URI;
import java.security.PrivilegedAction;
import java.text.MessageFormat;
@@ -38,6 +39,13 @@ import java.util.List;
@Data
@Slf4j
public class HdfsUtil {
+
+ /**
+ * Create directory on hdfs if not exist
+ *
+ * @param user the system user to create the directory, which will
infect the directory permission
+ * @param directory the directory path on hdfs
+ */
public static void createDirectory(String user, String directory) {
UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
try {
@@ -46,6 +54,7 @@ public class HdfsUtil {
// Create dest dir if not exist
Path destDirPath = new Path(directory);
if (!fs.exists(destDirPath)) {
+ log.info("Creating directory [{}] on hdfs",
destDirPath);
fs.mkdirs(destDirPath);
}
} catch (Exception e) {
@@ -61,10 +70,25 @@ public class HdfsUtil {
}
}
+ /**
+ * Upload file to hdfs, this will keep original filename on hdfs
+ *
+ * @param user the system user to upload the file, which will
infect the file permission
+ * @param localFilePath the local file path
+ * @param destDir the destination directory on hdfs
+ */
public static void uploadFile(String user, String localFilePath, String
destDir) {
uploadFile(user, localFilePath, destDir, null);
}
+ /**
+ * Upload file to hdfs
+ *
+ * @param user the system user to upload the file, which will
infect the file permission
+ * @param localFilePath the local file path
+ * @param destDir the destination directory on hdfs
+ * @param destFilename the destination filename on hdfs, if null, use the
original filename
+ */
public static void uploadFile(String user, String localFilePath, String
destDir, String destFilename) {
UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
try {
@@ -73,12 +97,19 @@ public class HdfsUtil {
// Create dest dir if not exist
Path destDirPath = new Path(destDir);
if (!fs.exists(destDirPath)) {
+ log.info("Creating directory [{}] on hdfs",
destDirPath);
fs.mkdirs(destDirPath);
}
// upload file
- Path destFilePath = destFilename == null ? new
Path(destDir) : new Path(destDir, destFilename);
- fs.copyFromLocalFile(new Path(localFilePath),
destFilePath);
+ String filename = destFilename == null
+ ?
localFilePath.substring(localFilePath.lastIndexOf(File.separator) + 1)
+ : destFilename;
+ Path destFilePath = new Path(destDir, filename);
+ if (!fs.exists(destFilePath)) {
+ log.info("Uploading [{}] to hdfs [{}]", localFilePath,
destFilePath);
+ fs.copyFromLocalFile(new Path(localFilePath),
destFilePath);
+ }
} catch (Exception e) {
log.error("Error while uploading file to hdfs", e);
throw new StackException(e);
@@ -92,6 +123,12 @@ public class HdfsUtil {
}
}
+ /**
+ * Get the hdfs FileSystem instance
+ *
+ * @return the hdfs FileSystem instance
+ * @throws Exception if any error occurs
+ */
private static FileSystem getFileSystem() throws Exception {
Configuration conf = new Configuration();
conf.addResource(new Path("/etc/hadoop/conf/core-site.xml"));
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsSetup.java
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsSetup.java
index 1cae445..ba8e813 100644
---
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsSetup.java
+++
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsSetup.java
@@ -150,9 +150,6 @@ public class HdfsSetup {
Constants.PERMISSION_644,
hdfsParams.getGlobalParamsMap());
- // log.info("Creating /apps on hdfs");
- // HdfsUtil.createDirectory(hdfsUser, "/apps");
-
log.info("Successfully configured HDFS");
return ShellResult.success("HDFS Configure success!");
}
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java
index 5709ace..aaf4ac9 100644
---
a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java
+++
b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java
@@ -20,8 +20,10 @@ package org.apache.bigtop.manager.stack.bigtop.v3_3_0.tez;
import org.apache.bigtop.manager.common.constants.Constants;
import org.apache.bigtop.manager.common.shell.ShellResult;
+import org.apache.bigtop.manager.stack.bigtop.utils.HdfsUtil;
import org.apache.bigtop.manager.stack.core.enums.ConfigType;
import org.apache.bigtop.manager.stack.core.param.Params;
+import org.apache.bigtop.manager.stack.core.utils.LocalSettings;
import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils;
import lombok.AccessLevel;
@@ -40,6 +42,7 @@ public class TezSetup {
TezParams tezParams = (TezParams) params;
String confDir = tezParams.confDir();
+ String hdfsUser = LocalSettings.users().get("hdfs");
String tezUser = tezParams.user();
String tezGroup = tezParams.group();
@@ -64,9 +67,8 @@ public class TezSetup {
PERMISSION_755,
tezParams.getGlobalParamsMap());
- // maybe we should upload tez.tar.gz to HDFS here?
- // log.info("Uploading tez.tar.gz to HDFS");
- // HdfsUtil.uploadFile(tezUser, tezParams.serviceHome() +
"/tez.tar.gz", "/apps/tez");
+ HdfsUtil.createDirectory(hdfsUser, "/apps");
+ HdfsUtil.uploadFile(tezUser, tezParams.serviceHome() +
"/lib/tez.tar.gz", "/apps/tez");
log.info("Successfully configured Tez");
return ShellResult.success("Tez Configure success!");
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/executor/StackExecutor.java
b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/executor/StackExecutor.java
index 179bd65..47f1015 100644
---
a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/executor/StackExecutor.java
+++
b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/executor/StackExecutor.java
@@ -23,6 +23,7 @@ import
org.apache.bigtop.manager.common.message.entity.payload.CommandPayload;
import org.apache.bigtop.manager.common.message.entity.pojo.CustomCommandInfo;
import org.apache.bigtop.manager.common.shell.ShellResult;
import org.apache.bigtop.manager.common.utils.CaseUtils;
+import org.apache.bigtop.manager.common.utils.Environments;
import org.apache.bigtop.manager.stack.core.exception.StackException;
import org.apache.bigtop.manager.stack.core.param.Params;
import org.apache.bigtop.manager.stack.core.spi.PrioritySPIFactory;
@@ -67,6 +68,10 @@ public class StackExecutor {
}
private static void runBeforeHook(String command, Params params) {
+ if (Environments.isDevMode()) {
+ return;
+ }
+
Hook hook = HOOK_MAP.get(command.toLowerCase());
if (hook != null) {
hook.before(params);
@@ -74,6 +79,10 @@ public class StackExecutor {
}
private static void runAfterHook(String command, Params params) {
+ if (Environments.isDevMode()) {
+ return;
+ }
+
Hook hook = HOOK_MAP.get(command.toLowerCase());
if (hook != null) {
hook.after(params);
diff --git a/pom.xml b/pom.xml
index 62b3b4a..173c2ef 100644
--- a/pom.xml
+++ b/pom.xml
@@ -50,8 +50,8 @@
<java.version>17</java.version>
<flatten-maven-plugin.version>1.4.0</flatten-maven-plugin.version>
<maven-assembly-plugin.version>3.3.0</maven-assembly-plugin.version>
- <maven-assembly-plugin.version>3.6.0</maven-assembly-plugin.version>
<maven-compiler-plugin.version>3.8.1</maven-compiler-plugin.version>
+
<build-helper-maven-plugin.version>3.6.0</build-helper-maven-plugin.version>
<maven-jar-plugin.version>3.2.0</maven-jar-plugin.version>
<spring-boot-maven-plugin.version>3.1.1</spring-boot-maven-plugin.version>
<spotless-maven-plugin.version>2.43.0</spotless-maven-plugin.version>
@@ -177,7 +177,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
- <version>${maven-assembly-plugin.version}</version>
+ <version>${build-helper-maven-plugin.version}</version>
</plugin>
<plugin>
