This is an automated email from the ASF dual-hosted git repository.
wuzhiguo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/bigtop-manager.git
The following commit(s) were added to refs/heads/main by this push:
new 7d3c388d BIGTOP-4287: Support Agent remotely installed by Server (#112)
7d3c388d is described below
commit 7d3c388d62a9cc4930b2eb3538cadb477407d411
Author: Zhiguo Wu <[email protected]>
AuthorDate: Thu Nov 28 09:59:08 2024 +0800
BIGTOP-4287: Support Agent remotely installed by Server (#112)
---
.../src/main/resources/assembly/agent.xml | 6 +-
.../manager/server/controller/HostController.java | 7 ++
.../manager/server/enums/ApiExceptionEnum.java | 3 +-
.../bigtop/manager/server/enums/LocaleKeys.java | 1 +
.../manager/server/model/req/HostPathReq.java | 34 ++++++
.../bigtop/manager/server/service/HostService.java | 9 ++
.../server/service/impl/HostServiceImpl.java | 116 ++++++++++++++++++---
.../src/main/resources/assembly/server.xml | 6 +-
.../src/main/resources/ddl/MySQL-DDL-CREATE.sql | 2 +-
.../main/resources/ddl/PostgreSQL-DDL-CREATE.sql | 2 +-
.../main/resources/i18n/messages_en_US.properties | 1 +
.../main/resources/i18n/messages_zh_CN.properties | 1 +
dev-support/docker/containers/build.sh | 6 +-
13 files changed, 166 insertions(+), 28 deletions(-)
diff --git a/bigtop-manager-agent/src/main/resources/assembly/agent.xml
b/bigtop-manager-agent/src/main/resources/assembly/agent.xml
index a8d51bb9..e464a927 100644
--- a/bigtop-manager-agent/src/main/resources/assembly/agent.xml
+++ b/bigtop-manager-agent/src/main/resources/assembly/agent.xml
@@ -22,10 +22,10 @@
xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.1.0
http://maven.apache.org/xsd/assembly-2.1.0.xsd";>
<id>agent</id>
<formats>
- <format>dir</format>
+ <format>tar.gz</format>
</formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <baseDirectory>agent</baseDirectory>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <baseDirectory>bigtop-manager-agent</baseDirectory>
<fileSets>
<fileSet>
<directory>${basedir}/src/main/resources</directory>
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/controller/HostController.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/controller/HostController.java
index 5f6890eb..b8fdafcf 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/controller/HostController.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/controller/HostController.java
@@ -21,6 +21,7 @@ package org.apache.bigtop.manager.server.controller;
import org.apache.bigtop.manager.dao.query.HostQuery;
import org.apache.bigtop.manager.server.model.converter.HostConverter;
import org.apache.bigtop.manager.server.model.dto.HostDTO;
+import org.apache.bigtop.manager.server.model.req.HostPathReq;
import org.apache.bigtop.manager.server.model.req.HostReq;
import org.apache.bigtop.manager.server.model.vo.HostVO;
import org.apache.bigtop.manager.server.model.vo.PageVO;
@@ -103,4 +104,10 @@ public class HostController {
HostDTO hostDTO = HostConverter.INSTANCE.fromReq2DTO(hostReq);
return ResponseEntity.success(hostService.checkConnection(hostDTO));
}
+
+ @Operation(summary = "Install dependencies", description = "Install
dependencies on a host")
+ @PostMapping("/install-dependencies")
+ public ResponseEntity<Boolean> checkConnection(@RequestBody @Validated
HostPathReq hostPathReq) {
+ return
ResponseEntity.success(hostService.installDependencies(hostPathReq.getHostIds(),
hostPathReq.getPath()));
+ }
}
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/enums/ApiExceptionEnum.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/enums/ApiExceptionEnum.java
index 2613fc47..26d432bb 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/enums/ApiExceptionEnum.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/enums/ApiExceptionEnum.java
@@ -39,7 +39,8 @@ public enum ApiExceptionEnum {
HOST_ASSIGNED(12001, LocaleKeys.HOST_ASSIGNED),
HOST_NOT_CONNECTED(12002, LocaleKeys.HOST_NOT_CONNECTED),
HOST_UNABLE_TO_CONNECT(12003, LocaleKeys.HOST_UNABLE_TO_CONNECT),
- HOST_HAS_COMPONENTS(12004, LocaleKeys.HOST_HAS_COMPONENTS),
+ HOST_UNABLE_TO_EXEC_COMMAND(12004, LocaleKeys.HOST_UNABLE_TO_EXEC_COMMAND),
+ HOST_HAS_COMPONENTS(12005, LocaleKeys.HOST_HAS_COMPONENTS),
// Stack Exceptions -- 13000 ~ 13999
STACK_NOT_FOUND(13000, LocaleKeys.STACK_NOT_FOUND),
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/enums/LocaleKeys.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/enums/LocaleKeys.java
index 94314912..e42d177f 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/enums/LocaleKeys.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/enums/LocaleKeys.java
@@ -41,6 +41,7 @@ public enum LocaleKeys {
HOST_ASSIGNED("host.assigned"),
HOST_NOT_CONNECTED("host.not.connected"),
HOST_UNABLE_TO_CONNECT("host.unable.to.connect"),
+ HOST_UNABLE_TO_EXEC_COMMAND("host.unable.to.exec.command"),
HOST_HAS_COMPONENTS("host.has.components"),
STACK_NOT_FOUND("stack.not.found"),
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/HostPathReq.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/HostPathReq.java
new file mode 100644
index 00000000..a3d39bd4
--- /dev/null
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/HostPathReq.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bigtop.manager.server.model.req;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class HostPathReq {
+
+ @Schema(example = "[1, 2]")
+ private List<Long> hostIds;
+
+ @Schema(example = "/opt")
+ private String path;
+}
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/HostService.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/HostService.java
index 4de47e85..210cae4b 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/HostService.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/HostService.java
@@ -71,4 +71,13 @@ public interface HostService {
* @return true if all hosts are able to connect
*/
Boolean checkConnection(HostDTO hostDTO);
+
+ /**
+ * Install dependencies
+ *
+ * @param hostIds host ids
+ * @param path remote host path
+ * @return true if all dependencies are installed
+ */
+ Boolean installDependencies(List<Long> hostIds, String path);
}
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/HostServiceImpl.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/HostServiceImpl.java
index 30de0974..bf8cdb06 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/HostServiceImpl.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/HostServiceImpl.java
@@ -18,11 +18,14 @@
*/
package org.apache.bigtop.manager.server.service.impl;
+import org.apache.bigtop.manager.common.constants.MessageConstants;
import org.apache.bigtop.manager.common.shell.ShellResult;
import org.apache.bigtop.manager.dao.po.HostPO;
+import org.apache.bigtop.manager.dao.po.RepoPO;
import org.apache.bigtop.manager.dao.query.HostQuery;
import org.apache.bigtop.manager.dao.repository.ComponentDao;
import org.apache.bigtop.manager.dao.repository.HostDao;
+import org.apache.bigtop.manager.dao.repository.RepoDao;
import org.apache.bigtop.manager.server.enums.ApiExceptionEnum;
import org.apache.bigtop.manager.server.enums.HealthyStatusEnum;
import org.apache.bigtop.manager.server.enums.HostAuthTypeEnum;
@@ -53,6 +56,9 @@ import java.util.stream.Collectors;
@Service
public class HostServiceImpl implements HostService {
+ @Resource
+ private RepoDao repoDao;
+
@Resource
private HostDao hostDao;
@@ -140,25 +146,9 @@ public class HostServiceImpl implements HostService {
@Override
public Boolean checkConnection(HostDTO hostDTO) {
String command = "hostname";
- HostAuthTypeEnum authType =
HostAuthTypeEnum.fromCode(hostDTO.getAuthType());
for (String hostname : hostDTO.getHostnames()) {
try {
- ShellResult result = null;
- switch (authType) {
- case PASSWORD -> result = RemoteSSHUtils.executeCommand(
- hostname, hostDTO.getSshPort(),
hostDTO.getSshUser(), hostDTO.getSshPassword(), command);
- case KEY -> result = RemoteSSHUtils.executeCommand(
- hostname,
- hostDTO.getSshPort(),
- hostDTO.getSshUser(),
- hostDTO.getSshKeyFilename(),
- hostDTO.getSshKeyString(),
- hostDTO.getSshKeyPassword(),
- command);
- case NO_AUTH -> result = RemoteSSHUtils.executeCommand(
- hostname, hostDTO.getSshPort(),
hostDTO.getSshUser(), command);
- }
-
+ ShellResult result = execCommandOnRemoteHost(hostDTO,
hostname, command);
if (result.getExitCode() != 0) {
log.error("Unable to connect to host, hostname: {}, msg:
{}", hostname, result.getErrMsg());
throw new
ApiException(ApiExceptionEnum.HOST_UNABLE_TO_CONNECT, hostname);
@@ -173,4 +163,96 @@ public class HostServiceImpl implements HostService {
return true;
}
+
+ @Override
+ public Boolean installDependencies(List<Long> hostIds, String path) {
+ List<RepoPO> repoPOList = repoDao.findAll();
+ Map<String, RepoPO> archRepoMap = repoPOList.stream()
+ .filter(repoPO -> repoPO.getType() == 2)
+ .collect(Collectors.toMap(RepoPO::getArch, repo -> repo));
+
+ List<HostPO> hostPOList = hostDao.findByIds(hostIds);
+ for (HostPO hostPO : hostPOList) {
+ HostDTO hostDTO = HostConverter.INSTANCE.fromPO2DTO(hostPO);
+
+ // Get host arch
+ String arch = execCommandOnRemoteHost(hostDTO,
hostDTO.getHostname(), "arch")
+ .getOutput()
+ .trim();
+ arch = arch.equals("arm64") ? "aarch64" : arch;
+
+ // Download & Extract agent tarball
+ String repoUrl = archRepoMap.get(arch).getBaseUrl();
+ String tarballUrl = repoUrl + "/bigtop-manager-agent.tar.gz";
+ String command = "curl -L " + tarballUrl + " | tar -xz -C " + path;
+ ShellResult result = execCommandOnRemoteHost(hostDTO,
hostDTO.getHostname(), command);
+ if (result.getExitCode() != MessageConstants.SUCCESS_CODE) {
+ hostPO.setErrInfo(result.getErrMsg());
+ hostDao.updateById(hostPO);
+
+ log.error(
+ "Unable to download & extract agent tarball, hostname:
{}, msg: {}",
+ hostDTO.getHostname(),
+ result.getErrMsg());
+ throw new
ApiException(ApiExceptionEnum.HOST_UNABLE_TO_EXEC_COMMAND,
hostDTO.getHostname());
+ }
+
+ // Run agent in background
+ command = "nohup " + path + "/bigtop-manager-agent/bin/start.sh >
/dev/null 2>&1 &";
+ result = execCommandOnRemoteHost(hostDTO, hostDTO.getHostname(),
command);
+ if (result.getExitCode() != MessageConstants.SUCCESS_CODE) {
+ hostPO.setErrInfo(result.getErrMsg());
+ hostDao.updateById(hostPO);
+
+ log.error("Unable to start agent, hostname: {}, msg: {}",
hostDTO.getHostname(), result.getErrMsg());
+ throw new
ApiException(ApiExceptionEnum.HOST_UNABLE_TO_EXEC_COMMAND,
hostDTO.getHostname());
+ }
+
+ // Check the process, the agent may encounter some errors and exit
when starting
+ // So we need to wait for a while before the check
+ try {
+ Thread.sleep(10 * 1000);
+ } catch (InterruptedException e) {
+ log.error("Thread sleep interrupted", e);
+ }
+ command = "ps -ef | grep bigtop-manager-agent | grep -v grep";
+ result = execCommandOnRemoteHost(hostDTO, hostDTO.getHostname(),
command);
+ if (result.getExitCode() != MessageConstants.SUCCESS_CODE
+ || !result.getOutput().contains("bigtop-manager-agent")) {
+ hostPO.setErrInfo("Unable to start agent process, please check
the log");
+ hostDao.updateById(hostPO);
+
+ log.error("Unable to start agent process, hostname: {}",
hostDTO.getHostname());
+ throw new
ApiException(ApiExceptionEnum.HOST_UNABLE_TO_EXEC_COMMAND,
hostDTO.getHostname());
+ }
+
+ hostPO.setStatus(HealthyStatusEnum.HEALTHY.getCode());
+ hostDao.updateById(hostPO);
+ }
+
+ return true;
+ }
+
+ private ShellResult execCommandOnRemoteHost(HostDTO hostDTO, String
hostname, String command) {
+ HostAuthTypeEnum authType =
HostAuthTypeEnum.fromCode(hostDTO.getAuthType());
+ try {
+ return switch (authType) {
+ case PASSWORD -> RemoteSSHUtils.executeCommand(
+ hostname, hostDTO.getSshPort(), hostDTO.getSshUser(),
hostDTO.getSshPassword(), command);
+ case KEY -> RemoteSSHUtils.executeCommand(
+ hostname,
+ hostDTO.getSshPort(),
+ hostDTO.getSshUser(),
+ hostDTO.getSshKeyFilename(),
+ hostDTO.getSshKeyString(),
+ hostDTO.getSshKeyPassword(),
+ command);
+ case NO_AUTH -> RemoteSSHUtils.executeCommand(
+ hostname, hostDTO.getSshPort(), hostDTO.getSshUser(),
command);
+ };
+ } catch (Exception e) {
+ log.error("Unable to exec command on host, hostname: {}, command:
{}", hostname, command, e);
+ throw new
ApiException(ApiExceptionEnum.HOST_UNABLE_TO_EXEC_COMMAND, hostname);
+ }
+ }
}
diff --git a/bigtop-manager-server/src/main/resources/assembly/server.xml
b/bigtop-manager-server/src/main/resources/assembly/server.xml
index c8f303f4..d8310722 100644
--- a/bigtop-manager-server/src/main/resources/assembly/server.xml
+++ b/bigtop-manager-server/src/main/resources/assembly/server.xml
@@ -22,10 +22,10 @@
xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.1.0
http://maven.apache.org/xsd/assembly-2.1.0.xsd";>
<id>server</id>
<formats>
- <format>dir</format>
+ <format>tar.gz</format>
</formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <baseDirectory>server</baseDirectory>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <baseDirectory>bigtop-manager-server</baseDirectory>
<fileSets>
<fileSet>
<directory>${basedir}/src/main/resources</directory>
diff --git a/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql
b/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql
index db5dcbd7..761abea4 100644
--- a/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql
+++ b/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql
@@ -109,7 +109,7 @@ CREATE TABLE `host`
`total_memory_size` BIGINT,
`desc` VARCHAR(255) DEFAULT NULL,
`status` INTEGER DEFAULT NULL COMMENT '1-healthy,
2-unhealthy, 3-unknown',
- `err_info` VARCHAR(255) DEFAULT NULL,
+ `err_info` TEXT DEFAULT NULL,
`create_time` DATETIME DEFAULT CURRENT_TIMESTAMP,
`update_time` DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE
CURRENT_TIMESTAMP,
`create_by` BIGINT,
diff --git
a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql
b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql
index 9cdff18d..060893b9 100644
--- a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql
+++ b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql
@@ -97,7 +97,7 @@ CREATE TABLE host
total_memory_size BIGINT,
"desc" VARCHAR(255) DEFAULT NULL,
status INT DEFAULT NULL,
- err_info VARCHAR(255) DEFAULT NULL,
+ err_info TEXT DEFAULT NULL,
create_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP,
update_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP,
create_by BIGINT,
diff --git
a/bigtop-manager-server/src/main/resources/i18n/messages_en_US.properties
b/bigtop-manager-server/src/main/resources/i18n/messages_en_US.properties
index 03f44141..d18f087f 100644
--- a/bigtop-manager-server/src/main/resources/i18n/messages_en_US.properties
+++ b/bigtop-manager-server/src/main/resources/i18n/messages_en_US.properties
@@ -35,6 +35,7 @@ host.not.found=Host not exist
host.assigned=Hosts [{0}] already assigned to another cluster
host.not.connected=Hosts [{0}] not connected
host.unable.to.connect=Unable to connect to host [{0}]
+host.unable.to.exec.command=Unable to execute command on host [{0}]
host.has.components=Host still has components, please remove them first
stack.not.found=Stack not exist
diff --git
a/bigtop-manager-server/src/main/resources/i18n/messages_zh_CN.properties
b/bigtop-manager-server/src/main/resources/i18n/messages_zh_CN.properties
index df922bb3..a12bbc2b 100644
--- a/bigtop-manager-server/src/main/resources/i18n/messages_zh_CN.properties
+++ b/bigtop-manager-server/src/main/resources/i18n/messages_zh_CN.properties
@@ -35,6 +35,7 @@ host.not.found=主机不存在
host.assigned=主机 [{0}] 已属于其他集群
host.not.connected=主机 [{0}] 未连接
host.unable.to.connect=无法连接到主机 [{0}]
+host.unable.to.exec.command=无法在主机 [{0}] 上执行命令
host.has.components=主机上仍有组件，请先移除
stack.not.found=组件栈不存在
diff --git a/dev-support/docker/containers/build.sh
b/dev-support/docker/containers/build.sh
index afe8090d..4fc0f4cc 100644
--- a/dev-support/docker/containers/build.sh
+++ b/dev-support/docker/containers/build.sh
@@ -67,12 +67,14 @@ create_container() {
log "Create ${container_name}"
if [ $i -eq 1 ]; then
docker run -itd -p 15005:5005 -p 15006:5006 -p 18080:8080 --name
${container_name} --hostname ${container_name} --network bigtop-manager
--cap-add=SYS_TIME bigtop-manager/develop:${OS}
- docker cp ../../../bigtop-manager-server/target/bigtop-manager-server
${container_name}:/opt/
+ docker cp
../../../bigtop-manager-server/target/bigtop-manager-server.tar.gz
${container_name}:/opt/
+ docker exec ${container_name} bash -c "cd /opt && tar -zxvf
bigtop-manager-server.tar.gz"
SERVER_PUB_KEY=`docker exec ${container_name} /bin/cat
/root/.ssh/id_rsa.pub`
else
docker run -itd --name ${container_name} --hostname ${container_name}
--network bigtop-manager --cap-add=SYS_TIME bigtop-manager/develop:${OS}
fi
- docker cp ../../../bigtop-manager-agent/target/bigtop-manager-agent
${container_name}:/opt/
+ docker cp
../../../bigtop-manager-agent/target/bigtop-manager-agent.tar.gz
${container_name}:/opt/
+ docker exec ${container_name} bash -c "cd /opt && tar -zxvf
bigtop-manager-agent.tar.gz"
docker exec ${container_name} bash -c "echo '$SERVER_PUB_KEY' >
/root/.ssh/authorized_keys"
docker exec ${container_name} ssh-keygen -N '' -t rsa -b 2048 -f
/etc/ssh/ssh_host_rsa_key
docker exec ${container_name} ssh-keygen -N '' -t ecdsa -b 256 -f
/etc/ssh/ssh_host_ecdsa_key
