This is an automated email from the ASF dual-hosted git repository.
wuzhiguo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/bigtop-manager.git
The following commit(s) were added to refs/heads/main by this push:
new bad99f18 BIGTOP-4319: Add clusterHosts info to infra services (#141)
bad99f18 is described below
commit bad99f18114c2e88a8d4d7200dccc5bb3f3ad64d
Author: haopeng <[email protected]>
AuthorDate: Tue Jan 7 11:31:51 2025 +0800
BIGTOP-4319: Add clusterHosts info to infra services (#141)
---
.../message/entity/payload/CommandPayload.java | 7 +++
.../org/apache/bigtop/manager/dao/po/HostPO.java | 3 ++
.../src/main/resources/mapper/mysql/HostMapper.xml | 6 +--
.../resources/mapper/postgresql/HostMapper.xml | 6 +--
.../command/stage/AbstractComponentStage.java | 17 +++++++
.../server/command/task/AbstractComponentTask.java | 4 +-
.../bigtop/manager/server/model/vo/HostVO.java | 2 +
.../manager/stack/infra/param/InfraParams.java | 7 +++
.../infra/v1_0_0/prometheus/PrometheusParams.java | 52 +++++++++++++---------
.../infra/v1_0_0/prometheus/PrometheusSetup.java | 8 +---
10 files changed, 78 insertions(+), 34 deletions(-)
diff --git
a/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/payload/CommandPayload.java
b/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/payload/CommandPayload.java
index 6f9402ae..65214b9f 100644
---
a/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/payload/CommandPayload.java
+++
b/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/payload/CommandPayload.java
@@ -27,6 +27,7 @@ import lombok.NoArgsConstructor;
import lombok.ToString;
import java.util.List;
+import java.util.Map;
@Data
@ToString(callSuper = true)
@@ -49,4 +50,10 @@ public class CommandPayload extends BasePayload {
private String componentName;
private List<PackageSpecificInfo> packageSpecifics;
+
+ /**
+ * This field is exclusively used for Prometheus and Grafana within the
infra services.
+ * Includes cluster and corresponding hostname.
+ */
+ private Map<String, List<String>> clusterHosts;
}
diff --git
a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/HostPO.java
b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/HostPO.java
index 9d685ab0..93f0bf86 100644
---
a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/HostPO.java
+++
b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/HostPO.java
@@ -108,6 +108,9 @@ public class HostPO extends BasePO implements Serializable {
@Transient
private String clusterName;
+ @Transient
+ private String clusterDisplayName;
+
@Transient
private Integer componentNum;
}
diff --git a/bigtop-manager-dao/src/main/resources/mapper/mysql/HostMapper.xml
b/bigtop-manager-dao/src/main/resources/mapper/mysql/HostMapper.xml
index 9a16c413..2d27411f 100644
--- a/bigtop-manager-dao/src/main/resources/mapper/mysql/HostMapper.xml
+++ b/bigtop-manager-dao/src/main/resources/mapper/mysql/HostMapper.xml
@@ -42,7 +42,7 @@
select
<include refid="baseColumnsV2">
<property name="alias" value="h"/>
- </include>, c.display_name as cluster_name, count(comp.id) as
component_num
+ </include>, c.name as cluster_name, c.display_name as
cluster_display_name, count(comp.id) as component_num
from host h
left join cluster c on h.cluster_id = c.id
left join component comp on h.id = comp.host_id
@@ -67,7 +67,7 @@
select
<include refid="baseColumnsV2">
<property name="alias" value="h"/>
- </include>, c.display_name as cluster_name, count(comp.id) as
component_num
+ </include>, c.name as cluster_name, c.display_name as
cluster_display_name, count(comp.id) as component_num
from host h
left join cluster c on h.cluster_id = c.id
left join component comp on h.id = comp.host_id
@@ -95,7 +95,7 @@
<include refid="baseColumnsV2">
<property name="alias" value="h"/>
</include>
- ,clus.display_name as cluster_name
+ , clus.name as cluster_name, clus.display_name as cluster_display_name
from host h
left join cluster clus
on h.cluster_id = clus.id
diff --git
a/bigtop-manager-dao/src/main/resources/mapper/postgresql/HostMapper.xml
b/bigtop-manager-dao/src/main/resources/mapper/postgresql/HostMapper.xml
index 42e2235e..2b1fdb9c 100644
--- a/bigtop-manager-dao/src/main/resources/mapper/postgresql/HostMapper.xml
+++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/HostMapper.xml
@@ -42,7 +42,7 @@
select
<include refid="baseColumnsV2">
<property name="alias" value="h"/>
- </include>, c.display_name as cluster_name, count(comp.id) as
component_num
+ </include>, c.name as cluster_name, c.display_name as
cluster_display_name, count(comp.id) as component_num
from host h
left join cluster c on h.cluster_id = c.id
left join component comp on h.id = comp.host_id
@@ -67,7 +67,7 @@
select
<include refid="baseColumnsV2">
<property name="alias" value="h"/>
- </include>, c.display_name as cluster_name, count(comp.id) as
component_num
+ </include>, c.name as cluster_name, c.display_name as
cluster_display_name, count(comp.id) as component_num
from host h
left join cluster c on h.cluster_id = c.id
left join component comp on h.id = comp.host_id
@@ -95,7 +95,7 @@
<include refid="baseColumnsV2">
<property name="alias" value="h"/>
</include>
- ,clus.display_name as cluster_name
+ , clus.name as cluster_name, clus.display_name as cluster_display_name
from
host h
left join cluster clus
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractComponentStage.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractComponentStage.java
index 29c86a8d..634b8554 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractComponentStage.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractComponentStage.java
@@ -19,13 +19,16 @@
package org.apache.bigtop.manager.server.command.stage;
import org.apache.bigtop.manager.dao.po.ClusterPO;
+import org.apache.bigtop.manager.dao.po.HostPO;
import org.apache.bigtop.manager.dao.repository.ClusterDao;
import org.apache.bigtop.manager.server.command.task.TaskContext;
import org.apache.bigtop.manager.server.holder.SpringContextHolder;
import org.apache.bigtop.manager.server.model.dto.ComponentDTO;
import org.apache.bigtop.manager.server.model.dto.ServiceDTO;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
public abstract class AbstractComponentStage extends AbstractStage {
@@ -77,7 +80,21 @@ public abstract class AbstractComponentStage extends
AbstractStage {
Map<String, Object> properties = new HashMap<>();
properties.put("packageSpecifics", serviceDTO.getPackageSpecifics());
+ properties.put("clusterHosts", getClusterHosts());
taskContext.setProperties(properties);
return taskContext;
}
+
+ private Map<String, List<String>> getClusterHosts() {
+ Map<String, List<String>> clusterHosts = new HashMap<>();
+ for (ClusterPO clusterPO : clusterDao.findAll()) {
+ List<String> hosts = new ArrayList<>();
+ for (HostPO hostPO :
hostDao.findAllByClusterId(clusterPO.getId())) {
+ String host = hostPO.getHostname();
+ hosts.add(host);
+ }
+ clusterHosts.put(clusterPO.getName(), hosts);
+ }
+ return clusterHosts;
+ }
}
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractComponentTask.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractComponentTask.java
index 4071ae66..b2d0f377 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractComponentTask.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractComponentTask.java
@@ -67,7 +67,9 @@ public abstract class AbstractComponentTask extends
AbstractTask {
commandPayload.setPackageSpecifics(
convertPackageSpecificInfo((List<PackageSpecificDTO>)
properties.get("packageSpecifics")));
-
+ if (stackDTO.getStackName().equals("infra")) {
+ commandPayload.setClusterHosts((Map<String, List<String>>)
properties.get("clusterHosts"));
+ }
CommandRequest.Builder builder = CommandRequest.newBuilder();
builder.setType(CommandType.COMPONENT);
builder.setPayload(JsonUtils.writeAsString(commandPayload));
diff --git
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/HostVO.java
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/HostVO.java
index f79cab2f..92c77f67 100644
---
a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/HostVO.java
+++
b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/HostVO.java
@@ -71,5 +71,7 @@ public class HostVO {
private String clusterName;
+ private String clusterDisplayName;
+
private Integer componentNum;
}
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/param/InfraParams.java
b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/param/InfraParams.java
index 94934abb..a5701b84 100644
---
a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/param/InfraParams.java
+++
b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/param/InfraParams.java
@@ -28,6 +28,8 @@ import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import java.io.File;
+import java.util.List;
+import java.util.Map;
@Slf4j
@NoArgsConstructor
@@ -72,4 +74,9 @@ public abstract class InfraParams extends BaseParams {
return parentPath + "/infras";
}
+
+ public Map<String, List<String>> getClusterHosts() {
+ // In Component Status stage, clusterHosts is null
+ return commandPayload.getClusterHosts();
+ }
}
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusParams.java
b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusParams.java
index d9af54a3..e8efa718 100644
---
a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusParams.java
+++
b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusParams.java
@@ -44,6 +44,7 @@ public class PrometheusParams extends InfraParams {
protected final String PROMETHEUS_SELF_JOB_NAME = "prometheus";
protected final String BM_AGENT_JOB_NAME = "bm-agent";
protected final String BM_AGENT_PORT = "8081";
+ protected final String AGENT_TARGET_LABEL = "cluster";
private Map<String, Object> prometheusScrapeJob;
private Map<String, Object> agentScrapeJob;
@@ -55,6 +56,7 @@ public class PrometheusParams extends InfraParams {
public PrometheusParams(CommandPayload commandPayload) {
super(commandPayload);
+ setAgentScrapeJob();
scrapeJobs = new ArrayList<>();
scrapeJobs.add(prometheusScrapeJob);
scrapeJobs.add(agentScrapeJob);
@@ -75,15 +77,6 @@ public class PrometheusParams extends InfraParams {
return "prometheus";
}
- protected List<String> getAllHost() {
- List<String> ips = LocalSettings.hosts().get("all");
- List<String> hosts = new ArrayList<>();
- for (String ip : ips) {
- hosts.add(MessageFormat.format("{0}:{1}", ip, BM_AGENT_PORT));
- }
- return hosts;
- }
-
@GlobalParams
public Map<String, Object> prometheusJob() {
Map<String, Object> configuration =
LocalSettings.configurations(getServiceName(), "prometheus");
@@ -91,22 +84,15 @@ public class PrometheusParams extends InfraParams {
Map<String, Object> job = new HashMap<>();
job.put("name", PROMETHEUS_SELF_JOB_NAME);
job.put("targets_file", targetsConfigFile(PROMETHEUS_SELF_JOB_NAME));
- job.put("targets_list", List.of(MessageFormat.format("localhost:{0}",
prometheusPort)));
+
+ Map<String, Object> target = new HashMap<>();
+ target.put("targets", List.of(MessageFormat.format("localhost:{0}",
prometheusPort)));
+ job.put("targets_list", List.of(target));
+
prometheusScrapeJob = job;
return configuration;
}
- @GlobalParams
- public Map<String, Object> agentJob() {
- Map<String, Object> job = new HashMap<>();
- job.put("name", BM_AGENT_JOB_NAME);
- job.put("targets_file", targetsConfigFile(BM_AGENT_JOB_NAME));
- job.put("targets_list", getAllHost());
- job.put("metrics_path", "/actuator/prometheus");
- agentScrapeJob = job;
- return LocalSettings.configurations(getServiceName(), "prometheus");
- }
-
@GlobalParams
public Map<String, Object> configs() {
Map<String, Object> configuration =
LocalSettings.configurations(getServiceName(), "prometheus");
@@ -127,4 +113,28 @@ public class PrometheusParams extends InfraParams {
public String listenAddress() {
return MessageFormat.format("0.0.0.0:{0}", prometheusPort);
}
+
+ public void setAgentScrapeJob() {
+ agentScrapeJob = new HashMap<>();
+ agentScrapeJob.put("name", BM_AGENT_JOB_NAME);
+ agentScrapeJob.put("targets_file",
targetsConfigFile(BM_AGENT_JOB_NAME));
+ agentScrapeJob.put("metrics_path", "/actuator/prometheus");
+
+ List<Map<String, Object>> agentTargets = new ArrayList<>();
+ Map<String, List<String>> clusterHosts = getClusterHosts();
+ if (clusterHosts != null) {
+ clusterHosts.forEach((cluster, hosts) -> {
+ Map<String, Object> agentTarget = new HashMap<>();
+ List<String> targets = new ArrayList<>();
+ for (String host : hosts) {
+ targets.add(MessageFormat.format("{0}:{1}", host,
BM_AGENT_PORT));
+ }
+ agentTarget.put("targets", targets);
+ agentTarget.put("labels", Map.of(AGENT_TARGET_LABEL, cluster));
+ agentTargets.add(agentTarget);
+ });
+ }
+
+ agentScrapeJob.put("targets_list", agentTargets);
+ }
}
diff --git
a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusSetup.java
b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusSetup.java
index 89d015de..a67e7f49 100644
---
a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusSetup.java
+++
b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusSetup.java
@@ -29,15 +29,12 @@ import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import java.text.MessageFormat;
-import java.util.HashMap;
-import java.util.List;
import java.util.Map;
@Slf4j
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class PrometheusSetup {
- @SuppressWarnings("unchecked")
public static ShellResult config(Params params) {
PrometheusParams prometheusParams = (PrometheusParams) params;
String user = prometheusParams.user();
@@ -65,16 +62,15 @@ public class PrometheusSetup {
for (int i = 0; i < prometheusParams.getScrapeJobs().size(); i++) {
Map<String, Object> job = prometheusParams.getScrapeJobs().get(i);
- Map<String, List<String>> targets = new HashMap<>();
- targets.put("targets", (List<String>) job.get("targets_list"));
LinuxFileUtils.toFile(
ConfigType.JSON,
(String) job.get("targets_file"),
user,
group,
Constants.PERMISSION_644,
- List.of(targets));
+ job.get("targets_list"));
}
+
return ShellResult.success("Prometheus Configure success!");
}
}
