This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new c25807604 [#960][part-1] feat(dashboard): Add some dashboard
interfaces. (#1053)
c25807604 is described below
commit c25807604fec4b683c8b105843897a9b54a8d860
Author: yl09099 <[email protected]>
AuthorDate: Thu Nov 9 18:56:24 2023 +0800
[#960][part-1] feat(dashboard): Add some dashboard interfaces. (#1053)
### What changes were proposed in this pull request?
Add a web dashboard for uniffle, providing some dashboard apis
### Why are the changes needed?
Provide data support for dashboard
Fix: #960
### Does this PR introduce _any_ user-facing change?
Users can obtain relevant metrics information through the new API
### How was this patch tested?
Existing UT.
---
.../org/apache/uniffle/common/config/RssConf.java | 4 +
.../uniffle/coordinator/CoordinatorServer.java | 1 +
.../apache/uniffle/coordinator/web/Response.java | 12 +--
.../coordinator/web/resource/APIResource.java | 10 ++
.../web/resource/ApplicationResource.java | 104 +++++++++++++++++++++
.../web/resource/CoordinatorServerResource.java | 87 +++++++++++++++++
.../coordinator/web/resource/ServerResource.java | 20 ++++
.../uniffle/coordinator/web/vo/AppInfoVO.java | 80 ++++++++++++++++
.../APIResource.java => vo/CoordinatorConfVO.java} | 34 ++++---
.../uniffle/coordinator/web/vo/UserAppNumVO.java | 69 ++++++++++++++
10 files changed, 402 insertions(+), 19 deletions(-)
diff --git a/common/src/main/java/org/apache/uniffle/common/config/RssConf.java
b/common/src/main/java/org/apache/uniffle/common/config/RssConf.java
index 805dc30a8..78c56f96e 100644
--- a/common/src/main/java/org/apache/uniffle/common/config/RssConf.java
+++ b/common/src/main/java/org/apache/uniffle/common/config/RssConf.java
@@ -663,6 +663,10 @@ public class RssConf implements Cloneable {
return this.settings.toString();
}
+ public Set<Map.Entry<String, Object>> getAll() {
+ return this.settings.entrySet();
+ }
+
public String getEnv(String key) {
return System.getenv(key);
}
diff --git
a/coordinator/src/main/java/org/apache/uniffle/coordinator/CoordinatorServer.java
b/coordinator/src/main/java/org/apache/uniffle/coordinator/CoordinatorServer.java
index 2c22d7cf6..befbb550b 100644
---
a/coordinator/src/main/java/org/apache/uniffle/coordinator/CoordinatorServer.java
+++
b/coordinator/src/main/java/org/apache/uniffle/coordinator/CoordinatorServer.java
@@ -190,6 +190,7 @@ public class CoordinatorServer extends ReconfigurableBase {
// register packages and instances for jersey
jettyServer.addResourcePackages(
"org.apache.uniffle.coordinator.web.resource",
"org.apache.uniffle.common.web.resource");
+ jettyServer.registerInstance(CoordinatorServer.class, this);
jettyServer.registerInstance(ClusterManager.class, clusterManager);
jettyServer.registerInstance(AccessManager.class, accessManager);
jettyServer.registerInstance(ApplicationManager.class, applicationManager);
diff --git
a/coordinator/src/main/java/org/apache/uniffle/coordinator/web/Response.java
b/coordinator/src/main/java/org/apache/uniffle/coordinator/web/Response.java
index 3b58d0db3..3313459af 100644
--- a/coordinator/src/main/java/org/apache/uniffle/coordinator/web/Response.java
+++ b/coordinator/src/main/java/org/apache/uniffle/coordinator/web/Response.java
@@ -22,18 +22,18 @@ public class Response<T> {
private static final int ERROR_CODE = -1;
private int code;
private T data;
- private String errMsg;
+ private String msg;
public Response() {}
- public Response(int code, T data, String errMsg) {
+ public Response(int code, T data, String msg) {
this.code = code;
this.data = data;
- this.errMsg = errMsg;
+ this.msg = msg;
}
public static <T> Response<T> success(T data) {
- return new Response<>(SUCCESS_CODE, data, null);
+ return new Response<>(SUCCESS_CODE, data, "success");
}
public static <T> Response<T> fail(String msg) {
@@ -61,10 +61,10 @@ public class Response<T> {
}
public String getErrMsg() {
- return errMsg;
+ return msg;
}
public void setErrMsg(String errMsg) {
- this.errMsg = errMsg;
+ this.msg = errMsg;
}
}
diff --git
a/coordinator/src/main/java/org/apache/uniffle/coordinator/web/resource/APIResource.java
b/coordinator/src/main/java/org/apache/uniffle/coordinator/web/resource/APIResource.java
index b01eb0d64..324820e65 100644
---
a/coordinator/src/main/java/org/apache/uniffle/coordinator/web/resource/APIResource.java
+++
b/coordinator/src/main/java/org/apache/uniffle/coordinator/web/resource/APIResource.java
@@ -33,4 +33,14 @@ public class APIResource {
public Class<AdminResource> getAdminResource() {
return AdminResource.class;
}
+
+ @Path("coordinator")
+ public Class<CoordinatorServerResource> getCoordinatorServerResource() {
+ return CoordinatorServerResource.class;
+ }
+
+ @Path("app")
+ public Class<ApplicationResource> getApplicationResource() {
+ return ApplicationResource.class;
+ }
}
diff --git
a/coordinator/src/main/java/org/apache/uniffle/coordinator/web/resource/ApplicationResource.java
b/coordinator/src/main/java/org/apache/uniffle/coordinator/web/resource/ApplicationResource.java
new file mode 100644
index 000000000..f2e120522
--- /dev/null
+++
b/coordinator/src/main/java/org/apache/uniffle/coordinator/web/resource/ApplicationResource.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.uniffle.coordinator.web.resource;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import javax.servlet.ServletContext;
+
+import org.apache.hbase.thirdparty.javax.ws.rs.GET;
+import org.apache.hbase.thirdparty.javax.ws.rs.Path;
+import org.apache.hbase.thirdparty.javax.ws.rs.Produces;
+import org.apache.hbase.thirdparty.javax.ws.rs.core.Context;
+import org.apache.hbase.thirdparty.javax.ws.rs.core.MediaType;
+
+import org.apache.uniffle.coordinator.ApplicationManager;
+import org.apache.uniffle.coordinator.web.Response;
+import org.apache.uniffle.coordinator.web.vo.AppInfoVO;
+import org.apache.uniffle.coordinator.web.vo.UserAppNumVO;
+
+@Produces({MediaType.APPLICATION_JSON})
+public class ApplicationResource extends BaseResource {
+
+ @Context protected ServletContext servletContext;
+
+ @GET
+ @Path("/total")
+ public Response<Map<String, Integer>> getAppTotality() {
+ return execute(
+ () -> {
+ Set<String> appIds = getApplicationManager().getAppIds();
+ Map<String, Integer> appTotalityMap = new HashMap<>();
+ appTotalityMap.put("appTotality", appIds.size());
+ return appTotalityMap;
+ });
+ }
+
+ @GET
+ @Path("/userTotal")
+ public Response<List<UserAppNumVO>> getUserApps() {
+ return execute(
+ () -> {
+ Map<String, Map<String, Long>> currentUserAndApp =
+ getApplicationManager().getQuotaManager().getCurrentUserAndApp();
+ List<UserAppNumVO> usercnt = new ArrayList<>();
+ for (Map.Entry<String, Map<String, Long>> stringMapEntry :
currentUserAndApp.entrySet()) {
+ String userName = stringMapEntry.getKey();
+ usercnt.add(new UserAppNumVO(userName,
stringMapEntry.getValue().size()));
+ }
+ // Display inverted by the number of user applications.
+ usercnt.sort(Comparator.reverseOrder());
+ return usercnt;
+ });
+ }
+
+ @GET
+ @Path("/appInfos")
+ public Response<List<AppInfoVO>> getAppInfoList() {
+ return execute(
+ () -> {
+ List<AppInfoVO> userToAppList = new ArrayList<>();
+ Map<String, Map<String, Long>> currentUserAndApp = new HashMap<>();
+ if (getApplicationManager().getQuotaManager() != null) {
+ currentUserAndApp =
getApplicationManager().getQuotaManager().getCurrentUserAndApp();
+ }
+ for (Map.Entry<String, Map<String, Long>> userAppIdTimestampMap :
+ currentUserAndApp.entrySet()) {
+ String userName = userAppIdTimestampMap.getKey();
+ for (Map.Entry<String, Long> appIdTimestampMap :
+ userAppIdTimestampMap.getValue().entrySet()) {
+ userToAppList.add(
+ new AppInfoVO(
+ userName, appIdTimestampMap.getKey(),
appIdTimestampMap.getValue()));
+ }
+ }
+ // Display is inverted by the submission time of the application.
+ userToAppList.sort(Comparator.reverseOrder());
+ return userToAppList;
+ });
+ }
+
+ private ApplicationManager getApplicationManager() {
+ return (ApplicationManager)
+
servletContext.getAttribute(ApplicationManager.class.getCanonicalName());
+ }
+}
diff --git
a/coordinator/src/main/java/org/apache/uniffle/coordinator/web/resource/CoordinatorServerResource.java
b/coordinator/src/main/java/org/apache/uniffle/coordinator/web/resource/CoordinatorServerResource.java
new file mode 100644
index 000000000..f6822366c
--- /dev/null
+++
b/coordinator/src/main/java/org/apache/uniffle/coordinator/web/resource/CoordinatorServerResource.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.uniffle.coordinator.web.resource;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import javax.servlet.ServletContext;
+
+import org.apache.hbase.thirdparty.javax.ws.rs.GET;
+import org.apache.hbase.thirdparty.javax.ws.rs.Path;
+import org.apache.hbase.thirdparty.javax.ws.rs.Produces;
+import org.apache.hbase.thirdparty.javax.ws.rs.core.Context;
+import org.apache.hbase.thirdparty.javax.ws.rs.core.MediaType;
+
+import org.apache.uniffle.common.util.RssUtils;
+import org.apache.uniffle.coordinator.CoordinatorConf;
+import org.apache.uniffle.coordinator.CoordinatorServer;
+import org.apache.uniffle.coordinator.util.CoordinatorUtils;
+import org.apache.uniffle.coordinator.web.Response;
+import org.apache.uniffle.coordinator.web.vo.CoordinatorConfVO;
+
+import static org.apache.uniffle.common.config.RssBaseConf.JETTY_HTTP_PORT;
+
+@Produces({MediaType.APPLICATION_JSON})
+public class CoordinatorServerResource extends BaseResource {
+
+ @Context protected ServletContext servletContext;
+
+ @GET
+ @Path("/conf")
+ public Response<List<CoordinatorConfVO>> getCoordinatorConf() {
+ return execute(
+ () -> {
+ CoordinatorConf coordinatorConf =
getCoordinatorServer().getCoordinatorConf();
+ Set<Map.Entry<String, Object>> allEntry = coordinatorConf.getAll();
+ List<CoordinatorConfVO> coordinatorConfVOs = new ArrayList<>();
+ for (Map.Entry<String, Object> stringObjectEntry : allEntry) {
+ CoordinatorConfVO result =
+ new CoordinatorConfVO(
+ stringObjectEntry.getKey(),
String.valueOf(stringObjectEntry.getValue()));
+ coordinatorConfVOs.add(result);
+ }
+ return coordinatorConfVOs;
+ });
+ }
+
+ @GET
+ @Path("/info")
+ public Response<Map<String, String>> getCoordinatorInfo() {
+ return execute(
+ () -> {
+ final CoordinatorConf coordinatorConf =
getCoordinatorServer().getCoordinatorConf();
+ Map<String, String> coordinatorServerInfo = new HashMap<>();
+ coordinatorServerInfo.put(
+ "coordinatorId",
coordinatorConf.getString(CoordinatorUtils.COORDINATOR_ID, "none"));
+ coordinatorServerInfo.put("serverIp", RssUtils.getHostIp());
+ coordinatorServerInfo.put(
+ "serverPort",
String.valueOf(coordinatorConf.getInteger("rss.rpc.server.port", 0)));
+ coordinatorServerInfo.put(
+ "serverWebPort",
String.valueOf(coordinatorConf.get(JETTY_HTTP_PORT)));
+ return coordinatorServerInfo;
+ });
+ }
+
+ private CoordinatorServer getCoordinatorServer() {
+ return (CoordinatorServer)
+
servletContext.getAttribute(CoordinatorServer.class.getCanonicalName());
+ }
+}
diff --git
a/coordinator/src/main/java/org/apache/uniffle/coordinator/web/resource/ServerResource.java
b/coordinator/src/main/java/org/apache/uniffle/coordinator/web/resource/ServerResource.java
index f4d193fe5..b6cd5b855 100644
---
a/coordinator/src/main/java/org/apache/uniffle/coordinator/web/resource/ServerResource.java
+++
b/coordinator/src/main/java/org/apache/uniffle/coordinator/web/resource/ServerResource.java
@@ -17,11 +17,14 @@
package org.apache.uniffle.coordinator.web.resource;
+import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import javax.servlet.ServletContext;
import org.apache.commons.collections.CollectionUtils;
@@ -162,6 +165,23 @@ public class ServerResource extends BaseResource {
}
}
+ @GET
+ @Path("/nodes/summary")
+ public Response<Map<String, Integer>> getNodeStatusTotal() {
+ return execute(
+ () -> {
+ ClusterManager clusterManager = getClusterManager();
+ Map<String, Integer> stringIntegerHash =
+ Stream.concat(
+
clusterManager.getServerList(Collections.emptySet()).stream(),
+ clusterManager.getLostServerList().stream())
+ .collect(
+ Collectors.groupingBy(
+ n -> n.getStatus().name(), Collectors.reducing(0, n
-> 1, Integer::sum)));
+ return stringIntegerHash;
+ });
+ }
+
private ClusterManager getClusterManager() {
return (ClusterManager)
servletContext.getAttribute(ClusterManager.class.getCanonicalName());
}
diff --git
a/coordinator/src/main/java/org/apache/uniffle/coordinator/web/vo/AppInfoVO.java
b/coordinator/src/main/java/org/apache/uniffle/coordinator/web/vo/AppInfoVO.java
new file mode 100644
index 000000000..87ee82683
--- /dev/null
+++
b/coordinator/src/main/java/org/apache/uniffle/coordinator/web/vo/AppInfoVO.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.uniffle.coordinator.web.vo;
+
+import java.util.Objects;
+
+public class AppInfoVO implements Comparable<AppInfoVO> {
+ private String userName;
+ private String appId;
+ private long updateTime;
+
+ public AppInfoVO(String userName, String appId, long updateTime) {
+ this.userName = userName;
+ this.appId = appId;
+ this.updateTime = updateTime;
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+
+ public void setUserName(String userName) {
+ this.userName = userName;
+ }
+
+ public String getAppId() {
+ return appId;
+ }
+
+ public void setAppId(String appId) {
+ this.appId = appId;
+ }
+
+ public long getUpdateTime() {
+ return updateTime;
+ }
+
+ public void setUpdateTime(long updateTime) {
+ this.updateTime = updateTime;
+ }
+
+ @Override
+ public int compareTo(AppInfoVO appInfoVO) {
+ return Long.compare(updateTime, appInfoVO.getUpdateTime());
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof AppInfoVO)) {
+ return false;
+ }
+ AppInfoVO appInfoVO = (AppInfoVO) o;
+ return updateTime == appInfoVO.updateTime
+ && userName.equals(appInfoVO.userName)
+ && appId.equals(appInfoVO.appId);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(userName, appId, updateTime);
+ }
+}
diff --git
a/coordinator/src/main/java/org/apache/uniffle/coordinator/web/resource/APIResource.java
b/coordinator/src/main/java/org/apache/uniffle/coordinator/web/vo/CoordinatorConfVO.java
similarity index 57%
copy from
coordinator/src/main/java/org/apache/uniffle/coordinator/web/resource/APIResource.java
copy to
coordinator/src/main/java/org/apache/uniffle/coordinator/web/vo/CoordinatorConfVO.java
index b01eb0d64..b3137ca8a 100644
---
a/coordinator/src/main/java/org/apache/uniffle/coordinator/web/resource/APIResource.java
+++
b/coordinator/src/main/java/org/apache/uniffle/coordinator/web/vo/CoordinatorConfVO.java
@@ -15,22 +15,30 @@
* limitations under the License.
*/
-package org.apache.uniffle.coordinator.web.resource;
+package org.apache.uniffle.coordinator.web.vo;
-import org.apache.hbase.thirdparty.javax.ws.rs.Path;
-import org.apache.hbase.thirdparty.javax.ws.rs.Produces;
-import org.apache.hbase.thirdparty.javax.ws.rs.core.MediaType;
+public class CoordinatorConfVO {
+ private String argumentKey;
+ private String argumentValue;
-@Path("api")
-@Produces({MediaType.APPLICATION_JSON})
-public class APIResource {
- @Path("server")
- public Class<ServerResource> getServerResource() {
- return ServerResource.class;
+ public CoordinatorConfVO(String argumentKey, String argumentValue) {
+ this.argumentKey = argumentKey;
+ this.argumentValue = argumentValue;
}
- @Path("admin")
- public Class<AdminResource> getAdminResource() {
- return AdminResource.class;
+ public String getArgumentKey() {
+ return argumentKey;
+ }
+
+ public void setArgumentKey(String argumentKey) {
+ this.argumentKey = argumentKey;
+ }
+
+ public String getArgumentValue() {
+ return argumentValue;
+ }
+
+ public void setArgumentValue(String argumentValue) {
+ this.argumentValue = argumentValue;
}
}
diff --git
a/coordinator/src/main/java/org/apache/uniffle/coordinator/web/vo/UserAppNumVO.java
b/coordinator/src/main/java/org/apache/uniffle/coordinator/web/vo/UserAppNumVO.java
new file mode 100644
index 000000000..6d5b56bb2
--- /dev/null
+++
b/coordinator/src/main/java/org/apache/uniffle/coordinator/web/vo/UserAppNumVO.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.uniffle.coordinator.web.vo;
+
+import java.util.Objects;
+
+public class UserAppNumVO implements Comparable<UserAppNumVO> {
+
+ private String userName;
+ private Integer appNum;
+
+ public UserAppNumVO(String userName, Integer appNum) {
+ this.userName = userName;
+ this.appNum = appNum;
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+
+ public void setUserName(String userName) {
+ this.userName = userName;
+ }
+
+ public Integer getAppNum() {
+ return appNum;
+ }
+
+ public void setAppNum(Integer appNum) {
+ this.appNum = appNum;
+ }
+
+ @Override
+ public int compareTo(UserAppNumVO userAppNumVO) {
+ return Integer.compare(appNum, userAppNumVO.getAppNum());
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof UserAppNumVO)) {
+ return false;
+ }
+ UserAppNumVO that = (UserAppNumVO) o;
+ return userName.equals(that.userName) && appNum.equals(that.appNum);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(userName, appNum);
+ }
+}
