This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new 3d9a042e6 [MINOR] feat(client-spark/coordinator/sever/dashboard):Show
version and commit id for each Process for easy troubleshooting (#2010)
3d9a042e6 is described below
commit 3d9a042e6ac8000f0461d74cd9a0987c34c82500
Author: maobaolong <[email protected]>
AuthorDate: Tue Aug 6 16:36:32 2024 +0800
[MINOR] feat(client-spark/coordinator/sever/dashboard):Show version and
commit id for each Process for easy troubleshooting (#2010)
### What changes were proposed in this pull request?
Display the version and commitId for each process.
### Why are the changes needed?
- I don't know clearly about the version of uniffle client current running
within spark from spark.log, we cost much time to check the version and related
code.
- After our colleague send us a log file, we don't know the version and
related code, we have to ask him and wait reply.
It is easy to print the version to each process, and make it convenient for
troubleshooting.
### Does this PR introduce _any_ user-facing change?
User can know the version of uniffle client from spark log
### How was this patch tested?
Run each Process and find the version output.
```
[2024-08-03 10:00:53.377] [main] [INFO] CoordinatorServer -
CoordinatorServer version: 0.10.0-SNAPSHOT-4473d13d
[2024-08-03 10:26:25.865] [main] [INFO] ShuffleServer - ShuffleServer
version: 0.10.0-SNAPSHOT-4473d13d
[2024-08-03 10:27:20.404] [main] [INFO] JettyServerFront - JettyServerFront
version: 0.10.0-SNAPSHOT-4473d13d
24/08/03 11:50:33 INFO RssShuffleManagerBase: Uniffle
org.apache.spark.shuffle.RssShuffleManager version: 0.10.0-SNAPSHOT-9e6be424
```
---
.../apache/uniffle/shuffle/manager/RssShuffleManagerBase.java | 6 ++++++
.../org/apache/spark/shuffle/DelegationRssShuffleManager.java | 2 ++
.../org/apache/spark/shuffle/DelegationRssShuffleManager.java | 2 ++
.../main/java/org/apache/uniffle/common/util/Constants.java | 11 +++++++++++
.../org/apache/uniffle/coordinator/CoordinatorServer.java | 3 +++
.../coordinator/web/resource/CoordinatorServerResource.java | 6 +++---
.../main/java/org/apache/uniffle/dashboard/web/Dashboard.java | 3 +++
.../uniffle/client/impl/grpc/CoordinatorGrpcClient.java | 10 +++++-----
.../main/java/org/apache/uniffle/server/ShuffleServer.java | 2 ++
9 files changed, 37 insertions(+), 8 deletions(-)
diff --git
a/client-spark/common/src/main/java/org/apache/uniffle/shuffle/manager/RssShuffleManagerBase.java
b/client-spark/common/src/main/java/org/apache/uniffle/shuffle/manager/RssShuffleManagerBase.java
index 5d5fcdfa2..d7bcebf49 100644
---
a/client-spark/common/src/main/java/org/apache/uniffle/shuffle/manager/RssShuffleManagerBase.java
+++
b/client-spark/common/src/main/java/org/apache/uniffle/shuffle/manager/RssShuffleManagerBase.java
@@ -83,6 +83,7 @@ import org.apache.uniffle.common.config.RssClientConf;
import org.apache.uniffle.common.config.RssConf;
import org.apache.uniffle.common.exception.RssException;
import org.apache.uniffle.common.rpc.StatusCode;
+import org.apache.uniffle.common.util.Constants;
import org.apache.uniffle.common.util.ExpiringCloseableSupplier;
import org.apache.uniffle.common.util.RetryUtils;
import org.apache.uniffle.shuffle.BlockIdManager;
@@ -126,6 +127,11 @@ public abstract class RssShuffleManagerBase implements
RssShuffleManagerInterfac
protected boolean shuffleManagerRpcServiceEnabled;
+ public RssShuffleManagerBase() {
+ LOG.info(
+ "Uniffle {} version: {}", this.getClass().getName(),
Constants.VERSION_AND_REVISION_SHORT);
+ }
+
public BlockIdManager getBlockIdManager() {
if (blockIdManager == null) {
synchronized (blockIdManagerLock) {
diff --git
a/client-spark/spark2/src/main/java/org/apache/spark/shuffle/DelegationRssShuffleManager.java
b/client-spark/spark2/src/main/java/org/apache/spark/shuffle/DelegationRssShuffleManager.java
index 7987077f7..cf19877de 100644
---
a/client-spark/spark2/src/main/java/org/apache/spark/shuffle/DelegationRssShuffleManager.java
+++
b/client-spark/spark2/src/main/java/org/apache/spark/shuffle/DelegationRssShuffleManager.java
@@ -53,6 +53,8 @@ public class DelegationRssShuffleManager implements
ShuffleManager {
private String uuid;
public DelegationRssShuffleManager(SparkConf sparkConf, boolean isDriver)
throws Exception {
+ LOG.info(
+ "Uniffle {} version: {}", this.getClass().getName(),
Constants.VERSION_AND_REVISION_SHORT);
this.sparkConf = sparkConf;
accessTimeoutMs = sparkConf.get(RssSparkConfig.RSS_ACCESS_TIMEOUT_MS);
if (isDriver) {
diff --git
a/client-spark/spark3/src/main/java/org/apache/spark/shuffle/DelegationRssShuffleManager.java
b/client-spark/spark3/src/main/java/org/apache/spark/shuffle/DelegationRssShuffleManager.java
index c99f9b057..9978f9381 100644
---
a/client-spark/spark3/src/main/java/org/apache/spark/shuffle/DelegationRssShuffleManager.java
+++
b/client-spark/spark3/src/main/java/org/apache/spark/shuffle/DelegationRssShuffleManager.java
@@ -53,6 +53,8 @@ public class DelegationRssShuffleManager implements
ShuffleManager {
private String uuid;
public DelegationRssShuffleManager(SparkConf sparkConf, boolean isDriver)
throws Exception {
+ LOG.info(
+ "Uniffle {} version: {}", this.getClass().getName(),
Constants.VERSION_AND_REVISION_SHORT);
this.sparkConf = sparkConf;
accessTimeoutMs = sparkConf.get(RssSparkConfig.RSS_ACCESS_TIMEOUT_MS);
if (isDriver) {
diff --git a/common/src/main/java/org/apache/uniffle/common/util/Constants.java
b/common/src/main/java/org/apache/uniffle/common/util/Constants.java
index ccaa51970..2d1de3551 100644
--- a/common/src/main/java/org/apache/uniffle/common/util/Constants.java
+++ b/common/src/main/java/org/apache/uniffle/common/util/Constants.java
@@ -17,10 +17,21 @@
package org.apache.uniffle.common.util;
+import org.apache.uniffle.common.ProjectConstants;
+
public final class Constants {
private Constants() {}
+ /** The version of this Uniffle instance. */
+ public static final String VERSION = ProjectConstants.VERSION;
+
+ public static final String REVISION_SHORT =
+ ProjectConstants.REVISION.length() > 8
+ ? ProjectConstants.REVISION.substring(0, 8)
+ : ProjectConstants.REVISION;
+ public static final String VERSION_AND_REVISION_SHORT = VERSION + "-" +
REVISION_SHORT;
+
// the value is used for client/server compatible, eg, online upgrade
public static final String SHUFFLE_SERVER_VERSION = "ss_v5";
public static final String METRICS_TAG_LABEL_NAME = "tags";
diff --git
a/coordinator/src/main/java/org/apache/uniffle/coordinator/CoordinatorServer.java
b/coordinator/src/main/java/org/apache/uniffle/coordinator/CoordinatorServer.java
index baa42dabc..a4e74f5d5 100644
---
a/coordinator/src/main/java/org/apache/uniffle/coordinator/CoordinatorServer.java
+++
b/coordinator/src/main/java/org/apache/uniffle/coordinator/CoordinatorServer.java
@@ -36,6 +36,7 @@ import
org.apache.uniffle.common.metrics.MetricReporterFactory;
import org.apache.uniffle.common.rpc.ServerInterface;
import org.apache.uniffle.common.security.SecurityConfig;
import org.apache.uniffle.common.security.SecurityContextFactory;
+import org.apache.uniffle.common.util.Constants;
import org.apache.uniffle.common.util.RssUtils;
import org.apache.uniffle.common.web.CoalescedCollectorRegistry;
import org.apache.uniffle.common.web.JettyServer;
@@ -100,6 +101,8 @@ public class CoordinatorServer {
}
public void start() throws Exception {
+ LOG.info(
+ "{} version: {}", this.getClass().getSimpleName(),
Constants.VERSION_AND_REVISION_SHORT);
jettyServer.start();
server.start();
if (metricReporter != null) {
diff --git
a/coordinator/src/main/java/org/apache/uniffle/coordinator/web/resource/CoordinatorServerResource.java
b/coordinator/src/main/java/org/apache/uniffle/coordinator/web/resource/CoordinatorServerResource.java
index 47500be18..f0e4fc91f 100644
---
a/coordinator/src/main/java/org/apache/uniffle/coordinator/web/resource/CoordinatorServerResource.java
+++
b/coordinator/src/main/java/org/apache/uniffle/coordinator/web/resource/CoordinatorServerResource.java
@@ -30,7 +30,7 @@ import org.apache.hbase.thirdparty.javax.ws.rs.Produces;
import org.apache.hbase.thirdparty.javax.ws.rs.core.Context;
import org.apache.hbase.thirdparty.javax.ws.rs.core.MediaType;
-import org.apache.uniffle.common.ProjectConstants;
+import org.apache.uniffle.common.util.Constants;
import org.apache.uniffle.common.util.RssUtils;
import org.apache.uniffle.common.util.ThreadUtils;
import org.apache.uniffle.common.web.resource.BaseResource;
@@ -81,8 +81,8 @@ public class CoordinatorServerResource extends BaseResource {
"serverPort",
String.valueOf(coordinatorConf.getInteger("rss.rpc.server.port", 0)));
coordinatorServerInfo.put(
"serverWebPort",
String.valueOf(coordinatorConf.get(JETTY_HTTP_PORT)));
- coordinatorServerInfo.put("version", ProjectConstants.VERSION);
- coordinatorServerInfo.put("gitCommitId",
ProjectConstants.getGitCommitId());
+ coordinatorServerInfo.put("version", Constants.VERSION);
+ coordinatorServerInfo.put("gitCommitId", Constants.REVISION_SHORT);
return coordinatorServerInfo;
});
}
diff --git
a/dashboard/src/main/java/org/apache/uniffle/dashboard/web/Dashboard.java
b/dashboard/src/main/java/org/apache/uniffle/dashboard/web/Dashboard.java
index 03cabef72..f3d75c524 100644
--- a/dashboard/src/main/java/org/apache/uniffle/dashboard/web/Dashboard.java
+++ b/dashboard/src/main/java/org/apache/uniffle/dashboard/web/Dashboard.java
@@ -43,6 +43,7 @@ import org.slf4j.LoggerFactory;
import picocli.CommandLine;
import org.apache.uniffle.common.Arguments;
+import org.apache.uniffle.common.util.Constants;
import org.apache.uniffle.common.util.ExitUtils;
import org.apache.uniffle.common.util.ThreadUtils;
import org.apache.uniffle.dashboard.web.config.DashboardConf;
@@ -146,6 +147,8 @@ public class Dashboard {
}
public void start() {
+ LOG.info(
+ "{} version: {}", this.getClass().getSimpleName(),
Constants.VERSION_AND_REVISION_SHORT);
try {
server.start();
server.join();
diff --git
a/internal-client/src/main/java/org/apache/uniffle/client/impl/grpc/CoordinatorGrpcClient.java
b/internal-client/src/main/java/org/apache/uniffle/client/impl/grpc/CoordinatorGrpcClient.java
index 4e4e3cecd..2577ae3ef 100644
---
a/internal-client/src/main/java/org/apache/uniffle/client/impl/grpc/CoordinatorGrpcClient.java
+++
b/internal-client/src/main/java/org/apache/uniffle/client/impl/grpc/CoordinatorGrpcClient.java
@@ -48,7 +48,6 @@ import
org.apache.uniffle.client.response.RssFetchRemoteStorageResponse;
import org.apache.uniffle.client.response.RssGetShuffleAssignmentsResponse;
import org.apache.uniffle.client.response.RssSendHeartBeatResponse;
import org.apache.uniffle.common.PartitionRange;
-import org.apache.uniffle.common.ProjectConstants;
import org.apache.uniffle.common.RemoteStorageInfo;
import org.apache.uniffle.common.ServerStatus;
import org.apache.uniffle.common.ShuffleServerInfo;
@@ -56,6 +55,7 @@ import org.apache.uniffle.common.exception.RssException;
import org.apache.uniffle.common.rpc.StatusCode;
import org.apache.uniffle.common.storage.StorageInfo;
import org.apache.uniffle.common.storage.StorageInfoUtils;
+import org.apache.uniffle.common.util.Constants;
import org.apache.uniffle.proto.CoordinatorServerGrpc;
import
org.apache.uniffle.proto.CoordinatorServerGrpc.CoordinatorServerBlockingStub;
import org.apache.uniffle.proto.RssProtos;
@@ -147,8 +147,8 @@ public class CoordinatorGrpcClient extends GrpcClient
implements CoordinatorClie
.setStatusValue(serverStatus.ordinal())
.putAllStorageInfo(StorageInfoUtils.toProto(storageInfo))
.setStartTimeMs(startTimeMs)
- .setVersion(ProjectConstants.VERSION)
- .setGitCommitId(ProjectConstants.getGitCommitId())
+ .setVersion(Constants.VERSION)
+ .setGitCommitId(Constants.REVISION_SHORT)
.build();
RssProtos.StatusCode status;
@@ -268,8 +268,8 @@ public class CoordinatorGrpcClient extends GrpcClient
implements CoordinatorClie
ApplicationInfoRequest.newBuilder()
.setAppId(request.getAppId())
.setUser(request.getUser())
- .setVersion(ProjectConstants.VERSION)
- .setGitCommitId(ProjectConstants.getGitCommitId())
+ .setVersion(Constants.VERSION)
+ .setGitCommitId(Constants.REVISION_SHORT)
.build();
ApplicationInfoResponse rpcResponse =
blockingStub
diff --git a/server/src/main/java/org/apache/uniffle/server/ShuffleServer.java
b/server/src/main/java/org/apache/uniffle/server/ShuffleServer.java
index 88ddb04fe..2cd06eeaf 100644
--- a/server/src/main/java/org/apache/uniffle/server/ShuffleServer.java
+++ b/server/src/main/java/org/apache/uniffle/server/ShuffleServer.java
@@ -135,6 +135,8 @@ public class ShuffleServer {
}
public void start() throws Exception {
+ LOG.info(
+ "{} version: {}", this.getClass().getSimpleName(),
Constants.VERSION_AND_REVISION_SHORT);
jettyServer.start();
grpcPort = server.start();
if (nettyServerEnabled) {
