This is an automated email from the ASF dual-hosted git repository.
mridulm80 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 651cbebc1 [CELEBORN-1525] Bump Ratis version from 3.1.0 to 3.1.1
651cbebc1 is described below
commit 651cbebc1a93cd7f3f980cbd097b47f0af36868f
Author: SteNicholas <[email protected]>
AuthorDate: Thu Sep 26 10:45:38 2024 -0500
[CELEBORN-1525] Bump Ratis version from 3.1.0 to 3.1.1
### What changes were proposed in this pull request?
Bump Ratis version from 3.1.0 to 3.1.1 including:
- Remove `address2String` and use `setAddress(ratisAddr)` with the release
of https://github.com/apache/ratis/pull/1125.
- Support `raft.grpc.message.size.max` must be 1m larger than
`raft.server.log.appender.buffer.byte-limit` for
https://github.com/apache/ratis/pull/1132.
### Why are the changes needed?
Bump Ratis version from 3.1.0 to 3.1.1. Ratis has released v3.1.1, of which
release note refers to [3.1.1](https://ratis.apache.org/post/3.1.1.html). The
3.1.1 version is a minor release with multiple improvements and bugfixes
including [[RATIS-2116] Fix the issue where RaftServerImpl.appendEntries may be
blocked indefinitely](https://issues.apache.org/jira/browse/RATIS-2116),
[[RATIS-2131] Configuring Ratis fails when hostname is used, and is an IPv6
host](https://issues.apache.org/j [...]
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
CI.
Closes #2759 from SteNicholas/CELEBORN-1525.
Authored-by: SteNicholas <[email protected]>
Signed-off-by: Mridul Muralidharan <mridul<at>gmail.com>
---
dev/deps/dependencies-server | 20 +++++++--------
.../deploy/master/clustermeta/ha/HARaftServer.java | 29 +++++-----------------
pom.xml | 2 +-
project/CelebornBuild.scala | 2 +-
4 files changed, 18 insertions(+), 35 deletions(-)
diff --git a/dev/deps/dependencies-server b/dev/deps/dependencies-server
index 61413d4c1..24b858182 100644
--- a/dev/deps/dependencies-server
+++ b/dev/deps/dependencies-server
@@ -122,16 +122,16 @@
osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
paranamer/2.8//paranamer-2.8.jar
picocli/4.7.6//picocli-4.7.6.jar
protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.1.0//ratis-client-3.1.0.jar
-ratis-common/3.1.0//ratis-common-3.1.0.jar
-ratis-grpc/3.1.0//ratis-grpc-3.1.0.jar
-ratis-metrics-api/3.1.0//ratis-metrics-api-3.1.0.jar
-ratis-metrics-default/3.1.0//ratis-metrics-default-3.1.0.jar
-ratis-netty/3.1.0//ratis-netty-3.1.0.jar
-ratis-proto/3.1.0//ratis-proto-3.1.0.jar
-ratis-server-api/3.1.0//ratis-server-api-3.1.0.jar
-ratis-server/3.1.0//ratis-server-3.1.0.jar
-ratis-shell/3.1.0//ratis-shell-3.1.0.jar
+ratis-client/3.1.1//ratis-client-3.1.1.jar
+ratis-common/3.1.1//ratis-common-3.1.1.jar
+ratis-grpc/3.1.1//ratis-grpc-3.1.1.jar
+ratis-metrics-api/3.1.1//ratis-metrics-api-3.1.1.jar
+ratis-metrics-default/3.1.1//ratis-metrics-default-3.1.1.jar
+ratis-netty/3.1.1//ratis-netty-3.1.1.jar
+ratis-proto/3.1.1//ratis-proto-3.1.1.jar
+ratis-server-api/3.1.1//ratis-server-api-3.1.1.jar
+ratis-server/3.1.1//ratis-server-3.1.1.jar
+ratis-shell/3.1.1//ratis-shell-3.1.1.jar
ratis-thirdparty-misc/1.0.6//ratis-thirdparty-misc-1.0.6.jar
rocksdbjni/9.5.2//rocksdbjni-9.5.2.jar
scala-library/2.12.18//scala-library-2.12.18.jar
diff --git
a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HARaftServer.java
b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HARaftServer.java
index d9141f034..4dfa74c66 100644
---
a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HARaftServer.java
+++
b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HARaftServer.java
@@ -19,7 +19,6 @@ package
org.apache.celeborn.service.deploy.master.clustermeta.ha;
import java.io.File;
import java.io.IOException;
-import java.net.Inet6Address;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
@@ -35,7 +34,6 @@ import javax.net.ssl.TrustManager;
import scala.Tuple2;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.net.InetAddresses;
import com.google.protobuf.InvalidProtocolBufferException;
import org.apache.ratis.RaftConfigKeys;
import org.apache.ratis.client.RaftClientConfigKeys;
@@ -174,22 +172,6 @@ public class HARaftServer {
TimeUnit.MILLISECONDS);
}
- // copy of org.apache.ratis.util.NetUtils.address2String
- private static String address2String(InetSocketAddress address) {
- if (address == null) {
- return null;
- }
- String hostName = address.getHostName();
- final StringBuilder b = new StringBuilder(hostName);
- // Surround with '[', ']' only if it is a IPv6 ip - not for a IPv6 host
- if (address.getAddress() instanceof Inet6Address
- && InetAddresses.isInetAddress(hostName)
- && InetAddresses.forString(hostName).getAddress().length == 16) {
- b.insert(0, '[').append(']');
- }
- return b.append(':').append(address.getPort()).toString();
- }
-
public static HARaftServer newMasterRatisServer(
MetaHandler metaHandler, CelebornConf conf, MasterNode localNode,
List<MasterNode> peerNodes)
throws IOException {
@@ -200,9 +182,7 @@ public class HARaftServer {
RaftPeer localRaftPeer =
RaftPeer.newBuilder()
.setId(localRaftPeerId)
- // TODO(CELEBORN-1525): Once Ratis 3.2.0 is released and we move
to it, we
- // should remove address2String and change this to
.setAddress(ratisAddr)
- .setAddress(address2String(ratisAddr))
+ .setAddress(ratisAddr)
.setClientAddress(localNode.rpcEndpoint())
// We use admin address to host the internal rpc address
.setAdminAddress(localNode.internalRpcEndpoint())
@@ -229,7 +209,7 @@ public class HARaftServer {
raftPeer =
RaftPeer.newBuilder()
.setId(raftPeerId)
- .setAddress(address2String(peerRatisAddr))
+ .setAddress(peerRatisAddr)
.setClientAddress(peer.rpcEndpoint())
// We use admin address to host the internal rpc address
.setAdminAddress(peer.internalRpcEndpoint())
@@ -359,7 +339,10 @@ public class HARaftServer {
RaftServerConfigKeys.Log.setPurgeGap(properties, logPurgeGap);
// For grpc set the maximum message size
- GrpcConfigKeys.setMessageSizeMax(properties,
SizeInBytes.valueOf(logAppenderQueueByteLimit));
+ // RATIS-2135. The leader keeps sending inconsistent entries repeatedly to
followers.
+ // raft.grpc.message.size.max must be 1m larger than
raft.server.log.appender.buffer.byte-limit
+ GrpcConfigKeys.setMessageSizeMax(
+ properties, SizeInBytes.valueOf(logAppenderQueueByteLimit +
SizeInBytes.ONE_MB.getSize()));
// Set the server request timeout
TimeDuration serverRequestTimeout =
diff --git a/pom.xml b/pom.xml
index ae3429e79..c2627a9a6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -94,7 +94,7 @@
<netty.version>4.1.109.Final</netty.version>
<bouncycastle.version>1.77</bouncycastle.version>
<protobuf.version>3.21.7</protobuf.version>
- <ratis.version>3.1.0</ratis.version>
+ <ratis.version>3.1.1</ratis.version>
<scalatest.version>3.2.16</scalatest.version>
<slf4j.version>1.7.36</slf4j.version>
<roaringbitmap.version>1.0.6</roaringbitmap.version>
diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
index 7c5a94972..502799cb7 100644
--- a/project/CelebornBuild.scala
+++ b/project/CelebornBuild.scala
@@ -59,7 +59,7 @@ object Dependencies {
val metricsVersion = "4.2.25"
val mockitoVersion = "4.11.0"
val nettyVersion = "4.1.109.Final"
- val ratisVersion = "3.1.0"
+ val ratisVersion = "3.1.1"
val roaringBitmapVersion = "1.0.6"
val rocksdbJniVersion = "9.5.2"
val jacksonVersion = "2.15.3"
