This is an automated email from the ASF dual-hosted git repository.
zaynt4606 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new cfaac4e076 [CELEBORN-2454] Add config
celeborn.master.ha.ratis.raft.server.close.threshold and docs for Ratis
JvmPauseMonitor
cfaac4e076 is described below
commit cfaac4e076e1d8eb23b147a485c0171ea1efb837
Author: yew1eb <[email protected]>
AuthorDate: Sat Oct 10 16:29:22 2026 +0800
[CELEBORN-2454] Add config
celeborn.master.ha.ratis.raft.server.close.threshold and docs for Ratis
JvmPauseMonitor
### What changes were proposed in this pull request?
Add a config `celeborn.master.ha.ratis.raft.server.close.threshold`
(default 60s) for the Ratis `raft.server.close.threshold`, and document the
JvmPauseMonitor behavior in the deploy guide.
The dedicated config takes precedence over the
`celeborn.ratis.raft.server.close.threshold` passthrough when both are set; the
passthrough alone still applies for backward compatibility.
### Why are the changes needed?
Related PR: https://github.com/apache/celeborn/pull/3838
Ratis `JvmPauseMonitor` closes the local RaftServer when a JVM pause
exceeds `raft.server.close.threshold` (default 60s), e.g. caused by `jmap -F`,
long GC, or disk I/O hang. Today this threshold can only be set via the generic
`celeborn.ratis.*` custom config passthrough, which is undocumented and easy to
miss — operators hit the unexpected master shutdown without knowing why or how
to tune it.
### Does this PR resolve a correctness bug?
- [ ] Yes
### Does this PR introduce _any_ user-facing change?
- [x] Yes - adds a new configuration
`celeborn.master.ha.ratis.raft.server.close.threshold` and documents the Ratis
JvmPauseMonitor behavior (including low-intrusion GC diagnostics guidance) in
the deployment guide.
### How was this patch tested?
Existing master HA tests pass.
Closes #3839 from yew1eb/CELEBORN-2454.
Authored-by: yew1eb <[email protected]>
Signed-off-by: zhengtao <[email protected]>
AI-Contributed/Feature: 0/39
AI-Contributed/UT: 0/0
---
.../scala/org/apache/celeborn/common/CelebornConf.scala | 13 +++++++++++++
docs/configuration/ha.md | 1 +
docs/deploy.md | 8 ++++++++
.../deploy/master/clustermeta/ha/HARaftServer.java | 17 +++++++++++++++++
4 files changed, 39 insertions(+)
diff --git
a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
index 1e34ac34f9..95ebf4f5e4 100644
--- a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
@@ -802,6 +802,7 @@ class CelebornConf(loadDefaults: Boolean) extends Cloneable
with Logging with Se
def haMasterRatisLogInstallSnapshotEnabled: Boolean =
get(HA_MASTER_RATIS_LOG_INSTALL_SNAPSHOT_ENABLED)
def haMasterRatisRpcRequestTimeout: Long =
get(HA_MASTER_RATIS_RPC_REQUEST_TIMEOUT)
+ def haMasterRatisCloseThreshold: Long = get(HA_MASTER_RATIS_CLOSE_THRESHOLD)
def haMasterRatisRetryCacheExpiryTime: Long =
get(HA_MASTER_RATIS_SERVER_RETRY_CACHE_EXPIRY_TIME)
def haMasterRatisRpcTimeoutMin: Long = get(HA_MASTER_RATIS_RPC_TIMEOUT_MIN)
def haMasterRatisRpcTimeoutMax: Long = get(HA_MASTER_RATIS_RPC_TIMEOUT_MAX)
@@ -2969,6 +2970,18 @@ object CelebornConf extends Logging {
.timeConf(TimeUnit.SECONDS)
.createWithDefaultString("3s")
+ val HA_MASTER_RATIS_CLOSE_THRESHOLD: ConfigEntry[Long] =
+ buildConf("celeborn.master.ha.ratis.raft.server.close.threshold")
+ .categories("ha")
+ .version("1.0.0")
+ .doc("Threshold for the Ratis JvmPauseMonitor to close the local raft
server after " +
+ "a JVM/host pause, e.g. long GC, VM live migration or jmap -F. Maps to
Ratis " +
+ "raft.server.close.threshold. Once exceeded, the master leaves the
raft group " +
+ "until the process is restarted. Takes precedence over the " +
+ "celeborn.ratis.raft.server.close.threshold passthrough when both are
set.")
+ .timeConf(TimeUnit.SECONDS)
+ .createWithDefaultString("60s")
+
val HA_MASTER_RATIS_SERVER_RETRY_CACHE_EXPIRY_TIME: ConfigEntry[Long] =
buildConf("celeborn.master.ha.ratis.raft.server.retrycache.expirytime")
.withAlternative("celeborn.ha.master.ratis.raft.server.retrycache.expirytime")
diff --git a/docs/configuration/ha.md b/docs/configuration/ha.md
index ed8ec7dbec..b0ead6b110 100644
--- a/docs/configuration/ha.md
+++ b/docs/configuration/ha.md
@@ -27,6 +27,7 @@ license: |
| celeborn.master.ha.node.<id>.port | 9097 | false | Port to bind of
master node <id> in HA mode. | 0.3.0 | celeborn.ha.master.node.<id>.port
|
| celeborn.master.ha.node.<id>.ratis.port | 9872 | false | Ratis port to
bind of master node <id> in HA mode. | 0.3.0 |
celeborn.ha.master.node.<id>.ratis.port |
| celeborn.master.ha.ratis.raft.rpc.type | netty | false | RPC type for Ratis,
available options: netty, grpc. | 0.3.0 |
celeborn.ha.master.ratis.raft.rpc.type |
+| celeborn.master.ha.ratis.raft.server.close.threshold | 60s | false |
Threshold for the Ratis JvmPauseMonitor to close the local raft server after a
JVM/host pause, e.g. long GC, VM live migration or jmap -F. Maps to Ratis
raft.server.close.threshold. Once exceeded, the master leaves the raft group
until the process is restarted. Takes precedence over the
celeborn.ratis.raft.server.close.threshold passthrough when both are set. |
1.0.0 | |
| celeborn.master.ha.ratis.raft.server.storage.dir | /tmp/ratis | false | Root
storage directory to hold RaftServer data. | 0.3.0 |
celeborn.ha.master.ratis.raft.server.storage.dir |
| celeborn.master.ha.ratis.raft.server.storage.startup.option | RECOVER |
false | Startup option of RaftServer storage. Available options: RECOVER,
FORMAT. | 0.5.0 | |
<!--end-include-->
diff --git a/docs/deploy.md b/docs/deploy.md
index 5a1524df1e..fef96b6146 100644
--- a/docs/deploy.md
+++ b/docs/deploy.md
@@ -112,6 +112,14 @@ celeborn.worker.replicate.fastFail.duration 240s
celeborn.worker.monitor.disk.enabled false
```
+Note: In an HA cluster, each master runs a Ratis JvmPauseMonitor. If the JVM
or the host is paused
+longer than `celeborn.master.ha.ratis.raft.server.close.threshold` (default
60s), the local raft
+server is closed and the master leaves the raft group; the process must be
restarted to recover.
+Avoid operations that suspend the JVM for a long time on production masters,
e.g. `jmap -F`,
+heap dumps, or `jcmd <pid> GC.class_histogram` (which forces a full GC by
default; the `all=true`
+form skips the forced GC but still walks the whole heap). Prefer low-intrusion
diagnostics such as
+`jstat -gcutil <pid> <interval>` for GC observation.
+
Flink engine related configurations:
```properties
# If you are using Celeborn for flink, these settings will be needed.
diff --git
a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HARaftServer.java
b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HARaftServer.java
index 3054f8a03b..597d6bf46a 100644
---
a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HARaftServer.java
+++
b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HARaftServer.java
@@ -374,6 +374,10 @@ public class HARaftServer {
TimeDuration.valueOf(conf.haMasterRatisRpcRequestTimeout(),
TimeUnit.SECONDS);
RaftServerConfigKeys.Rpc.setRequestTimeout(properties,
serverRequestTimeout);
+ // Set the threshold for JvmPauseMonitor to close the local raft server
+ RaftServerConfigKeys.setCloseThreshold(
+ properties, TimeDuration.valueOf(conf.haMasterRatisCloseThreshold(),
TimeUnit.SECONDS));
+
// Set timeout for server retry cache entry
TimeDuration retryCacheExpiryTime =
TimeDuration.valueOf(conf.haMasterRatisRetryCacheExpiryTime(),
TimeUnit.SECONDS);
@@ -434,6 +438,19 @@ public class HARaftServer {
properties.set(ratisEntry.getKey().replace("celeborn.ratis.", ""),
ratisEntry.getValue());
}
+ // The dedicated config takes precedence over the
celeborn.ratis.raft.server.close.threshold
+ // passthrough above when explicitly set; otherwise the passthrough (if
any) still applies.
+ if (conf.contains(CelebornConf.HA_MASTER_RATIS_CLOSE_THRESHOLD().key())) {
+ if
(conf.haRatisCustomConfigs().containsKey("celeborn.ratis.raft.server.close.threshold"))
{
+ LOG.warn(
+ "Both {} and celeborn.ratis.raft.server.close.threshold are set; "
+ + "the dedicated config takes precedence.",
+ CelebornConf.HA_MASTER_RATIS_CLOSE_THRESHOLD().key());
+ }
+ RaftServerConfigKeys.setCloseThreshold(
+ properties, TimeDuration.valueOf(conf.haMasterRatisCloseThreshold(),
TimeUnit.SECONDS));
+ }
+
return properties;
}
