This is an automated email from the ASF dual-hosted git repository.
shoothzj pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new f2fdb5b5f7 BP-66: support throttling for zookeeper read during
rereplication (#4256)
f2fdb5b5f7 is described below
commit f2fdb5b5f7bd9c1b7de416450d6251180b400c32
Author: Wenzhi Feng <[email protected]>
AuthorDate: Thu May 9 14:29:23 2024 +0800
BP-66: support throttling for zookeeper read during rereplication (#4256)
---
.../bookkeeper/conf/AbstractConfiguration.java | 21 +++++++++++++++++++++
.../meta/ZkLedgerUnderreplicationManager.java | 13 +++++++++++++
conf/bk_server.conf | 6 ++++++
site3/website/docs/reference/config.md | 2 +-
4 files changed, 41 insertions(+), 1 deletion(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java
index 6c55e580f3..cecd3773e7 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java
@@ -84,6 +84,7 @@ public abstract class AbstractConfiguration<T extends
AbstractConfiguration>
protected static final String METADATA_SERVICE_URI = "metadataServiceUri";
protected static final String ZK_LEDGERS_ROOT_PATH = "zkLedgersRootPath";
protected static final String ZK_REQUEST_RATE_LIMIT = "zkRequestRateLimit";
+ protected static final String ZK_REPLICATION_TASK_RATE_LIMIT =
"zkReplicationTaskRateLimit";
protected static final String AVAILABLE_NODE = "available";
protected static final String REREPLICATION_ENTRY_BATCH_SIZE =
"rereplicationEntryBatchSize";
protected static final String
STORE_SYSTEMTIME_AS_LEDGER_UNDERREPLICATED_MARK_TIME =
@@ -1254,6 +1255,26 @@ public abstract class AbstractConfiguration<T extends
AbstractConfiguration>
return getThis();
}
+ /**
+ * get the max tasks can be acquired per second of re-replication.
+ * @return max tasks can be acquired per second of re-replication.
+ */
+ public double getZkReplicationTaskRateLimit() {
+ return getDouble(ZK_REPLICATION_TASK_RATE_LIMIT, 0);
+ }
+
+ /**
+ * set the max tasks can be acquired per second of re-replication, default
is 0, which means no limit.
+ * Value greater than 0 will enable the rate limiting. Decimal value is
allowed.
+ * For example, 0.5 means 1 task per 2 seconds, 1 means 1 task per second.
+ * @param zkReplicationTaskRateLimit
+ * @return ClientConfiguration
+ */
+ public T setZkReplicationTaskRateLimit(double zkReplicationTaskRateLimit) {
+ setProperty(ZK_REPLICATION_TASK_RATE_LIMIT,
zkReplicationTaskRateLimit);
+ return getThis();
+ }
+
/**
* Trickery to allow inheritance with fluent style.
*/
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/ZkLedgerUnderreplicationManager.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/ZkLedgerUnderreplicationManager.java
index a8c30c4327..fd43fcfa3e 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/ZkLedgerUnderreplicationManager.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/ZkLedgerUnderreplicationManager.java
@@ -21,6 +21,7 @@ package org.apache.bookkeeper.meta;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.common.base.Joiner;
+import com.google.common.util.concurrent.RateLimiter;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.TextFormat;
import com.google.protobuf.TextFormat.ParseException;
@@ -124,10 +125,19 @@ public class ZkLedgerUnderreplicationManager implements
LedgerUnderreplicationMa
private final String replicasCheckCtimeZnode;
private final ZooKeeper zkc;
private final SubTreeCache subTreeCache;
+ private final RateLimiter rateLimiter;
public ZkLedgerUnderreplicationManager(AbstractConfiguration conf,
ZooKeeper zkc)
throws UnavailableException, InterruptedException,
ReplicationException.CompatibilityException {
this.conf = conf;
+ if (conf.getZkReplicationTaskRateLimit() > 0) {
+ LOG.info("Throttling acquire task rate is configured to {} permits
per second",
+ conf.getZkReplicationTaskRateLimit());
+ rateLimiter =
RateLimiter.create(conf.getZkReplicationTaskRateLimit());
+ } else {
+ LOG.info("Throttling acquire task rate is disabled");
+ rateLimiter = null;
+ }
rootPath = ZKMetadataDriverBase.resolveZkLedgersRootPath(conf);
basePath = getBasePath(rootPath);
layoutZNode = basePath + '/' + BookKeeperConstants.LAYOUT_ZNODE;
@@ -593,6 +603,9 @@ public class ZkLedgerUnderreplicationManager implements
LedgerUnderreplicationMa
LOG.debug("getLedgerToRereplicate()");
}
while (true) {
+ if (rateLimiter != null) {
+ rateLimiter.acquire();
+ }
final CountDownLatch changedLatch = new CountDownLatch(1);
Watcher w = new Watcher() {
@Override
diff --git a/conf/bk_server.conf b/conf/bk_server.conf
index 175dca7334..ebedf206eb 100644
--- a/conf/bk_server.conf
+++ b/conf/bk_server.conf
@@ -1067,6 +1067,12 @@ reorderReadSequenceEnabled=true
# The time to backoff when replication worker encounters exceptions on
replicating a ledger, in milliseconds.
# rwRereplicateBackoffMs=5000
+# The rate limit for replicators trying to acquire the re-replication task
from ZooKeeper.
+# Used to relieve the pressure on ZooKeeper in AutoRecovery.
+# It is only enabled when setting a positive value. Default value is 0.
+# Decimals are also allowed. For example:
+# 0.5 means 1 task per 2 seconds, 1 means 1 task per second.
+# zkReplicationTaskRateLimit=0
##################################################################
##################################################################
diff --git a/site3/website/docs/reference/config.md
b/site3/website/docs/reference/config.md
index 6215f0be29..cf83d4358b 100644
--- a/site3/website/docs/reference/config.md
+++ b/site3/website/docs/reference/config.md
@@ -271,7 +271,6 @@ The table below lists parameters that you can set to
configure bookies. All conf
| zkRequestRateLimit | The Zookeeper request limit. It is only enabled when
setting a postivie value. | |
| zkEnableSecurity | Set ACLs on every node written on ZooKeeper, this way
only allowed users will be able to read and write BookKeeper metadata stored on
ZooKeeper. In order to make ACLs work you need to setup ZooKeeper JAAS
authentication all the bookies and Client need to share the same user, and this
is usually done using Kerberos authentication. See ZooKeeper documentation |
false |
-
## Statistics
| Parameter | Description | Default
@@ -352,6 +351,7 @@ The table below lists parameters that you can set to
configure bookies. All conf
| openLedgerRereplicationGracePeriod | The grace period, in milliseconds, that
the replication worker waits before fencing and replicating a ledger fragment
that's still being written to upon bookie failure. | 30000 |
| lockReleaseOfFailedLedgerGracePeriod | Set the grace period, in
milliseconds, which the replication worker has to wait before releasing the
lock after it failed to replicate a ledger. For the first
ReplicationWorker.NUM_OF_EXPONENTIAL_BACKOFF_RETRIALS failures it will do
exponential backoff then it will bound at lockReleaseOfFailedLedgerGracePeriod.
| 300000 |
| rwRereplicateBackoffMs | The time to backoff when replication worker
encounters exceptions on replicating a ledger, in milliseconds. | 5000 |
+| zkReplicationTaskRateLimit | The rate limit for the replication task, used
to relieve the pressure on ZooKeeper in AutoRecovery. | 0 |
## Memory allocator settings
