This is an automated email from the ASF dual-hosted git repository.
eolivelli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new a3401a2139 make rocksdb delete entries batch size configurable (#3646)
a3401a2139 is described below
commit a3401a2139a554a8fbe52c8a398eeb4e51a8dad1
Author: Hang Chen <[email protected]>
AuthorDate: Thu Nov 17 21:20:38 2022 +0800
make rocksdb delete entries batch size configurable (#3646)
---
.../bookie/storage/ldb/EntryLocationIndex.java | 6 +++---
.../bookkeeper/conf/ServerConfiguration.java | 22 ++++++++++++++++++++++
conf/bk_server.conf | 4 ++++
3 files changed, 29 insertions(+), 3 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/EntryLocationIndex.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/EntryLocationIndex.java
index 710206e068..0fd48291f7 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/EntryLocationIndex.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/EntryLocationIndex.java
@@ -47,12 +47,14 @@ public class EntryLocationIndex implements Closeable {
private final KeyValueStorage locationsDb;
private final ConcurrentLongHashSet deletedLedgers =
ConcurrentLongHashSet.newBuilder().build();
+ private final int deleteEntriesBatchSize;
private final EntryLocationIndexStats stats;
public EntryLocationIndex(ServerConfiguration conf, KeyValueStorageFactory
storageFactory, String basePath,
StatsLogger stats) throws IOException {
locationsDb = storageFactory.newKeyValueStorage(basePath, "locations",
DbConfigType.EntryLocation, conf);
+ deleteEntriesBatchSize = conf.getRocksDBDeleteEntriesBatchSize();
this.stats = new EntryLocationIndexStats(
stats,
@@ -190,8 +192,6 @@ public class EntryLocationIndex implements Closeable {
deletedLedgers.add(ledgerId);
}
- private static final int DELETE_ENTRIES_BATCH_SIZE = 100000;
-
public void removeOffsetFromDeletedLedgers() throws IOException {
LongPairWrapper firstKeyWrapper = LongPairWrapper.get(-1, -1);
LongPairWrapper lastKeyWrapper = LongPairWrapper.get(-1, -1);
@@ -254,7 +254,7 @@ public class EntryLocationIndex implements Closeable {
++deletedEntries;
}
- if (deletedEntriesInBatch > DELETE_ENTRIES_BATCH_SIZE) {
+ if (deletedEntriesInBatch > deleteEntriesBatchSize) {
batch.flush();
batch.clear();
deletedEntriesInBatch = 0;
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
index 05a4dec907..5c2ebb7146 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
@@ -337,6 +337,8 @@ public class ServerConfiguration extends
AbstractConfiguration<ServerConfigurati
// Used for location index, lots of writes and much bigger dataset
protected static final String LEDGER_METADATA_ROCKSDB_CONF =
"ledgerMetadataRocksdbConf";
+ protected static final String ROCKSDB_DELETE_ENTRIES_BATCH_SIZE =
"rocksDBDeleteEntriesBatchSize";
+
/**
* Construct a default configuration object.
*/
@@ -4046,4 +4048,24 @@ public class ServerConfiguration extends
AbstractConfiguration<ServerConfigurati
this.setProperty(LEDGER_METADATA_ROCKSDB_CONF,
ledgerMetadataRocksdbConf);
return this;
}
+
+ /**
+ * Get entry log location index delete entries batch size from RocksDB.
+ *
+ * @return Int rocksDB delete entries batch size configured in Service
configuration.
+ */
+ public int getRocksDBDeleteEntriesBatchSize() {
+ return getInt(ROCKSDB_DELETE_ENTRIES_BATCH_SIZE, 100000);
+ }
+
+ /**
+ * Set entry log location index delete entries batch size from RocksDB.
+ *
+ * @param rocksDBDeleteEntriesBatchSize
+ * @return
+ */
+ public ServerConfiguration setRocksDBDeleteEntriesBatchSize(int
rocksDBDeleteEntriesBatchSize) {
+ this.setProperty(ROCKSDB_DELETE_ENTRIES_BATCH_SIZE,
rocksDBDeleteEntriesBatchSize);
+ return this;
+ }
}
diff --git a/conf/bk_server.conf b/conf/bk_server.conf
index c1696ca3be..94cfe72736 100755
--- a/conf/bk_server.conf
+++ b/conf/bk_server.conf
@@ -756,6 +756,10 @@ gcEntryLogMetadataCacheEnabled=false
# Default is to use 10% / numberOfLedgers of the direct memory size
# dbStorage_rocksDB_blockCacheSize=
+# entry log location index delete entries batch size from RocksDB.
+# Default is 100000
+# rocksDBDeleteEntriesBatchSize=100000
+
# Other RocksDB specific tunables
# dbStorage_rocksDB_writeBufferSizeMB=64
# dbStorage_rocksDB_sstSizeInMB=64
