This is an automated email from the ASF dual-hosted git repository.
hulk pushed a commit to branch unstable
in repository https://gitbox.apache.org/repos/asf/kvrocks.git
The following commit(s) were added to refs/heads/unstable by this push:
new e00fcf10 feat(config): allow to config the rocksdb's max compaction
bytes (#2802)
e00fcf10 is described below
commit e00fcf10621f242f6de4bf4602d79da8b9cfcc10
Author: hulk <[email protected]>
AuthorDate: Mon Feb 24 23:44:58 2025 +0800
feat(config): allow to config the rocksdb's max compaction bytes (#2802)
---
kvrocks.conf | 6 ++++++
src/config/config.cc | 2 ++
src/config/config.h | 1 +
src/storage/storage.cc | 1 +
tests/gocase/unit/config/config_test.go | 24 ++++++++++++++++++++++++
5 files changed, 34 insertions(+)
diff --git a/kvrocks.conf b/kvrocks.conf
index 0d78d794..2d230b9f 100644
--- a/kvrocks.conf
+++ b/kvrocks.conf
@@ -1046,5 +1046,11 @@ rocksdb.partition_filters yes
# Default: 0
# rocksdb.write_options.write_batch_max_bytes 0
+# RocksDB will try to limit number of bytes in one compaction to be lower than
this threshold.
+# If set to 0, it will be sanitized to [25 * target_file_size_base]
+#
+# Default: 0
+rocksdb.max_compaction_bytes 0
+
################################ NAMESPACE
#####################################
# namespace.test change.me
diff --git a/src/config/config.cc b/src/config/config.cc
index bc1518b3..8728fabe 100644
--- a/src/config/config.cc
+++ b/src/config/config.cc
@@ -299,6 +299,7 @@ Config::Config() {
{"rocksdb.rate_limiter_auto_tuned", true, new
YesNoField(&rocks_db.rate_limiter_auto_tuned, true)},
{"rocksdb.avoid_unnecessary_blocking_io", true, new
YesNoField(&rocks_db.avoid_unnecessary_blocking_io, true)},
{"rocksdb.partition_filters", true, new
YesNoField(&rocks_db.partition_filters, true)},
+ {"rocksdb.max_compaction_bytes", false, new
Int64Field(&rocks_db.max_compaction_bytes, 0, 0, INT64_MAX)},
/* rocksdb write options */
{"rocksdb.write_options.sync", true, new
YesNoField(&rocks_db.write_options.sync, false)},
@@ -734,6 +735,7 @@ void Config::initFieldCallback() {
{"rocksdb.compaction_readahead_size", set_db_option_cb},
{"rocksdb.max_background_jobs", set_db_option_cb},
+ {"rocksdb.max_compaction_bytes", set_cf_option_cb},
{"rocksdb.max_write_buffer_number", set_cf_option_cb},
{"rocksdb.level0_slowdown_writes_trigger", set_cf_option_cb},
{"rocksdb.level0_stop_writes_trigger", set_cf_option_cb},
diff --git a/src/config/config.h b/src/config/config.h
index 3a9a7db4..6323b3b0 100644
--- a/src/config/config.h
+++ b/src/config/config.h
@@ -223,6 +223,7 @@ struct Config {
bool rate_limiter_auto_tuned;
bool avoid_unnecessary_blocking_io = true;
bool partition_filters;
+ int64_t max_compaction_bytes;
struct WriteOptions {
bool sync;
diff --git a/src/storage/storage.cc b/src/storage/storage.cc
index adbe131a..e6bc881c 100644
--- a/src/storage/storage.cc
+++ b/src/storage/storage.cc
@@ -221,6 +221,7 @@ rocksdb::Options Storage::InitRocksDBOptions() {
options.max_bytes_for_level_multiplier =
config_->rocks_db.max_bytes_for_level_multiplier;
options.level_compaction_dynamic_level_bytes =
config_->rocks_db.level_compaction_dynamic_level_bytes;
options.max_background_jobs = config_->rocks_db.max_background_jobs;
+ options.max_compaction_bytes =
static_cast<uint64_t>(config_->rocks_db.max_compaction_bytes);
// avoid blocking io on iteration
// see https://github.com/facebook/rocksdb/wiki/IO#avoid-blocking-io
diff --git a/tests/gocase/unit/config/config_test.go
b/tests/gocase/unit/config/config_test.go
index 827cd291..4d463618 100644
--- a/tests/gocase/unit/config/config_test.go
+++ b/tests/gocase/unit/config/config_test.go
@@ -347,3 +347,27 @@ func TestGetConfigSkipBlockCacheDeallocationOnClose(t
*testing.T) {
val = rdb.ConfigGet(ctx, "skip-block-cache-deallocation-on-close").Val()
require.EqualValues(t, "no",
val["skip-block-cache-deallocation-on-close"])
}
+
+func TestConfigRocksDBOptions(t *testing.T) {
+ t.Parallel()
+ srv := util.StartServer(t, map[string]string{})
+ defer srv.Close()
+
+ rdb := srv.NewClient()
+ defer func() { require.NoError(t, rdb.Close()) }()
+
+ t.Run("Get and Set rocksdb.max_compaction_bytes", func(t *testing.T) {
+ ctx := context.Background()
+ parameter := "rocksdb.max_compaction_bytes"
+ result, err := rdb.ConfigGet(ctx, parameter).Result()
+ require.NoError(t, err)
+ require.EqualValues(t, "0", result[parameter])
+
+ util.ErrorRegexp(t, rdb.ConfigSet(ctx, parameter, "-1").Err(),
".*out of numeric range")
+
+ require.NoError(t, rdb.ConfigSet(ctx, parameter,
"1073741824").Err())
+ result, err = rdb.ConfigGet(ctx, parameter).Result()
+ require.NoError(t, err)
+ require.EqualValues(t, "1073741824", result[parameter])
+ })
+}
