This is an automated email from the ASF dual-hosted git repository.
twice pushed a commit to branch unstable
in repository https://gitbox.apache.org/repos/asf/kvrocks.git
The following commit(s) were added to refs/heads/unstable by this push:
new 47097341 feat(config): add `rocksdb.wal_compression` to allow enable
wal compression (#2607)
47097341 is described below
commit 470973411a31f6efae1785fd52fae3a78fc9f987
Author: Egor <[email protected]>
AuthorDate: Fri Oct 18 12:20:28 2024 +0700
feat(config): add `rocksdb.wal_compression` to allow enable wal compression
(#2607)
Co-authored-by: Twice <[email protected]>
Co-authored-by: hulk <[email protected]>
---
kvrocks.conf | 10 ++++++++++
src/config/config.cc | 12 ++++++++++++
src/config/config.h | 1 +
src/storage/storage.cc | 1 +
src/storage/storage.h | 5 +++++
tests/cppunit/config_test.cc | 1 +
6 files changed, 30 insertions(+)
diff --git a/kvrocks.conf b/kvrocks.conf
index b6a2e027..3c460c88 100644
--- a/kvrocks.conf
+++ b/kvrocks.conf
@@ -718,6 +718,16 @@ rocksdb.max_background_flushes -1
# Default: 2
rocksdb.max_subcompactions 2
+# If enabled WAL records will be compressed before they are written. Only
+# ZSTD (= kZSTD) is supported (until streaming support is adapted for other
+# compression types). Compressed WAL records will be read in supported
+# versions (>= RocksDB 7.4.0 for ZSTD) regardless of this setting when
+# the WAL is read.
+#
+# Accept value: "no", "zstd"
+# Default is no
+rocksdb.wal_compression no
+
# In order to limit the size of WALs, RocksDB uses
DBOptions::max_total_wal_size
# as the trigger of column family flush. Once WALs exceed this size, RocksDB
# will start forcing the flush of column families to allow deletion of some
diff --git a/src/config/config.cc b/src/config/config.cc
index 38b8fbac..93866bca 100644
--- a/src/config/config.cc
+++ b/src/config/config.cc
@@ -79,6 +79,15 @@ const std::vector<ConfigEnum<rocksdb::CompressionType>>
compression_types{[] {
return res;
}()};
+const std::vector<ConfigEnum<rocksdb::CompressionType>>
wal_compression_types{[] {
+ std::vector<ConfigEnum<rocksdb::CompressionType>> res;
+ res.reserve(engine::WalCompressionOptions.size());
+ for (const auto &e : engine::WalCompressionOptions) {
+ res.push_back({e.name, e.type});
+ }
+ return res;
+}()};
+
const std::vector<ConfigEnum<BlockCacheType>> cache_types{[] {
std::vector<ConfigEnum<BlockCacheType>> res;
res.reserve(engine::CacheOptions.size());
@@ -211,6 +220,9 @@ Config::Config() {
{"rocksdb.max_background_flushes", true, new
IntField(&rocks_db.max_background_flushes, 2, -1, 32)},
{"rocksdb.max_subcompactions", false, new
IntField(&rocks_db.max_subcompactions, 2, 0, 16)},
{"rocksdb.delayed_write_rate", false, new
Int64Field(&rocks_db.delayed_write_rate, 0, 0, INT64_MAX)},
+ {"rocksdb.wal_compression", true,
+ new EnumField<rocksdb::CompressionType>(&rocks_db.wal_compression,
wal_compression_types,
+
rocksdb::CompressionType::kNoCompression)},
{"rocksdb.wal_ttl_seconds", true, new
IntField(&rocks_db.wal_ttl_seconds, 3 * 3600, 0, INT_MAX)},
{"rocksdb.wal_size_limit_mb", true, new
IntField(&rocks_db.wal_size_limit_mb, 16384, 0, INT_MAX)},
{"rocksdb.max_total_wal_size", false, new
IntField(&rocks_db.max_total_wal_size, 64 * 4 * 2, 0, INT_MAX)},
diff --git a/src/config/config.h b/src/config/config.h
index 1b7311e4..5969ef55 100644
--- a/src/config/config.h
+++ b/src/config/config.h
@@ -192,6 +192,7 @@ struct Config {
int64_t delayed_write_rate;
int compaction_readahead_size;
int target_file_size_base;
+ rocksdb::CompressionType wal_compression;
int wal_ttl_seconds;
int wal_size_limit_mb;
int max_total_wal_size;
diff --git a/src/storage/storage.cc b/src/storage/storage.cc
index 116b4527..9e1b315d 100644
--- a/src/storage/storage.cc
+++ b/src/storage/storage.cc
@@ -168,6 +168,7 @@ rocksdb::Options Storage::InitRocksDBOptions() {
options.num_levels = 7;
options.compression_opts.level = config_->rocks_db.compression_level;
options.compression_per_level.resize(options.num_levels);
+ options.wal_compression = config_->rocks_db.wal_compression;
// only compress levels >= 2
for (int i = 0; i < options.num_levels; ++i) {
if (i < 2) {
diff --git a/src/storage/storage.h b/src/storage/storage.h
index 3ce9b78a..c29b4689 100644
--- a/src/storage/storage.h
+++ b/src/storage/storage.h
@@ -94,6 +94,11 @@ inline const std::vector<CompressionOption>
CompressionOptions = {
{rocksdb::kZSTD, "zstd", "kZSTD"},
};
+inline const std::vector<CompressionOption> WalCompressionOptions = {
+ {rocksdb::kNoCompression, "no", "kNoCompression"},
+ {rocksdb::kZSTD, "zstd", "kZSTD"},
+};
+
struct CacheOption {
BlockCacheType type;
const std::string name;
diff --git a/tests/cppunit/config_test.cc b/tests/cppunit/config_test.cc
index 68919ed8..dfe63118 100644
--- a/tests/cppunit/config_test.cc
+++ b/tests/cppunit/config_test.cc
@@ -128,6 +128,7 @@ TEST(Config, GetAndSet) {
{"rocksdb.row_cache_size", "100"},
{"rocksdb.rate_limiter_auto_tuned", "yes"},
{"rocksdb.compression_level", "32767"},
+ {"rocksdb.wal_compression", "no"},
};
for (const auto &iter : immutable_cases) {
s = config.Set(nullptr, iter.first, iter.second);
