This is an automated email from the ASF dual-hosted git repository.
hulk pushed a commit to branch unstable
in repository https://gitbox.apache.org/repos/asf/kvrocks.git
The following commit(s) were added to refs/heads/unstable by this push:
new 9dc0dc26 Add cron to recalculate db size (#2217)
9dc0dc26 is described below
commit 9dc0dc268385301b823506f4d834b766ac984447
Author: Yann Defretin <[email protected]>
AuthorDate: Wed Apr 3 12:31:26 2024 +0200
Add cron to recalculate db size (#2217)
---
kvrocks.conf | 7 +++++++
src/config/config.cc | 6 ++++++
src/config/config.h | 2 ++
src/server/server.cc | 18 ++++++++++++++++++
tests/cppunit/config_test.cc | 1 +
5 files changed, 34 insertions(+)
diff --git a/kvrocks.conf b/kvrocks.conf
index 2949ef39..e7009e44 100644
--- a/kvrocks.conf
+++ b/kvrocks.conf
@@ -519,6 +519,13 @@ compaction-checker-range 0-7
# e.g. bgsave-cron 0 3 * * * 0 4 * * *
# would bgsave the db at 3am and 4am every day
+# Kvrocks doesn't store the key number directly. It needs to scan the DB and
+# then retrieve the key number by using the dbsize scan command.
+# The Dbsize scan scheduler auto-recalculates the estimated keys at scheduled
time.
+# Time expression format is the same as crontab (currently only support * and
int)
+# e.g. dbsize-scan-cron 0 * * * *
+# would recalculate the keyspace infos of the db every hour.
+
# Command renaming.
#
# It is possible to change the name of dangerous commands in a shared
diff --git a/src/config/config.cc b/src/config/config.cc
index 41408e8b..6d812ac6 100644
--- a/src/config/config.cc
+++ b/src/config/config.cc
@@ -137,6 +137,7 @@ Config::Config() {
{"slaveof", true, new StringField(&slaveof_, "")},
{"compact-cron", false, new StringField(&compact_cron_str_, "")},
{"bgsave-cron", false, new StringField(&bgsave_cron_str_, "")},
+ {"dbsize-scan-cron", false, new StringField(&dbsize_scan_cron_str_, "")},
{"replica-announce-ip", false, new StringField(&replica_announce_ip,
"")},
{"replica-announce-port", false, new UInt32Field(&replica_announce_port,
0, 0, PORT_LIMIT)},
{"compaction-checker-range", false, new
StringField(&compaction_checker_range_str_, "")},
@@ -292,6 +293,11 @@ void Config::initFieldValidator() {
std::vector<std::string> args = util::Split(v, " \t");
return bgsave_cron.SetScheduleTime(args);
}},
+ {"dbsize-scan-cron",
+ [this](const std::string &k, const std::string &v) -> Status {
+ std::vector<std::string> args = util::Split(v, " \t");
+ return dbsize_scan_cron.SetScheduleTime(args);
+ }},
{"compaction-checker-range",
[this](const std::string &k, const std::string &v) -> Status {
if (v.empty()) {
diff --git a/src/config/config.h b/src/config/config.h
index fc62b517..c4bc705b 100644
--- a/src/config/config.h
+++ b/src/config/config.h
@@ -137,6 +137,7 @@ struct Config {
uint32_t master_port = 0;
Cron compact_cron;
Cron bgsave_cron;
+ Cron dbsize_scan_cron;
CompactionCheckerRange compaction_checker_range{-1, -1};
int64_t force_compact_file_age;
int force_compact_file_min_deleted_percentage;
@@ -246,6 +247,7 @@ struct Config {
std::string slaveof_;
std::string compact_cron_str_;
std::string bgsave_cron_str_;
+ std::string dbsize_scan_cron_str_;
std::string compaction_checker_range_str_;
std::string profiling_sample_commands_str_;
std::map<std::string, std::unique_ptr<ConfigField>> fields_;
diff --git a/src/server/server.cc b/src/server/server.cc
index ca0b2acd..5485487c 100644
--- a/src/server/server.cc
+++ b/src/server/server.cc
@@ -755,6 +755,24 @@ void Server::cron() {
Status s = AsyncBgSaveDB();
LOG(INFO) << "[server] Schedule to bgsave the db, result: " << s.Msg();
}
+ if (config_->dbsize_scan_cron.IsEnabled() &&
config_->dbsize_scan_cron.IsTimeMatch(&now)) {
+ auto tokens = namespace_.List();
+ std::vector<std::string> namespaces;
+
+ // Number of namespaces (custom namespaces + default one)
+ namespaces.reserve(tokens.size() + 1);
+ for (auto &token : tokens) {
+ namespaces.emplace_back(token.second); // namespace
+ }
+
+ // add default namespace as fallback
+ namespaces.emplace_back(kDefaultNamespace);
+
+ for (auto &ns : namespaces) {
+ Status s = AsyncScanDBSize(ns);
+ LOG(INFO) << "[server] Schedule to recalculate the db size on
namespace: " << ns << ", result: " << s.Msg();
+ }
+ }
}
// check every 10s
if (counter != 0 && counter % 100 == 0) {
diff --git a/tests/cppunit/config_test.cc b/tests/cppunit/config_test.cc
index cca20bf1..f9252127 100644
--- a/tests/cppunit/config_test.cc
+++ b/tests/cppunit/config_test.cc
@@ -46,6 +46,7 @@ TEST(Config, GetAndSet) {
{"masterauth", "mytest_masterauth"},
{"compact-cron", "1 2 3 4 5"},
{"bgsave-cron", "5 4 3 2 1"},
+ {"dbsize-scan-cron", "1 2 3 2 1"},
{"max-io-mb", "5000"},
{"max-db-size", "6000"},
{"max-replication-mb", "7000"},
