This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 50eaddfa9 KUDU-3591 a few extra options to speed up RocksDB
bootstrapping
50eaddfa9 is described below
commit 50eaddfa9448c188008939a4b8c1439e9d4c95d6
Author: Alexey Serbin <[email protected]>
AuthorDate: Tue Dec 17 19:50:18 2024 -0800
KUDU-3591 a few extra options to speed up RocksDB bootstrapping
Even with changelist [1], RocksDB-specific flavors of the
FsManagerTestBase.TestAddRemoveDataDirsFuzz scenario still fail from
time to time even in non-sanitized DEBUG builds due to running out of
time allowance.
This patch introduces a couple of extra flags for the "logr" block
manager to use in test scenarios like that and tweaks already existing
flags to speed up bootstrapping of the embedded RocksDB instance.
Also, the number of iterations for the scenario has decreased 4 times.
As a quick validation of the scenario's runtime before and after this
patch, I ran the following multiple times:
time KUDU_ALLOW_SLOW_TESTS=1 ./bin/fs_manager-test
--gtest_filter='BlockManagerTypes/FsManagerTestBase.TestAddRemoveDataDirsFuzz/4'
--gtest_repeat=10 > /dev/null 2>&1
There is high variation in run times due to the stochastic nature
of the test scenario, but after running it many times I found that the
numbers below provide a good sense of the corresponding run times.
Before:
real 1m1.831s
user 0m36.494s
sys 0m16.228s
After:
real 0m10.237s
user 0m5.945s
sys 0m2.872s
This is a follow-up to [1].
[1] https://github.com/apache/kudu/commit/4116941c0
Change-Id: If95282d0e5ef3f5c1bd20e04dd24ac0a14374cee
Reviewed-on: http://gerrit.cloudera.org:8080/22232
Reviewed-by: Abhishek Chennaka <[email protected]>
Tested-by: Alexey Serbin <[email protected]>
---
src/kudu/fs/dir_manager.cc | 15 +++++++++++++++
src/kudu/fs/fs_manager-test.cc | 23 ++++++++++++++++++++---
2 files changed, 35 insertions(+), 3 deletions(-)
diff --git a/src/kudu/fs/dir_manager.cc b/src/kudu/fs/dir_manager.cc
index 9bc71bfe8..368e32762 100644
--- a/src/kudu/fs/dir_manager.cc
+++ b/src/kudu/fs/dir_manager.cc
@@ -159,6 +159,18 @@
DEFINE_int32(log_container_rdb_level0_file_num_compaction_trigger, 4,
"effective when --block_manager='logr'");
TAG_FLAG(log_container_rdb_level0_file_num_compaction_trigger, advanced);
TAG_FLAG(log_container_rdb_level0_file_num_compaction_trigger, experimental);
+
+DEFINE_bool(log_container_rdb_paranoid_checks, true,
+ "If true, RocksDB aggressively checks consistency of its data. "
+ "Effective only when --block_manager='logr'");
+TAG_FLAG(log_container_rdb_paranoid_checks, advanced);
+TAG_FLAG(log_container_rdb_paranoid_checks, experimental);
+
+DEFINE_bool(log_container_rdb_skip_stats_update_on_db_open, false,
+ "Whether to skip updating the RocksDB's stats for compaction "
+ "decision upon startup. Effective only when
--block_manager='logr'");
+TAG_FLAG(log_container_rdb_skip_stats_update_on_db_open, advanced);
+TAG_FLAG(log_container_rdb_skip_stats_update_on_db_open, experimental);
#endif
namespace kudu {
@@ -353,6 +365,9 @@ Status RdbDir::InitRocksDBInstance(bool newly_created) {
opts.max_manifest_file_size = FLAGS_log_container_rdb_max_manifest_file_size;
opts.level0_file_num_compaction_trigger =
FLAGS_log_container_rdb_level0_file_num_compaction_trigger;
+ opts.paranoid_checks = FLAGS_log_container_rdb_paranoid_checks;
+ opts.skip_stats_update_on_db_open =
+ FLAGS_log_container_rdb_skip_stats_update_on_db_open;
static std::once_flag flag;
std::call_once(flag, [&]() {
diff --git a/src/kudu/fs/fs_manager-test.cc b/src/kudu/fs/fs_manager-test.cc
index 3ab3e2fa8..b13e87a21 100644
--- a/src/kudu/fs/fs_manager-test.cc
+++ b/src/kudu/fs/fs_manager-test.cc
@@ -82,14 +82,20 @@ using std::unordered_set;
using std::vector;
using strings::Substitute;
+
+
DECLARE_bool(crash_on_eio);
+DECLARE_bool(encrypt_data_at_rest);
+DECLARE_bool(enable_multi_tenancy);
+DECLARE_bool(log_container_rdb_paranoid_checks);
+DECLARE_bool(log_container_rdb_skip_stats_update_on_db_open);
DECLARE_double(env_inject_eio);
DECLARE_string(block_manager);
DECLARE_string(env_inject_eio_globs);
DECLARE_string(env_inject_lock_failure_globs);
DECLARE_string(umask);
-DECLARE_bool(encrypt_data_at_rest);
-DECLARE_bool(enable_multi_tenancy);
+DECLARE_uint32(log_container_rdb_max_background_jobs);
+DECLARE_uint64(log_container_rdb_write_buffer_size);
namespace kudu {
namespace fs {
@@ -1265,9 +1271,20 @@ TEST_P(FsManagerTestBase, TestAddRemoveDataDirsFuzz) {
// runs if slow tests are not allowed. Since logr will open multiple RocksDB
instances and it
// takes longer than log block manager, it will only run the loop 100 times
if slow
// tests are allowed.
- const int kNumAttempts = AllowSlowTests() ? (FLAGS_block_manager == "logr" ?
100 : 1000) : 10;
+ const int kNumAttempts = AllowSlowTests() ? (FLAGS_block_manager == "logr" ?
25 : 1000) : 10;
#endif
+ // In case of the "logr" block manager, it's quite expensive to run paranoid
+ // checks, allocate big RocksDB memtables, spawn many threads, and update
+ // compaction stats on startup just to do that again next iteration
+ // when almost no data is being written. To speed up the test, let's change
+ // a few configuration settings to speed up this test scenario while still
+ // having a meaningful configuration for the embedded RocksDB instance.
+ FLAGS_log_container_rdb_paranoid_checks = false;
+ FLAGS_log_container_rdb_skip_stats_update_on_db_open = true;
+ FLAGS_log_container_rdb_max_background_jobs = 2;
+ FLAGS_log_container_rdb_write_buffer_size = 1 << 20;
+
Random rng_(SeedRandom());
FsManagerOpts fs_opts;
