This is an automated email from the ASF dual-hosted git repository. awong pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kudu.git
commit cac9b48ccee922b5d88cac20f496cd77b409f0bc Author: Andrew Wong <[email protected]> AuthorDate: Thu Nov 18 18:38:43 2021 -0800 [dense_node-itest] enable on macOS This patch addresses a TODO to enable this test on macOS once the file cache is used for WAL segments. This occurred in commit fc4ab691a502067bc4d5bdff30507cac7feb7cfe. Since macOS doesn't currently support the LogBlockManager, this also updates the test to look for some BlockManager-agnostic metrics if not using the LogBlockManager. I also found we can add more block-related stress to the test scenario by disabling compactions, which I've done optionally (though leaving the default behavior the same). Change-Id: I240331c4c9293d4f449d52b6a4ed2392a6ec9f3c Reviewed-on: http://gerrit.cloudera.org:8080/18055 Tested-by: Kudu Jenkins Reviewed-by: Alexey Serbin <[email protected]> --- src/kudu/integration-tests/CMakeLists.txt | 5 +--- src/kudu/integration-tests/dense_node-itest.cc | 36 +++++++++++++++++++++----- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/kudu/integration-tests/CMakeLists.txt b/src/kudu/integration-tests/CMakeLists.txt index c687930..9e2f8ad 100644 --- a/src/kudu/integration-tests/CMakeLists.txt +++ b/src/kudu/integration-tests/CMakeLists.txt @@ -76,6 +76,7 @@ ADD_KUDU_TEST(create-table-stress-test RUN_SERIAL true) ADD_KUDU_TEST(decimal-itest) ADD_KUDU_TEST(delete_table-itest NUM_SHARDS 8 PROCESSORS 4) ADD_KUDU_TEST(delete_tablet-itest PROCESSORS 2) +ADD_KUDU_TEST(dense_node-itest RUN_SERIAL true) ADD_KUDU_TEST(disk_failure-itest PROCESSORS 2) ADD_KUDU_TEST(disk_reservation-itest) ADD_KUDU_TEST(dns_alias-itest) @@ -151,10 +152,6 @@ ADD_KUDU_TEST(write_throttling-itest) if (NOT APPLE) ADD_KUDU_TEST(minidump_generation-itest) - # Opens too many files on macOS because we're not using a file cache for - # reading log segments. - # TODO(wdb): Re-enable once a file cache is being used. - ADD_KUDU_TEST(dense_node-itest RUN_SERIAL true) endif() # Tests that should not be run automatically by ctest. diff --git a/src/kudu/integration-tests/dense_node-itest.cc b/src/kudu/integration-tests/dense_node-itest.cc index b75cd09..2e37625 100644 --- a/src/kudu/integration-tests/dense_node-itest.cc +++ b/src/kudu/integration-tests/dense_node-itest.cc @@ -19,7 +19,6 @@ #include <algorithm> #include <cstdint> -#include <initializer_list> #include <memory> #include <ostream> #include <string> @@ -31,6 +30,7 @@ #include <gtest/gtest.h> #include "kudu/client/schema.h" +#include "kudu/gutil/integral_types.h" #include "kudu/gutil/strings/substitute.h" #include "kudu/integration-tests/cluster_itest_util.h" #include "kudu/integration-tests/external_mini_cluster-itest-base.h" @@ -44,12 +44,18 @@ #include "kudu/util/test_macros.h" METRIC_DECLARE_entity(server); +METRIC_DECLARE_gauge_uint64(block_manager_total_blocks_created); +METRIC_DECLARE_gauge_uint64(block_manager_total_bytes_written); +METRIC_DECLARE_gauge_uint64(block_manager_total_disk_sync); METRIC_DECLARE_gauge_uint64(log_block_manager_blocks_under_management); METRIC_DECLARE_gauge_uint64(log_block_manager_bytes_under_management); METRIC_DECLARE_gauge_uint64(log_block_manager_containers); METRIC_DECLARE_gauge_uint64(log_block_manager_full_containers); METRIC_DECLARE_gauge_uint64(threads_running); +DECLARE_bool(enable_rowset_compaction); +DECLARE_string(block_manager); + DEFINE_bool(measure_startup_drop_caches, false, "Whether to drop kernel caches before measuring startup time. Must be root"); DEFINE_bool(measure_startup_sync, false, @@ -109,9 +115,15 @@ TEST_F(DenseNodeTest, RunTest) { FLAGS_max_blocks_per_container), // UNDO delta block GC runs a lot to eagerly open newly created cfiles. - // Disable it so we can maximize flushes and compactions. + // Disable it so we can maximize flushes. "--enable_undo_delta_block_gc=false", + // Compactions aim to reduce the number of rowsets and thus reduce the + // number of cfiles. Potentially disable them so we can maximize the + // number of files. NOTE: this also means we don't exercise the block + // deletions that come with compactions. + Substitute("--enable_rowset_compaction=$0", FLAGS_enable_rowset_compaction), + // Allow our single tserver to service many, many RPCs. "--rpc_service_queue_length=1000", @@ -177,11 +189,21 @@ TEST_F(DenseNodeTest, RunTest) { // Collect some interesting metrics. The cluster is shut down before the // metrics are logged so that they're easier to find in the log output. vector<pair<string, int64_t>> metrics; - for (const auto* m : { &METRIC_log_block_manager_blocks_under_management, - &METRIC_log_block_manager_bytes_under_management, - &METRIC_log_block_manager_containers, - &METRIC_log_block_manager_full_containers, - &METRIC_threads_running }) { + vector<GaugePrototype<uint64>*> metric_prototypes; + if (FLAGS_block_manager == "log") { + metric_prototypes = { &METRIC_log_block_manager_blocks_under_management, + &METRIC_log_block_manager_bytes_under_management, + &METRIC_log_block_manager_containers, + &METRIC_log_block_manager_full_containers, + &METRIC_block_manager_total_disk_sync, + &METRIC_threads_running }; + } else { + metric_prototypes = { &METRIC_block_manager_total_bytes_written, + &METRIC_block_manager_total_blocks_created, + &METRIC_block_manager_total_disk_sync, + &METRIC_threads_running }; + } + for (const auto* m : metric_prototypes) { int64_t value; ASSERT_OK(itest::GetInt64Metric( cluster_->tablet_server(0)->bound_http_hostport(), &METRIC_ENTITY_server,
