This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 1e5fadb KUDU-3340 [compaction] Disable compaction on the specified
table
1e5fadb is described below
commit 1e5fadb029310f87193550126d8f6e448cddc27a
Author: kedeng <[email protected]>
AuthorDate: Fri Nov 26 14:53:21 2021 +0800
KUDU-3340 [compaction] Disable compaction on the specified table
For tables with only inserts but no updates, we can disable the
compaction of the table's data, which can improve the search
efficiency (the effect is more obvious in the search scenario
involving multiple tables).
Change-Id: Ia8452bd9151f345fcad72bb9e0f07cd78432757e
Reviewed-on: http://gerrit.cloudera.org:8080/18054
Tested-by: Alexey Serbin <[email protected]>
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin <[email protected]>
---
src/kudu/common/common.proto | 3 ++
src/kudu/common/wire_protocol.cc | 25 +++++++++-
src/kudu/integration-tests/alter_table-test.cc | 30 ++++++++++++
src/kudu/tablet/tablet.cc | 7 +++
src/kudu/tablet/tablet.h | 4 ++
src/kudu/tablet/tablet_mm_ops-test.cc | 67 ++++++++++++++++++++++++++
src/kudu/tablet/tablet_mm_ops.cc | 19 +++++---
src/kudu/tablet/tablet_mm_ops.h | 4 ++
8 files changed, 152 insertions(+), 7 deletions(-)
diff --git a/src/kudu/common/common.proto b/src/kudu/common/common.proto
index 923b020..cf41ec3 100644
--- a/src/kudu/common/common.proto
+++ b/src/kudu/common/common.proto
@@ -472,6 +472,9 @@ message TableExtraConfigPB {
// range [-FLAGS_max_priority_range, FLAGS_max_priority_range] when
// calculate maintenance priority score.
optional int32 maintenance_priority = 2;
+
+ // If set true, the table's data on disk is not compacted.
+ optional bool disable_compaction = 3;
}
// The type of a given table. This is useful in determining whether a
diff --git a/src/kudu/common/wire_protocol.cc b/src/kudu/common/wire_protocol.cc
index 8665fc0..f1b8dc2 100644
--- a/src/kudu/common/wire_protocol.cc
+++ b/src/kudu/common/wire_protocol.cc
@@ -58,6 +58,7 @@
#include "kudu/util/pb_util.h"
#include "kudu/util/safe_math.h"
#include "kudu/util/slice.h"
+#include "kudu/util/string_case.h"
namespace kudu {
class BlockBloomFilterPB;
@@ -666,12 +667,25 @@ Status ParseInt32Config(const string& name, const string&
value, int32_t* result
return Status::OK();
}
+Status ParseBoolConfig(const string& name, const string& value, bool* result) {
+ CHECK(result);
+ bool true_flag = iequals(value, "TRUE");
+ bool false_flag = iequals(value, "FALSE");
+ if (!(true_flag || false_flag)) {
+ return Status::InvalidArgument(Substitute("unable to parse $0", name),
value);
+ }
+ *result = true_flag;
+ return Status::OK();
+}
+
static const std::string kTableHistoryMaxAgeSec =
"kudu.table.history_max_age_sec";
static const std::string kTableMaintenancePriority =
"kudu.table.maintenance_priority";
+static const std::string kTableDisableCompaction =
"kudu.table.disable_compaction";
Status ExtraConfigPBFromPBMap(const Map<string, string>& configs,
TableExtraConfigPB* pb) {
static const unordered_set<string> kSupportedConfigs({kTableHistoryMaxAgeSec,
-
kTableMaintenancePriority});
+
kTableMaintenancePriority,
+
kTableDisableCompaction});
TableExtraConfigPB result;
for (const auto& config : configs) {
const string& name = config.first;
@@ -693,6 +707,12 @@ Status ExtraConfigPBFromPBMap(const Map<string, string>&
configs, TableExtraConf
RETURN_NOT_OK(ParseInt32Config(name, value, &maintenance_priority));
result.set_maintenance_priority(maintenance_priority);
}
+ } else if (name == kTableDisableCompaction) {
+ if (!value.empty()) {
+ bool disable_compaction = false;
+ RETURN_NOT_OK(ParseBoolConfig(name, value, &disable_compaction));
+ result.set_disable_compaction(disable_compaction);
+ }
} else {
LOG(FATAL) << "Unknown extra configuration property: " << name;
}
@@ -709,6 +729,9 @@ Status ExtraConfigPBToPBMap(const TableExtraConfigPB& pb,
Map<string, string>* c
if (pb.has_maintenance_priority()) {
result[kTableMaintenancePriority] =
std::to_string(pb.maintenance_priority());
}
+ if (pb.has_disable_compaction()) {
+ result[kTableDisableCompaction] = std::to_string(pb.disable_compaction());
+ }
*configs = std::move(result);
return Status::OK();
}
diff --git a/src/kudu/integration-tests/alter_table-test.cc
b/src/kudu/integration-tests/alter_table-test.cc
index d3735d4..0c2e79f 100644
--- a/src/kudu/integration-tests/alter_table-test.cc
+++ b/src/kudu/integration-tests/alter_table-test.cc
@@ -376,6 +376,10 @@ class AlterTableTest : public KuduTest {
}
}
+ void CheckDisableCompaction(const bool& disable_compaction) {
+ ASSERT_EQ(tablet_replica_->tablet()->disable_compaction(),
disable_compaction);
+ }
+
protected:
virtual int num_replicas() const { return 1; }
virtual int num_tservers() const { return 1; }
@@ -2513,4 +2517,30 @@ TEST_F(AlterTableTest, TestMaintenancePriorityAlter) {
NO_FATALS(CheckMaintenancePriority(0));
}
+TEST_F(AlterTableTest, TestDisableCompactAlter) {
+ // Default disable_compaction.
+ NO_FATALS(CheckDisableCompaction(false));
+
+ // Set disable_compaction true.
+ unique_ptr<KuduTableAlterer>
table_alterer(client_->NewTableAlterer(kTableName));
+ ASSERT_OK(table_alterer->AlterExtraConfig(
+ {{"kudu.table.disable_compaction",
"true"}})->Alter());
+ NO_FATALS(CheckDisableCompaction(true));
+
+ // Reset to default disable_compaction.
+ ASSERT_OK(table_alterer->AlterExtraConfig(
+ {{"kudu.table.disable_compaction", ""}})->Alter());
+ NO_FATALS(CheckDisableCompaction(false));
+
+ // Set disable_compaction false.
+ ASSERT_OK(table_alterer->AlterExtraConfig(
+ {{"kudu.table.disable_compaction",
"false"}})->Alter());
+ NO_FATALS(CheckDisableCompaction(false));
+
+ // Set to invalid string.
+ Status s = table_alterer->AlterExtraConfig(
+ {{"kudu.table.disable_compaction", "ok"}})->Alter();
+ ASSERT_TRUE(s.IsInvalidArgument());
+}
+
} // namespace kudu
diff --git a/src/kudu/tablet/tablet.cc b/src/kudu/tablet/tablet.cc
index 903698b..cbeb2ff 100644
--- a/src/kudu/tablet/tablet.cc
+++ b/src/kudu/tablet/tablet.cc
@@ -1743,6 +1743,13 @@ Status Tablet::PickRowSetsToCompact(RowSetsInCompaction
*picked,
return Status::OK();
}
+bool Tablet::disable_compaction() const {
+ if (metadata_->extra_config() &&
metadata_->extra_config()->has_disable_compaction()) {
+ return metadata_->extra_config()->disable_compaction();
+ }
+ return false;
+}
+
void Tablet::GetRowSetsForTests(RowSetVector* out) {
shared_ptr<RowSetTree> rowsets_copy;
{
diff --git a/src/kudu/tablet/tablet.h b/src/kudu/tablet/tablet.h
index cc156ca..4d72cb9 100644
--- a/src/kudu/tablet/tablet.h
+++ b/src/kudu/tablet/tablet.h
@@ -503,6 +503,10 @@ class Tablet {
std::string LogPrefix() const;
+ // Return false if the tablets need to compact,
+ // otherwise return true.
+ bool disable_compaction() const;
+
// Return the default bloom filter sizing parameters, configured by server
flags.
static BloomFilterSizing DefaultBloomSizing();
diff --git a/src/kudu/tablet/tablet_mm_ops-test.cc
b/src/kudu/tablet/tablet_mm_ops-test.cc
index 3a13386..710cdbb 100644
--- a/src/kudu/tablet/tablet_mm_ops-test.cc
+++ b/src/kudu/tablet/tablet_mm_ops-test.cc
@@ -19,16 +19,19 @@
#include <unordered_set>
#include <vector>
+#include <boost/optional/optional.hpp>
#include <gtest/gtest.h>
#include "kudu/common/common.pb.h"
#include "kudu/gutil/map-util.h"
#include "kudu/gutil/port.h"
#include "kudu/gutil/ref_counted.h"
+#include "kudu/tablet/tablet-harness.h"
#include "kudu/tablet/tablet-test-base.h"
#include "kudu/tablet/tablet.h"
#include "kudu/tablet/tablet_metrics.h"
#include "kudu/tablet/tablet_mm_ops.h"
+#include "kudu/util/hdr_histogram.h"
#include "kudu/util/maintenance_manager.h"
#include "kudu/util/metrics.h"
#include "kudu/util/monotime.h"
@@ -54,6 +57,9 @@ class KuduTabletMmOpsTest : public
TabletTestBase<IntKeyTestSetup<INT64>> {
all_possible_metrics_.push_back(metrics->compact_rs_duration);
all_possible_metrics_.push_back(metrics->delta_minor_compact_rs_duration);
all_possible_metrics_.push_back(metrics->delta_major_compact_rs_duration);
+ compaction_metrics_.push_back(metrics->flush_mrs_duration);
+ compaction_metrics_.push_back(metrics->flush_dms_duration);
+ compaction_metrics_.push_back(metrics->compact_rs_duration);
}
// Functions that call MaintenanceOp::UpdateStats() first sleep for a nominal
@@ -95,20 +101,62 @@ class KuduTabletMmOpsTest : public
TabletTestBase<IntKeyTestSetup<INT64>> {
}
}
+ void UpsertData() {
+ // Upsert new rows.
+ this->UpsertTestRows(0, 1000, 1000);
+ // Upsert rows that are in the MRS.
+ this->UpsertTestRows(0, 2000, 1001);
+
+ // Flush it.
+ ASSERT_OK(this->tablet()->Flush());
+ // Upsert rows that are in the DRS.
+ this->UpsertTestRows(0, 3000, 1002);
+ }
+
+ void TestNoAffectedMetrics(MaintenanceOp* op) {
+ for (const scoped_refptr<Histogram>& c : compaction_metrics_) {
+ auto min_value = c->histogram()->MinValue();
+ auto mean_value = c->histogram()->MeanValue();
+ auto max_value = c->histogram()->MaxValue();
+ auto total_count = c->histogram()->TotalCount();
+ auto total_sum = c->histogram()->TotalSum();
+ UpsertData();
+ op->UpdateStats(&stats_);
+ ASSERT_FALSE(stats_.runnable());
+ ASSERT_EQ(min_value, c->histogram()->MinValue());
+ ASSERT_EQ(mean_value, c->histogram()->MeanValue());
+ ASSERT_EQ(max_value, c->histogram()->MaxValue());
+ ASSERT_EQ(total_count, c->histogram()->TotalCount());
+ ASSERT_EQ(total_sum, c->histogram()->TotalSum());
+ }
+ }
+
MaintenanceOpStats stats_;
MonoTime next_time_;
std::vector<scoped_refptr<Histogram> > all_possible_metrics_;
+ std::vector<scoped_refptr<Histogram> > compaction_metrics_;
};
TEST_F(KuduTabletMmOpsTest, TestCompactRowSetsOpCacheStats) {
CompactRowSetsOp op(tablet().get());
+ ASSERT_FALSE(op.DisableCompaction());
NO_FATALS(TestFirstCall(&op));
NO_FATALS(TestAffectedMetrics(&op, { tablet()->metrics()->flush_mrs_duration,
tablet()->metrics()->compact_rs_duration }));
}
+TEST_F(KuduTabletMmOpsTest, TestDisableCompactRowSetsOp) {
+ CompactRowSetsOp op(tablet().get());
+ TableExtraConfigPB extra_config;
+ extra_config.set_disable_compaction(true);
+ NO_FATALS(AlterSchema(*harness_->tablet()->schema(),
boost::make_optional(extra_config)));
+ ASSERT_TRUE(op.DisableCompaction());
+ NO_FATALS(TestNoAffectedMetrics(&op));
+}
+
TEST_F(KuduTabletMmOpsTest, TestMinorDeltaCompactionOpCacheStats) {
MinorDeltaCompactionOp op(tablet().get());
+ ASSERT_FALSE(op.DisableCompaction());
NO_FATALS(TestFirstCall(&op));
NO_FATALS(TestAffectedMetrics(&op, { tablet()->metrics()->flush_mrs_duration,
tablet()->metrics()->flush_dms_duration,
@@ -116,8 +164,18 @@ TEST_F(KuduTabletMmOpsTest,
TestMinorDeltaCompactionOpCacheStats) {
tablet()->metrics()->delta_minor_compact_rs_duration }));
}
+TEST_F(KuduTabletMmOpsTest, TestDisableMinorDeltaCompactionOp) {
+ MinorDeltaCompactionOp op(tablet().get());
+ TableExtraConfigPB extra_config;
+ extra_config.set_disable_compaction(true);
+ NO_FATALS(AlterSchema(*harness_->tablet()->schema(),
boost::make_optional(extra_config)));
+ ASSERT_TRUE(op.DisableCompaction());
+ NO_FATALS(TestNoAffectedMetrics(&op));
+}
+
TEST_F(KuduTabletMmOpsTest, TestMajorDeltaCompactionOpCacheStats) {
MajorDeltaCompactionOp op(tablet().get());
+ ASSERT_FALSE(op.DisableCompaction());
NO_FATALS(TestFirstCall(&op));
NO_FATALS(TestAffectedMetrics(&op, { tablet()->metrics()->flush_mrs_duration,
tablet()->metrics()->flush_dms_duration,
@@ -125,5 +183,14 @@ TEST_F(KuduTabletMmOpsTest,
TestMajorDeltaCompactionOpCacheStats) {
tablet()->metrics()->delta_minor_compact_rs_duration,
tablet()->metrics()->delta_major_compact_rs_duration }));
}
+
+TEST_F(KuduTabletMmOpsTest, TestDisableMajorDeltaCompactionOp) {
+ MajorDeltaCompactionOp op(tablet().get());
+ TableExtraConfigPB extra_config;
+ extra_config.set_disable_compaction(true);
+ NO_FATALS(AlterSchema(*harness_->tablet()->schema(),
boost::make_optional(extra_config)));
+ ASSERT_TRUE(op.DisableCompaction());
+ NO_FATALS(TestNoAffectedMetrics(&op));
+}
} // namespace tablet
} // namespace kudu
diff --git a/src/kudu/tablet/tablet_mm_ops.cc b/src/kudu/tablet/tablet_mm_ops.cc
index d5c83f5..6a38680 100644
--- a/src/kudu/tablet/tablet_mm_ops.cc
+++ b/src/kudu/tablet/tablet_mm_ops.cc
@@ -113,6 +113,10 @@ int32_t TabletOpBase::priority() const {
return priority;
}
+bool TabletOpBase::DisableCompaction() const {
+ return tablet_->disable_compaction();
+}
+
////////////////////////////////////////////////////////////
// CompactRowSetsOp
////////////////////////////////////////////////////////////
@@ -125,9 +129,10 @@ CompactRowSetsOp::CompactRowSetsOp(Tablet* tablet)
}
void CompactRowSetsOp::UpdateStats(MaintenanceOpStats* stats) {
- if (PREDICT_FALSE(!FLAGS_enable_rowset_compaction)) {
+ if (PREDICT_FALSE(!FLAGS_enable_rowset_compaction || DisableCompaction())) {
KLOG_EVERY_N_SECS(WARNING, 300)
- << "Rowset compaction is disabled (check --enable_rowset_compaction)";
+ << Substitute("Rowset compaction is disabled (check
--enable_rowset_compaction "
+ "and disable_compaction in extra_config for tablet:$0)",
tablet_->tablet_id());
stats->set_runnable(false);
return;
}
@@ -200,9 +205,10 @@ MinorDeltaCompactionOp::MinorDeltaCompactionOp(Tablet*
tablet)
}
void MinorDeltaCompactionOp::UpdateStats(MaintenanceOpStats* stats) {
- if (PREDICT_FALSE(!FLAGS_enable_minor_delta_compaction)) {
+ if (PREDICT_FALSE(!FLAGS_enable_minor_delta_compaction ||
DisableCompaction())) {
KLOG_EVERY_N_SECS(WARNING, 300)
- << "Minor delta compaction is disabled (check
--enable_minor_delta_compaction)";
+ << Substitute("Minor delta compaction is disabled (check
--enable_minor_delta_compaction "
+ "and disable_compaction in extra_config for tablet:$0)",
tablet_->tablet_id());
stats->set_runnable(false);
return;
}
@@ -283,9 +289,10 @@ MajorDeltaCompactionOp::MajorDeltaCompactionOp(Tablet*
tablet)
}
void MajorDeltaCompactionOp::UpdateStats(MaintenanceOpStats* stats) {
- if (PREDICT_FALSE(!FLAGS_enable_major_delta_compaction)) {
+ if (PREDICT_FALSE(!FLAGS_enable_major_delta_compaction ||
DisableCompaction())) {
KLOG_EVERY_N_SECS(WARNING, 300)
- << "Major delta compaction is disabled (check
--enable_major_delta_compaction)";
+ << Substitute("Major delta compaction is disabled (check
--enable_major_delta_compaction "
+ "and disable_compaction in extra_config for tablet:$0)",
tablet_->tablet_id());
stats->set_runnable(false);
return;
}
diff --git a/src/kudu/tablet/tablet_mm_ops.h b/src/kudu/tablet/tablet_mm_ops.h
index 114e551..23e3399 100644
--- a/src/kudu/tablet/tablet_mm_ops.h
+++ b/src/kudu/tablet/tablet_mm_ops.h
@@ -40,6 +40,10 @@ class TabletOpBase : public MaintenanceOp {
TabletOpBase(std::string name, IOUsage io_usage, Tablet* tablet);
std::string LogPrefix() const;
+ // Return false if the tablets need to compact,
+ // otherwise return true.
+ bool DisableCompaction() const;
+
protected:
int32_t priority() const override;
