This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new f239f44 [Compaction][Bug-Fix] Fix bug that meta lock need to be held
when calculating compaction score (#4829)
f239f44 is described below
commit f239f44b37618c1f2e040a7fc640fa7cfb71b257
Author: Mingyu Chen <[email protected]>
AuthorDate: Thu Nov 5 20:29:01 2020 +0800
[Compaction][Bug-Fix] Fix bug that meta lock need to be held when
calculating compaction score (#4829)
* [Compaction][Buf] Fix bug that meta lock need to be held when calucating
compaction score
* fix
Co-authored-by: morningman <[email protected]>
---
be/src/olap/base_compaction.cpp | 2 +-
be/src/olap/cumulative_compaction.cpp | 2 +-
be/src/olap/tablet.cpp | 10 ++++++----
be/src/olap/tablet.h | 5 +++--
be/src/olap/tablet_manager.cpp | 10 +---------
be/test/olap/cumulative_compaction_policy_test.cpp | 6 +++---
6 files changed, 15 insertions(+), 20 deletions(-)
diff --git a/be/src/olap/base_compaction.cpp b/be/src/olap/base_compaction.cpp
index d938c58..03bb6eb 100644
--- a/be/src/olap/base_compaction.cpp
+++ b/be/src/olap/base_compaction.cpp
@@ -45,7 +45,7 @@ OLAPStatus BaseCompaction::compact() {
TRACE_COUNTER_INCREMENT("input_rowsets_count", _input_rowsets.size());
// 2. do base compaction, merge rowsets
- int64_t permits = _tablet->calc_base_compaction_score();
+ int64_t permits =
_tablet->calc_compaction_score(CompactionType::BASE_COMPACTION);
RETURN_NOT_OK(do_compaction(permits));
TRACE("compaction finished");
diff --git a/be/src/olap/cumulative_compaction.cpp
b/be/src/olap/cumulative_compaction.cpp
index e89e6ed..4b26e3f 100755
--- a/be/src/olap/cumulative_compaction.cpp
+++ b/be/src/olap/cumulative_compaction.cpp
@@ -53,7 +53,7 @@ OLAPStatus CumulativeCompaction::compact() {
TRACE_COUNTER_INCREMENT("input_rowsets_count", _input_rowsets.size());
// 3. do cumulative compaction, merge rowsets
- int64_t permits = _tablet->calc_cumulative_compaction_score();
+ int64_t permits =
_tablet->calc_compaction_score(CompactionType::CUMULATIVE_COMPACTION);
RETURN_NOT_OK(do_compaction(permits));
TRACE("compaction finished");
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 2b7ac86..9ad2739 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -751,22 +751,24 @@ bool Tablet::can_do_compaction() {
}
const uint32_t Tablet::calc_compaction_score(CompactionType compaction_type)
const {
+ // Need meta lock, because it will iterator "all_rs_metas" of tablet meta.
+ ReadLock rdlock(&_meta_lock);
if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) {
- return calc_cumulative_compaction_score();
+ return _calc_cumulative_compaction_score();
} else {
DCHECK_EQ(compaction_type, CompactionType::BASE_COMPACTION);
- return calc_base_compaction_score();
+ return _calc_base_compaction_score();
}
}
-const uint32_t Tablet::calc_cumulative_compaction_score() const {
+const uint32_t Tablet::_calc_cumulative_compaction_score() const {
uint32_t score = 0;
_cumulative_compaction_policy->calc_cumulative_compaction_score(
_tablet_meta->all_rs_metas(), cumulative_layer_point(), &score);
return score;
}
-const uint32_t Tablet::calc_base_compaction_score() const {
+const uint32_t Tablet::_calc_base_compaction_score() const {
uint32_t score = 0;
const int64_t point = cumulative_layer_point();
bool base_rowset_exist = false;
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index 266b83c..7262e39 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -164,8 +164,6 @@ public:
// operation for compaction
bool can_do_compaction();
const uint32_t calc_compaction_score(CompactionType compaction_type) const;
- const uint32_t calc_cumulative_compaction_score() const;
- const uint32_t calc_base_compaction_score() const;
static void compute_version_hash_from_rowsets(const
std::vector<RowsetSharedPtr>& rowsets,
VersionHash* version_hash);
@@ -249,6 +247,9 @@ private:
OLAPStatus _capture_consistent_rowsets_unlocked(const vector<Version>&
version_path,
vector<RowsetSharedPtr>*
rowsets) const;
+ const uint32_t _calc_cumulative_compaction_score() const;
+ const uint32_t _calc_base_compaction_score() const;
+
public:
static const int64_t K_INVALID_CUMULATIVE_POINT = -1;
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 4fa653e..d5bda7e 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -745,15 +745,7 @@ TabletSharedPtr
TabletManager::find_best_tablet_to_compaction(
}
}
- uint32_t table_score = 0;
- {
- ReadLock rdlock(tablet_ptr->get_header_lock_ptr());
- if (compaction_type == CompactionType::BASE_COMPACTION) {
- table_score = tablet_ptr->calc_base_compaction_score();
- } else if (compaction_type ==
CompactionType::CUMULATIVE_COMPACTION) {
- table_score =
tablet_ptr->calc_cumulative_compaction_score();
- }
- }
+ uint32_t table_score =
tablet_ptr->calc_compaction_score(compaction_type);
if (table_score > highest_score) {
highest_score = table_score;
best_tablet = tablet_ptr;
diff --git a/be/test/olap/cumulative_compaction_policy_test.cpp
b/be/test/olap/cumulative_compaction_policy_test.cpp
index 4d5d0d1..e331056 100644
--- a/be/test/olap/cumulative_compaction_policy_test.cpp
+++ b/be/test/olap/cumulative_compaction_policy_test.cpp
@@ -212,7 +212,7 @@ TEST_F(TestNumBasedCumulativeCompactionPolicy,
calc_cumulative_compaction_score)
TabletSharedPtr _tablet(new Tablet(_tablet_meta, nullptr,
CUMULATIVE_NUM_BASED_POLICY));
_tablet->init();
- const uint32_t score = _tablet->calc_cumulative_compaction_score();
+ const uint32_t score =
_tablet->calc_compaction_score(CompactionType::CUMULATIVE_COMPACTION);
ASSERT_EQ(15, score);
}
@@ -675,7 +675,7 @@ TEST_F(TestSizeBasedCumulativeCompactionPolicy,
calc_cumulative_compaction_score
_tablet->init();
_tablet->calculate_cumulative_point();
- const uint32_t score = _tablet->calc_cumulative_compaction_score();
+ const uint32_t score =
_tablet->calc_compaction_score(CompactionType::CUMULATIVE_COMPACTION);
ASSERT_EQ(15, score);
}
@@ -692,7 +692,7 @@ TEST_F(TestSizeBasedCumulativeCompactionPolicy,
calc_cumulative_compaction_score
TabletSharedPtr _tablet(new Tablet(_tablet_meta, nullptr,
CUMULATIVE_SIZE_BASED_POLICY));
_tablet->init();
_tablet->calculate_cumulative_point();
- const uint32_t score = _tablet->calc_cumulative_compaction_score();
+ const uint32_t score =
_tablet->calc_compaction_score(CompactionType::CUMULATIVE_COMPACTION);
ASSERT_EQ(7, score);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
