This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new b3c00561118 [fix](clone) fix calculate merge missing version (#35627)
b3c00561118 is described below
commit b3c00561118c1e025da0d93786b871feceeaa1e3
Author: yujun <[email protected]>
AuthorDate: Thu May 30 23:30:57 2024 +0800
[fix](clone) fix calculate merge missing version (#35627)
BUG: when calculate missing version, the snapshot need single versions.
For example, if missing version 4 ~ 6, the missing version should be [4,
4], [5, 5], [6, 6]. But pr #30128 merge missing versions, it will give
[4, 6], then clone will be fail.
FIX: don't merge missing versions.
---
be/src/cloud/cloud_tablet.cpp | 30 ++++++++
be/src/cloud/cloud_tablet.h | 7 ++
be/src/olap/base_tablet.cpp | 30 --------
be/src/olap/base_tablet.h | 6 +-
be/src/olap/tablet.cpp | 33 ++++++++
be/src/olap/tablet.h | 7 ++
.../clone_p0/test_clone_missing_version.groovy | 88 ++++++++++++++++++++++
7 files changed, 169 insertions(+), 32 deletions(-)
diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp
index 3d9f31d78f6..3505a12a50d 100644
--- a/be/src/cloud/cloud_tablet.cpp
+++ b/be/src/cloud/cloud_tablet.cpp
@@ -627,6 +627,36 @@ Status CloudTablet::save_delete_bitmap(const
TabletTxnInfo* txn_info, int64_t tx
return Status::OK();
}
+Versions CloudTablet::calc_missed_versions(int64_t spec_version, Versions
existing_versions) const {
+ DCHECK(spec_version > 0) << "invalid spec_version: " << spec_version;
+
+ // sort the existing versions in ascending order
+ std::sort(existing_versions.begin(), existing_versions.end(),
+ [](const Version& a, const Version& b) {
+ // simple because 2 versions are certainly not overlapping
+ return a.first < b.first;
+ });
+
+ // From the first version(=0), find the missing version until spec_version
+ int64_t last_version = -1;
+ Versions missed_versions;
+ for (const Version& version : existing_versions) {
+ if (version.first > last_version + 1) {
+ // there is a hole between versions
+ missed_versions.emplace_back(last_version + 1,
std::min(version.first, spec_version));
+ }
+ last_version = version.second;
+ if (last_version >= spec_version) {
+ break;
+ }
+ }
+ if (last_version < spec_version) {
+ // there is a hole between the last version and the specificed version.
+ missed_versions.emplace_back(last_version + 1, spec_version);
+ }
+ return missed_versions;
+}
+
Status CloudTablet::calc_delete_bitmap_for_compaction(
const std::vector<RowsetSharedPtr>& input_rowsets, const
RowsetSharedPtr& output_rowset,
const RowIdConversion& rowid_conversion, ReaderType compaction_type,
int64_t merged_rows,
diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h
index e0ab3519787..ca05759cdbf 100644
--- a/be/src/cloud/cloud_tablet.h
+++ b/be/src/cloud/cloud_tablet.h
@@ -189,6 +189,13 @@ public:
DeleteBitmapPtr&
output_rowset_delete_bitmap,
bool
allow_delete_in_cumu_compaction);
+ // Find the missed versions until the spec_version.
+ //
+ // for example:
+ // [0-4][5-5][8-8][9-9][14-14]
+ // if spec_version = 12, it will return [6-7],[10-12]
+ Versions calc_missed_versions(int64_t spec_version, Versions
existing_versions) const override;
+
std::mutex& get_rowset_update_lock() { return _rowset_update_lock; }
const auto& rowset_map() const { return _rs_version_map; }
diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp
index 9494ce5fd94..12352058f8a 100644
--- a/be/src/olap/base_tablet.cpp
+++ b/be/src/olap/base_tablet.cpp
@@ -340,36 +340,6 @@ Versions BaseTablet::get_missed_versions_unlocked(int64_t
spec_version) const {
return calc_missed_versions(spec_version, std::move(existing_versions));
}
-Versions BaseTablet::calc_missed_versions(int64_t spec_version, Versions
existing_versions) {
- DCHECK(spec_version > 0) << "invalid spec_version: " << spec_version;
-
- // sort the existing versions in ascending order
- std::sort(existing_versions.begin(), existing_versions.end(),
- [](const Version& a, const Version& b) {
- // simple because 2 versions are certainly not overlapping
- return a.first < b.first;
- });
-
- // From the first version(=0), find the missing version until spec_version
- int64_t last_version = -1;
- Versions missed_versions;
- for (const Version& version : existing_versions) {
- if (version.first > last_version + 1) {
- // there is a hole between versions
- missed_versions.emplace_back(last_version + 1,
std::min(version.first, spec_version));
- }
- last_version = version.second;
- if (last_version >= spec_version) {
- break;
- }
- }
- if (last_version < spec_version) {
- // there is a hole between the last version and the specificed version.
- missed_versions.emplace_back(last_version + 1, spec_version);
- }
- return missed_versions;
-}
-
void BaseTablet::_print_missed_versions(const Versions& missed_versions) const
{
std::stringstream ss;
ss << tablet_id() << " has " << missed_versions.size() << " missed
version:";
diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h
index 10ef2f52db8..7043df6a25a 100644
--- a/be/src/olap/base_tablet.h
+++ b/be/src/olap/base_tablet.h
@@ -254,8 +254,10 @@ protected:
//
// for example:
// [0-4][5-5][8-8][9-9][14-14]
- // if spec_version = 12, it will return [6-7],[10-12]
- static Versions calc_missed_versions(int64_t spec_version, Versions
existing_versions);
+ // for cloud, if spec_version = 12, it will return [6-7],[10-12]
+ // for local, if spec_version = 12, it will return [6, 6], [7, 7], [10,
10], [11, 11], [12, 12]
+ virtual Versions calc_missed_versions(int64_t spec_version,
+ Versions existing_versions) const =
0;
void _print_missed_versions(const Versions& missed_versions) const;
bool _reconstruct_version_tracker_if_necessary();
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index ec4b6a26013..d8b1021302f 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -909,6 +909,39 @@ Status Tablet::capture_rs_readers(const Version&
spec_version, std::vector<RowSe
return Status::OK();
}
+Versions Tablet::calc_missed_versions(int64_t spec_version, Versions
existing_versions) const {
+ DCHECK(spec_version > 0) << "invalid spec_version: " << spec_version;
+
+ // sort the existing versions in ascending order
+ std::sort(existing_versions.begin(), existing_versions.end(),
+ [](const Version& a, const Version& b) {
+ // simple because 2 versions are certainly not overlapping
+ return a.first < b.first;
+ });
+
+ // From the first version(=0), find the missing version until spec_version
+ int64_t last_version = -1;
+ Versions missed_versions;
+ for (const Version& version : existing_versions) {
+ if (version.first > last_version + 1) {
+ for (int64_t i = last_version + 1; i < version.first && i <=
spec_version; ++i) {
+ // Don't merge missed_versions because clone & snapshot use
single version.
+ // For example, if miss 4 ~ 6, clone need [4, 4], [5, 5], [6,
6], but not [4, 6].
+ missed_versions.emplace_back(i, i);
+ }
+ }
+ last_version = version.second;
+ if (last_version >= spec_version) {
+ break;
+ }
+ }
+ for (int64_t i = last_version + 1; i <= spec_version; ++i) {
+ missed_versions.emplace_back(i, i);
+ }
+
+ return missed_versions;
+}
+
bool Tablet::can_do_compaction(size_t path_hash, CompactionType
compaction_type) {
if (compaction_type == CompactionType::BASE_COMPACTION && tablet_state()
!= TABLET_RUNNING) {
// base compaction can only be done for tablet in TABLET_RUNNING state.
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index 8735b07653c..3fe48c32349 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -190,6 +190,13 @@ public:
Status capture_rs_readers(const Version& spec_version,
std::vector<RowSetSplits>* rs_splits,
bool skip_missing_version) override;
+ // Find the missed versions until the spec_version.
+ //
+ // for example:
+ // [0-4][5-5][8-8][9-9][14-14]
+ // if spec_version = 12, it will return [6, 6], [7, 7], [10, 10], [11,
11], [12, 12]
+ Versions calc_missed_versions(int64_t spec_version, Versions
existing_versions) const override;
+
// meta lock
std::shared_mutex& get_header_lock() { return _meta_lock; }
std::mutex& get_rowset_update_lock() { return _rowset_update_lock; }
diff --git a/regression-test/suites/clone_p0/test_clone_missing_version.groovy
b/regression-test/suites/clone_p0/test_clone_missing_version.groovy
new file mode 100644
index 00000000000..144d45bc5f9
--- /dev/null
+++ b/regression-test/suites/clone_p0/test_clone_missing_version.groovy
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.doris.regression.suite.ClusterOptions
+import org.apache.doris.regression.util.NodeType
+
+suite('test_clone_missing_version') {
+ def options = new ClusterOptions()
+ options.feConfigs += [
+ 'disable_tablet_scheduler=true',
+ 'tablet_checker_interval_ms=500',
+ 'schedule_batch_size=1000',
+ 'schedule_slot_num_per_hdd_path=1000',
+ ]
+ options.beConfigs += [
+ 'disable_auto_compaction=true',
+ 'report_tablet_interval_seconds=1',
+ ]
+
+ options.enableDebugPoints()
+ options.cloudMode = false
+ docker(options) {
+ def injectName = 'TxnManager.prepare_txn.random_failed'
+ def be = sql_return_maparray('show backends').get(0)
+ def addInject = { ->
+ GetDebugPoint().enableDebugPoint(be.Host, be.HttpPort as int,
NodeType.BE, injectName, [percent:1.0])
+ }
+ def deleteInject = { ->
+ GetDebugPoint().disableDebugPoint(be.Host, be.HttpPort as int,
NodeType.BE, injectName)
+ }
+
+ sql 'CREATE TABLE t (k INT) DISTRIBUTED BY HASH(k) BUCKETS 1'
+
+ sql 'INSERT INTO t VALUES(2)'
+
+ addInject()
+
+ sql 'INSERT INTO t VALUES(3)'
+ sql 'INSERT INTO t VALUES(4)'
+ sql 'INSERT INTO t VALUES(5)'
+
+ deleteInject()
+
+ sql 'INSERT INTO t VALUES(6)'
+
+ addInject()
+
+ sql 'INSERT INTO t VALUES(7)'
+ sql 'INSERT INTO t VALUES(8)'
+ sql 'INSERT INTO t VALUES(9)'
+
+ deleteInject()
+
+ sql 'INSERT INTO t VALUES(10)'
+
+ sleep 5000
+
+ def replica = sql_return_maparray('show tablets from t').find {
it.BackendId.toLong().equals(be.BackendId.toLong()) }
+ assertNotNull(replica)
+ assertEquals(4, replica.VersionCount.toInteger())
+ assertEquals(2, replica.Version.toInteger())
+ assertEquals(9, replica.LstFailedVersion.toInteger())
+
+ setFeConfig('disable_tablet_scheduler', false)
+
+ sleep 10000
+
+ replica = sql_return_maparray('show tablets from t').find {
it.BackendId.toLong().equals(be.BackendId.toLong()) }
+ assertNotNull(replica)
+ assertEquals(10, replica.VersionCount.toInteger())
+ assertEquals(10, replica.Version.toInteger())
+ assertEquals(-1, replica.LstFailedVersion.toInteger())
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
