This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 3da175bf964 [branch-2.0](be-ut)Add compaction commit delete bitmap
unit test #25569 (#25771)
3da175bf964 is described below
commit 3da175bf964c06a833800ec804d322cfa40a2965
Author: abmdocrt <[email protected]>
AuthorDate: Mon Oct 23 23:29:07 2023 +0800
[branch-2.0](be-ut)Add compaction commit delete bitmap unit test #25569
(#25771)
---
.../compaction_delete_bitmap_calculator_test.cpp | 279 +++++++++++++++++++++
1 file changed, 279 insertions(+)
diff --git a/be/test/olap/compaction_delete_bitmap_calculator_test.cpp
b/be/test/olap/compaction_delete_bitmap_calculator_test.cpp
new file mode 100644
index 00000000000..0f95bb202c7
--- /dev/null
+++ b/be/test/olap/compaction_delete_bitmap_calculator_test.cpp
@@ -0,0 +1,279 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gen_cpp/AgentService_types.h>
+#include <gen_cpp/olap_file.pb.h>
+#include <gtest/gtest-message.h>
+#include <gtest/gtest-test-part.h>
+
+#include <memory>
+
+#include "gtest/gtest_pred_impl.h"
+#include "olap/cumulative_compaction_policy.h"
+#include "olap/olap_common.h"
+#include "olap/olap_meta.h"
+#include "olap/rowset/rowset_factory.h"
+#include "olap/rowset/rowset_meta.h"
+#include "olap/rowset/rowset_meta_manager.h"
+#include "olap/storage_engine.h"
+#include "olap/tablet.h"
+#include "olap/tablet_meta.h"
+#include "olap/task/engine_publish_version_task.h"
+#include "olap/txn_manager.h"
+#include "util/uid_util.h"
+
+namespace doris {
+
+static StorageEngine* k_engine = nullptr;
+
+const std::string json_rowset_meta1 = R"({
+ "rowset_id": 10000,
+ "tablet_id": 11111,
+ "txn_id": 1,
+ "tablet_schema_hash": 567997577,
+ "rowset_type": "BETA_ROWSET",
+ "rowset_state": "VISIBLE",
+ "start_version": 0,
+ "end_version": 1,
+ "num_rows": 3929,
+ "total_disk_size": 41,
+ "data_disk_size": 41,
+ "index_disk_size": 235,
+ "empty": false,
+ "load_id": {
+ "hi": 1,
+ "lo": 1
+ },
+ "creation_time": 1553765670,
+ "num_segments": 3
+ })";
+
+const std::string json_rowset_meta2 = R"({
+ "rowset_id": 10001,
+ "tablet_id": 11111,
+ "txn_id": 2,
+ "tablet_schema_hash": 567997577,
+ "rowset_type": "BETA_ROWSET",
+ "rowset_state": "VISIBLE",
+ "start_version": 2,
+ "end_version": 2,
+ "num_rows": 3929,
+ "total_disk_size": 41,
+ "data_disk_size": 41,
+ "index_disk_size": 235,
+ "empty": false,
+ "load_id": {
+ "hi": 2,
+ "lo": 2
+ },
+ "creation_time": 1553765677,
+ "num_segments": 3
+ })";
+
+const std::string json_rowset_meta3 = R"({
+ "rowset_id": 10002,
+ "tablet_id": 11111,
+ "txn_id": 3,
+ "tablet_schema_hash": 567997577,
+ "rowset_type": "BETA_ROWSET",
+ "rowset_state": "VISIBLE",
+ "start_version": 3,
+ "end_version": 4,
+ "num_rows": 3929,
+ "total_disk_size": 41,
+ "data_disk_size": 41,
+ "index_disk_size": 235,
+ "empty": false,
+ "load_id": {
+ "hi": 3,
+ "lo": 3
+ },
+ "creation_time": 1553765679,
+ "num_segments": 3
+ })";
+
+class CompactionDeleteBitmapCalculatorTest : public testing::Test {
+public:
+ CompactionDeleteBitmapCalculatorTest() = default;
+ void init_tablet_schema() {
+ TabletSchemaPB tablet_schema_pb;
+ tablet_schema_pb.set_keys_type(UNIQUE_KEYS);
+ tablet_schema_pb.set_num_short_key_columns(3);
+ tablet_schema_pb.set_num_rows_per_row_block(1024);
+ tablet_schema_pb.set_compress_kind(COMPRESS_LZ4);
+ tablet_schema_pb.set_next_column_unique_id(4);
+
+ ColumnPB* column_1 = tablet_schema_pb.add_column();
+ column_1->set_unique_id(1);
+ column_1->set_name("k1");
+ column_1->set_type("INT");
+ column_1->set_is_key(true);
+ column_1->set_length(4);
+ column_1->set_index_length(4);
+ column_1->set_is_nullable(true);
+ column_1->set_is_bf_column(false);
+
+ ColumnPB* column_2 = tablet_schema_pb.add_column();
+ column_2->set_unique_id(2);
+ column_2->set_name("k2");
+ column_2->set_type("INT");
+ column_2->set_length(4);
+ column_2->set_index_length(4);
+ column_2->set_is_nullable(true);
+ column_2->set_is_key(true);
+ column_2->set_is_nullable(true);
+ column_2->set_is_bf_column(false);
+
+ ColumnPB* column_3 = tablet_schema_pb.add_column();
+ column_3->set_unique_id(3);
+ column_3->set_name("v1");
+ column_3->set_type("VARCHAR");
+ column_3->set_length(10);
+ column_3->set_index_length(10);
+ column_3->set_is_key(false);
+ column_3->set_is_nullable(false);
+ column_3->set_is_bf_column(false);
+
+ _schema.reset(new TabletSchema);
+ _schema->init_from_pb(tablet_schema_pb);
+ }
+
+ void SetUp() override {
+ config::max_runnings_transactions_per_txn_map = 500;
+ _txn_mgr.reset(new TxnManager(1, 1));
+
+ config::tablet_map_shard_size = 1;
+ config::txn_map_shard_size = 1;
+ config::txn_shard_size = 1;
+ EngineOptions options;
+ // won't open engine, options.path is needless
+ options.backend_uid = UniqueId::gen_uid();
+ if (k_engine == nullptr) {
+ k_engine = new StorageEngine(options);
+ }
+ ExecEnv::GetInstance()->set_storage_engine(k_engine);
+
+ std::string meta_path = "./meta";
+ std::filesystem::remove_all("./meta");
+ EXPECT_TRUE(std::filesystem::create_directory(meta_path));
+ _meta = new (std::nothrow) OlapMeta(meta_path);
+ EXPECT_NE(nullptr, _meta);
+ Status st = _meta->init();
+ EXPECT_TRUE(st == Status::OK());
+ EXPECT_TRUE(std::filesystem::exists("./meta"));
+ load_id.set_hi(0);
+ load_id.set_lo(0);
+
+ init_tablet_schema();
+
+ // init rowset meta 1
+ RowsetId rowset_id;
+ rowset_id.init(10000);
+ RowsetMetaSharedPtr rowset_meta1(new RowsetMeta());
+ rowset_meta1->init_from_json(json_rowset_meta1);
+ EXPECT_EQ(rowset_meta1->rowset_id(), rowset_id);
+ EXPECT_EQ(Status::OK(),
+ RowsetFactory::create_rowset(_schema, meta_path,
rowset_meta1, &_rowset1));
+
+ // init rowset meta 2
+ rowset_id.init(10001);
+ RowsetMetaSharedPtr rowset_meta2(new RowsetMeta());
+ rowset_meta2->init_from_json(json_rowset_meta2);
+ EXPECT_EQ(rowset_meta2->rowset_id(), rowset_id);
+ EXPECT_EQ(Status::OK(),
+ RowsetFactory::create_rowset(_schema, meta_path,
rowset_meta2, &_rowset2));
+
+ // init rowset meta 3
+ rowset_id.init(10002);
+ RowsetMetaSharedPtr rowset_meta3(new RowsetMeta());
+ rowset_meta3->init_from_json(json_rowset_meta3);
+ EXPECT_EQ(rowset_meta3->rowset_id(), rowset_id);
+ EXPECT_EQ(Status::OK(),
+ RowsetFactory::create_rowset(_schema, meta_path,
rowset_meta3, &_rowset3));
+ _tablet_uid = TabletUid(10, 10);
+ // init tablet meta
+ _tablet_meta = std::shared_ptr<TabletMeta>(new TabletMeta(
+ 1, partition_id, tablet_id, 15674, schema_hash, 5,
TTabletSchema(), 6, {{7, 8}},
+ _tablet_uid, TTabletType::TABLET_TYPE_DISK,
TCompressionType::LZ4F, 0, true, {},
+ "size_based", 1024, 2000, 3600));
+ static_cast<void>(_tablet_meta->add_rs_meta(rowset_meta1));
+ static_cast<void>(_tablet_meta->add_rs_meta(rowset_meta2));
+ static_cast<void>(_tablet_meta->add_rs_meta(rowset_meta3));
+ _tablet = std::make_shared<Tablet>(_tablet_meta, nullptr,
CUMULATIVE_SIZE_BASED_POLICY);
+ static_cast<void>(_tablet->init());
+ }
+
+ void TearDown() override {
+ delete _meta;
+ EXPECT_TRUE(std::filesystem::remove_all("./meta"));
+ }
+
+private:
+ OlapMeta* _meta;
+ std::string _json_rowset_meta;
+ std::unique_ptr<TxnManager> _txn_mgr;
+ TPartitionId partition_id = 111;
+ TTabletId tablet_id = 11111;
+ SchemaHash schema_hash = 333;
+ TabletUid _tablet_uid {0, 0};
+ PUniqueId load_id;
+ TabletSchemaSPtr _schema;
+ RowsetSharedPtr _rowset1;
+ RowsetSharedPtr _rowset2;
+ RowsetSharedPtr _rowset3;
+ TabletSharedPtr _tablet;
+ TabletMetaSharedPtr _tablet_meta;
+};
+
+TEST_F(CompactionDeleteBitmapCalculatorTest, test) {
+ // publish rowset 1
+ DeleteBitmapPtr delete_bitmap =
std::make_shared<DeleteBitmap>(_tablet->tablet_id());
+ RowsetIdUnorderedSet set;
+ set.insert(_tablet_meta->all_rs_metas()[0]->rowset_id());
+ _txn_mgr->set_txn_related_delete_bitmap(partition_id, 1,
_tablet->tablet_id(), schema_hash,
+ _tablet->tablet_uid(), true,
delete_bitmap, set);
+
+ // commit rowset 2
+ load_id.set_hi(2);
+ load_id.set_lo(2);
+ Status status =
+ _txn_mgr->prepare_txn(partition_id, 2, tablet_id, schema_hash,
_tablet_uid, load_id);
+ EXPECT_TRUE(status == Status::OK());
+ status = _txn_mgr->commit_txn(_meta, partition_id, 2, tablet_id,
schema_hash, _tablet_uid,
+ load_id, _rowset2, false);
+ EXPECT_TRUE(status == Status::OK());
+ set.insert(_tablet_meta->all_rs_metas()[1]->rowset_id());
+ _txn_mgr->set_txn_related_delete_bitmap(partition_id, 2, tablet_id,
schema_hash, _tablet_uid,
+ true, delete_bitmap, set);
+
+ // prepare rowset 3
+ load_id.set_hi(3);
+ load_id.set_lo(3);
+ status = _txn_mgr->prepare_txn(partition_id, 3, tablet_id, schema_hash,
_tablet_uid, load_id);
+ EXPECT_TRUE(status == Status::OK());
+
+ RowsetMetaSharedPtr rowset_meta(new RowsetMeta());
+ status = RowsetMetaManager::get_rowset_meta(_meta, _tablet_uid,
_rowset2->rowset_id(),
+ rowset_meta);
+ EXPECT_TRUE(status == Status::OK());
+ EXPECT_TRUE(rowset_meta->rowset_id() == _rowset2->rowset_id());
+
+ CommitTabletTxnInfoVec commit_tablet_txn_info_vec {};
+ _txn_mgr->get_all_commit_tablet_txn_info_by_tablet(_tablet,
&commit_tablet_txn_info_vec);
+ EXPECT_EQ(commit_tablet_txn_info_vec.size(), 1);
+}
+} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
