wyxxxcat commented on code in PR #68594:
URL: https://github.com/apache/doris/pull/68594#discussion_r4130566480
##########
cloud/test/recycler_test.cpp:
##########
@@ -7766,81 +7766,335 @@ TEST(RecyclerTest,
delete_rowset_data_packed_file_respects_recycled_tablet) {
EXPECT_EQ(TxnErrorCode::TXN_KEY_NOT_FOUND, txn->get(merged_key,
&updated_val));
}
+TEST(RecyclerTest, recycle_partial_update_rowsets) {
+ RecyclePrepareRowsetConfigGuard config_guard(false);
+ auto sp = SyncPoint::get_instance();
+ DORIS_CLOUD_DEFER {
+ sp->clear_all_call_backs();
+ sp->disable_processing();
+ };
+ std::atomic<int> prefix_deletes = 0;
+ sp->set_call_back("MockAccessor::delete_prefix", [&](auto&&) {
++prefix_deletes; });
+ sp->enable_processing();
+
+ for (bool abort_before_delete : {false, true}) {
+ for (auto type :
+ {RecycleRowsetPB::PREPARE, RecycleRowsetPB::COMPACT,
RecycleRowsetPB::DROP}) {
+ for (bool packed : {false, true}) {
+ SCOPED_TRACE(fmt::format("type={}, packed={}, abort={}", type,
packed,
+ abort_before_delete));
+ prefix_deletes = 0;
+ auto txn_kv = std::make_shared<MemTxnKv>();
+ ASSERT_EQ(txn_kv->init(), 0);
+ const std::string resource_id = "partial_update_recycle";
+ auto instance = create_recycler_test_instance(resource_id);
+ InstanceRecycler recycler(txn_kv, instance, thread_group,
+
std::make_shared<TxnLazyCommitter>(txn_kv));
+ ASSERT_EQ(recycler.init(), 0);
+ auto accessor = recycler.accessor_map_.begin()->second;
+ doris::TabletSchemaCloudPB schema;
+ schema.set_schema_version(1);
+ auto rowset = create_rowset(resource_id, 8642, 512, 1, schema,
+
RowsetStatePB::BEGIN_PARTIAL_UPDATE);
+ rowset.set_end_version(1);
+ if (abort_before_delete && type == RecycleRowsetPB::PREPARE) {
+ constexpr int64_t txn_id = 70001;
+ ASSERT_EQ(create_prepared_txn(txn_kv.get(), db_id,
rowset.tablet_id(), txn_id),
+ 0);
+ rowset.set_txn_id(txn_id);
+ rowset.set_start_version(2);
+ rowset.set_end_version(2);
+ rowset.mutable_load_id()->set_hi(123);
+ rowset.mutable_load_id()->set_lo(456);
+ }
+ const auto segment = segment_path(rowset.tablet_id(),
rowset.rowset_id_v2(), 0);
+ const auto extra_segment =
+ segment_path(rowset.tablet_id(),
rowset.rowset_id_v2(), 1);
+ const auto packed_path = fmt::format("packed/{}.dat",
rowset.rowset_id_v2());
+ const auto packed_key = packed_file_key({instance_id,
packed_path});
+ if (packed) {
+ auto& location =
(*rowset.mutable_packed_slice_locations())[segment];
+ location.set_packed_file_path(packed_path);
+ location.set_offset(0);
+ location.set_size(256);
+ PackedFileInfoPB info;
+ info.set_ref_cnt(1);
+ info.set_total_slice_num(1);
+ info.set_total_slice_bytes(256);
+ info.set_remaining_slice_bytes(256);
+ info.set_state(PackedFileInfoPB::NORMAL);
+ info.set_resource_id(resource_id);
+ auto* slice = info.add_slices();
+ slice->set_path(segment);
+ slice->set_offset(0);
+ slice->set_size(256);
+ slice->set_deleted(false);
+ slice->set_txn_id(next_small_file_txn_id());
+ slice->set_rowset_id(rowset.rowset_id_v2());
+ slice->set_tablet_id(rowset.tablet_id());
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK);
+ txn->put(packed_key, info.SerializeAsString());
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+ ASSERT_EQ(accessor->put_file(packed_path, ""), 0);
+ }
+ ASSERT_EQ(create_recycle_rowset(txn_kv.get(), accessor.get(),
rowset, type, true),
+ 0);
+ ASSERT_EQ(accessor->put_file(extra_segment, ""), 0);
+ ASSERT_EQ(recycler.recycle_rowsets(), 0);
+ const bool by_prefix = type == RecycleRowsetPB::PREPARE;
+ EXPECT_EQ(prefix_deletes.load(), by_prefix ? 1 : 0);
+ EXPECT_EQ(accessor->exists(extra_segment), by_prefix ? 1 : 0);
+ EXPECT_EQ(accessor->exists(segment), by_prefix || !packed ? 1
: 0);
+ EXPECT_EQ(count_recycle_rowsets(txn_kv.get(),
rowset.tablet_id()), 0);
+ if (abort_before_delete && type == RecycleRowsetPB::PREPARE) {
+ TxnInfoPB txn_info;
+ ASSERT_EQ(get_txn_info(txn_kv, instance_id, db_id,
rowset.txn_id(), txn_info),
+ 0);
+ EXPECT_EQ(txn_info.status(),
TxnStatusPB::TXN_STATUS_ABORTED);
+ }
+ // PREPARE's prefix-only path does not clean packed references
yet.
Review Comment:
This could cause a data leak, but it wasn't introduced in this PR; a
separate PR is planned to fix it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
