This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new dcecaf0c371 branch-4.0: Revert "[Fix](mow) seg compaction should wait
for calc delete bitmap tasks on input segments before converting delete bitmaps
on them (#53198)" (#58855)
dcecaf0c371 is described below
commit dcecaf0c3715f4f5722aad596c3c328c3c3350b6
Author: bobhan1 <[email protected]>
AuthorDate: Tue Dec 9 17:15:10 2025 +0800
branch-4.0: Revert "[Fix](mow) seg compaction should wait for calc delete
bitmap tasks on input segments before converting delete bitmaps on them
(#53198)" (#58855)
This reverts commit 9b6115d7c9675c4d1ead8f063e26f9d27e8e1ded.
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/src/common/config.cpp | 2 -
be/src/common/config.h | 2 -
be/src/olap/base_tablet.cpp | 24 ++---
be/src/olap/base_tablet.h | 15 ++-
be/src/olap/calc_delete_bitmap_executor.cpp | 16 ++--
be/src/olap/calc_delete_bitmap_executor.h | 3 +-
be/src/olap/olap_common.h | 36 -------
be/src/olap/rowset/beta_rowset_writer.cpp | 22 ++---
be/src/olap/rowset/beta_rowset_writer.h | 8 +-
be/src/olap/rowset/segcompaction.cpp | 40 ++------
be/src/olap/rowset/segcompaction.h | 11 +--
be/test/olap/segcompaction_mow_test.cpp | 20 ----
.../test_mow_seq_seg_compaction.out | 23 -----
.../test_mow_seq_seg_compaction.groovy | 103 ---------------------
14 files changed, 41 insertions(+), 284 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 7a184b7e5c4..b38c05e5cab 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1068,8 +1068,6 @@ DEFINE_Int64(segcompaction_task_max_bytes, "157286400");
// Global segcompaction thread pool size.
DEFINE_mInt32(segcompaction_num_threads, "5");
-DEFINE_mInt32(segcompaction_wait_for_dbm_task_timeout_s, "3600"); // 1h
-
// enable java udf and jdbc scannode
DEFINE_Bool(enable_java_support, "true");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index e7844e02395..25c95391968 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1113,8 +1113,6 @@ DECLARE_Int32(segcompaction_task_max_rows);
// Max total file size allowed in a single segcompaction task.
DECLARE_Int64(segcompaction_task_max_bytes);
-DECLARE_Int32(segcompaction_wait_for_dbm_task_timeout_s);
-
// Global segcompaction thread pool size.
DECLARE_mInt32(segcompaction_num_threads);
diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp
index 2c858e557bb..4ad411262ed 100644
--- a/be/src/olap/base_tablet.cpp
+++ b/be/src/olap/base_tablet.cpp
@@ -525,13 +525,12 @@ Status BaseTablet::lookup_row_key(const Slice&
encoded_key, TabletSchema* latest
// user can get all delete bitmaps from that token.
// if `token` is nullptr, the calculation will run in local, and user can get
the result
// delete bitmap from `delete_bitmap` directly.
-Status BaseTablet::calc_delete_bitmap(
- const BaseTabletSPtr& tablet, RowsetSharedPtr rowset,
- const std::vector<segment_v2::SegmentSharedPtr>& segments,
- const std::vector<RowsetSharedPtr>& specified_rowsets, DeleteBitmapPtr
delete_bitmap,
- int64_t end_version, CalcDeleteBitmapToken* token, RowsetWriter*
rowset_writer,
- DeleteBitmapPtr tablet_delete_bitmap,
- std::function<void(segment_v2::SegmentSharedPtr, Status)> callback) {
+Status BaseTablet::calc_delete_bitmap(const BaseTabletSPtr& tablet,
RowsetSharedPtr rowset,
+ const
std::vector<segment_v2::SegmentSharedPtr>& segments,
+ const std::vector<RowsetSharedPtr>&
specified_rowsets,
+ DeleteBitmapPtr delete_bitmap, int64_t
end_version,
+ CalcDeleteBitmapToken* token,
RowsetWriter* rowset_writer,
+ DeleteBitmapPtr tablet_delete_bitmap) {
if (specified_rowsets.empty() || segments.empty()) {
return Status::OK();
}
@@ -541,8 +540,7 @@ Status BaseTablet::calc_delete_bitmap(
const auto& seg = segment;
if (token != nullptr) {
RETURN_IF_ERROR(token->submit(tablet, rowset, seg,
specified_rowsets, end_version,
- delete_bitmap, rowset_writer,
tablet_delete_bitmap,
- callback));
+ delete_bitmap, rowset_writer,
tablet_delete_bitmap));
} else {
RETURN_IF_ERROR(tablet->calc_segment_delete_bitmap(
rowset, segment, specified_rowsets, delete_bitmap,
end_version, rowset_writer,
@@ -584,14 +582,6 @@ Status
BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset,
}
}
- DBUG_EXECUTE_IF("BaseTablet::calc_segment_delete_bitmap.sleep", {
- auto target_tablet_id = dp->param<int64_t>("tablet_id", -1);
- auto sleep = dp->param<int64_t>("sleep", 10);
- if (target_tablet_id == tablet_id()) {
- std::this_thread::sleep_for(std::chrono::seconds(sleep));
- }
- });
-
if (rowset_schema->num_variant_columns() > 0) {
// During partial updates, the extracted columns of a variant should
not be included in the rowset schema.
// This is because the partial update for a variant needs to ignore
the extracted columns.
diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h
index d2762a8a53e..acd31ec2ff7 100644
--- a/be/src/olap/base_tablet.h
+++ b/be/src/olap/base_tablet.h
@@ -176,14 +176,13 @@ public:
// for rowset 6-7. Also, if a compaction happens between commit_txn and
// publish_txn, we should remove compaction input rowsets' delete_bitmap
// and build newly generated rowset's delete_bitmap
- static Status calc_delete_bitmap(
- const BaseTabletSPtr& tablet, RowsetSharedPtr rowset,
- const std::vector<segment_v2::SegmentSharedPtr>& segments,
- const std::vector<RowsetSharedPtr>& specified_rowsets,
DeleteBitmapPtr delete_bitmap,
- int64_t version, CalcDeleteBitmapToken* token, RowsetWriter*
rowset_writer = nullptr,
- DeleteBitmapPtr tablet_delete_bitmap = nullptr,
- std::function<void(segment_v2::SegmentSharedPtr, Status)> callback
=
- [](segment_v2::SegmentSharedPtr, Status) {});
+ static Status calc_delete_bitmap(const BaseTabletSPtr& tablet,
RowsetSharedPtr rowset,
+ const
std::vector<segment_v2::SegmentSharedPtr>& segments,
+ const std::vector<RowsetSharedPtr>&
specified_rowsets,
+ DeleteBitmapPtr delete_bitmap, int64_t
version,
+ CalcDeleteBitmapToken* token,
+ RowsetWriter* rowset_writer = nullptr,
+ DeleteBitmapPtr tablet_delete_bitmap =
nullptr);
Status calc_segment_delete_bitmap(RowsetSharedPtr rowset,
const segment_v2::SegmentSharedPtr& seg,
diff --git a/be/src/olap/calc_delete_bitmap_executor.cpp
b/be/src/olap/calc_delete_bitmap_executor.cpp
index 28cb0038771..bd197ccb4ed 100644
--- a/be/src/olap/calc_delete_bitmap_executor.cpp
+++ b/be/src/olap/calc_delete_bitmap_executor.cpp
@@ -29,20 +29,19 @@
namespace doris {
using namespace ErrorCode;
-Status CalcDeleteBitmapToken::submit(
- BaseTabletSPtr tablet, RowsetSharedPtr cur_rowset,
- const segment_v2::SegmentSharedPtr& cur_segment,
- const std::vector<RowsetSharedPtr>& target_rowsets, int64_t
end_version,
- DeleteBitmapPtr delete_bitmap, RowsetWriter* rowset_writer,
- DeleteBitmapPtr tablet_delete_bitmap,
- std::function<void(segment_v2::SegmentSharedPtr, Status)> callback) {
+Status CalcDeleteBitmapToken::submit(BaseTabletSPtr tablet, RowsetSharedPtr
cur_rowset,
+ const segment_v2::SegmentSharedPtr&
cur_segment,
+ const std::vector<RowsetSharedPtr>&
target_rowsets,
+ int64_t end_version, DeleteBitmapPtr
delete_bitmap,
+ RowsetWriter* rowset_writer,
+ DeleteBitmapPtr tablet_delete_bitmap) {
{
std::shared_lock rlock(_lock);
RETURN_IF_ERROR(_status);
_resource_ctx = thread_context()->resource_ctx();
}
- return _thread_token->submit_func([=, this, callback =
std::move(callback)]() {
+ return _thread_token->submit_func([=, this]() {
SCOPED_ATTACH_TASK(_resource_ctx);
auto st = tablet->calc_segment_delete_bitmap(cur_rowset, cur_segment,
target_rowsets,
delete_bitmap,
end_version, rowset_writer,
@@ -57,7 +56,6 @@ Status CalcDeleteBitmapToken::submit(
_status = st;
}
}
- callback(cur_segment, st);
});
}
diff --git a/be/src/olap/calc_delete_bitmap_executor.h
b/be/src/olap/calc_delete_bitmap_executor.h
index 040fe1d40e5..a9faf019a97 100644
--- a/be/src/olap/calc_delete_bitmap_executor.h
+++ b/be/src/olap/calc_delete_bitmap_executor.h
@@ -54,8 +54,7 @@ public:
const segment_v2::SegmentSharedPtr& cur_segment,
const std::vector<RowsetSharedPtr>& target_rowsets, int64_t
end_version,
DeleteBitmapPtr delete_bitmap, RowsetWriter* rowset_writer,
- DeleteBitmapPtr tablet_delete_bitmap,
- std::function<void(segment_v2::SegmentSharedPtr, Status)>
callback);
+ DeleteBitmapPtr tablet_delete_bitmap);
// calculate delete bitmap between `segments`
Status submit(BaseTabletSPtr tablet, TabletSchemaSPtr schema, RowsetId
rowset_id,
diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h
index d018a315ac3..2bcc6645c22 100644
--- a/be/src/olap/olap_common.h
+++ b/be/src/olap/olap_common.h
@@ -27,7 +27,6 @@
#include <list>
#include <map>
#include <memory>
-#include <mutex>
#include <ostream>
#include <sstream>
#include <string>
@@ -43,7 +42,6 @@
#include "olap/inverted_index_stats.h"
#include "olap/olap_define.h"
#include "olap/rowset/rowset_fwd.h"
-#include "util/countdown_latch.h"
#include "util/hash_util.hpp"
#include "util/time.h"
#include "util/uid_util.h"
@@ -566,30 +564,6 @@ inline RowsetId extract_rowset_id(std::string_view
filename) {
}
class DeleteBitmap;
-
-struct CalcDeleteBitmapTask {
- std::mutex m;
- Status status {Status::OK()};
- CountDownLatch latch {1};
-
- void set_status(Status st) {
- {
- std::unique_lock l(m);
- status = std::move(st);
- }
- latch.count_down(1);
- }
-
- Status get_status() {
- if (!latch.wait_for(
-
std::chrono::seconds(config::segcompaction_wait_for_dbm_task_timeout_s))) {
- return Status::InternalError<false>("wait for calc delete bitmap
task timeout");
- };
- std::unique_lock l(m);
- return status;
- }
-};
-
// merge on write context
struct MowContext {
MowContext(int64_t version, int64_t txnid,
std::shared_ptr<RowsetIdUnorderedSet> ids,
@@ -599,21 +573,11 @@ struct MowContext {
rowset_ids(std::move(ids)),
rowset_ptrs(std::move(rowset_ptrs)),
delete_bitmap(std::move(db)) {}
-
- CalcDeleteBitmapTask* get_calc_dbm_task(int32_t segment_id) {
- std::lock_guard l(m);
- return &calc_dbm_tasks[segment_id];
- }
-
int64_t max_version;
int64_t txn_id;
std::shared_ptr<RowsetIdUnorderedSet> rowset_ids;
std::vector<RowsetSharedPtr> rowset_ptrs;
std::shared_ptr<DeleteBitmap> delete_bitmap;
-
- std::mutex m;
- // status of calc delete bitmap task in flush phase
- std::unordered_map<int32_t /* origin seg id*/, CalcDeleteBitmapTask>
calc_dbm_tasks;
};
// used for controll compaction
diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp
b/be/src/olap/rowset/beta_rowset_writer.cpp
index 71760b343f7..05f14af1ddd 100644
--- a/be/src/olap/rowset/beta_rowset_writer.cpp
+++ b/be/src/olap/rowset/beta_rowset_writer.cpp
@@ -336,14 +336,10 @@ Status
BaseBetaRowsetWriter::_generate_delete_bitmap(int32_t segment_id) {
_context.tablet->get_rowset_by_ids(_context.mow_context->rowset_ids.get());
}
OlapStopWatch watch;
- auto finish_callback = [this](segment_v2::SegmentSharedPtr segment, Status
st) {
- auto* task = calc_delete_bitmap_task(segment->id());
- task->set_status(st);
- };
RETURN_IF_ERROR(BaseTablet::calc_delete_bitmap(
_context.tablet, rowset_ptr, segments, specified_rowsets,
_context.mow_context->delete_bitmap,
_context.mow_context->max_version,
- _calc_delete_bitmap_token.get(), nullptr, nullptr,
std::move(finish_callback)));
+ _calc_delete_bitmap_token.get()));
size_t total_rows = std::accumulate(
segments.begin(), segments.end(), 0,
[](size_t sum, const segment_v2::SegmentSharedPtr& s) { return sum
+= s->num_rows(); });
@@ -426,8 +422,8 @@ Status
BetaRowsetWriter::_find_longest_consecutive_small_segment(
auto dst_seg_id = _num_segcompacted.load();
RETURN_IF_ERROR(_rename_compacted_segment_plain(_segcompacted_point++));
if (_segcompaction_worker->need_convert_delete_bitmap()) {
-
RETURN_IF_ERROR(_segcompaction_worker->convert_segment_delete_bitmap(
- segment, _context.mow_context->delete_bitmap,
segid, dst_seg_id));
+ _segcompaction_worker->convert_segment_delete_bitmap(
+ _context.mow_context->delete_bitmap, segid,
dst_seg_id);
}
continue;
} else {
@@ -455,12 +451,10 @@ Status
BetaRowsetWriter::_find_longest_consecutive_small_segment(
VLOG_DEBUG << "only one candidate segment";
auto src_seg_id = _segcompacted_point.load();
auto dst_seg_id = _num_segcompacted.load();
- segment_v2::SegmentSharedPtr segment;
- RETURN_IF_ERROR(_load_noncompacted_segment(segment, src_seg_id));
RETURN_IF_ERROR(_rename_compacted_segment_plain(_segcompacted_point++));
if (_segcompaction_worker->need_convert_delete_bitmap()) {
-
RETURN_IF_ERROR(_segcompaction_worker->convert_segment_delete_bitmap(
- segment, _context.mow_context->delete_bitmap, src_seg_id,
dst_seg_id));
+ _segcompaction_worker->convert_segment_delete_bitmap(
+ _context.mow_context->delete_bitmap, src_seg_id,
dst_seg_id);
}
segments->clear();
return Status::OK();
@@ -659,12 +653,10 @@ Status
BetaRowsetWriter::_segcompaction_rename_last_segments() {
VLOG_DEBUG << "segcompaction last few segments";
for (int32_t segid = _segcompacted_point; segid < _num_segment; segid++) {
auto dst_segid = _num_segcompacted.load();
- segment_v2::SegmentSharedPtr segment;
- RETURN_IF_ERROR(_load_noncompacted_segment(segment, segid));
RETURN_IF_ERROR(_rename_compacted_segment_plain(_segcompacted_point++));
if (_segcompaction_worker->need_convert_delete_bitmap()) {
-
RETURN_IF_ERROR(_segcompaction_worker->convert_segment_delete_bitmap(
- segment, _context.mow_context->delete_bitmap, segid,
dst_segid));
+ _segcompaction_worker->convert_segment_delete_bitmap(
+ _context.mow_context->delete_bitmap, segid, dst_segid);
}
}
return Status::OK();
diff --git a/be/src/olap/rowset/beta_rowset_writer.h
b/be/src/olap/rowset/beta_rowset_writer.h
index e21bdf4009d..1c62853c8b0 100644
--- a/be/src/olap/rowset/beta_rowset_writer.h
+++ b/be/src/olap/rowset/beta_rowset_writer.h
@@ -193,13 +193,6 @@ public:
return this->_idx_files.get_file_writers();
}
- CalcDeleteBitmapToken* calc_delete_bitmap_token() { return
_calc_delete_bitmap_token.get(); }
-
- CalcDeleteBitmapTask* calc_delete_bitmap_task(int32_t segment_id) {
- DCHECK(_context.mow_context != nullptr);
- return _context.mow_context->get_calc_dbm_task(segment_id);
- }
-
private:
// build a tmp rowset for load segment to calc delete_bitmap
// for this segment
@@ -259,6 +252,7 @@ protected:
fmt::memory_buffer vlog_buffer;
+ std::shared_ptr<MowContext> _mow_context;
std::unique_ptr<CalcDeleteBitmapToken> _calc_delete_bitmap_token;
int64_t _delete_bitmap_ns = 0;
diff --git a/be/src/olap/rowset/segcompaction.cpp
b/be/src/olap/rowset/segcompaction.cpp
index 456d64d82b5..c5478499bc0 100644
--- a/be/src/olap/rowset/segcompaction.cpp
+++ b/be/src/olap/rowset/segcompaction.cpp
@@ -340,8 +340,8 @@ Status
SegcompactionWorker::_do_compact_segments(SegCompactionCandidatesSharedPt
RETURN_IF_ERROR(_delete_original_segments(begin, end));
if (_rowid_conversion != nullptr) {
- RETURN_IF_ERROR(convert_segment_delete_bitmap(segments,
ctx.mow_context->delete_bitmap,
- begin, end,
_writer->_num_segcompacted));
+ convert_segment_delete_bitmap(ctx.mow_context->delete_bitmap, begin,
end,
+ _writer->_num_segcompacted);
}
RETURN_IF_ERROR(_writer->_rename_compacted_segments(begin, end));
if (_index_file_writer != nullptr) {
@@ -370,21 +370,6 @@ Status
SegcompactionWorker::_do_compact_segments(SegCompactionCandidatesSharedPt
return Status::OK();
}
-Status SegcompactionWorker::_wait_calc_delete_bitmap(const
SegCompactionCandidates& segments) {
- for (const auto& segment : segments) {
- auto* calc_task = _writer->calc_delete_bitmap_task(segment->id());
- if (auto st = calc_task->get_status() /* blocking */; !st.ok()) {
- LOG_WARNING("failed to wait for segment calc dbm task")
- .tag("tablet_id", _writer->context().tablet_id)
- .tag("rowset_id", segment->rowset_id().to_string())
- .tag("segment_id", segment->id())
- .tag("error", st.to_string());
- return st;
- }
- }
- return Status::OK();
-}
-
void SegcompactionWorker::compact_segments(SegCompactionCandidatesSharedPtr
segments) {
Status status = Status::OK();
if (_is_compacting_state_mutable.exchange(false)) {
@@ -430,12 +415,8 @@ bool SegcompactionWorker::need_convert_delete_bitmap() {
tablet->tablet_schema()->has_sequence_col();
}
-Status
SegcompactionWorker::convert_segment_delete_bitmap(segment_v2::SegmentSharedPtr
segment,
- DeleteBitmapPtr
src_delete_bitmap,
- uint32_t src_seg_id,
- uint32_t
dest_seg_id) {
- // should wait until delete bitmaps on input segments are generated before
converting them
- RETURN_IF_ERROR(_wait_calc_delete_bitmap({segment}));
+void SegcompactionWorker::convert_segment_delete_bitmap(DeleteBitmapPtr
src_delete_bitmap,
+ uint32_t src_seg_id,
uint32_t dest_seg_id) {
// lazy init
if (nullptr == _converted_delete_bitmap) {
_converted_delete_bitmap =
std::make_shared<DeleteBitmap>(_writer->context().tablet_id);
@@ -447,16 +428,11 @@ Status
SegcompactionWorker::convert_segment_delete_bitmap(segment_v2::SegmentSha
_converted_delete_bitmap->set({rowset_id, dest_seg_id,
DeleteBitmap::TEMP_VERSION_COMMON},
*seg_map);
}
-
DBUG_EXECUTE_IF("SegcompactionWorker::convert_segment_delete_bitmap.after",
DBUG_BLOCK);
- return Status::OK();
}
-Status
SegcompactionWorker::convert_segment_delete_bitmap(SegCompactionCandidatesSharedPtr
segments,
- DeleteBitmapPtr
src_delete_bitmap,
- uint32_t src_begin,
uint32_t src_end,
- uint32_t dst_seg_id)
{
- // should wait until delete bitmaps on input segments are generated before
converting them
- RETURN_IF_ERROR(_wait_calc_delete_bitmap(*segments));
+void SegcompactionWorker::convert_segment_delete_bitmap(DeleteBitmapPtr
src_delete_bitmap,
+ uint32_t src_begin,
uint32_t src_end,
+ uint32_t dst_seg_id) {
// lazy init
if (nullptr == _converted_delete_bitmap) {
_converted_delete_bitmap =
std::make_shared<DeleteBitmap>(_writer->context().tablet_id);
@@ -480,8 +456,6 @@ Status
SegcompactionWorker::convert_segment_delete_bitmap(SegCompactionCandidate
{rowset_id, dst_seg_id,
DeleteBitmap::TEMP_VERSION_COMMON}, dst_row_id);
}
}
-
DBUG_EXECUTE_IF("SegcompactionWorker::convert_segment_delete_bitmap.after",
DBUG_BLOCK);
- return Status::OK();
}
bool SegcompactionWorker::cancel() {
diff --git a/be/src/olap/rowset/segcompaction.h
b/be/src/olap/rowset/segcompaction.h
index bcd800f78ac..121fec8d100 100644
--- a/be/src/olap/rowset/segcompaction.h
+++ b/be/src/olap/rowset/segcompaction.h
@@ -63,12 +63,10 @@ public:
bool need_convert_delete_bitmap();
- Status convert_segment_delete_bitmap(segment_v2::SegmentSharedPtr segment,
- DeleteBitmapPtr src_delete_bitmap,
uint32_t src_seg_id,
- uint32_t dest_seg_id);
- Status convert_segment_delete_bitmap(SegCompactionCandidatesSharedPtr
segments,
- DeleteBitmapPtr src_delete_bitmap,
uint32_t src_begin,
- uint32_t src_end, uint32_t
dest_seg_id);
+ void convert_segment_delete_bitmap(DeleteBitmapPtr src_delete_bitmap,
uint32_t src_seg_id,
+ uint32_t dest_seg_id);
+ void convert_segment_delete_bitmap(DeleteBitmapPtr src_delete_bitmap,
uint32_t src_begin,
+ uint32_t src_end, uint32_t dest_seg_id);
DeleteBitmapPtr get_converted_delete_bitmap() { return
_converted_delete_bitmap; }
io::FileWriterPtr& get_file_writer() { return _file_writer; }
@@ -95,7 +93,6 @@ private:
Status _check_correctness(OlapReaderStatistics& reader_stat,
Merger::Statistics& merger_stat,
uint32_t begin, uint32_t end, bool
is_mow_with_cluster_keys);
Status _do_compact_segments(SegCompactionCandidatesSharedPtr segments);
- Status _wait_calc_delete_bitmap(const SegCompactionCandidates& segments);
private:
//TODO(zhengyu): current impl depends heavily on the access to feilds of
BetaRowsetWriter
diff --git a/be/test/olap/segcompaction_mow_test.cpp
b/be/test/olap/segcompaction_mow_test.cpp
index d8bee2b9ce5..d1e2b44ce4d 100644
--- a/be/test/olap/segcompaction_mow_test.cpp
+++ b/be/test/olap/segcompaction_mow_test.cpp
@@ -359,7 +359,6 @@ TEST_P(SegCompactionMoWTest, SegCompactionThenRead) {
}
}
}
-
writer_context.mow_context->get_calc_dbm_task(i)->set_status(Status::OK());
s = rowset_writer->add_block(&block);
EXPECT_TRUE(s.ok());
s = rowset_writer->flush();
@@ -420,8 +419,6 @@ TEST_F(SegCompactionMoWTest,
SegCompactionInterleaveWithBig_ooooOOoOooooooooO) {
DeleteBitmapPtr delete_bitmap = std::make_shared<DeleteBitmap>(TABLET_ID);
uint32_t rows_mark_deleted = 0;
uint32_t total_written_rows = 0;
-
- uint32_t cur_seg_id = 0;
{ // write `num_segments * rows_per_segment` rows to rowset
RowsetWriterContext writer_context;
create_rowset_writer_context(20048, tablet_schema, &writer_context);
@@ -462,7 +459,6 @@ TEST_F(SegCompactionMoWTest,
SegCompactionInterleaveWithBig_ooooOOoOooooooooO) {
rows_mark_deleted++;
}
}
-
writer_context.mow_context->get_calc_dbm_task(cur_seg_id++)->set_status(Status::OK());
s = rowset_writer->add_block(&block);
EXPECT_TRUE(s.ok());
s = rowset_writer->flush();
@@ -470,7 +466,6 @@ TEST_F(SegCompactionMoWTest,
SegCompactionInterleaveWithBig_ooooOOoOooooooooO) {
segid++;
total_written_rows += rows_per_segment;
}
-
num_segments = 2;
rows_per_segment = 6400;
for (int i = 0; i < num_segments; ++i) {
@@ -492,7 +487,6 @@ TEST_F(SegCompactionMoWTest,
SegCompactionInterleaveWithBig_ooooOOoOooooooooO) {
rows_mark_deleted++;
}
}
-
writer_context.mow_context->get_calc_dbm_task(cur_seg_id++)->set_status(Status::OK());
s = rowset_writer->add_block(&block);
EXPECT_TRUE(s.ok());
s = rowset_writer->flush();
@@ -500,7 +494,6 @@ TEST_F(SegCompactionMoWTest,
SegCompactionInterleaveWithBig_ooooOOoOooooooooO) {
segid++;
total_written_rows += rows_per_segment;
}
-
num_segments = 1;
rows_per_segment = 4096;
for (int i = 0; i < num_segments; ++i) {
@@ -522,7 +515,6 @@ TEST_F(SegCompactionMoWTest,
SegCompactionInterleaveWithBig_ooooOOoOooooooooO) {
rows_mark_deleted++;
}
}
-
writer_context.mow_context->get_calc_dbm_task(cur_seg_id++)->set_status(Status::OK());
s = rowset_writer->add_block(&block);
EXPECT_TRUE(s.ok());
s = rowset_writer->flush();
@@ -530,7 +522,6 @@ TEST_F(SegCompactionMoWTest,
SegCompactionInterleaveWithBig_ooooOOoOooooooooO) {
segid++;
total_written_rows += rows_per_segment;
}
-
num_segments = 1;
rows_per_segment = 6400;
for (int i = 0; i < num_segments; ++i) {
@@ -552,7 +543,6 @@ TEST_F(SegCompactionMoWTest,
SegCompactionInterleaveWithBig_ooooOOoOooooooooO) {
rows_mark_deleted++;
}
}
-
writer_context.mow_context->get_calc_dbm_task(cur_seg_id++)->set_status(Status::OK());
s = rowset_writer->add_block(&block);
EXPECT_TRUE(s.ok());
s = rowset_writer->flush();
@@ -560,7 +550,6 @@ TEST_F(SegCompactionMoWTest,
SegCompactionInterleaveWithBig_ooooOOoOooooooooO) {
segid++;
total_written_rows += rows_per_segment;
}
-
num_segments = 8;
rows_per_segment = 4096;
std::map<uint32_t, uint32_t> unique_keys;
@@ -585,7 +574,6 @@ TEST_F(SegCompactionMoWTest,
SegCompactionInterleaveWithBig_ooooOOoOooooooooO) {
}
unique_keys.emplace(k1, rid);
}
-
writer_context.mow_context->get_calc_dbm_task(cur_seg_id++)->set_status(Status::OK());
s = rowset_writer->add_block(&block);
EXPECT_TRUE(s.ok());
s = rowset_writer->flush();
@@ -623,7 +611,6 @@ TEST_F(SegCompactionMoWTest,
SegCompactionInterleaveWithBig_ooooOOoOooooooooO) {
rows_mark_deleted++;
}
}
-
writer_context.mow_context->get_calc_dbm_task(cur_seg_id++)->set_status(Status::OK());
s = rowset_writer->add_block(&block);
EXPECT_TRUE(s.ok());
s = rowset_writer->flush();
@@ -662,7 +649,6 @@ TEST_F(SegCompactionMoWTest,
SegCompactionInterleaveWithBig_OoOoO) {
DeleteBitmapPtr delete_bitmap = std::make_shared<DeleteBitmap>(TABLET_ID);
uint32_t rows_mark_deleted = 0;
uint32_t total_written_rows = 0;
- uint32_t cur_seg_id = 0;
{ // write `num_segments * rows_per_segment` rows to rowset
RowsetWriterContext writer_context;
create_rowset_writer_context(20049, tablet_schema, &writer_context);
@@ -703,7 +689,6 @@ TEST_F(SegCompactionMoWTest,
SegCompactionInterleaveWithBig_OoOoO) {
rows_mark_deleted++;
}
}
-
writer_context.mow_context->get_calc_dbm_task(cur_seg_id++)->set_status(Status::OK());
s = rowset_writer->add_block(&block);
EXPECT_TRUE(s.ok());
s = rowset_writer->flush();
@@ -732,7 +717,6 @@ TEST_F(SegCompactionMoWTest,
SegCompactionInterleaveWithBig_OoOoO) {
rows_mark_deleted++;
}
}
-
writer_context.mow_context->get_calc_dbm_task(cur_seg_id++)->set_status(Status::OK());
s = rowset_writer->add_block(&block);
EXPECT_TRUE(s.ok());
s = rowset_writer->flush();
@@ -761,7 +745,6 @@ TEST_F(SegCompactionMoWTest,
SegCompactionInterleaveWithBig_OoOoO) {
rows_mark_deleted++;
}
}
-
writer_context.mow_context->get_calc_dbm_task(cur_seg_id++)->set_status(Status::OK());
s = rowset_writer->add_block(&block);
EXPECT_TRUE(s.ok());
s = rowset_writer->flush();
@@ -790,7 +773,6 @@ TEST_F(SegCompactionMoWTest,
SegCompactionInterleaveWithBig_OoOoO) {
rows_mark_deleted++;
}
}
-
writer_context.mow_context->get_calc_dbm_task(cur_seg_id++)->set_status(Status::OK());
s = rowset_writer->add_block(&block);
EXPECT_TRUE(s.ok());
s = rowset_writer->flush();
@@ -819,7 +801,6 @@ TEST_F(SegCompactionMoWTest,
SegCompactionInterleaveWithBig_OoOoO) {
rows_mark_deleted++;
}
}
-
writer_context.mow_context->get_calc_dbm_task(cur_seg_id++)->set_status(Status::OK());
s = rowset_writer->add_block(&block);
EXPECT_TRUE(s.ok());
s = rowset_writer->flush();
@@ -893,7 +874,6 @@ TEST_F(SegCompactionMoWTest, SegCompactionNotTrigger) {
rows_mark_deleted++;
}
}
-
writer_context.mow_context->get_calc_dbm_task(i)->set_status(Status::OK());
s = rowset_writer->add_block(&block);
EXPECT_TRUE(s.ok());
s = rowset_writer->flush();
diff --git
a/regression-test/data/fault_injection_p0/test_mow_seq_seg_compaction.out
b/regression-test/data/fault_injection_p0/test_mow_seq_seg_compaction.out
deleted file mode 100644
index ad5c6fa5903..00000000000
--- a/regression-test/data/fault_injection_p0/test_mow_seq_seg_compaction.out
+++ /dev/null
@@ -1,23 +0,0 @@
--- This file is automatically generated. You should know what you did if you
want to edit this
--- !sql --
-4065
-
--- !sql --
-8130
-
--- !sql --
-0 0 99999 2 0 99999
-1 1 99999 2 0 99999
-2 2 99999 2 0 99999
-3 3 99999 2 0 99999
-4 4 99999 2 0 99999
-5 5 99999 2 0 99999
-6 6 99999 2 0 99999
-7 7 99999 2 0 99999
-8 8 99999 2 0 99999
-9 9 99999 2 0 99999
-10 10 99999 2 0 99999
-
--- !dup_key_count --
-0
-
diff --git
a/regression-test/suites/fault_injection_p0/test_mow_seq_seg_compaction.groovy
b/regression-test/suites/fault_injection_p0/test_mow_seq_seg_compaction.groovy
deleted file mode 100644
index 375e3627db9..00000000000
---
a/regression-test/suites/fault_injection_p0/test_mow_seq_seg_compaction.groovy
+++ /dev/null
@@ -1,103 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-suite("test_mow_seq_seg_compaction", "nonConcurrent") {
- def table1 = "test_mow_seq_seg_compaction"
- sql "DROP TABLE IF EXISTS ${table1} FORCE;"
- sql """ CREATE TABLE IF NOT EXISTS ${table1} (
- `k1` int NOT NULL,
- `c1` int,
- `c2` int
- )UNIQUE KEY(k1)
- DISTRIBUTED BY HASH(k1) BUCKETS 1
- PROPERTIES (
- "enable_unique_key_merge_on_write" = "true",
- "disable_auto_compaction" = "true",
- "function_column.sequence_col" = "c2",
- "replication_num" = "1"); """
-
- sql """insert into ${table1} select number,number,99999 from
numbers("number"="4065");"""
- sql "sync;"
- qt_sql "select count() from ${table1};"
-
- def tabletStats = sql_return_maparray """ show tablets from ${table1}; """
- assert tabletStats.size() == 1
- def tabletId = tabletStats[0].TabletId
-
- // to cause multi segments and segment compaction
- def customBeConfig = [
- doris_scanner_row_bytes : 1,
- segcompaction_batch_size: 2
- ]
-
- setBeConfigTemporary(customBeConfig) {
- try {
- GetDebugPoint().clearDebugPointsForAllBEs()
- GetDebugPoint().clearDebugPointsForAllFEs()
- // batch_size is 4164 in csv_reader.cpp
- // _batch_size is 8192 in vtablet_writer.cpp
- // to cause multi segments
- GetDebugPoint().enableDebugPointForAllBEs("MemTable.need_flush")
-
-
GetDebugPoint().enableDebugPointForAllBEs("BaseTablet::calc_segment_delete_bitmap.sleep",
[tablet_id: "${tabletId}"])
- //
GetDebugPoint().enableDebugPointForAllBEs("SegcompactionWorker::convert_segment_delete_bitmap.after")
- Thread.sleep(1000)
-
- int rows = 4064
- // load data that will have multi segments and there are duplicate
keys between segments
- String content = ""
- (1..rows).each {
- int x = 100000 + it
- content += "${x},${x},${x}\n"
- }
- (1..rows).each {
- int x = it
- content += "${x},${x},${1}\n"
- }
- content += "200000,200000,200000"
- def t1 = Thread.start {
- streamLoad {
- table "${table1}"
- set 'column_separator', ','
- inputStream new ByteArrayInputStream(content.getBytes())
- set 'memtable_on_sink_node', 'false'
- time 30000
-
- check { result, exception, startTime, endTime ->
- if (exception != null) {
- throw exception
- }
- def json = parseJson(result)
- assert "success" == json.Status.toLowerCase()
- }
- }
- }
- // Thread.sleep(2000)
- //
GetDebugPoint().disableDebugPointForAllBEs("SegcompactionWorker::convert_segment_delete_bitmap.after")
- t1.join()
- qt_sql "select count() from ${table1};"
- qt_sql "select *,__DORIS_VERSION_COL__ as ver,
__DORIS_DELETE_SIGN__ as del,__DORIS_SEQUENCE_COL__ as seq from ${table1} where
k1<=10 order by k1,__DORIS_VERSION_COL__;"
- qt_dup_key_count "select count() from (select k1,count() as cnt
from ${table1} group by k1 having cnt > 1) A;"
- } catch(Exception e) {
- logger.info(e.getMessage())
- throw e
- } finally {
- GetDebugPoint().clearDebugPointsForAllBEs()
- GetDebugPoint().clearDebugPointsForAllFEs()
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
