This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3b4cc06a62 [feature](move-memtable)[1/8] support creating segments
externally in rowset writer (#23085)
3b4cc06a62 is described below
commit 3b4cc06a626060d71e41e827b78202d2b7cd173e
Author: Kaijie Chen <[email protected]>
AuthorDate: Mon Aug 21 21:02:34 2023 +0800
[feature](move-memtable)[1/8] support creating segments externally in
rowset writer (#23085)
---
be/src/olap/rowset/beta_rowset_writer.cpp | 20 +++++++++++---------
be/src/olap/rowset/beta_rowset_writer.h | 11 ++---------
be/src/olap/rowset/rowset_writer.h | 15 ++++++++++++++-
3 files changed, 27 insertions(+), 19 deletions(-)
diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp
b/be/src/olap/rowset/beta_rowset_writer.cpp
index d3b0ee3d2f..e62e182b3d 100644
--- a/be/src/olap/rowset/beta_rowset_writer.cpp
+++ b/be/src/olap/rowset/beta_rowset_writer.cpp
@@ -446,8 +446,6 @@ Status BetaRowsetWriter::flush_memtable(vectorized::Block*
block, int32_t segmen
RETURN_IF_ERROR(
_segment_creator.flush_single_block(block, segment_id,
flush_size, flush_schema));
}
- RETURN_IF_ERROR(_generate_delete_bitmap(segment_id));
- RETURN_IF_ERROR(_segcompaction_if_necessary());
return Status::OK();
}
@@ -714,18 +712,18 @@ Status BetaRowsetWriter::_check_segment_number_limit() {
return Status::OK();
}
-Status BetaRowsetWriter::add_segment(uint32_t segid, SegmentStatistics&
segstat) {
- uint32_t segid_offset = segid - _segment_start_id;
+Status BetaRowsetWriter::add_segment(uint32_t segment_id, SegmentStatistics&
segstat) {
+ uint32_t segid_offset = segment_id - _segment_start_id;
{
std::lock_guard<std::mutex> lock(_segid_statistics_map_mutex);
- CHECK_EQ(_segid_statistics_map.find(segid) ==
_segid_statistics_map.end(), true);
- _segid_statistics_map.emplace(segid, segstat);
- if (segid >= _segment_num_rows.size()) {
- _segment_num_rows.resize(segid + 1);
+ CHECK_EQ(_segid_statistics_map.find(segment_id) ==
_segid_statistics_map.end(), true);
+ _segid_statistics_map.emplace(segment_id, segstat);
+ if (segment_id >= _segment_num_rows.size()) {
+ _segment_num_rows.resize(segment_id + 1);
}
_segment_num_rows[segid_offset] = segstat.row_num;
}
- VLOG_DEBUG << "_segid_statistics_map add new record. segid:" << segid
+ VLOG_DEBUG << "_segid_statistics_map add new record. segment_id:" <<
segment_id
<< " row_num:" << segstat.row_num << " data_size:" <<
segstat.data_size
<< " index_size:" << segstat.index_size;
@@ -736,6 +734,10 @@ Status BetaRowsetWriter::add_segment(uint32_t segid,
SegmentStatistics& segstat)
_num_segment++;
}
}
+ if (_context.mow_context != nullptr) {
+ RETURN_IF_ERROR(_generate_delete_bitmap(segment_id));
+ }
+ RETURN_IF_ERROR(_segcompaction_if_necessary());
return Status::OK();
}
diff --git a/be/src/olap/rowset/beta_rowset_writer.h
b/be/src/olap/rowset/beta_rowset_writer.h
index 3ab1e176e4..38cb6d24f9 100644
--- a/be/src/olap/rowset/beta_rowset_writer.h
+++ b/be/src/olap/rowset/beta_rowset_writer.h
@@ -62,13 +62,6 @@ namespace vectorized::schema_util {
class LocalSchemaChangeRecorder;
}
-struct SegmentStatistics {
- int64_t row_num;
- int64_t data_size;
- int64_t index_size;
- KeyBoundsPB key_bounds;
-};
-
class BetaRowsetWriter : public RowsetWriter {
friend class SegcompactionWorker;
@@ -86,9 +79,9 @@ public:
Status add_rowset_for_linked_schema_change(RowsetSharedPtr rowset)
override;
- Status create_file_writer(uint32_t segment_id, io::FileWriterPtr& writer);
+ Status create_file_writer(uint32_t segment_id, io::FileWriterPtr& writer)
override;
- Status add_segment(uint32_t segid, SegmentStatistics& segstat);
+ Status add_segment(uint32_t segment_id, SegmentStatistics& segstat)
override;
Status flush() override;
diff --git a/be/src/olap/rowset/rowset_writer.h
b/be/src/olap/rowset/rowset_writer.h
index 16287aa273..61cb20498a 100644
--- a/be/src/olap/rowset/rowset_writer.h
+++ b/be/src/olap/rowset/rowset_writer.h
@@ -33,7 +33,12 @@
namespace doris {
-class MemTable;
+struct SegmentStatistics {
+ int64_t row_num;
+ int64_t data_size;
+ int64_t index_size;
+ KeyBoundsPB key_bounds;
+};
class RowsetWriter {
public:
@@ -58,6 +63,10 @@ public:
// Precondition: the input `rowset` should have the same type of the
rowset we're building
virtual Status add_rowset_for_linked_schema_change(RowsetSharedPtr rowset)
= 0;
+ virtual Status create_file_writer(uint32_t segment_id, io::FileWriterPtr&
writer) {
+ return Status::NotSupported("RowsetWriter does not support
create_file_writer");
+ }
+
// explicit flush all buffered rows into segment file.
// note that `add_row` could also trigger flush when certain conditions
are met
virtual Status flush() = 0;
@@ -81,6 +90,10 @@ public:
"RowsetWriter not support flush_single_block");
}
+ virtual Status add_segment(uint32_t segment_id, SegmentStatistics&
segstat) {
+ return Status::NotSupported("RowsetWriter does not support
add_segment");
+ }
+
// finish building and return pointer to the built rowset (guaranteed to
be inited).
// return nullptr when failed
virtual RowsetSharedPtr build() = 0;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
