github-actions[bot] commented on code in PR #16731:
URL: https://github.com/apache/doris/pull/16731#discussion_r1112928298
##########
be/src/olap/rowset/beta_rowset_writer.cpp:
##########
@@ -116,35 +118,46 @@ Status BetaRowsetWriter::add_block(const
vectorized::Block* block) {
return _add_block(block, &_segment_writer);
}
-RowwiseIteratorUPtr BetaRowsetWriter::_get_segcompaction_reader(
- SegCompactionCandidatesSharedPtr segments, std::shared_ptr<Schema>
schema,
- OlapReaderStatistics* stat, uint64_t* merged_row_stat) {
+std::unique_ptr<vectorized::VerticalBlockReader>
BetaRowsetWriter::_get_segcompaction_reader(
Review Comment:
warning: method '_get_segcompaction_reader' can be made const
[readability-make-member-function-const]
be/src/olap/rowset/beta_rowset_writer.cpp:122:
```diff
- std::shared_ptr<Schema> schema, OlapReaderStatistics* stat,
uint64_t* merged_row_stat, vectorized::RowSourcesBuffer& row_sources_buf, bool
is_key, std::vector<uint32_t>& return_columns) {
+ std::shared_ptr<Schema> schema, OlapReaderStatistics* stat,
uint64_t* merged_row_stat, vectorized::RowSourcesBuffer& row_sources_buf, bool
is_key, std::vector<uint32_t>& return_columns) const {
```
be/src/olap/rowset/beta_rowset_writer.h:121:
```diff
-
vectorized::RowSourcesBuffer& row_sources_buf, bool is_key,
std::vector<uint32_t>& return_columns);
+
vectorized::RowSourcesBuffer& row_sources_buf, bool is_key,
std::vector<uint32_t>& return_columns) const;
```
##########
be/src/vec/olap/vertical_block_reader.cpp:
##########
@@ -86,9 +86,26 @@ Status VerticalBlockReader::_init_collect_iter(const
ReaderParams& read_params)
std::vector<RowwiseIteratorUPtr> segment_iters;
std::vector<bool> iterator_init_flag;
std::vector<RowsetId> rowset_ids;
- RETURN_IF_ERROR(
- _get_segment_iterators(read_params, &segment_iters,
&iterator_init_flag, &rowset_ids));
- CHECK(segment_iters.size() == iterator_init_flag.size());
+ std::vector<RowwiseIterator*>* segment_iters_ptr =
read_params.segment_iters_ptr;
+ std::vector<RowwiseIterator*> segment_iters;
Review Comment:
warning: redefinition of 'segment_iters' with a different type:
'vector<doris::RowwiseIterator *>' vs 'vector<doris::RowwiseIteratorUPtr>'
[clang-diagnostic-error]
```cpp
std::vector<RowwiseIterator*> segment_iters;
^
```
**be/src/vec/olap/vertical_block_reader.cpp:85:** previous definition is here
```cpp
std::vector<RowwiseIteratorUPtr> segment_iters;
^
```
##########
be/test/olap/segcompaction_test.cpp:
##########
@@ -200,12 +223,124 @@
std::unique_ptr<DataDir> _data_dir;
};
+TEST_F(SegCompactionTest, SegCompactionThenRead) {
+ config::enable_segcompaction = true;
+ config::enable_storage_vectorization = true;
+ Status s;
+ TabletSchemaSPtr tablet_schema = std::make_shared<TabletSchema>();
+ create_tablet_schema(tablet_schema, DUP_KEYS);
+
+ RowsetSharedPtr rowset;
+ const int num_segments = 15;
+ const uint32_t rows_per_segment = 4096;
+ config::segcompaction_small_threshold = 6000; // set threshold above
+ // rows_per_segment
+ config::segcompaction_threshold_segment_num = 10;
+ std::vector<uint32_t> segment_num_rows;
+ { // write `num_segments * rows_per_segment` rows to rowset
+ RowsetWriterContext writer_context;
+ create_rowset_writer_context(10047, tablet_schema, &writer_context);
+
+ std::unique_ptr<RowsetWriter> rowset_writer;
+ s = RowsetFactory::create_rowset_writer(writer_context, false,
&rowset_writer);
+ EXPECT_EQ(Status::OK(), s);
+
+ RowCursor input_row;
+ input_row.init(tablet_schema);
+
+ // for segment "i", row "rid"
+ // k1 := rid*10 + i
+ // k2 := k1 * 10
+ // k3 := rid
+ for (int i = 0; i < num_segments; ++i) {
+ MemPool mem_pool;
+ for (int rid = 0; rid < rows_per_segment; ++rid) {
+ uint32_t k1 = rid * 100 + i;
+ uint32_t k2 = i;
+ uint32_t k3 = rid;
+ input_row.set_field_content(0, reinterpret_cast<char*>(&k1),
&mem_pool);
+ input_row.set_field_content(1, reinterpret_cast<char*>(&k2),
&mem_pool);
+ input_row.set_field_content(2, reinterpret_cast<char*>(&k3),
&mem_pool);
+ s = rowset_writer->add_row(input_row);
Review Comment:
warning: no member named 'set_field_content' in 'doris::RowCursor'
[clang-diagnostic-error]
```cpp
input_row.set_field_content(2, reinterpret_cast<char*>(&k3),
&mem_pool);
^
```
##########
be/test/olap/segcompaction_test.cpp:
##########
@@ -184,6 +186,27 @@
rowset_writer_context->tablet_schema = tablet_schema;
rowset_writer_context->version.first = 10;
rowset_writer_context->version.second = 10;
+
+#if 0
+ TCreateTabletReq req;
+ req.table_id =
+ req.tablet_id =
+ req.tablet_scheme =
+ req.partition_id =
+ l_engine->create_tablet(req);
+ rowset_writer_context->tablet =
l_engine->tablet_manager()->get_tablet(TTabletId tablet_id);
+#endif
+ std::shared_ptr<DataDir> data_dir =
std::make_shared<DataDir>(lTestDir);
+ TabletMetaSharedPtr tablet_meta = std::make_shared<TabletMeta>();
+ tablet_meta->_tablet_id = 1;
+ tablet_meta->_schema = tablet_schema;
+ auto tablet = std::make_shared<Tablet>(tablet_meta, data_dir.get(),
"test_str");
Review Comment:
warning: '_schema' is a private member of 'doris::TabletMeta'
[clang-diagnostic-error]
```cpp
tablet_meta->_schema = tablet_schema;
^
```
**be/src/olap/tablet_meta.h:229:** declared private here
```cpp
TabletSchemaSPtr _schema;
^
```
##########
be/test/olap/segcompaction_test.cpp:
##########
@@ -200,12 +223,124 @@
std::unique_ptr<DataDir> _data_dir;
};
+TEST_F(SegCompactionTest, SegCompactionThenRead) {
+ config::enable_segcompaction = true;
+ config::enable_storage_vectorization = true;
+ Status s;
+ TabletSchemaSPtr tablet_schema = std::make_shared<TabletSchema>();
+ create_tablet_schema(tablet_schema, DUP_KEYS);
+
+ RowsetSharedPtr rowset;
+ const int num_segments = 15;
+ const uint32_t rows_per_segment = 4096;
+ config::segcompaction_small_threshold = 6000; // set threshold above
+ // rows_per_segment
+ config::segcompaction_threshold_segment_num = 10;
+ std::vector<uint32_t> segment_num_rows;
+ { // write `num_segments * rows_per_segment` rows to rowset
+ RowsetWriterContext writer_context;
+ create_rowset_writer_context(10047, tablet_schema, &writer_context);
+
+ std::unique_ptr<RowsetWriter> rowset_writer;
+ s = RowsetFactory::create_rowset_writer(writer_context, false,
&rowset_writer);
+ EXPECT_EQ(Status::OK(), s);
+
+ RowCursor input_row;
+ input_row.init(tablet_schema);
+
+ // for segment "i", row "rid"
+ // k1 := rid*10 + i
+ // k2 := k1 * 10
+ // k3 := rid
+ for (int i = 0; i < num_segments; ++i) {
+ MemPool mem_pool;
+ for (int rid = 0; rid < rows_per_segment; ++rid) {
+ uint32_t k1 = rid * 100 + i;
+ uint32_t k2 = i;
+ uint32_t k3 = rid;
+ input_row.set_field_content(0, reinterpret_cast<char*>(&k1),
&mem_pool);
+ input_row.set_field_content(1, reinterpret_cast<char*>(&k2),
&mem_pool);
+ input_row.set_field_content(2, reinterpret_cast<char*>(&k3),
&mem_pool);
+ s = rowset_writer->add_row(input_row);
+ EXPECT_EQ(Status::OK(), s);
Review Comment:
warning: no member named 'add_row' in 'doris::RowsetWriter'
[clang-diagnostic-error]
```cpp
s = rowset_writer->add_row(input_row);
^
```
##########
be/src/vec/olap/vertical_block_reader.cpp:
##########
@@ -98,11 +115,11 @@
seq_col_idx =
read_params.tablet->tablet_schema()->sequence_col_idx();
}
_vcollect_iter = new_vertical_heap_merge_iterator(
Review Comment:
warning: no matching function for call to 'new_vertical_heap_merge_iterator'
[clang-diagnostic-error]
```cpp
_vcollect_iter = new_vertical_heap_merge_iterator(
^
```
**be/src/vec/olap/vertical_merge_iterator.h:330:** candidate function not
viable: no known conversion from 'vector<doris::RowwiseIterator *>' to
'vector<doris::RowwiseIteratorUPtr>' for 1st argument
```cpp
std::shared_ptr<RowwiseIterator> new_vertical_heap_merge_iterator(
^
```
##########
be/test/olap/segcompaction_test.cpp:
##########
@@ -184,6 +186,27 @@ class SegCompactionTest : public testing::Test {
rowset_writer_context->tablet_schema = tablet_schema;
rowset_writer_context->version.first = 10;
rowset_writer_context->version.second = 10;
+
+#if 0
+ TCreateTabletReq req;
+ req.table_id =
+ req.tablet_id =
+ req.tablet_scheme =
+ req.partition_id =
+ l_engine->create_tablet(req);
+ rowset_writer_context->tablet =
l_engine->tablet_manager()->get_tablet(TTabletId tablet_id);
+#endif
+ std::shared_ptr<DataDir> data_dir =
std::make_shared<DataDir>(lTestDir);
+ TabletMetaSharedPtr tablet_meta = std::make_shared<TabletMeta>();
+ tablet_meta->_tablet_id = 1;
+ tablet_meta->_schema = tablet_schema;
Review Comment:
warning: '_tablet_id' is a private member of 'doris::TabletMeta'
[clang-diagnostic-error]
```cpp
tablet_meta->_tablet_id = 1;
^
```
**be/src/olap/tablet_meta.h:217:** declared private here
```cpp
int64_t _tablet_id = 0;
^
```
##########
be/src/vec/olap/vertical_block_reader.cpp:
##########
@@ -86,9 +86,26 @@
std::vector<RowwiseIteratorUPtr> segment_iters;
std::vector<bool> iterator_init_flag;
std::vector<RowsetId> rowset_ids;
- RETURN_IF_ERROR(
- _get_segment_iterators(read_params, &segment_iters,
&iterator_init_flag, &rowset_ids));
- CHECK(segment_iters.size() == iterator_init_flag.size());
+ std::vector<RowwiseIterator*>* segment_iters_ptr =
read_params.segment_iters_ptr;
+ std::vector<RowwiseIterator*> segment_iters;
+
+ if (!segment_iters_ptr) {
+ RETURN_IF_ERROR(_get_segment_iterators(read_params, &segment_iters,
&iterator_init_flag,
+ &rowset_ids));
+ CHECK(segment_iters.size() == iterator_init_flag.size());
+ segment_iters_ptr = &segment_iters;
Review Comment:
warning: incompatible pointer types assigning to
'std::vector<RowwiseIterator *> *' from 'std::vector<RowwiseIteratorUPtr> *'
(aka 'vector<unique_ptr<doris::RowwiseIterator>> *') [clang-diagnostic-error]
```cpp
segment_iters_ptr = &segment_iters;
^
```
##########
be/src/vec/olap/vertical_block_reader.cpp:
##########
@@ -98,11 +115,11 @@
seq_col_idx =
read_params.tablet->tablet_schema()->sequence_col_idx();
}
_vcollect_iter = new_vertical_heap_merge_iterator(
- std::move(segment_iters), iterator_init_flag, rowset_ids,
ori_return_col_size,
+ *segment_iters_ptr, iterator_init_flag, rowset_ids,
ori_return_col_size,
read_params.tablet->keys_type(), seq_col_idx,
_row_sources_buffer);
} else {
- _vcollect_iter =
new_vertical_mask_merge_iterator(std::move(segment_iters),
- ori_return_col_size,
_row_sources_buffer);
+ _vcollect_iter = new_vertical_mask_merge_iterator(*segment_iters_ptr,
ori_return_col_size,
Review Comment:
warning: no matching function for call to 'new_vertical_mask_merge_iterator'
[clang-diagnostic-error]
```cpp
_vcollect_iter =
new_vertical_mask_merge_iterator(*segment_iters_ptr, ori_return_col_size,
^
```
**be/src/vec/olap/vertical_merge_iterator.h:335:** candidate function not
viable: no known conversion from 'vector<doris::RowwiseIterator *>' to
'vector<doris::RowwiseIteratorUPtr>' for 1st argument
```cpp
std::shared_ptr<RowwiseIterator> new_vertical_mask_merge_iterator(
^
```
##########
be/test/olap/segcompaction_test.cpp:
##########
@@ -200,12 +223,124 @@
std::unique_ptr<DataDir> _data_dir;
};
+TEST_F(SegCompactionTest, SegCompactionThenRead) {
+ config::enable_segcompaction = true;
+ config::enable_storage_vectorization = true;
+ Status s;
+ TabletSchemaSPtr tablet_schema = std::make_shared<TabletSchema>();
+ create_tablet_schema(tablet_schema, DUP_KEYS);
+
+ RowsetSharedPtr rowset;
+ const int num_segments = 15;
+ const uint32_t rows_per_segment = 4096;
+ config::segcompaction_small_threshold = 6000; // set threshold above
+ // rows_per_segment
+ config::segcompaction_threshold_segment_num = 10;
+ std::vector<uint32_t> segment_num_rows;
+ { // write `num_segments * rows_per_segment` rows to rowset
+ RowsetWriterContext writer_context;
+ create_rowset_writer_context(10047, tablet_schema, &writer_context);
+
+ std::unique_ptr<RowsetWriter> rowset_writer;
+ s = RowsetFactory::create_rowset_writer(writer_context, false,
&rowset_writer);
+ EXPECT_EQ(Status::OK(), s);
+
+ RowCursor input_row;
+ input_row.init(tablet_schema);
+
+ // for segment "i", row "rid"
+ // k1 := rid*10 + i
+ // k2 := k1 * 10
+ // k3 := rid
+ for (int i = 0; i < num_segments; ++i) {
+ MemPool mem_pool;
+ for (int rid = 0; rid < rows_per_segment; ++rid) {
+ uint32_t k1 = rid * 100 + i;
+ uint32_t k2 = i;
+ uint32_t k3 = rid;
+ input_row.set_field_content(0, reinterpret_cast<char*>(&k1),
&mem_pool);
+ input_row.set_field_content(1, reinterpret_cast<char*>(&k2),
&mem_pool);
Review Comment:
warning: no member named 'set_field_content' in 'doris::RowCursor'
[clang-diagnostic-error]
```cpp
input_row.set_field_content(0, reinterpret_cast<char*>(&k1),
&mem_pool);
^
```
##########
be/test/olap/segcompaction_test.cpp:
##########
@@ -184,6 +186,27 @@
rowset_writer_context->tablet_schema = tablet_schema;
rowset_writer_context->version.first = 10;
rowset_writer_context->version.second = 10;
+
+#if 0
+ TCreateTabletReq req;
+ req.table_id =
+ req.tablet_id =
+ req.tablet_scheme =
+ req.partition_id =
+ l_engine->create_tablet(req);
+ rowset_writer_context->tablet =
l_engine->tablet_manager()->get_tablet(TTabletId tablet_id);
+#endif
+ std::shared_ptr<DataDir> data_dir =
std::make_shared<DataDir>(lTestDir);
+ TabletMetaSharedPtr tablet_meta = std::make_shared<TabletMeta>();
+ tablet_meta->_tablet_id = 1;
+ tablet_meta->_schema = tablet_schema;
+ auto tablet = std::make_shared<Tablet>(tablet_meta, data_dir.get(),
"test_str");
+ char* tmp_str = (char*)malloc(20);
+ strncpy(tmp_str, "test_tablet_name", 20);
+
+ tablet->_full_name = tmp_str;
+ // tablet->key
Review Comment:
warning: '_full_name' is a protected member of 'doris::BaseTablet'
[clang-diagnostic-error]
```cpp
tablet->_full_name = tmp_str;
^
```
**be/src/olap/base_tablet.h:84:** declared protected here
```cpp
std::string _full_name;
^
```
##########
be/test/olap/segcompaction_test.cpp:
##########
@@ -200,12 +223,124 @@
std::unique_ptr<DataDir> _data_dir;
};
+TEST_F(SegCompactionTest, SegCompactionThenRead) {
+ config::enable_segcompaction = true;
+ config::enable_storage_vectorization = true;
+ Status s;
+ TabletSchemaSPtr tablet_schema = std::make_shared<TabletSchema>();
+ create_tablet_schema(tablet_schema, DUP_KEYS);
+
+ RowsetSharedPtr rowset;
+ const int num_segments = 15;
+ const uint32_t rows_per_segment = 4096;
+ config::segcompaction_small_threshold = 6000; // set threshold above
+ // rows_per_segment
+ config::segcompaction_threshold_segment_num = 10;
+ std::vector<uint32_t> segment_num_rows;
+ { // write `num_segments * rows_per_segment` rows to rowset
+ RowsetWriterContext writer_context;
+ create_rowset_writer_context(10047, tablet_schema, &writer_context);
+
+ std::unique_ptr<RowsetWriter> rowset_writer;
+ s = RowsetFactory::create_rowset_writer(writer_context, false,
&rowset_writer);
+ EXPECT_EQ(Status::OK(), s);
+
+ RowCursor input_row;
+ input_row.init(tablet_schema);
+
+ // for segment "i", row "rid"
+ // k1 := rid*10 + i
+ // k2 := k1 * 10
+ // k3 := rid
+ for (int i = 0; i < num_segments; ++i) {
+ MemPool mem_pool;
+ for (int rid = 0; rid < rows_per_segment; ++rid) {
+ uint32_t k1 = rid * 100 + i;
+ uint32_t k2 = i;
+ uint32_t k3 = rid;
+ input_row.set_field_content(0, reinterpret_cast<char*>(&k1),
&mem_pool);
+ input_row.set_field_content(1, reinterpret_cast<char*>(&k2),
&mem_pool);
+ input_row.set_field_content(2, reinterpret_cast<char*>(&k3),
&mem_pool);
Review Comment:
warning: no member named 'set_field_content' in 'doris::RowCursor'
[clang-diagnostic-error]
```cpp
input_row.set_field_content(1, reinterpret_cast<char*>(&k2),
&mem_pool);
^
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
