imay commented on a change in pull request #2547: [Segment V2] Support lazy
materialization read
URL: https://github.com/apache/incubator-doris/pull/2547#discussion_r361301431
##########
File path: be/src/olap/rowset/segment_v2/segment_iterator.cpp
##########
@@ -353,31 +418,56 @@ Status SegmentIterator::_seek_columns(const
std::vector<ColumnId>& column_ids, r
return Status::OK();
}
+Status SegmentIterator::_read_columns(const std::vector<ColumnId>& column_ids,
+ RowBlockV2* block,
+ size_t row_offset,
+ size_t nrows) {
+ for (auto cid : column_ids) {
+ auto column_block = block->column_block(cid);
+ ColumnBlockView dst(&column_block, row_offset);
+ size_t rows_read = nrows;
+ RETURN_IF_ERROR(_column_iterators[cid]->next_batch(&rows_read, &dst));
+ block->set_delete_state(column_block.delete_state());
+ DCHECK_EQ(nrows, rows_read);
+ }
+ return Status::OK();
+}
+
Status SegmentIterator::next_batch(RowBlockV2* block) {
SCOPED_RAW_TIMER(&_opts.stats->block_load_ns);
if (UNLIKELY(!_inited)) {
RETURN_IF_ERROR(_init());
+ if (_lazy_materialization_read) {
+ _block_rowids.reserve(block->capacity());
+ }
_inited = true;
}
uint32_t total_read = 0;
uint32_t remaining = block->capacity();
- // trying to fill in block
+ _block_rowids.resize(0);
+ const auto& read_columns = _lazy_materialization_read ? _predicate_columns
: block->schema()->column_ids();
+
+ // phase 1: read rows selected by various index (indicated by _row_bitmap)
into block
+ // when using lazy-materialization-read, only columns with predicates are
read
do {
uint32_t range_from;
uint32_t range_to;
bool has_next_range = _range_iter->next_range(remaining, &range_from,
&range_to);
if (!has_next_range) {
break;
}
+ if (_lazy_materialization_read) {
+ for (uint32_t rid = range_from; rid < range_to; rid++) {
+ _block_rowids.push_back(rid);
Review comment:
Yes, this is what I mean.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
