hubgeter commented on code in PR #42004:
URL: https://github.com/apache/doris/pull/42004#discussion_r1809738009
##########
be/src/vec/exec/format/orc/vorc_reader.cpp:
##########
@@ -853,6 +853,58 @@ Status OrcReader::set_fill_columns(
_lazy_read_ctx.can_lazy_read = false;
}
+ _row_reader_options.range(_range_start_offset, _range_size);
+ _row_reader_options.setTimezoneName(_ctz == "CST" ? "Asia/Shanghai" :
_ctz);
+ _row_reader_options.include(_read_cols);
+ _row_reader_options.setEnableLazyDecoding(true);
+
+ uint64_t number_of_stripes = _reader->getNumberOfStripes();
+ auto allStripesNeeded = _reader->getNeedReadStripes(_row_reader_options);
+
+ int64_t range_end_offset = _range_start_offset + _range_size;
+
+ // 三个参数 todo
+ int orc_tiny_stripe_threshold = 8L * 1024L * 1024L;
+ int orc_once_max_read_size = 8L * 1024L * 1024L;
+ int orc_max_merge_distance = 1L * 1024L * 1024L;
+
+ bool all_tiny_stripes = true;
+ std::vector<io::PrefetchRange> tiny_stripe_ranges;
+
+ for (uint64_t i = 0; i < number_of_stripes; i++) {
+ std::unique_ptr<orc::StripeInformation> strip_info =
_reader->getStripe(i);
+ uint64_t strip_start_offset = strip_info->getOffset();
+ uint64_t strip_end_offset = strip_start_offset +
strip_info->getLength();
+
+ if (strip_start_offset >= range_end_offset || strip_end_offset <
_range_start_offset ||
+ !allStripesNeeded[i]) {
+ continue;
+ }
+ if (strip_info->getLength() > orc_tiny_stripe_threshold) {
+ all_tiny_stripes = false;
+ break;
+ }
+
+ tiny_stripe_ranges.emplace_back(strip_start_offset, strip_end_offset);
+ }
+ if (all_tiny_stripes && number_of_stripes > 0) {
+ std::vector<io::PrefetchRange> prefetch_merge_ranges =
+ io::PrefetchRange::mergeAdjacentSeqRanges(
+ tiny_stripe_ranges, orc_max_merge_distance,
orc_once_max_read_size);
+
+ auto range_finder =
+
std::make_shared<io::LinearProbeRangeFinder>(std::move(prefetch_merge_ranges));
+
+ auto* orcInputStreamPtr =
static_cast<ORCFileInputStream*>(_reader->getStream());
+ orcInputStreamPtr->set_all_tiny_stripes();
+ auto& orc_file_reader = orcInputStreamPtr->get_file_reader();
+ orc_file_reader->collect_profile_before_close();
Review Comment:
I suddenly realized that I didn't need to call this thing
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
