This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new af11693f48f [opt](profile) add index page profile for io (#45675)
af11693f48f is described below
commit af11693f48f7406438471a79718b67b3f8851731
Author: airborne12 <[email protected]>
AuthorDate: Fri Dec 20 20:27:18 2024 +0800
[opt](profile) add index page profile for io (#45675)
---
be/src/olap/base_tablet.cpp | 6 +--
be/src/olap/delete_bitmap_calculator.cpp | 5 +-
be/src/olap/primary_key_index.cpp | 12 +++--
be/src/olap/primary_key_index.h | 12 ++---
.../segment_v2/bloom_filter_index_reader.cpp | 15 +++---
.../rowset/segment_v2/bloom_filter_index_reader.h | 12 ++---
be/src/olap/rowset/segment_v2/column_reader.cpp | 62 +++++++++++++---------
be/src/olap/rowset/segment_v2/column_reader.h | 26 +++++----
.../rowset/segment_v2/indexed_column_reader.cpp | 11 ++--
.../olap/rowset/segment_v2/indexed_column_reader.h | 4 +-
.../olap/rowset/segment_v2/ordinal_page_index.cpp | 16 +++---
be/src/olap/rowset/segment_v2/ordinal_page_index.h | 5 +-
be/src/olap/rowset/segment_v2/segment.cpp | 37 +++++++------
be/src/olap/rowset/segment_v2/segment.h | 12 ++---
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 2 +-
be/src/olap/rowset/segment_v2/zone_map_index.cpp | 15 +++---
be/src/olap/rowset/segment_v2/zone_map_index.h | 6 ++-
be/test/olap/date_bloom_filter_test.cpp | 8 +--
be/test/olap/primary_key_index_test.cpp | 20 +++----
.../bloom_filter_index_reader_writer_test.cpp | 4 +-
.../rowset/segment_v2/ordinal_page_index_test.cpp | 4 +-
be/test/olap/segment_cache_test.cpp | 2 +-
22 files changed, 164 insertions(+), 132 deletions(-)
diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp
index 33275a2663b..a4720f89d19 100644
--- a/be/src/olap/base_tablet.cpp
+++ b/be/src/olap/base_tablet.cpp
@@ -499,7 +499,7 @@ Status BaseTablet::lookup_row_key(const Slice& encoded_key,
TabletSchema* latest
for (auto id : picked_segments) {
Status s = segments[id]->lookup_row_key(encoded_key, schema,
with_seq_col, with_rowid,
- &loc, encoded_seq_value,
stats);
+ &loc, stats,
encoded_seq_value);
if (s.is<KEY_NOT_FOUND>()) {
continue;
}
@@ -615,7 +615,7 @@ Status
BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset,
vectorized::Block ordered_block = block.clone_empty();
uint32_t pos = 0;
- RETURN_IF_ERROR(seg->load_pk_index_and_bf()); // We need index blocks to
iterate
+ RETURN_IF_ERROR(seg->load_pk_index_and_bf(nullptr)); // We need index
blocks to iterate
const auto* pk_idx = seg->get_primary_key_index();
int total = pk_idx->num_rows();
uint32_t row_id = 0;
@@ -629,7 +629,7 @@ Status
BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset,
std::vector<std::unique_ptr<SegmentCacheHandle>>
segment_caches(specified_rowsets.size());
while (remaining > 0) {
std::unique_ptr<segment_v2::IndexedColumnIterator> iter;
- RETURN_IF_ERROR(pk_idx->new_iterator(&iter));
+ RETURN_IF_ERROR(pk_idx->new_iterator(&iter, nullptr));
size_t num_to_read = std::min(batch_size, remaining);
auto index_type =
vectorized::DataTypeFactory::instance().create_data_type(
diff --git a/be/src/olap/delete_bitmap_calculator.cpp
b/be/src/olap/delete_bitmap_calculator.cpp
index 017e3cff3d0..8ac05a1e393 100644
--- a/be/src/olap/delete_bitmap_calculator.cpp
+++ b/be/src/olap/delete_bitmap_calculator.cpp
@@ -145,12 +145,11 @@ Status MergeIndexDeleteBitmapCalculator::init(RowsetId
rowset_id,
MergeIndexDeleteBitmapCalculatorContext::Comparator(seq_col_length,
_rowid_length);
_contexts.reserve(segments.size());
_heap = std::make_unique<Heap>(_comparator);
-
for (auto& segment : segments) {
- RETURN_IF_ERROR(segment->load_index());
+ RETURN_IF_ERROR(segment->load_index(nullptr));
auto pk_idx = segment->get_primary_key_index();
std::unique_ptr<segment_v2::IndexedColumnIterator> index;
- RETURN_IF_ERROR(pk_idx->new_iterator(&index));
+ RETURN_IF_ERROR(pk_idx->new_iterator(&index, nullptr));
auto index_type =
vectorized::DataTypeFactory::instance().create_data_type(
pk_idx->type_info()->type(), 1, 0);
_contexts.emplace_back(std::move(index), index_type,
segment->id(), pk_idx->num_rows());
diff --git a/be/src/olap/primary_key_index.cpp
b/be/src/olap/primary_key_index.cpp
index 5f7bedb01fc..00b72832ee6 100644
--- a/be/src/olap/primary_key_index.cpp
+++ b/be/src/olap/primary_key_index.cpp
@@ -95,27 +95,29 @@ Status
PrimaryKeyIndexBuilder::finalize(segment_v2::PrimaryKeyIndexMetaPB* meta)
}
Status PrimaryKeyIndexReader::parse_index(io::FileReaderSPtr file_reader,
- const
segment_v2::PrimaryKeyIndexMetaPB& meta) {
+ const
segment_v2::PrimaryKeyIndexMetaPB& meta,
+ OlapReaderStatistics*
pk_index_load_stats) {
// parse primary key index
_index_reader.reset(new segment_v2::IndexedColumnReader(file_reader,
meta.primary_key_index()));
_index_reader->set_is_pk_index(true);
RETURN_IF_ERROR(_index_reader->load(!config::disable_pk_storage_page_cache,
false,
- _pk_index_load_stats));
+ pk_index_load_stats));
_index_parsed = true;
return Status::OK();
}
Status PrimaryKeyIndexReader::parse_bf(io::FileReaderSPtr file_reader,
- const
segment_v2::PrimaryKeyIndexMetaPB& meta) {
+ const
segment_v2::PrimaryKeyIndexMetaPB& meta,
+ OlapReaderStatistics*
pk_index_load_stats) {
// parse bloom filter
segment_v2::ColumnIndexMetaPB column_index_meta =
meta.bloom_filter_index();
segment_v2::BloomFilterIndexReader bf_index_reader(std::move(file_reader),
column_index_meta.bloom_filter_index());
RETURN_IF_ERROR(bf_index_reader.load(!config::disable_pk_storage_page_cache,
false,
- _pk_index_load_stats));
+ pk_index_load_stats));
std::unique_ptr<segment_v2::BloomFilterIndexIterator> bf_iter;
- RETURN_IF_ERROR(bf_index_reader.new_iterator(&bf_iter));
+ RETURN_IF_ERROR(bf_index_reader.new_iterator(&bf_iter,
pk_index_load_stats));
RETURN_IF_ERROR(bf_iter->read_bloom_filter(0, &_bf));
segment_v2::g_pk_total_bloom_filter_num << 1;
segment_v2::g_pk_total_bloom_filter_total_bytes << _bf->size();
diff --git a/be/src/olap/primary_key_index.h b/be/src/olap/primary_key_index.h
index dcbbc5f3062..f74d3e42030 100644
--- a/be/src/olap/primary_key_index.h
+++ b/be/src/olap/primary_key_index.h
@@ -98,8 +98,7 @@ private:
class PrimaryKeyIndexReader {
public:
- PrimaryKeyIndexReader(OlapReaderStatistics* pk_index_load_stats = nullptr)
- : _index_parsed(false), _bf_parsed(false),
_pk_index_load_stats(pk_index_load_stats) {}
+ PrimaryKeyIndexReader() : _index_parsed(false), _bf_parsed(false) {}
~PrimaryKeyIndexReader() {
segment_v2::g_pk_total_bloom_filter_num <<
-static_cast<int64_t>(_bf_num);
@@ -109,12 +108,14 @@ public:
}
Status parse_index(io::FileReaderSPtr file_reader,
- const segment_v2::PrimaryKeyIndexMetaPB& meta);
+ const segment_v2::PrimaryKeyIndexMetaPB& meta,
+ OlapReaderStatistics* pk_index_load_stats);
- Status parse_bf(io::FileReaderSPtr file_reader, const
segment_v2::PrimaryKeyIndexMetaPB& meta);
+ Status parse_bf(io::FileReaderSPtr file_reader, const
segment_v2::PrimaryKeyIndexMetaPB& meta,
+ OlapReaderStatistics* pk_index_load_stats);
Status new_iterator(std::unique_ptr<segment_v2::IndexedColumnIterator>*
index_iterator,
- OlapReaderStatistics* stats = nullptr) const {
+ OlapReaderStatistics* stats) const {
DCHECK(_index_parsed);
index_iterator->reset(new
segment_v2::IndexedColumnIterator(_index_reader.get(), stats));
return Status::OK();
@@ -155,7 +156,6 @@ private:
std::unique_ptr<segment_v2::BloomFilter> _bf;
size_t _bf_num = 0;
uint64 _bf_bytes = 0;
- OlapReaderStatistics* _pk_index_load_stats = nullptr;
};
} // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp
b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp
index 8c63c25d20a..7c51f0a24c1 100644
--- a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp
@@ -34,9 +34,8 @@ namespace segment_v2 {
Status BloomFilterIndexReader::load(bool use_page_cache, bool kept_in_memory,
OlapReaderStatistics* index_load_stats) {
// TODO yyq: implement a new once flag to avoid status construct.
- _index_load_stats = index_load_stats;
- return _load_once.call([this, use_page_cache, kept_in_memory] {
- return _load(use_page_cache, kept_in_memory);
+ return _load_once.call([this, use_page_cache, kept_in_memory,
index_load_stats] {
+ return _load(use_page_cache, kept_in_memory, index_load_stats);
});
}
@@ -45,20 +44,22 @@ int64_t BloomFilterIndexReader::get_metadata_size() const {
(_bloom_filter_index_meta ?
_bloom_filter_index_meta->ByteSizeLong() : 0);
}
-Status BloomFilterIndexReader::_load(bool use_page_cache, bool kept_in_memory)
{
+Status BloomFilterIndexReader::_load(bool use_page_cache, bool kept_in_memory,
+ OlapReaderStatistics* index_load_stats) {
const IndexedColumnMetaPB& bf_index_meta =
_bloom_filter_index_meta->bloom_filter();
_bloom_filter_reader.reset(new IndexedColumnReader(_file_reader,
bf_index_meta));
- RETURN_IF_ERROR(_bloom_filter_reader->load(use_page_cache, kept_in_memory,
_index_load_stats));
+ RETURN_IF_ERROR(_bloom_filter_reader->load(use_page_cache, kept_in_memory,
index_load_stats));
update_metadata_size();
return Status::OK();
}
-Status
BloomFilterIndexReader::new_iterator(std::unique_ptr<BloomFilterIndexIterator>*
iterator) {
+Status
BloomFilterIndexReader::new_iterator(std::unique_ptr<BloomFilterIndexIterator>*
iterator,
+ OlapReaderStatistics*
index_load_stats) {
DBUG_EXECUTE_IF("BloomFilterIndexReader::new_iterator.fail", {
return Status::InternalError("new_iterator for bloom filter index
failed");
});
- iterator->reset(new BloomFilterIndexIterator(this));
+ iterator->reset(new BloomFilterIndexIterator(this, index_load_stats));
return Status::OK();
}
diff --git a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h
b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h
index fcb0239a244..fb53af89c0f 100644
--- a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h
+++ b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h
@@ -48,17 +48,18 @@ public:
}
Status load(bool use_page_cache, bool kept_in_memory,
- OlapReaderStatistics* _bf_index_load_stats = nullptr);
+ OlapReaderStatistics* bf_index_load_stats);
BloomFilterAlgorithmPB algorithm() { return
_bloom_filter_index_meta->algorithm(); }
// create a new column iterator.
- Status new_iterator(std::unique_ptr<BloomFilterIndexIterator>* iterator);
+ Status new_iterator(std::unique_ptr<BloomFilterIndexIterator>* iterator,
+ OlapReaderStatistics* index_load_stats);
const TypeInfo* type_info() const { return _type_info; }
private:
- Status _load(bool use_page_cache, bool kept_in_memory);
+ Status _load(bool use_page_cache, bool kept_in_memory,
OlapReaderStatistics* index_load_stats);
int64_t get_metadata_size() const override;
@@ -70,13 +71,12 @@ private:
const TypeInfo* _type_info = nullptr;
std::unique_ptr<BloomFilterIndexPB> _bloom_filter_index_meta = nullptr;
std::unique_ptr<IndexedColumnReader> _bloom_filter_reader;
- OlapReaderStatistics* _index_load_stats = nullptr;
};
class BloomFilterIndexIterator {
public:
- explicit BloomFilterIndexIterator(BloomFilterIndexReader* reader)
- : _reader(reader),
_bloom_filter_iter(reader->_bloom_filter_reader.get()) {}
+ explicit BloomFilterIndexIterator(BloomFilterIndexReader* reader,
OlapReaderStatistics* stats)
+ : _reader(reader),
_bloom_filter_iter(reader->_bloom_filter_reader.get(), stats) {}
// Read bloom filter at the given ordinal into `bf`.
Status read_bloom_filter(rowid_t ordinal, std::unique_ptr<BloomFilter>*
bf);
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp
b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 9d5328de869..78c415530cd 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -374,10 +374,12 @@ Status ColumnReader::read_page(const
ColumnIteratorOptions& iter_opts, const Pag
Status ColumnReader::get_row_ranges_by_zone_map(
const AndBlockColumnPredicate* col_predicates,
- const std::vector<const ColumnPredicate*>* delete_predicates,
RowRanges* row_ranges) {
+ const std::vector<const ColumnPredicate*>* delete_predicates,
RowRanges* row_ranges,
+ const ColumnIteratorOptions& iter_opts) {
std::vector<uint32_t> page_indexes;
- RETURN_IF_ERROR(_get_filtered_pages(col_predicates, delete_predicates,
&page_indexes));
- RETURN_IF_ERROR(_calculate_row_ranges(page_indexes, row_ranges));
+ RETURN_IF_ERROR(
+ _get_filtered_pages(col_predicates, delete_predicates,
&page_indexes, iter_opts));
+ RETURN_IF_ERROR(_calculate_row_ranges(page_indexes, row_ranges,
iter_opts));
return Status::OK();
}
@@ -514,8 +516,8 @@ bool ColumnReader::_zone_map_match_condition(const
ZoneMapPB& zone_map,
Status ColumnReader::_get_filtered_pages(
const AndBlockColumnPredicate* col_predicates,
const std::vector<const ColumnPredicate*>* delete_predicates,
- std::vector<uint32_t>* page_indexes) {
- RETURN_IF_ERROR(_load_zone_map_index(_use_index_page_cache,
_opts.kept_in_memory));
+ std::vector<uint32_t>* page_indexes, const ColumnIteratorOptions&
iter_opts) {
+ RETURN_IF_ERROR(_load_zone_map_index(_use_index_page_cache,
_opts.kept_in_memory, iter_opts));
FieldType type = _type_info->type();
const std::vector<ZoneMapPB>& zone_maps =
_zone_map_index->page_zone_maps();
@@ -553,9 +555,10 @@ Status ColumnReader::_get_filtered_pages(
}
Status ColumnReader::_calculate_row_ranges(const std::vector<uint32_t>&
page_indexes,
- RowRanges* row_ranges) {
+ RowRanges* row_ranges,
+ const ColumnIteratorOptions&
iter_opts) {
row_ranges->clear();
- RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache,
_opts.kept_in_memory));
+ RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache,
_opts.kept_in_memory, iter_opts));
for (auto i : page_indexes) {
ordinal_t page_first_id = _ordinal_index->get_first_ordinal(i);
ordinal_t page_last_id = _ordinal_index->get_last_ordinal(i);
@@ -566,12 +569,14 @@ Status ColumnReader::_calculate_row_ranges(const
std::vector<uint32_t>& page_ind
}
Status ColumnReader::get_row_ranges_by_bloom_filter(const
AndBlockColumnPredicate* col_predicates,
- RowRanges* row_ranges) {
- RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache,
_opts.kept_in_memory));
- RETURN_IF_ERROR(_load_bloom_filter_index(_use_index_page_cache,
_opts.kept_in_memory));
+ RowRanges* row_ranges,
+ const
ColumnIteratorOptions& iter_opts) {
+ RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache,
_opts.kept_in_memory, iter_opts));
+ RETURN_IF_ERROR(
+ _load_bloom_filter_index(_use_index_page_cache,
_opts.kept_in_memory, iter_opts));
RowRanges bf_row_ranges;
std::unique_ptr<BloomFilterIndexIterator> bf_iter;
- RETURN_IF_ERROR(_bloom_filter_index->new_iterator(&bf_iter));
+ RETURN_IF_ERROR(_bloom_filter_index->new_iterator(&bf_iter,
iter_opts.stats));
size_t range_size = row_ranges->range_size();
// get covered page ids
std::set<uint32_t> page_ids;
@@ -598,16 +603,18 @@ Status ColumnReader::get_row_ranges_by_bloom_filter(const
AndBlockColumnPredicat
return Status::OK();
}
-Status ColumnReader::_load_ordinal_index(bool use_page_cache, bool
kept_in_memory) {
+Status ColumnReader::_load_ordinal_index(bool use_page_cache, bool
kept_in_memory,
+ const ColumnIteratorOptions&
iter_opts) {
if (!_ordinal_index) {
return Status::InternalError("ordinal_index not inited");
}
- return _ordinal_index->load(use_page_cache, kept_in_memory);
+ return _ordinal_index->load(use_page_cache, kept_in_memory,
iter_opts.stats);
}
-Status ColumnReader::_load_zone_map_index(bool use_page_cache, bool
kept_in_memory) {
+Status ColumnReader::_load_zone_map_index(bool use_page_cache, bool
kept_in_memory,
+ const ColumnIteratorOptions&
iter_opts) {
if (_zone_map_index != nullptr) {
- return _zone_map_index->load(use_page_cache, kept_in_memory);
+ return _zone_map_index->load(use_page_cache, kept_in_memory,
iter_opts.stats);
}
return Status::OK();
}
@@ -681,15 +688,17 @@ bool ColumnReader::has_bloom_filter_index(bool ngram)
const {
}
}
-Status ColumnReader::_load_bloom_filter_index(bool use_page_cache, bool
kept_in_memory) {
+Status ColumnReader::_load_bloom_filter_index(bool use_page_cache, bool
kept_in_memory,
+ const ColumnIteratorOptions&
iter_opts) {
if (_bloom_filter_index != nullptr) {
- return _bloom_filter_index->load(use_page_cache, kept_in_memory);
+ return _bloom_filter_index->load(use_page_cache, kept_in_memory,
iter_opts.stats);
}
return Status::OK();
}
-Status ColumnReader::seek_to_first(OrdinalPageIndexIterator* iter) {
- RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache,
_opts.kept_in_memory));
+Status ColumnReader::seek_to_first(OrdinalPageIndexIterator* iter,
+ const ColumnIteratorOptions& iter_opts) {
+ RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache,
_opts.kept_in_memory, iter_opts));
*iter = _ordinal_index->begin();
if (!iter->valid()) {
return Status::NotFound("Failed to seek to first rowid");
@@ -697,8 +706,9 @@ Status
ColumnReader::seek_to_first(OrdinalPageIndexIterator* iter) {
return Status::OK();
}
-Status ColumnReader::seek_at_or_before(ordinal_t ordinal,
OrdinalPageIndexIterator* iter) {
- RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache,
_opts.kept_in_memory));
+Status ColumnReader::seek_at_or_before(ordinal_t ordinal,
OrdinalPageIndexIterator* iter,
+ const ColumnIteratorOptions& iter_opts)
{
+ RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache,
_opts.kept_in_memory, iter_opts));
*iter = _ordinal_index->seek_at_or_before(ordinal);
if (!iter->valid()) {
return Status::NotFound("Failed to seek to ordinal {}, ", ordinal);
@@ -1172,7 +1182,7 @@ Status FileColumnIterator::init(const
ColumnIteratorOptions& opts) {
FileColumnIterator::~FileColumnIterator() = default;
Status FileColumnIterator::seek_to_first() {
- RETURN_IF_ERROR(_reader->seek_to_first(&_page_iter));
+ RETURN_IF_ERROR(_reader->seek_to_first(&_page_iter, _opts));
RETURN_IF_ERROR(_read_data_page(_page_iter));
_seek_to_pos_in_page(&_page, 0);
@@ -1183,7 +1193,7 @@ Status FileColumnIterator::seek_to_first() {
Status FileColumnIterator::seek_to_ordinal(ordinal_t ord) {
// if current page contains this row, we don't need to seek
if (!_page || !_page.contains(ord) || !_page_iter.valid()) {
- RETURN_IF_ERROR(_reader->seek_at_or_before(ord, &_page_iter));
+ RETURN_IF_ERROR(_reader->seek_at_or_before(ord, &_page_iter, _opts));
RETURN_IF_ERROR(_read_data_page(_page_iter));
}
_seek_to_pos_in_page(&_page, ord - _page.first_ordinal);
@@ -1431,8 +1441,8 @@ Status FileColumnIterator::get_row_ranges_by_zone_map(
const AndBlockColumnPredicate* col_predicates,
const std::vector<const ColumnPredicate*>* delete_predicates,
RowRanges* row_ranges) {
if (_reader->has_zone_map()) {
- RETURN_IF_ERROR(
- _reader->get_row_ranges_by_zone_map(col_predicates,
delete_predicates, row_ranges));
+ RETURN_IF_ERROR(_reader->get_row_ranges_by_zone_map(col_predicates,
delete_predicates,
+ row_ranges,
_opts));
}
return Status::OK();
}
@@ -1441,7 +1451,7 @@ Status FileColumnIterator::get_row_ranges_by_bloom_filter(
const AndBlockColumnPredicate* col_predicates, RowRanges* row_ranges) {
if ((col_predicates->can_do_bloom_filter(false) &&
_reader->has_bloom_filter_index(false)) ||
(col_predicates->can_do_bloom_filter(true) &&
_reader->has_bloom_filter_index(true))) {
-
RETURN_IF_ERROR(_reader->get_row_ranges_by_bloom_filter(col_predicates,
row_ranges));
+
RETURN_IF_ERROR(_reader->get_row_ranges_by_bloom_filter(col_predicates,
row_ranges, _opts));
}
return Status::OK();
}
diff --git a/be/src/olap/rowset/segment_v2/column_reader.h
b/be/src/olap/rowset/segment_v2/column_reader.h
index d72d802f977..7e32b3a09b3 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.h
+++ b/be/src/olap/rowset/segment_v2/column_reader.h
@@ -148,8 +148,9 @@ public:
std::unique_ptr<InvertedIndexIterator>*
iterator);
// Seek to the first entry in the column.
- Status seek_to_first(OrdinalPageIndexIterator* iter);
- Status seek_at_or_before(ordinal_t ordinal, OrdinalPageIndexIterator*
iter);
+ Status seek_to_first(OrdinalPageIndexIterator* iter, const
ColumnIteratorOptions& iter_opts);
+ Status seek_at_or_before(ordinal_t ordinal, OrdinalPageIndexIterator* iter,
+ const ColumnIteratorOptions& iter_opts);
// read a page from file into a page handle
Status read_page(const ColumnIteratorOptions& iter_opts, const
PagePointer& pp,
@@ -175,11 +176,13 @@ public:
// - delete_condition is a delete predicate of one version
Status get_row_ranges_by_zone_map(const AndBlockColumnPredicate*
col_predicates,
const std::vector<const
ColumnPredicate*>* delete_predicates,
- RowRanges* row_ranges);
+ RowRanges* row_ranges,
+ const ColumnIteratorOptions& iter_opts);
// get row ranges with bloom filter index
Status get_row_ranges_by_bloom_filter(const AndBlockColumnPredicate*
col_predicates,
- RowRanges* row_ranges);
+ RowRanges* row_ranges,
+ const ColumnIteratorOptions&
iter_opts);
PagePointer get_dict_page_pointer() const { return _meta_dict_page; }
@@ -219,13 +222,16 @@ private:
return Status::OK();
}
- [[nodiscard]] Status _load_zone_map_index(bool use_page_cache, bool
kept_in_memory);
- [[nodiscard]] Status _load_ordinal_index(bool use_page_cache, bool
kept_in_memory);
+ [[nodiscard]] Status _load_zone_map_index(bool use_page_cache, bool
kept_in_memory,
+ const ColumnIteratorOptions&
iter_opts);
+ [[nodiscard]] Status _load_ordinal_index(bool use_page_cache, bool
kept_in_memory,
+ const ColumnIteratorOptions&
iter_opts);
[[nodiscard]] Status _load_bitmap_index(bool use_page_cache, bool
kept_in_memory);
[[nodiscard]] Status _load_inverted_index_index(
std::shared_ptr<InvertedIndexFileReader> index_file_reader,
const TabletIndex* index_meta);
- [[nodiscard]] Status _load_bloom_filter_index(bool use_page_cache, bool
kept_in_memory);
+ [[nodiscard]] Status _load_bloom_filter_index(bool use_page_cache, bool
kept_in_memory,
+ const ColumnIteratorOptions&
iter_opts);
bool _zone_map_match_condition(const ZoneMapPB& zone_map, WrapperField*
min_value_container,
WrapperField* max_value_container,
@@ -239,9 +245,11 @@ private:
Status _get_filtered_pages(const AndBlockColumnPredicate* col_predicates,
const std::vector<const ColumnPredicate*>*
delete_predicates,
- std::vector<uint32_t>* page_indexes);
+ std::vector<uint32_t>* page_indexes,
+ const ColumnIteratorOptions& iter_opts);
- Status _calculate_row_ranges(const std::vector<uint32_t>& page_indexes,
RowRanges* row_ranges);
+ Status _calculate_row_ranges(const std::vector<uint32_t>& page_indexes,
RowRanges* row_ranges,
+ const ColumnIteratorOptions& iter_opts);
int64_t get_metadata_size() const override;
diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
index da6beff5d8d..3f582293ee4 100644
--- a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
@@ -66,7 +66,6 @@ Status IndexedColumnReader::load(bool use_page_cache, bool
kept_in_memory,
OlapReaderStatistics* index_load_stats) {
_use_page_cache = use_page_cache;
_kept_in_memory = kept_in_memory;
- _index_load_stats = index_load_stats;
_type_info = get_scalar_type_info((FieldType)_meta.data_type());
if (_type_info == nullptr) {
@@ -82,7 +81,7 @@ Status IndexedColumnReader::load(bool use_page_cache, bool
kept_in_memory,
} else {
RETURN_IF_ERROR(load_index_page(_meta.ordinal_index_meta().root_page(),
&_ordinal_index_page_handle,
- _ordinal_index_reader.get()));
+ _ordinal_index_reader.get(),
index_load_stats));
_has_index_page = true;
}
}
@@ -93,7 +92,8 @@ Status IndexedColumnReader::load(bool use_page_cache, bool
kept_in_memory,
_sole_data_page =
PagePointer(_meta.value_index_meta().root_page());
} else {
RETURN_IF_ERROR(load_index_page(_meta.value_index_meta().root_page(),
- &_value_index_page_handle,
_value_index_reader.get()));
+ &_value_index_page_handle,
_value_index_reader.get(),
+ index_load_stats));
_has_index_page = true;
}
}
@@ -104,13 +104,14 @@ Status IndexedColumnReader::load(bool use_page_cache,
bool kept_in_memory,
}
Status IndexedColumnReader::load_index_page(const PagePointerPB& pp,
PageHandle* handle,
- IndexPageReader* reader) {
+ IndexPageReader* reader,
+ OlapReaderStatistics*
index_load_stats) {
Slice body;
PageFooterPB footer;
BlockCompressionCodec* local_compress_codec;
RETURN_IF_ERROR(get_block_compression_codec(_meta.compression(),
&local_compress_codec));
RETURN_IF_ERROR(read_page(PagePointer(pp), handle, &body, &footer,
INDEX_PAGE,
- local_compress_codec, false, _index_load_stats));
+ local_compress_codec, false, index_load_stats));
RETURN_IF_ERROR(reader->parse(body, footer.index_page_footer()));
_mem_size += body.get_size();
return Status::OK();
diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.h
b/be/src/olap/rowset/segment_v2/indexed_column_reader.h
index c9640c0007c..6e62feaafdc 100644
--- a/be/src/olap/rowset/segment_v2/indexed_column_reader.h
+++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.h
@@ -76,7 +76,8 @@ public:
void set_is_pk_index(bool is_pk) { _is_pk_index = is_pk; }
private:
- Status load_index_page(const PagePointerPB& pp, PageHandle* handle,
IndexPageReader* reader);
+ Status load_index_page(const PagePointerPB& pp, PageHandle* handle,
IndexPageReader* reader,
+ OlapReaderStatistics* index_load_stats);
int64_t get_metadata_size() const override;
@@ -103,7 +104,6 @@ private:
const KeyCoder* _value_key_coder = nullptr;
uint64_t _mem_size = 0;
bool _is_pk_index = false;
- OlapReaderStatistics* _index_load_stats = nullptr;
};
class IndexedColumnIterator {
diff --git a/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp
b/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp
index 9ee82bacdd7..4995e779892 100644
--- a/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp
+++ b/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp
@@ -69,15 +69,17 @@ Status OrdinalIndexWriter::finish(io::FileWriter*
file_writer, ColumnIndexMetaPB
return Status::OK();
}
-Status OrdinalIndexReader::load(bool use_page_cache, bool kept_in_memory) {
+Status OrdinalIndexReader::load(bool use_page_cache, bool kept_in_memory,
+ OlapReaderStatistics* index_load_stats) {
// TODO yyq: implement a new once flag to avoid status construct.
- return _load_once.call([this, use_page_cache, kept_in_memory] {
- return _load(use_page_cache, kept_in_memory, std::move(_meta_pb));
+ return _load_once.call([this, use_page_cache, kept_in_memory,
index_load_stats] {
+ return _load(use_page_cache, kept_in_memory, std::move(_meta_pb),
index_load_stats);
});
}
Status OrdinalIndexReader::_load(bool use_page_cache, bool kept_in_memory,
- std::unique_ptr<OrdinalIndexPB> index_meta) {
+ std::unique_ptr<OrdinalIndexPB> index_meta,
+ OlapReaderStatistics* stats) {
if (index_meta->root_page().is_root_data_page()) {
// only one data page, no index page
_num_pages = 1;
@@ -88,6 +90,7 @@ Status OrdinalIndexReader::_load(bool use_page_cache, bool
kept_in_memory,
}
// need to read index page
OlapReaderStatistics tmp_stats;
+ OlapReaderStatistics* stats_ptr = stats != nullptr ? stats : &tmp_stats;
PageReadOptions opts {
.use_page_cache = use_page_cache,
.kept_in_memory = kept_in_memory,
@@ -96,8 +99,9 @@ Status OrdinalIndexReader::_load(bool use_page_cache, bool
kept_in_memory,
.page_pointer = PagePointer(index_meta->root_page().root_page()),
// ordinal index page uses NO_COMPRESSION right now
.codec = nullptr,
- .stats = &tmp_stats,
- .io_ctx = io::IOContext {.is_index_data = true},
+ .stats = stats_ptr,
+ .io_ctx = io::IOContext {.is_index_data = true,
+ .file_cache_stats =
&stats_ptr->file_cache_stats},
};
// read index page
diff --git a/be/src/olap/rowset/segment_v2/ordinal_page_index.h
b/be/src/olap/rowset/segment_v2/ordinal_page_index.h
index 1d74cf98952..df60edb12d1 100644
--- a/be/src/olap/rowset/segment_v2/ordinal_page_index.h
+++ b/be/src/olap/rowset/segment_v2/ordinal_page_index.h
@@ -75,7 +75,7 @@ public:
virtual ~OrdinalIndexReader();
// load and parse the index page into memory
- Status load(bool use_page_cache, bool kept_in_memory);
+ Status load(bool use_page_cache, bool kept_in_memory,
OlapReaderStatistics* index_load_stats);
// the returned iter points to the largest element which is less than
`ordinal`,
// or points to the first element if all elements are greater than
`ordinal`,
@@ -94,7 +94,8 @@ public:
private:
Status _load(bool use_page_cache, bool kept_in_memory,
- std::unique_ptr<OrdinalIndexPB> index_meta);
+ std::unique_ptr<OrdinalIndexPB> index_meta,
+ OlapReaderStatistics* index_load_stats);
int64_t get_metadata_size() const override;
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp
b/be/src/olap/rowset/segment_v2/segment.cpp
index d55d84901c2..b5ab3f0e873 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -290,7 +290,7 @@ Status Segment::new_iterator(SchemaSPtr schema, const
StorageReadOptions& read_o
{
SCOPED_RAW_TIMER(&read_options.stats->segment_load_index_timer_ns);
- RETURN_IF_ERROR(load_index());
+ RETURN_IF_ERROR(load_index(read_options.stats));
}
if (read_options.delete_condition_predicates->num_of_column_predicate() ==
0 &&
@@ -475,7 +475,7 @@ Status Segment::_parse_footer(SegmentFooterPB* footer) {
return Status::OK();
}
-Status Segment::_load_pk_bloom_filter() {
+Status Segment::_load_pk_bloom_filter(OlapReaderStatistics* stats) {
#ifdef BE_TEST
if (_pk_index_meta == nullptr) {
// for BE UT "segment_cache_test"
@@ -490,30 +490,30 @@ Status Segment::_load_pk_bloom_filter() {
DCHECK(_pk_index_meta != nullptr);
DCHECK(_pk_index_reader != nullptr);
- return _load_pk_bf_once.call([this] {
- RETURN_IF_ERROR(_pk_index_reader->parse_bf(_file_reader,
*_pk_index_meta));
+ return _load_pk_bf_once.call([this, stats] {
+ RETURN_IF_ERROR(_pk_index_reader->parse_bf(_file_reader,
*_pk_index_meta, stats));
// _meta_mem_usage += _pk_index_reader->get_bf_memory_size();
return Status::OK();
});
}
Status Segment::load_pk_index_and_bf(OlapReaderStatistics* index_load_stats) {
- _pk_index_load_stats = index_load_stats;
- RETURN_IF_ERROR(load_index());
- RETURN_IF_ERROR(_load_pk_bloom_filter());
+ RETURN_IF_ERROR(load_index(index_load_stats));
+ RETURN_IF_ERROR(_load_pk_bloom_filter(index_load_stats));
return Status::OK();
}
-Status Segment::load_index() {
- return _load_index_once.call([this] {
+Status Segment::load_index(OlapReaderStatistics* stats) {
+ return _load_index_once.call([this, stats] {
if (_tablet_schema->keys_type() == UNIQUE_KEYS && _pk_index_meta !=
nullptr) {
- _pk_index_reader =
std::make_unique<PrimaryKeyIndexReader>(_pk_index_load_stats);
- RETURN_IF_ERROR(_pk_index_reader->parse_index(_file_reader,
*_pk_index_meta));
+ _pk_index_reader = std::make_unique<PrimaryKeyIndexReader>();
+ RETURN_IF_ERROR(_pk_index_reader->parse_index(_file_reader,
*_pk_index_meta, stats));
// _meta_mem_usage += _pk_index_reader->get_memory_size();
return Status::OK();
} else {
// read and parse short key index page
OlapReaderStatistics tmp_stats;
+ OlapReaderStatistics* stats_ptr = stats != nullptr ? stats :
&tmp_stats;
PageReadOptions opts {
.use_page_cache = true,
.type = INDEX_PAGE,
@@ -522,7 +522,8 @@ Status Segment::load_index() {
// short key index page uses NO_COMPRESSION for now
.codec = nullptr,
.stats = &tmp_stats,
- .io_ctx = io::IOContext {.is_index_data = true},
+ .io_ctx = io::IOContext {.is_index_data = true,
+ .file_cache_stats =
&stats_ptr->file_cache_stats},
};
Slice body;
PageFooterPB footer;
@@ -970,8 +971,8 @@ Status Segment::new_inverted_index_iterator(const
TabletColumn& tablet_column,
Status Segment::lookup_row_key(const Slice& key, const TabletSchema*
latest_schema,
bool with_seq_col, bool with_rowid,
RowLocation* row_location,
- std::string* encoded_seq_value,
OlapReaderStatistics* stats) {
- RETURN_IF_ERROR(load_pk_index_and_bf());
+ OlapReaderStatistics* stats, std::string*
encoded_seq_value) {
+ RETURN_IF_ERROR(load_pk_index_and_bf(stats));
bool has_seq_col = latest_schema->has_sequence_col();
bool has_rowid = !latest_schema->cluster_key_uids().empty();
size_t seq_col_length = 0;
@@ -1071,9 +1072,10 @@ Status Segment::lookup_row_key(const Slice& key, const
TabletSchema* latest_sche
}
Status Segment::read_key_by_rowid(uint32_t row_id, std::string* key) {
- RETURN_IF_ERROR(load_pk_index_and_bf());
+ OlapReaderStatistics* null_stat = nullptr;
+ RETURN_IF_ERROR(load_pk_index_and_bf(null_stat));
std::unique_ptr<segment_v2::IndexedColumnIterator> iter;
- RETURN_IF_ERROR(_pk_index_reader->new_iterator(&iter));
+ RETURN_IF_ERROR(_pk_index_reader->new_iterator(&iter, null_stat));
auto index_type = vectorized::DataTypeFactory::instance().create_data_type(
_pk_index_reader->type_info()->type(), 1, 0);
@@ -1129,7 +1131,8 @@ Status Segment::seek_and_read_by_rowid(const
TabletSchema& schema, SlotDescripto
.use_page_cache = !config::disable_storage_page_cache,
.file_reader = file_reader().get(),
.stats = &stats,
- .io_ctx = io::IOContext {.reader_type = ReaderType::READER_QUERY},
+ .io_ctx = io::IOContext {.reader_type = ReaderType::READER_QUERY,
+ .file_cache_stats =
&stats.file_cache_stats},
};
std::vector<segment_v2::rowid_t> single_row_loc {row_id};
if (!slot->column_paths().empty()) {
diff --git a/be/src/olap/rowset/segment_v2/segment.h
b/be/src/olap/rowset/segment_v2/segment.h
index ca2fee0e77a..441ae3e85e9 100644
--- a/be/src/olap/rowset/segment_v2/segment.h
+++ b/be/src/olap/rowset/segment_v2/segment.h
@@ -134,9 +134,8 @@ public:
}
Status lookup_row_key(const Slice& key, const TabletSchema* latest_schema,
bool with_seq_col,
- bool with_rowid, RowLocation* row_location,
- std::string* encoded_seq_value = nullptr,
- OlapReaderStatistics* stats = nullptr);
+ bool with_rowid, RowLocation* row_location,
OlapReaderStatistics* stats,
+ std::string* encoded_seq_value = nullptr);
Status read_key_by_rowid(uint32_t row_id, std::string* key);
@@ -144,9 +143,9 @@ public:
vectorized::MutableColumnPtr& result,
OlapReaderStatistics& stats,
std::unique_ptr<ColumnIterator>&
iterator_hint);
- Status load_index();
+ Status load_index(OlapReaderStatistics* stats);
- Status load_pk_index_and_bf(OlapReaderStatistics* index_load_stats =
nullptr);
+ Status load_pk_index_and_bf(OlapReaderStatistics* stats);
void update_healthy_status(Status new_status) {
_healthy_status.update(new_status); }
// The segment is loaded into SegmentCache and then will load indices, if
there are something wrong
@@ -227,7 +226,7 @@ private:
Status _open();
Status _parse_footer(SegmentFooterPB* footer);
Status _create_column_readers(const SegmentFooterPB& footer);
- Status _load_pk_bloom_filter();
+ Status _load_pk_bloom_filter(OlapReaderStatistics* stats);
ColumnReader* _get_column_reader(const TabletColumn& col);
// Get Iterator which will read variant root column and extract with paths
and types info
@@ -305,7 +304,6 @@ private:
InvertedIndexFileInfo _idx_file_info;
int _be_exec_version = BeExecVersionManager::get_newest_version();
- OlapReaderStatistics* _pk_index_load_stats = nullptr;
};
} // namespace segment_v2
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 0c54eaa2d6c..5f50ffeea2d 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -1181,7 +1181,7 @@ Status
SegmentIterator::_lookup_ordinal_from_pk_index(const RowCursor& key, bool
bool exact_match = false;
std::unique_ptr<segment_v2::IndexedColumnIterator> index_iterator;
- RETURN_IF_ERROR(pk_index_reader->new_iterator(&index_iterator));
+ RETURN_IF_ERROR(pk_index_reader->new_iterator(&index_iterator,
_opts.stats));
Status status = index_iterator->seek_at_or_after(&index_key, &exact_match);
if (UNLIKELY(!status.ok())) {
diff --git a/be/src/olap/rowset/segment_v2/zone_map_index.cpp
b/be/src/olap/rowset/segment_v2/zone_map_index.cpp
index c2139ff0899..9249c82aedf 100644
--- a/be/src/olap/rowset/segment_v2/zone_map_index.cpp
+++ b/be/src/olap/rowset/segment_v2/zone_map_index.cpp
@@ -140,18 +140,21 @@ Status
TypedZoneMapIndexWriter<Type>::finish(io::FileWriter* file_writer,
return writer.finish(meta->mutable_page_zone_maps());
}
-Status ZoneMapIndexReader::load(bool use_page_cache, bool kept_in_memory) {
+Status ZoneMapIndexReader::load(bool use_page_cache, bool kept_in_memory,
+ OlapReaderStatistics* index_load_stats) {
// TODO yyq: implement a new once flag to avoid status construct.
- return _load_once.call([this, use_page_cache, kept_in_memory] {
- return _load(use_page_cache, kept_in_memory,
std::move(_page_zone_maps_meta));
+ return _load_once.call([this, use_page_cache, kept_in_memory,
index_load_stats] {
+ return _load(use_page_cache, kept_in_memory,
std::move(_page_zone_maps_meta),
+ index_load_stats);
});
}
Status ZoneMapIndexReader::_load(bool use_page_cache, bool kept_in_memory,
- std::unique_ptr<IndexedColumnMetaPB>
page_zone_maps_meta) {
+ std::unique_ptr<IndexedColumnMetaPB>
page_zone_maps_meta,
+ OlapReaderStatistics* index_load_stats) {
IndexedColumnReader reader(_file_reader, *page_zone_maps_meta);
- RETURN_IF_ERROR(reader.load(use_page_cache, kept_in_memory));
- IndexedColumnIterator iter(&reader);
+ RETURN_IF_ERROR(reader.load(use_page_cache, kept_in_memory,
index_load_stats));
+ IndexedColumnIterator iter(&reader, index_load_stats);
_page_zone_maps.resize(reader.num_values());
diff --git a/be/src/olap/rowset/segment_v2/zone_map_index.h
b/be/src/olap/rowset/segment_v2/zone_map_index.h
index 34869bbbfee..04cae12975c 100644
--- a/be/src/olap/rowset/segment_v2/zone_map_index.h
+++ b/be/src/olap/rowset/segment_v2/zone_map_index.h
@@ -154,14 +154,16 @@ public:
virtual ~ZoneMapIndexReader();
// load all page zone maps into memory
- Status load(bool use_page_cache, bool kept_in_memory);
+ Status load(bool use_page_cache, bool kept_in_memory,
+ OlapReaderStatistics* index_load_stats = nullptr);
const std::vector<ZoneMapPB>& page_zone_maps() const { return
_page_zone_maps; }
int32_t num_pages() const { return _page_zone_maps.size(); }
private:
- Status _load(bool use_page_cache, bool kept_in_memory,
std::unique_ptr<IndexedColumnMetaPB>);
+ Status _load(bool use_page_cache, bool kept_in_memory,
std::unique_ptr<IndexedColumnMetaPB>,
+ OlapReaderStatistics* index_load_stats);
int64_t get_metadata_size() const override;
diff --git a/be/test/olap/date_bloom_filter_test.cpp
b/be/test/olap/date_bloom_filter_test.cpp
index 715301419e2..51de4ebd8e7 100644
--- a/be/test/olap/date_bloom_filter_test.cpp
+++ b/be/test/olap/date_bloom_filter_test.cpp
@@ -155,8 +155,8 @@ TEST_F(DateBloomFilterTest, query_index_test) {
{
const auto& reader = segment->_column_readers[0];
std::unique_ptr<BloomFilterIndexIterator> bf_iter;
- EXPECT_TRUE(reader->_bloom_filter_index->load(true, true).ok());
- EXPECT_TRUE(reader->_bloom_filter_index->new_iterator(&bf_iter).ok());
+ EXPECT_TRUE(reader->_bloom_filter_index->load(true, true,
nullptr).ok());
+ EXPECT_TRUE(reader->_bloom_filter_index->new_iterator(&bf_iter,
nullptr).ok());
std::unique_ptr<BloomFilter> bf;
EXPECT_TRUE(bf_iter->read_bloom_filter(0, &bf).ok());
auto test = [&](const std::string& query_string, bool result) {
@@ -174,8 +174,8 @@ TEST_F(DateBloomFilterTest, query_index_test) {
{
const auto& reader = segment->_column_readers[1];
std::unique_ptr<BloomFilterIndexIterator> bf_iter;
- EXPECT_TRUE(reader->_bloom_filter_index->load(true, true).ok());
- EXPECT_TRUE(reader->_bloom_filter_index->new_iterator(&bf_iter).ok());
+ EXPECT_TRUE(reader->_bloom_filter_index->load(true, true,
nullptr).ok());
+ EXPECT_TRUE(reader->_bloom_filter_index->new_iterator(&bf_iter,
nullptr).ok());
std::unique_ptr<BloomFilter> bf;
EXPECT_TRUE(bf_iter->read_bloom_filter(0, &bf).ok());
auto test = [&](const std::string& query_string, bool result) {
diff --git a/be/test/olap/primary_key_index_test.cpp
b/be/test/olap/primary_key_index_test.cpp
index 72aae56cd09..9407be93886 100644
--- a/be/test/olap/primary_key_index_test.cpp
+++ b/be/test/olap/primary_key_index_test.cpp
@@ -80,12 +80,12 @@ TEST_F(PrimaryKeyIndexTest, builder) {
PrimaryKeyIndexReader index_reader;
io::FileReaderSPtr file_reader;
EXPECT_TRUE(fs->open_file(filename, &file_reader).ok());
- EXPECT_TRUE(index_reader.parse_index(file_reader, index_meta).ok());
- EXPECT_TRUE(index_reader.parse_bf(file_reader, index_meta).ok());
+ EXPECT_TRUE(index_reader.parse_index(file_reader, index_meta,
nullptr).ok());
+ EXPECT_TRUE(index_reader.parse_bf(file_reader, index_meta, nullptr).ok());
EXPECT_EQ(num_rows, index_reader.num_rows());
std::unique_ptr<segment_v2::IndexedColumnIterator> index_iterator;
- EXPECT_TRUE(index_reader.new_iterator(&index_iterator).ok());
+ EXPECT_TRUE(index_reader.new_iterator(&index_iterator, nullptr).ok());
bool exact_match = false;
uint32_t row_id;
for (size_t i = 0; i < keys.size(); i++) {
@@ -142,7 +142,7 @@ TEST_F(PrimaryKeyIndexTest, builder) {
int batch_size = 1024;
while (remaining > 0) {
std::unique_ptr<segment_v2::IndexedColumnIterator> iter;
- EXPECT_TRUE(index_reader.new_iterator(&iter).ok());
+ EXPECT_TRUE(index_reader.new_iterator(&iter, nullptr).ok());
size_t num_to_read = std::min(batch_size, remaining);
auto index_type =
vectorized::DataTypeFactory::instance().create_data_type(
@@ -199,12 +199,12 @@ TEST_F(PrimaryKeyIndexTest, multiple_pages) {
PrimaryKeyIndexReader index_reader;
io::FileReaderSPtr file_reader;
EXPECT_TRUE(fs->open_file(filename, &file_reader).ok());
- EXPECT_TRUE(index_reader.parse_index(file_reader, index_meta).ok());
- EXPECT_TRUE(index_reader.parse_bf(file_reader, index_meta).ok());
+ EXPECT_TRUE(index_reader.parse_index(file_reader, index_meta,
nullptr).ok());
+ EXPECT_TRUE(index_reader.parse_bf(file_reader, index_meta, nullptr).ok());
EXPECT_EQ(num_rows, index_reader.num_rows());
std::unique_ptr<segment_v2::IndexedColumnIterator> index_iterator;
- EXPECT_TRUE(index_reader.new_iterator(&index_iterator).ok());
+ EXPECT_TRUE(index_reader.new_iterator(&index_iterator, nullptr).ok());
bool exact_match = false;
uint32_t row_id;
for (size_t i = 0; i < keys.size(); i++) {
@@ -283,12 +283,12 @@ TEST_F(PrimaryKeyIndexTest, single_page) {
PrimaryKeyIndexReader index_reader;
io::FileReaderSPtr file_reader;
EXPECT_TRUE(fs->open_file(filename, &file_reader).ok());
- EXPECT_TRUE(index_reader.parse_index(file_reader, index_meta).ok());
- EXPECT_TRUE(index_reader.parse_bf(file_reader, index_meta).ok());
+ EXPECT_TRUE(index_reader.parse_index(file_reader, index_meta,
nullptr).ok());
+ EXPECT_TRUE(index_reader.parse_bf(file_reader, index_meta, nullptr).ok());
EXPECT_EQ(num_rows, index_reader.num_rows());
std::unique_ptr<segment_v2::IndexedColumnIterator> index_iterator;
- EXPECT_TRUE(index_reader.new_iterator(&index_iterator).ok());
+ EXPECT_TRUE(index_reader.new_iterator(&index_iterator, nullptr).ok());
bool exact_match = false;
uint32_t row_id;
for (size_t i = 0; i < keys.size(); i++) {
diff --git
a/be/test/olap/rowset/segment_v2/bloom_filter_index_reader_writer_test.cpp
b/be/test/olap/rowset/segment_v2/bloom_filter_index_reader_writer_test.cpp
index 813952595ef..e561f8ce944 100644
--- a/be/test/olap/rowset/segment_v2/bloom_filter_index_reader_writer_test.cpp
+++ b/be/test/olap/rowset/segment_v2/bloom_filter_index_reader_writer_test.cpp
@@ -124,10 +124,10 @@ void get_bloom_filter_reader_iter(const std::string&
file_name, const ColumnInde
io::FileReaderSPtr file_reader;
ASSERT_EQ(io::global_local_filesystem()->open_file(fname, &file_reader),
Status::OK());
*reader = new BloomFilterIndexReader(std::move(file_reader),
meta.bloom_filter_index());
- auto st = (*reader)->load(true, false);
+ auto st = (*reader)->load(true, false, nullptr);
EXPECT_TRUE(st.ok());
- st = (*reader)->new_iterator(iter);
+ st = (*reader)->new_iterator(iter, nullptr);
EXPECT_TRUE(st.ok());
}
diff --git a/be/test/olap/rowset/segment_v2/ordinal_page_index_test.cpp
b/be/test/olap/rowset/segment_v2/ordinal_page_index_test.cpp
index 33848c5959c..ffd9c92ee02 100644
--- a/be/test/olap/rowset/segment_v2/ordinal_page_index_test.cpp
+++ b/be/test/olap/rowset/segment_v2/ordinal_page_index_test.cpp
@@ -74,7 +74,7 @@ TEST_F(OrdinalPageIndexTest, normal) {
io::FileReaderSPtr file_reader;
EXPECT_TRUE(fs->open_file(filename, &file_reader).ok());
OrdinalIndexReader index(file_reader, 16 * 1024 * 4096 + 1,
index_meta.ordinal_index());
- EXPECT_TRUE(index.load(true, false).ok());
+ EXPECT_TRUE(index.load(true, false, nullptr).ok());
EXPECT_EQ(16 * 1024, index.num_data_pages());
EXPECT_EQ(1, index.get_first_ordinal(0));
EXPECT_EQ(4096, index.get_last_ordinal(0));
@@ -128,7 +128,7 @@ TEST_F(OrdinalPageIndexTest, one_data_page) {
}
OrdinalIndexReader index(nullptr, num_values, index_meta.ordinal_index());
- EXPECT_TRUE(index.load(true, false).ok());
+ EXPECT_TRUE(index.load(true, false, nullptr).ok());
EXPECT_EQ(1, index.num_data_pages());
EXPECT_EQ(0, index.get_first_ordinal(0));
EXPECT_EQ(num_values - 1, index.get_last_ordinal(0));
diff --git a/be/test/olap/segment_cache_test.cpp
b/be/test/olap/segment_cache_test.cpp
index b226bc6c229..c527ffddd42 100644
--- a/be/test/olap/segment_cache_test.cpp
+++ b/be/test/olap/segment_cache_test.cpp
@@ -323,7 +323,7 @@ TEST_F(SegmentCacheTest, vec_sequence_col) {
segment_v2::SegmentSharedPtr segment_ptr = handle.get_segments()[0];
// load index and bf second
- res = segment_ptr->load_pk_index_and_bf();
+ res = segment_ptr->load_pk_index_and_bf(nullptr);
ASSERT_TRUE(res.ok());
// check cache mem usage equals to segment mem usage
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
