This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 7711586c2d8 branch-3.1: [enhance](variant) Add some metrics for
querying variant in the profile #55941 (#56506)
7711586c2d8 is described below
commit 7711586c2d8320e80316bff52fee9990ccd1a081
Author: Sun Chenyang <[email protected]>
AuthorDate: Mon Sep 29 14:37:03 2025 +0800
branch-3.1: [enhance](variant) Add some metrics for querying variant in the
profile #55941 (#56506)
pick from master #55941
---
be/src/exec/rowid_fetcher.cpp | 10 +++++++---
be/src/olap/olap_common.h | 8 ++++++++
be/src/olap/rowset/segment_v2/segment.cpp | 19 ++++++++----------
be/src/olap/rowset/segment_v2/segment.h | 3 ++-
.../variant/hierarchical_data_iterator.cpp | 8 ++++++--
.../variant/hierarchical_data_iterator.h | 9 +++++++--
.../variant/sparse_column_extract_iterator.h | 23 ++++++++++++++--------
.../segment_v2/variant/variant_column_reader.cpp | 7 ++++++-
be/src/pipeline/exec/olap_scan_operator.cpp | 14 +++++++++++++
be/src/pipeline/exec/olap_scan_operator.h | 16 +++++++++++++++
be/src/service/point_query_executor.cpp | 6 +++++-
be/src/vec/exec/scan/new_olap_scanner.cpp | 16 ++++++++++++++-
.../variant_column_writer_reader_test.cpp | 23 ++++++++++++++++------
13 files changed, 126 insertions(+), 36 deletions(-)
diff --git a/be/src/exec/rowid_fetcher.cpp b/be/src/exec/rowid_fetcher.cpp
index e0b81283df1..60b0e4b22ef 100644
--- a/be/src/exec/rowid_fetcher.cpp
+++ b/be/src/exec/rowid_fetcher.cpp
@@ -324,6 +324,8 @@ struct IteratorItem {
std::unique_ptr<ColumnIterator> iterator;
// for holding the reference of segment to avoid use after release
SegmentSharedPtr segment;
+ // for holding the reference of storage read options to avoid use after
release
+ StorageReadOptions storage_read_options;
};
Status RowIdStorageReader::read_by_rowids(const PMultiGetRequest& request,
@@ -440,12 +442,14 @@ Status RowIdStorageReader::read_by_rowids(const
PMultiGetRequest& request,
if (iterator_item.segment == nullptr) {
// hold the reference
iterator_map[iterator_key].segment = segment;
+ iterator_item.storage_read_options.stats = &stats;
+ iterator_item.storage_read_options.io_ctx.reader_type =
ReaderType::READER_QUERY;
}
segment = iterator_item.segment;
try {
-
RETURN_IF_ERROR(segment->seek_and_read_by_rowid(full_read_schema,
desc.slots()[x],
- row_id,
column, stats,
-
iterator_item.iterator));
+ RETURN_IF_ERROR(segment->seek_and_read_by_rowid(
+ full_read_schema, desc.slots()[x], row_id, column,
+ iterator_item.storage_read_options,
iterator_item.iterator));
} catch (const Exception& e) {
return Status::Error<false>(e.code(), "Row id fetch failed
because {}", e.what());
}
diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h
index e8cc1e02198..07c6a6525fc 100644
--- a/be/src/olap/olap_common.h
+++ b/be/src/olap/olap_common.h
@@ -419,6 +419,14 @@ struct OlapReaderStatistics {
int64_t segment_create_column_readers_timer_ns = 0;
int64_t segment_load_index_timer_ns = 0;
+
+ int64_t variant_scan_sparse_column_timer_ns = 0;
+ int64_t variant_scan_sparse_column_bytes = 0;
+ int64_t variant_fill_path_from_sparse_column_timer_ns = 0;
+ int64_t variant_subtree_default_iter_count = 0;
+ int64_t variant_subtree_leaf_iter_count = 0;
+ int64_t variant_subtree_hierarchical_iter_count = 0;
+ int64_t variant_subtree_sparse_iter_count = 0;
};
using ColumnId = uint32_t;
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp
b/be/src/olap/rowset/segment_v2/segment.cpp
index dba2030f0bb..3091a8141d2 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -957,17 +957,15 @@ bool Segment::same_with_storage_type(int32_t cid, const
Schema& schema, bool rea
Status Segment::seek_and_read_by_rowid(const TabletSchema& schema,
SlotDescriptor* slot,
uint32_t row_id,
vectorized::MutableColumnPtr& result,
- OlapReaderStatistics& stats,
+ StorageReadOptions&
storage_read_options,
std::unique_ptr<ColumnIterator>&
iterator_hint) {
- StorageReadOptions storage_read_opt;
- storage_read_opt.stats = &stats;
- storage_read_opt.io_ctx.reader_type = ReaderType::READER_QUERY;
segment_v2::ColumnIteratorOptions opt {
.use_page_cache = !config::disable_storage_page_cache,
.file_reader = file_reader().get(),
- .stats = &stats,
+ .stats = storage_read_options.stats,
.io_ctx = io::IOContext {.reader_type = ReaderType::READER_QUERY,
- .file_cache_stats =
&stats.file_cache_stats},
+ .file_cache_stats =
+
&storage_read_options.stats->file_cache_stats},
};
std::vector<segment_v2::rowid_t> single_row_loc {row_id};
if (!slot->column_paths().empty()) {
@@ -976,14 +974,14 @@ Status Segment::seek_and_read_by_rowid(const
TabletSchema& schema, SlotDescripto
slot->col_unique_id(), slot->type().max_subcolumns_count());
// here need create column readers to make sure column reader is
created before seek_and_read_by_rowid
// if segment cache miss, column reader will be created to make sure
the variant column result not coredump
- RETURN_IF_ERROR(_create_column_meta_once(&stats));
+ RETURN_IF_ERROR(_create_column_meta_once(storage_read_options.stats));
auto storage_type = get_data_type_of(column, false);
vectorized::MutableColumnPtr file_storage_column =
storage_type->create_column();
DCHECK(storage_type != nullptr);
if (iterator_hint == nullptr) {
- RETURN_IF_ERROR(new_column_iterator(column, &iterator_hint,
&storage_read_opt));
+ RETURN_IF_ERROR(new_column_iterator(column, &iterator_hint,
&storage_read_options));
RETURN_IF_ERROR(iterator_hint->init(opt));
}
RETURN_IF_ERROR(
@@ -999,10 +997,9 @@ Status Segment::seek_and_read_by_rowid(const TabletSchema&
schema, SlotDescripto
<< ", field_name_to_index=" << schema.get_all_field_names();
return Status::InternalError(ss.str());
}
- storage_read_opt.io_ctx.reader_type = ReaderType::READER_QUERY;
if (iterator_hint == nullptr) {
- RETURN_IF_ERROR(
- new_column_iterator(schema.column(index), &iterator_hint,
&storage_read_opt));
+ RETURN_IF_ERROR(new_column_iterator(schema.column(index),
&iterator_hint,
+ &storage_read_options));
RETURN_IF_ERROR(iterator_hint->init(opt));
}
RETURN_IF_ERROR(iterator_hint->read_by_rowids(single_row_loc.data(),
1, result));
diff --git a/be/src/olap/rowset/segment_v2/segment.h
b/be/src/olap/rowset/segment_v2/segment.h
index d6d516b83e8..532a7e313ee 100644
--- a/be/src/olap/rowset/segment_v2/segment.h
+++ b/be/src/olap/rowset/segment_v2/segment.h
@@ -145,7 +145,8 @@ public:
Status read_key_by_rowid(uint32_t row_id, std::string* key);
Status seek_and_read_by_rowid(const TabletSchema& schema, SlotDescriptor*
slot, uint32_t row_id,
- vectorized::MutableColumnPtr& result,
OlapReaderStatistics& stats,
+ vectorized::MutableColumnPtr& result,
+ StorageReadOptions& storage_read_options,
std::unique_ptr<ColumnIterator>&
iterator_hint);
Status load_index(OlapReaderStatistics* stats);
diff --git
a/be/src/olap/rowset/segment_v2/variant/hierarchical_data_iterator.cpp
b/be/src/olap/rowset/segment_v2/variant/hierarchical_data_iterator.cpp
index 96e31010de8..7b728a377c8 100644
--- a/be/src/olap/rowset/segment_v2/variant/hierarchical_data_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/variant/hierarchical_data_iterator.cpp
@@ -44,7 +44,7 @@ Status HierarchicalDataIterator::create(ColumnIteratorUPtr*
reader, int32_t col_
ColumnReaderCache* column_reader_cache,
OlapReaderStatistics* stats) {
// None leave node need merge with root
- auto stream_iter = std::make_unique<HierarchicalDataIterator>(path);
+ std::unique_ptr<HierarchicalDataIterator> stream_iter(new
HierarchicalDataIterator(path));
if (node != nullptr) {
std::vector<const SubcolumnColumnMetaInfo::Node*> leaves;
vectorized::PathsInData leaves_paths;
@@ -62,6 +62,7 @@ Status HierarchicalDataIterator::create(ColumnIteratorUPtr*
reader, int32_t col_
stream_iter->_root_reader = std::move(root_column_reader);
// need read from sparse column if not null
stream_iter->_sparse_column_reader = std::move(sparse_reader);
+ stream_iter->_stats = stats;
*reader = std::move(stream_iter);
return Status::OK();
@@ -291,8 +292,11 @@ Status
HierarchicalDataIterator::_init_container(vectorized::MutableColumnPtr& c
RETURN_IF_ERROR(_process_sub_columns(container_variant,
non_nested_subcolumns));
RETURN_IF_ERROR(_process_nested_columns(container_variant,
nested_subcolumns, nrows));
+ {
+
SCOPED_RAW_TIMER(&_stats->variant_fill_path_from_sparse_column_timer_ns);
+ RETURN_IF_ERROR(_process_sparse_column(container_variant, nrows));
+ }
- RETURN_IF_ERROR(_process_sparse_column(container_variant, nrows));
container_variant.set_num_rows(nrows);
return Status::OK();
}
diff --git a/be/src/olap/rowset/segment_v2/variant/hierarchical_data_iterator.h
b/be/src/olap/rowset/segment_v2/variant/hierarchical_data_iterator.h
index 87892673233..36889e64267 100644
--- a/be/src/olap/rowset/segment_v2/variant/hierarchical_data_iterator.h
+++ b/be/src/olap/rowset/segment_v2/variant/hierarchical_data_iterator.h
@@ -67,8 +67,6 @@ public:
// Currently two types of read, merge sparse columns with root columns, or
read directly
enum class ReadType { MERGE_ROOT, READ_DIRECT };
- HierarchicalDataIterator(const vectorized::PathInData& path) : _path(path)
{}
-
static Status create(ColumnIteratorUPtr* reader, int32_t col_uid,
vectorized::PathInData path,
const SubcolumnColumnMetaInfo::Node* target_node,
std::unique_ptr<SubstreamIterator>&& sparse_reader,
@@ -97,6 +95,9 @@ private:
std::unique_ptr<SubstreamIterator> _sparse_column_reader;
size_t _rows_read = 0;
vectorized::PathInData _path;
+ OlapReaderStatistics* _stats = nullptr;
+
+ HierarchicalDataIterator(const vectorized::PathInData& path) : _path(path)
{}
template <typename NodeFunction>
Status tranverse(NodeFunction&& node_func) {
@@ -156,7 +157,11 @@ private:
// read sparse column
if (_sparse_column_reader) {
+ SCOPED_RAW_TIMER(&_stats->variant_scan_sparse_column_timer_ns);
+ int64_t curr_size = _sparse_column_reader->column->byte_size();
RETURN_IF_ERROR(read_func(*_sparse_column_reader, {}, nullptr));
+ _stats->variant_scan_sparse_column_bytes +=
+ _sparse_column_reader->column->byte_size() - curr_size;
}
MutableColumnPtr container;
diff --git
a/be/src/olap/rowset/segment_v2/variant/sparse_column_extract_iterator.h
b/be/src/olap/rowset/segment_v2/variant/sparse_column_extract_iterator.h
index 3ba87ea7259..de25018712c 100644
--- a/be/src/olap/rowset/segment_v2/variant/sparse_column_extract_iterator.h
+++ b/be/src/olap/rowset/segment_v2/variant/sparse_column_extract_iterator.h
@@ -57,7 +57,7 @@ protected:
vectorized::MutableColumnPtr _sparse_column;
StorageReadOptions* _read_opts; // Shared cache pointer
std::unique_ptr<ColumnIterator> _sparse_column_reader;
- const TabletColumn& _col;
+ int32_t _col_uid;
// Pure virtual method for data processing when encounter existing sparse
columns(to be implemented by subclasses)
virtual void
_process_data_with_existing_sparse_column(vectorized::MutableColumnPtr& dst,
size_t num_rows) =
0;
@@ -69,7 +69,9 @@ protected:
public:
BaseSparseColumnProcessor(std::unique_ptr<ColumnIterator>&& reader,
StorageReadOptions* opts,
const TabletColumn& col)
- : _read_opts(opts), _sparse_column_reader(std::move(reader)),
_col(col) {
+ : _read_opts(opts),
+ _sparse_column_reader(std::move(reader)),
+ _col_uid(col.parent_unique_id()) {
_sparse_column = vectorized::ColumnObject::create_sparse_column_fn();
}
@@ -83,7 +85,7 @@ public:
// So we need to cache the sparse column and reuse it.
// The cache is only used when the compaction reader is used.
bool has_sparse_column_cache() const {
- return _read_opts &&
_read_opts->sparse_column_cache[_col.parent_unique_id()] &&
+ return _read_opts && _read_opts->sparse_column_cache[_col_uid] &&
ColumnReader::is_compaction_reader_type(_read_opts->io_ctx.reader_type);
}
@@ -107,18 +109,23 @@ public:
vectorized::MutableColumnPtr& dst) {
// Cache check and population logic
if (has_sparse_column_cache()) {
- _sparse_column =
-
_read_opts->sparse_column_cache[_col.parent_unique_id()]->assume_mutable();
+ _sparse_column =
_read_opts->sparse_column_cache[_col_uid]->assume_mutable();
} else {
_sparse_column->clear();
- RETURN_IF_ERROR(read_method());
+ {
+
SCOPED_RAW_TIMER(&_read_opts->stats->variant_scan_sparse_column_timer_ns);
+ int64_t before_size =
_read_opts->stats->uncompressed_bytes_read;
+ RETURN_IF_ERROR(read_method());
+ _read_opts->stats->variant_scan_sparse_column_bytes +=
+ _read_opts->stats->uncompressed_bytes_read -
before_size;
+ }
// cache the sparse column
if (_read_opts) {
- _read_opts->sparse_column_cache[_col.parent_unique_id()] =
- _sparse_column->get_ptr();
+ _read_opts->sparse_column_cache[_col_uid] =
_sparse_column->get_ptr();
}
}
+
SCOPED_RAW_TIMER(&_read_opts->stats->variant_fill_path_from_sparse_column_timer_ns);
const auto& offsets =
assert_cast<const
vectorized::ColumnMap&>(*_sparse_column).get_offsets();
diff --git a/be/src/olap/rowset/segment_v2/variant/variant_column_reader.cpp
b/be/src/olap/rowset/segment_v2/variant/variant_column_reader.cpp
index f4a1431bcfe..e41159a0330 100644
--- a/be/src/olap/rowset/segment_v2/variant/variant_column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/variant/variant_column_reader.cpp
@@ -114,6 +114,7 @@ Status
VariantColumnReader::_create_hierarchical_reader(ColumnIteratorUPtr* read
const
SubcolumnColumnMetaInfo::Node* root,
ColumnReaderCache*
column_reader_cache,
OlapReaderStatistics*
stats) {
+ stats->variant_subtree_hierarchical_iter_count++;
// Node contains column with children columns or has correspoding sparse
columns
// Create reader with hirachical data.
std::unique_ptr<SubstreamIterator> sparse_iter;
@@ -372,7 +373,9 @@ Status
VariantColumnReader::new_iterator(ColumnIteratorUPtr* iterator,
DCHECK(opt);
// Sparse column exists or reached sparse size limit, read sparse
column
*iterator = std::make_unique<SparseColumnExtractIterator>(
- relative_path.get_path(), std::move(inner_iter), nullptr,
*target_col);
+ relative_path.get_path(), std::move(inner_iter),
+ const_cast<StorageReadOptions*>(opt), *target_col);
+ opt->stats->variant_subtree_sparse_iter_count++;
return Status::OK();
}
@@ -386,6 +389,7 @@ Status
VariantColumnReader::new_iterator(ColumnIteratorUPtr* iterator,
RETURN_IF_ERROR(column_reader_cache->get_path_column_reader(
col_uid, node->path, &column_reader, opt->stats, node));
RETURN_IF_ERROR(column_reader->new_iterator(iterator, nullptr));
+ opt->stats->variant_subtree_leaf_iter_count++;
} else {
RETURN_IF_ERROR(_create_hierarchical_reader(iterator, col_uid,
relative_path, node,
root,
column_reader_cache, opt->stats));
@@ -393,6 +397,7 @@ Status
VariantColumnReader::new_iterator(ColumnIteratorUPtr* iterator,
} else {
// Sparse column not exists and not reached stats limit, then the
target path is not exist, get a default iterator
RETURN_IF_ERROR(Segment::new_default_iterator(*target_col, iterator));
+ opt->stats->variant_subtree_default_iter_count++;
}
return Status::OK();
}
diff --git a/be/src/pipeline/exec/olap_scan_operator.cpp
b/be/src/pipeline/exec/olap_scan_operator.cpp
index 9f38f77b3d9..97fecc2a432 100644
--- a/be/src/pipeline/exec/olap_scan_operator.cpp
+++ b/be/src/pipeline/exec/olap_scan_operator.cpp
@@ -245,6 +245,20 @@ Status OlapScanLocalState::_init_profile() {
_index_filter_profile = std::make_unique<RuntimeProfile>("IndexFilter");
_scanner_profile->add_child(_index_filter_profile.get(), true, nullptr);
+ _variant_scan_sparse_column_timer = ADD_TIMER(_segment_profile,
"VariantScanSparseColumnTimer");
+ _variant_scan_sparse_column_bytes =
+ ADD_COUNTER(_segment_profile, "VariantScanSparseColumnBytes",
TUnit::BYTES);
+ _variant_fill_path_from_sparse_column_timer =
+ ADD_TIMER(_segment_profile,
"VariantFillPathFromSparseColumnTimer");
+ _variant_subtree_default_iter_count =
+ ADD_COUNTER(_segment_profile, "VariantSubtreeDefaultIterCount",
TUnit::UNIT);
+ _variant_subtree_leaf_iter_count =
+ ADD_COUNTER(_segment_profile, "VariantSubtreeLeafIterCount",
TUnit::UNIT);
+ _variant_subtree_hierarchical_iter_count =
+ ADD_COUNTER(_segment_profile,
"VariantSubtreeHierarchicalIterCount", TUnit::UNIT);
+ _variant_subtree_sparse_iter_count =
+ ADD_COUNTER(_segment_profile, "VariantSubtreeSparseIterCount",
TUnit::UNIT);
+
return Status::OK();
}
diff --git a/be/src/pipeline/exec/olap_scan_operator.h
b/be/src/pipeline/exec/olap_scan_operator.h
index 5c3d0f38b84..cfeb43c3e4b 100644
--- a/be/src/pipeline/exec/olap_scan_operator.h
+++ b/be/src/pipeline/exec/olap_scan_operator.h
@@ -247,6 +247,22 @@ private:
RuntimeProfile::Counter* _segment_load_index_timer = nullptr;
std::mutex _profile_mtx;
+ // total uncompressed bytes read when scanning sparse columns in variant
+ RuntimeProfile::Counter* _variant_scan_sparse_column_bytes = nullptr;
+
+ // total time spent scanning sparse subcolumns
+ RuntimeProfile::Counter* _variant_scan_sparse_column_timer = nullptr;
+ // time to build/resolve subcolumn paths from the sparse column
+ RuntimeProfile::Counter* _variant_fill_path_from_sparse_column_timer =
nullptr;
+ // Variant subtree: times falling back to default iterator due to missing
path
+ RuntimeProfile::Counter* _variant_subtree_default_iter_count = nullptr;
+ // Variant subtree: times selecting leaf iterator (target subcolumn is a
leaf)
+ RuntimeProfile::Counter* _variant_subtree_leaf_iter_count = nullptr;
+ // Variant subtree: times selecting hierarchical iterator (node has
children and sparse columns)
+ RuntimeProfile::Counter* _variant_subtree_hierarchical_iter_count =
nullptr;
+ // Variant subtree: times selecting sparse iterator (iterate over sparse
subcolumn)
+ RuntimeProfile::Counter* _variant_subtree_sparse_iter_count = nullptr;
+
std::vector<TabletWithVersion> _tablets;
std::vector<TabletReadSource> _read_sources;
};
diff --git a/be/src/service/point_query_executor.cpp
b/be/src/service/point_query_executor.cpp
index 4377e26fafd..ee923b27d64 100644
--- a/be/src/service/point_query_executor.cpp
+++ b/be/src/service/point_query_executor.cpp
@@ -503,8 +503,12 @@ Status PointQueryExecutor::_lookup_row_data() {
_result_block->get_by_position(pos).column->assume_mutable();
std::unique_ptr<ColumnIterator> iter;
SlotDescriptor* slot = _reusable->tuple_desc()->slots()[pos];
+ StorageReadOptions storage_read_options;
+ storage_read_options.stats = &_read_stats;
+ storage_read_options.io_ctx.reader_type =
ReaderType::READER_QUERY;
RETURN_IF_ERROR(segment->seek_and_read_by_rowid(*_tablet->tablet_schema(), slot,
- row_id,
column, _read_stats, iter));
+ row_id, column,
+
storage_read_options, iter));
if (_tablet->tablet_schema()
->column_by_uid(slot->col_unique_id())
.has_char_type()) {
diff --git a/be/src/vec/exec/scan/new_olap_scanner.cpp
b/be/src/vec/exec/scan/new_olap_scanner.cpp
index ddb17471e08..7711fd5e3ed 100644
--- a/be/src/vec/exec/scan/new_olap_scanner.cpp
+++ b/be/src/vec/exec/scan/new_olap_scanner.cpp
@@ -701,7 +701,21 @@ void NewOlapScanner::_collect_profile_before_close() {
COUNTER_UPDATE(Parent->_output_index_result_column_timer,
\
stats.output_index_result_column_timer);
\
COUNTER_UPDATE(Parent->_filtered_segment_counter,
stats.filtered_segment_number); \
- COUNTER_UPDATE(Parent->_total_segment_counter, stats.total_segment_number);
+ COUNTER_UPDATE(Parent->_total_segment_counter,
stats.total_segment_number); \
+ COUNTER_UPDATE(Parent->_variant_scan_sparse_column_timer,
\
+ stats.variant_scan_sparse_column_timer_ns);
\
+ COUNTER_UPDATE(Parent->_variant_scan_sparse_column_bytes,
\
+ stats.variant_scan_sparse_column_bytes);
\
+ COUNTER_UPDATE(Parent->_variant_fill_path_from_sparse_column_timer,
\
+ stats.variant_fill_path_from_sparse_column_timer_ns);
\
+ COUNTER_UPDATE(Parent->_variant_subtree_default_iter_count,
\
+ stats.variant_subtree_default_iter_count);
\
+ COUNTER_UPDATE(Parent->_variant_subtree_leaf_iter_count,
\
+ stats.variant_subtree_leaf_iter_count);
\
+ COUNTER_UPDATE(Parent->_variant_subtree_hierarchical_iter_count,
\
+ stats.variant_subtree_hierarchical_iter_count);
\
+ COUNTER_UPDATE(Parent->_variant_subtree_sparse_iter_count,
\
+ stats.variant_subtree_sparse_iter_count);
// Update counters for NewOlapScanner
// Update counters from tablet reader's stats
diff --git
a/be/test/olap/rowset/segment_v2/variant_column_writer_reader_test.cpp
b/be/test/olap/rowset/segment_v2/variant_column_writer_reader_test.cpp
index 117fc0579b5..1608ab9a6e6 100644
--- a/be/test/olap/rowset/segment_v2/variant_column_writer_reader_test.cpp
+++ b/be/test/olap/rowset/segment_v2/variant_column_writer_reader_test.cpp
@@ -294,13 +294,14 @@ TEST_F(VariantColumnWriterReaderTest,
test_write_data_normal) {
ColumnIteratorUPtr it;
TabletColumn parent_column = _tablet_schema->column(0);
StorageReadOptions storage_read_opts;
+ OlapReaderStatistics stats;
storage_read_opts.io_ctx.reader_type = ReaderType::READER_QUERY;
+ storage_read_opts.stats = &stats;
st = variant_column_reader->new_iterator(&it, &parent_column,
&storage_read_opts,
&column_reader_cache);
EXPECT_TRUE(st.ok()) << st.msg();
EXPECT_TRUE(assert_cast<HierarchicalDataIterator*>(it.get()) != nullptr);
ColumnIteratorOptions column_iter_opts;
- OlapReaderStatistics stats;
column_iter_opts.stats = &stats;
column_iter_opts.file_reader = file_reader.get();
st = it->init(column_iter_opts);
@@ -399,6 +400,7 @@ TEST_F(VariantColumnWriterReaderTest,
test_write_data_normal) {
// read with opt
auto* iter = assert_cast<SparseColumnExtractIterator*>(it.get());
StorageReadOptions storage_read_opts1;
+ storage_read_opts1.stats = &stats;
storage_read_opts1.io_ctx.reader_type = ReaderType::READER_QUERY;
iter->_read_opts = &storage_read_opts1;
st = iter->next_batch(&nrows, new_column_object, nullptr);
@@ -830,12 +832,13 @@ TEST_F(VariantColumnWriterReaderTest,
test_write_data_advanced) {
TabletColumn parent_column = _tablet_schema->column(0);
StorageReadOptions storage_read_opts;
storage_read_opts.io_ctx.reader_type = ReaderType::READER_QUERY;
+ OlapReaderStatistics stats;
+ storage_read_opts.stats = &stats;
st = variant_column_reader->new_iterator(&it, &parent_column,
&storage_read_opts,
&column_reader_cache);
EXPECT_TRUE(st.ok()) << st.msg();
EXPECT_TRUE(assert_cast<HierarchicalDataIterator*>(it.get()) != nullptr);
ColumnIteratorOptions column_iter_opts;
- OlapReaderStatistics stats;
column_iter_opts.stats = &stats;
column_iter_opts.file_reader = file_reader.get();
st = it->init(column_iter_opts);
@@ -1118,13 +1121,14 @@ TEST_F(VariantColumnWriterReaderTest,
test_write_data_nullable) {
ColumnIteratorUPtr it;
TabletColumn parent_column = _tablet_schema->column(0);
StorageReadOptions storage_read_opts;
+ OlapReaderStatistics stats;
+ storage_read_opts.stats = &stats;
storage_read_opts.io_ctx.reader_type = ReaderType::READER_QUERY;
st = variant_column_reader->new_iterator(&it, &parent_column,
&storage_read_opts,
&column_reader_cache);
EXPECT_TRUE(st.ok()) << st.msg();
EXPECT_TRUE(assert_cast<HierarchicalDataIterator*>(it.get()) != nullptr);
ColumnIteratorOptions column_iter_opts;
- OlapReaderStatistics stats;
column_iter_opts.stats = &stats;
column_iter_opts.file_reader = file_reader.get();
st = it->init(column_iter_opts);
@@ -1691,6 +1695,8 @@ TEST_F(VariantColumnWriterReaderTest,
test_no_sub_in_sparse_column) {
// 10. test hierarchical reader with empty statistics
ColumnIteratorUPtr iterator;
StorageReadOptions read_opts;
+ OlapReaderStatistics stats;
+ read_opts.stats = &stats;
st = variant_reader->new_iterator(&iterator, &column, &read_opts,
&column_reader_cache);
EXPECT_TRUE(st.ok()) << st.msg();
EXPECT_TRUE(iterator != nullptr);
@@ -1833,6 +1839,8 @@ TEST_F(VariantColumnWriterReaderTest,
test_prefix_in_sub_and_sparse) {
// 10. test hierarchical reader with empty statistics
ColumnIteratorUPtr iterator;
StorageReadOptions read_opts;
+ OlapReaderStatistics stats;
+ read_opts.stats = &stats;
st = variant_reader->new_iterator(&iterator, &column, &read_opts,
&column_reader_cache);
EXPECT_TRUE(st.ok()) << st.msg();
EXPECT_TRUE(iterator != nullptr);
@@ -2150,6 +2158,8 @@ TEST_F(VariantColumnWriterReaderTest, test_nested_iter) {
StorageReadOptions storageReadOptions;
storageReadOptions.io_ctx.reader_type = ReaderType::READER_QUERY;
+ OlapReaderStatistics stats;
+ storageReadOptions.stats = &stats;
ColumnIteratorUPtr nested_column_iter;
st = variant_column_reader->new_iterator(&nested_column_iter,
&_tablet_schema->column(0),
@@ -2159,7 +2169,6 @@ TEST_F(VariantColumnWriterReaderTest, test_nested_iter) {
auto* nested_iter =
assert_cast<HierarchicalDataIterator*>(nested_column_iter.get());
EXPECT_TRUE(nested_iter != nullptr);
ColumnIteratorOptions column_iter_opts;
- OlapReaderStatistics stats;
column_iter_opts.stats = &stats;
column_iter_opts.file_reader = file_reader.get();
st = nested_iter->init(column_iter_opts);
@@ -2304,6 +2313,8 @@ TEST_F(VariantColumnWriterReaderTest,
test_nested_iter_nullable) {
StorageReadOptions storageReadOptions;
storageReadOptions.io_ctx.reader_type = ReaderType::READER_QUERY;
+ OlapReaderStatistics stats;
+ storageReadOptions.stats = &stats;
MockColumnReaderCache column_reader_cache(footer, file_reader,
_tablet_schema);
@@ -2315,7 +2326,6 @@ TEST_F(VariantColumnWriterReaderTest,
test_nested_iter_nullable) {
auto* nested_iter =
assert_cast<HierarchicalDataIterator*>(nested_column_iter.get());
EXPECT_TRUE(nested_iter != nullptr);
ColumnIteratorOptions column_iter_opts;
- OlapReaderStatistics stats;
column_iter_opts.stats = &stats;
column_iter_opts.file_reader = file_reader.get();
st = nested_iter->init(column_iter_opts);
@@ -2470,6 +2480,8 @@ TEST_F(VariantColumnWriterReaderTest,
test_read_with_checksum) {
subcolumn.set_is_nullable(true);
_tablet_schema->append_column(subcolumn);
storage_read_opts.io_ctx.reader_type = ReaderType::READER_QUERY;
+ OlapReaderStatistics stats;
+ storage_read_opts.stats = &stats;
ColumnIteratorUPtr hierarchical_it;
MockColumnReaderCache column_reader_cache(footer, file_reader,
_tablet_schema);
@@ -2485,7 +2497,6 @@ TEST_F(VariantColumnWriterReaderTest,
test_read_with_checksum) {
EXPECT_TRUE(st.ok()) << st.msg();
EXPECT_TRUE(dynamic_cast<FileColumnIterator*>(it.get()) != nullptr);
ColumnIteratorOptions column_iter_opts;
- OlapReaderStatistics stats;
column_iter_opts.stats = &stats;
column_iter_opts.file_reader = file_reader.get();
st = it->init(column_iter_opts);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
