github-actions[bot] commented on code in PR #59263:
URL: https://github.com/apache/doris/pull/59263#discussion_r3488093319
##########
be/test/storage/segment/column_reader_test.cpp:
##########
@@ -23,27 +23,355 @@
#include <gtest/gtest.h>
#include <chrono>
+#include <cstdint>
+#include <iterator>
#include <memory>
+#include <string>
#include <thread>
+#include <utility>
#include <vector>
#include "agent/be_exec_version_manager.h"
#include "common/config.h"
#include "io/fs/file_reader.h"
+#include "io/fs/file_system.h"
+#include "io/fs/file_writer.h"
+#include "io/fs/local_file_system.h"
+#include "storage/olap_common.h"
#include "storage/segment/column_reader_cache.h"
+#include "storage/segment/column_writer.h"
#include "storage/segment/mock/mock_segment.h"
#include "storage/segment/segment.h"
#include "storage/segment/variant/variant_column_reader.h"
#include "storage/tablet/tablet_schema.h"
+#include "storage/types.h"
#include "util/json/path_in_data.h"
namespace doris::segment_v2 {
+namespace {
+class TestColumnIterator final : public ColumnIterator {
+public:
+ Status seek_to_ordinal(ordinal_t /* ord */) override { return
Status::OK(); }
+
+ Status next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null)
override {
+ dst->insert_many_defaults(*n);
+ if (has_null != nullptr) {
+ *has_null = false;
+ }
+ return Status::OK();
+ }
+
+ Status read_by_rowids(const rowid_t* /* rowids */, const size_t count,
+ MutableColumnPtr& dst) override {
+ dst->insert_many_defaults(count);
+ return Status::OK();
+ }
+
+ ordinal_t get_current_ordinal() const override { return 0; }
+
+ void force_set_read_requirement(ReadRequirement requirement) {
+ _read_requirement = requirement;
+ }
+
+ Result<TColumnAccessPaths> get_sub_access_paths(const TColumnAccessPaths&
access_paths,
+ bool is_predicate = false)
{
+ return _get_sub_access_paths(access_paths, is_predicate);
+ }
+
+ void check_and_set_meta_read_mode(ReadRequirement
requirement_before_access_path,
+ const TColumnAccessPaths& access_paths) {
+ _check_and_set_meta_read_mode(requirement_before_access_path,
access_paths);
+ }
+
+ void convert_to_place_holder_column(MutableColumnPtr& dst, size_t count) {
+ _convert_to_place_holder_column(dst, count);
+ }
+};
+
+TColumnAccessPath create_access_path(std::vector<std::string> path) {
+ TColumnAccessPath access_path;
+ TDataAccessPath data_access_path;
+ data_access_path.__set_path(std::move(path));
+ access_path.__set_data_access_path(std::move(data_access_path));
+ return access_path;
+}
+
+std::shared_ptr<ColumnReader> create_test_reader(
+ bool is_nullable = false, uint64_t num_rows = 0,
+ FieldType field_type = FieldType::OLAP_FIELD_TYPE_INT) {
+ auto reader = std::make_shared<ColumnReader>();
+ reader->_meta_is_nullable = is_nullable;
+ reader->_num_rows = num_rows;
+ reader->_meta_type = field_type;
+ return reader;
+}
+
+class TrackingColumnIterator final : public ColumnIterator {
+public:
+ Status seek_to_ordinal(ordinal_t ord) override {
+ seek_ordinals.emplace_back(ord);
+ _current_ordinal = ord;
+ return Status::OK();
+ }
+
+ Status next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null)
override {
+ next_batch_sizes.emplace_back(*n);
+ if (!need_to_read()) {
+ _convert_to_place_holder_column(dst, *n);
+ if (has_null != nullptr) {
+ *has_null = false;
+ }
+ return Status::OK();
+ }
+
+ _recovery_from_place_holder_column(dst);
+ dst->insert_many_defaults(*n);
+ _current_ordinal += *n;
+ if (has_null != nullptr) {
+ *has_null = false;
+ }
+ return Status::OK();
+ }
+
+ Status read_by_rowids(const rowid_t* rowids, const size_t count,
+ MutableColumnPtr& dst) override {
+ read_by_rowids_batches.emplace_back(rowids, rowids + count);
+ if (!need_to_read()) {
+ _convert_to_place_holder_column(dst, count);
+ return Status::OK();
+ }
+
+ _recovery_from_place_holder_column(dst);
+ dst->insert_many_defaults(count);
+ return Status::OK();
+ }
+
+ ordinal_t get_current_ordinal() const override { return _current_ordinal; }
+
+ void collect_prefetchers(
+ std::map<PrefetcherInitMethod, std::vector<SegmentPrefetcher*>>&
prefetchers,
+ PrefetcherInitMethod init_method) override {
+ record_collect_method(init_method);
+ prefetchers[init_method].emplace_back(prefetcher());
+ }
+
+ SegmentPrefetcher* prefetcher() const {
+ return
reinterpret_cast<SegmentPrefetcher*>(const_cast<TrackingColumnIterator*>(this));
+ }
+
+ void clear_tracking() {
+ seek_ordinals.clear();
+ next_batch_sizes.clear();
+ read_by_rowids_batches.clear();
+ collect_methods.clear();
+ }
+
+ std::vector<ordinal_t> seek_ordinals;
+ std::vector<size_t> next_batch_sizes;
+ std::vector<std::vector<rowid_t>> read_by_rowids_batches;
+ std::vector<PrefetcherInitMethod> collect_methods;
+
+private:
+ void record_collect_method(PrefetcherInitMethod init_method) {
+ collect_methods.emplace_back(init_method);
+ }
+
+ ordinal_t _current_ordinal = 0;
+};
+
+class TrackingFileColumnIterator final : public FileColumnIterator {
+public:
+ explicit TrackingFileColumnIterator(std::shared_ptr<ColumnReader> reader)
+ : FileColumnIterator(std::move(reader)) {}
+
+ Status seek_to_ordinal(ordinal_t ord) override {
+ seek_ordinals.emplace_back(ord);
+ _current_ordinal = ord;
+ return Status::OK();
+ }
+
+ Status next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null)
override {
+ next_batch_sizes.emplace_back(*n);
+ dst->insert_many_defaults(*n);
+ _current_ordinal += *n;
+ if (has_null != nullptr) {
+ *has_null = false;
+ }
+ return Status::OK();
+ }
+
+ Status read_by_rowids(const rowid_t* rowids, const size_t count,
+ MutableColumnPtr& dst) override {
+ read_by_rowids_batches.emplace_back(rowids, rowids + count);
+ dst->insert_many_defaults(count);
+ return Status::OK();
+ }
+
+ ordinal_t get_current_ordinal() const override { return _current_ordinal; }
+
+ void collect_prefetchers(
+ std::map<PrefetcherInitMethod, std::vector<SegmentPrefetcher*>>&
prefetchers,
+ PrefetcherInitMethod init_method) override {
+ record_collect_method(init_method);
+ prefetchers[init_method].emplace_back(prefetcher());
+ }
+
+ SegmentPrefetcher* prefetcher() const {
+ return
reinterpret_cast<SegmentPrefetcher*>(const_cast<TrackingFileColumnIterator*>(this));
+ }
+
+ std::vector<ordinal_t> seek_ordinals;
+ std::vector<size_t> next_batch_sizes;
+ std::vector<std::vector<rowid_t>> read_by_rowids_batches;
+ std::vector<PrefetcherInitMethod> collect_methods;
+
+private:
+ void record_collect_method(PrefetcherInitMethod init_method) {
+ collect_methods.emplace_back(init_method);
+ }
+
+ ordinal_t _current_ordinal = 0;
+};
+
+class NullMapOnlyFileColumnIterator final : public FileColumnIterator {
+public:
+ explicit NullMapOnlyFileColumnIterator(std::shared_ptr<ColumnReader>
reader)
+ : FileColumnIterator(std::move(reader)) {}
+
+ void force_null_map_only() { _meta_read_mode =
MetaReadMode::NULL_MAP_ONLY; }
+};
+
+MutableColumnPtr create_int_struct_column(size_t field_count) {
+ Columns columns;
+ for (size_t i = 0; i < field_count; ++i) {
+ columns.emplace_back(ColumnInt32::create());
+ }
+ return ColumnStruct::create(std::move(columns));
+}
+
+MutableColumnPtr create_nullable_int_struct_column(size_t field_count) {
+ return ColumnNullable::create(create_int_struct_column(field_count),
ColumnUInt8::create());
+}
+
+MutableColumnPtr create_nullable_int_array_column() {
+ return ColumnNullable::create(
+ ColumnArray::create(ColumnInt32::create(),
ColumnArray::ColumnOffsets::create()),
+ ColumnUInt8::create());
+}
+
+MutableColumnPtr create_nullable_int_map_column() {
+ return ColumnNullable::create(ColumnMap::create(ColumnInt32::create(),
ColumnInt32::create(),
+
ColumnArray::ColumnOffsets::create()),
+ ColumnUInt8::create());
+}
+
+struct TrackingOffsetIterator {
+ OffsetFileColumnIteratorUPtr iterator;
+ TrackingFileColumnIterator* tracker = nullptr;
+};
+
+TrackingOffsetIterator create_tracking_offset_iterator() {
+ auto file_iterator =
std::make_unique<TrackingFileColumnIterator>(create_test_reader());
+ auto* tracker = file_iterator.get();
+ return
{std::make_unique<OffsetFileColumnIterator>(std::move(file_iterator)), tracker};
+}
+} // namespace
+
+static const std::string COLUMN_READER_FILE_TEST_DIR =
"./ut_dir/column_reader_test";
+
class ColumnReaderTest : public ::testing::Test {
protected:
- void SetUp() override {}
- void TearDown() override {}
+ void SetUp() override {
+ config::disable_storage_page_cache = true;
Review Comment:
This fixture now forces the process-wide
`config::disable_storage_page_cache` flag to `true`, but `TearDown()` only
removes the test directory and never restores the old value. `doris_be_test`
runs storage tests in one process, and this flag defaults to `false` and
controls both `ColumnReader` page-cache usage and segment
`ColumnIteratorOptions::use_page_cache`, so any test that runs after this
fixture can silently inherit page cache disabled and become order-dependent.
Please save the previous value in the fixture and restore it in `TearDown()` or
scope the override to the single file-backed test that needs it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
