eldenmoon commented on code in PR #54181: URL: https://github.com/apache/doris/pull/54181#discussion_r2252923359
########## be/src/olap/rowset/segment_v2/variant/sparse_column_extract_iterator.h: ########## @@ -0,0 +1,198 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <parallel_hashmap/phmap.h> + +#include <memory> +#include <string_view> +#include <unordered_map> +#include <utility> + +#include "common/exception.h" +#include "common/status.h" +#include "io/io_common.h" +#include "olap/field.h" +#include "olap/iterators.h" +#include "olap/rowset/segment_v2/column_reader.h" +#include "olap/rowset/segment_v2/stream_reader.h" +#include "olap/schema.h" +#include "olap/tablet_schema.h" +#include "vec/columns/column.h" +#include "vec/columns/column_nullable.h" +#include "vec/columns/column_variant.h" +#include "vec/columns/subcolumn_tree.h" +#include "vec/common/assert_cast.h" +#include "vec/core/column_with_type_and_name.h" +#include "vec/core/columns_with_type_and_name.h" +#include "vec/core/types.h" +#include "vec/data_types/data_type.h" +#include "vec/data_types/data_type_array.h" +#include "vec/data_types/data_type_nullable.h" +#include "vec/data_types/data_type_string.h" +#include "vec/data_types/data_type_variant.h" +#include "vec/functions/function_helpers.h" +#include "vec/json/path_in_data.h" + +namespace doris::segment_v2 { + +#include "common/compile_check_begin.h" + +// Base class for sparse column processors with common functionality +class BaseSparseColumnProcessor : public ColumnIterator { +protected: + vectorized::MutableColumnPtr _sparse_column; + StorageReadOptions* _read_opts; // Shared cache pointer + std::unique_ptr<ColumnIterator> _sparse_column_reader; + const TabletColumn& _col; + // Pure virtual method for data processing when encounter existing sparse columns(to be implemented by subclasses) + virtual void _process_data_with_existing_sparse_column(vectorized::MutableColumnPtr& dst, + size_t num_rows) = 0; + + // Pure virtual method for data processing when no sparse columns(to be implemented by subclasses) + virtual void _process_data_without_sparse_column(vectorized::MutableColumnPtr& dst, + size_t num_rows) = 0; + +public: + BaseSparseColumnProcessor(std::unique_ptr<ColumnIterator>&& reader, StorageReadOptions* opts, + const TabletColumn& col) + : _read_opts(opts), _sparse_column_reader(std::move(reader)), _col(col) { + _sparse_column = vectorized::ColumnVariant::create_sparse_column_fn(); + } + + // Common initialization for all processors + Status init(const ColumnIteratorOptions& opts) override { + return _sparse_column_reader->init(opts); + } + + // When performing compaction, multiple columns are extracted from the sparse columns, + // and the sparse columns only need to be read once. + // So we need to cache the sparse column and reuse it. + // The cache is only used when the compaction reader is used. + bool has_sparse_column_cache() const { Review Comment: 如果不cache的话就需要读多次， 比如v['a'], v['b']都在稀疏列里，第一次v['a']读取放到cache里，v['b']就不需要读稀疏列了 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
