github-actions[bot] commented on code in PR #28460:
URL: https://github.com/apache/doris/pull/28460#discussion_r1427594619
##########
be/src/vec/exec/format/parquet/byte_array_dict_decoder.h:
##########
@@ -66,10 +66,97 @@ class ByteArrayDictDecoder final : public BaseDictDecoder {
MutableColumnPtr convert_dict_column_to_string_column(const ColumnInt32*
dict_column) override;
protected:
+ template <typename DecimalPrimitiveType, bool has_filter>
+ Status _decode_binary_decimal(MutableColumnPtr& doris_column, DataTypePtr&
data_type,
+ ColumnSelectVector& select_vector);
+
// For dictionary encoding
std::vector<StringRef> _dict_items;
std::vector<uint8_t> _dict_data;
size_t _max_value_length;
std::unordered_map<StringRef, int32_t> _dict_value_to_code;
+
+private:
+ template <typename DecimalPrimitiveType, bool has_filter,
+ DecimalScaleParams::ScaleType ScaleType>
+ Status _decode_binary_decimal_internal(MutableColumnPtr& doris_column,
DataTypePtr& data_type,
+ ColumnSelectVector& select_vector);
};
+
+template <typename DecimalPrimitiveType, bool has_filter>
+Status ByteArrayDictDecoder::_decode_binary_decimal(MutableColumnPtr&
doris_column,
+ DataTypePtr& data_type,
+ ColumnSelectVector&
select_vector) {
+ init_decimal_converter<DecimalPrimitiveType>(data_type);
+ DecimalScaleParams& scale_params = _decode_params->decimal_scale;
+ if (scale_params.scale_type == DecimalScaleParams::SCALE_UP) {
+ return _decode_binary_decimal_internal<DecimalPrimitiveType,
has_filter,
+ DecimalScaleParams::SCALE_UP>(
+ doris_column, data_type, select_vector);
+ } else if (scale_params.scale_type == DecimalScaleParams::SCALE_DOWN) {
+ return _decode_binary_decimal_internal<DecimalPrimitiveType,
has_filter,
+ DecimalScaleParams::SCALE_DOWN>(
+ doris_column, data_type, select_vector);
+ } else {
+ return _decode_binary_decimal_internal<DecimalPrimitiveType,
has_filter,
+ DecimalScaleParams::NO_SCALE>(
+ doris_column, data_type, select_vector);
+ }
+}
+
+template <typename DecimalPrimitiveType, bool has_filter,
DecimalScaleParams::ScaleType ScaleType>
+Status ByteArrayDictDecoder::_decode_binary_decimal_internal(MutableColumnPtr&
doris_column,
+ DataTypePtr&
data_type,
+
ColumnSelectVector& select_vector) {
+ auto& column_data =
+
static_cast<ColumnDecimal<Decimal<DecimalPrimitiveType>>&>(*doris_column).get_data();
+ size_t data_index = column_data.size();
+ column_data.resize(data_index + select_vector.num_values() -
select_vector.num_filtered());
+ size_t dict_index = 0;
+ DecimalScaleParams& scale_params = _decode_params->decimal_scale;
+ ColumnSelectVector::DataReadType read_type;
+ while (size_t run_length =
select_vector.get_next_run<has_filter>(&read_type)) {
+ switch (read_type) {
+ case ColumnSelectVector::CONTENT: {
+ for (size_t i = 0; i < run_length; ++i) {
+ StringRef& slice = _dict_items[_indexes[dict_index++]];
+ char* buf_start = const_cast<char*>(slice.data);
+ uint32_t length = (uint32_t)slice.size;
Review Comment:
warning: use auto when initializing with a cast to avoid duplicating the
type name [modernize-use-auto]
```suggestion
auto length = (uint32_t)slice.size;
```
##########
be/src/vec/exec/format/parquet/delta_bit_pack_decoder.cpp:
##########
@@ -0,0 +1,283 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "delta_bit_pack_decoder.h"
+
+#include <string.h>
+
+#include <algorithm>
+#include <string_view>
+
+#include "vec/columns/column.h"
+#include "vec/common/arithmetic_overflow.h"
+#include "vec/common/string_ref.h"
+#include "vec/core/types.h"
+#include "vec/data_types/data_type_nullable.h"
+
+namespace doris::vectorized {
+
+template <typename T>
+Status DeltaBitPackDecoder<T>::_init_header() {
+ if (!_bit_reader->GetVlqInt(&_values_per_block) ||
+ !_bit_reader->GetVlqInt(&_mini_blocks_per_block) ||
+ !_bit_reader->GetVlqInt(&_total_value_count) ||
+ !_bit_reader->GetZigZagVlqInt(&_last_value)) {
+ return Status::IOError("Init header eof");
+ }
+ if (_values_per_block == 0) {
+ return Status::InvalidArgument("Cannot have zero value per block");
+ }
+ if (_values_per_block % 128 != 0) {
+ return Status::InvalidArgument(
+ "the number of values in a block must be multiple of 128, but
it's " +
+ std::to_string(_values_per_block));
+ }
+ if (_mini_blocks_per_block == 0) {
+ return Status::InvalidArgument("Cannot have zero miniblock per block");
+ }
+ _values_per_mini_block = _values_per_block / _mini_blocks_per_block;
+ if (_values_per_mini_block == 0) {
+ return Status::InvalidArgument("Cannot have zero value per miniblock");
+ }
+ if (_values_per_mini_block % 32 != 0) {
+ return Status::InvalidArgument(
+ "The number of values in a miniblock must be multiple of 32,
but it's " +
+ std::to_string(_values_per_mini_block));
+ }
+ _total_values_remaining = _total_value_count;
+ _delta_bit_widths.resize(_mini_blocks_per_block);
+ // init as empty property
+ _block_initialized = false;
+ _values_remaining_current_mini_block = 0;
+ return Status::OK();
+}
+
+template <typename T>
+Status DeltaBitPackDecoder<T>::_init_block() {
+ DCHECK_GT(_total_values_remaining, 0) << "InitBlock called at EOF";
+ if (!_bit_reader->GetZigZagVlqInt(&_min_delta)) {
+ return Status::IOError("Init block eof");
+ }
+
+ // read the bitwidth of each miniblock
+ uint8_t* bit_width_data = _delta_bit_widths.data();
+ for (uint32_t i = 0; i < _mini_blocks_per_block; ++i) {
+ if (!_bit_reader->GetAligned<uint8_t>(1, bit_width_data + i)) {
+ return Status::IOError("Decode bit-width EOF");
+ }
+ // Note that non-conformant bitwidth entries are allowed by the
Parquet spec
+ // for extraneous miniblocks in the last block (GH-14923), so we check
+ // the bitwidths when actually using them (see InitMiniBlock()).
+ }
+ _mini_block_idx = 0;
+ _block_initialized = true;
+ RETURN_IF_ERROR(_init_mini_block(bit_width_data[0]));
+ return Status::OK();
+}
+
+template <typename T>
+Status DeltaBitPackDecoder<T>::_init_mini_block(int bit_width) {
+ if (PREDICT_FALSE(bit_width > kMaxDeltaBitWidth)) {
+ return Status::InvalidArgument("delta bit width larger than integer
bit width");
+ }
+ _delta_bit_width = bit_width;
+ _values_remaining_current_mini_block = _values_per_mini_block;
+ return Status::OK();
+}
+
+template <typename T>
+Status DeltaBitPackDecoder<T>::_get_internal(T* buffer, int num_values, int*
out_num_values) {
Review Comment:
warning: function '_get_internal' has cognitive complexity of 76 (threshold
50) [readability-function-cognitive-complexity]
```cpp
Status DeltaBitPackDecoder<T>::_get_internal(T* buffer, int num_values, int*
out_num_values) {
^
```
<details>
<summary>Additional context</summary>
**be/src/vec/exec/format/parquet/delta_bit_pack_decoder.cpp:104:** +1,
including nesting penalty of 0, nesting level increased to 1
```cpp
if (num_values == 0) {
^
```
**be/src/vec/exec/format/parquet/delta_bit_pack_decoder.cpp:109:** +1,
including nesting penalty of 0, nesting level increased to 1
```cpp
while (i < num_values) {
^
```
**be/src/vec/exec/format/parquet/delta_bit_pack_decoder.cpp:110:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
if (PREDICT_FALSE(_values_remaining_current_mini_block == 0)) {
^
```
**be/src/vec/exec/format/parquet/delta_bit_pack_decoder.cpp:111:** +3,
including nesting penalty of 2, nesting level increased to 3
```cpp
if (PREDICT_FALSE(!_block_initialized)) {
^
```
**be/src/vec/exec/format/parquet/delta_bit_pack_decoder.cpp:114:** +4,
including nesting penalty of 3, nesting level increased to 4
```cpp
if (i == num_values) {
^
```
**be/src/vec/exec/format/parquet/delta_bit_pack_decoder.cpp:122:** +5,
including nesting penalty of 4, nesting level increased to 5
```cpp
if (_total_value_count != 1) {
^
```
**be/src/vec/exec/format/parquet/delta_bit_pack_decoder.cpp:123:** +6,
including nesting penalty of 5, nesting level increased to 6
```cpp
RETURN_IF_ERROR(_init_block());
^
```
**be/src/common/status.h:532:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/vec/exec/format/parquet/delta_bit_pack_decoder.cpp:123:** +7,
including nesting penalty of 6, nesting level increased to 7
```cpp
RETURN_IF_ERROR(_init_block());
^
```
**be/src/common/status.h:534:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/vec/exec/format/parquet/delta_bit_pack_decoder.cpp:127:** +4,
including nesting penalty of 3, nesting level increased to 4
```cpp
RETURN_IF_ERROR(_init_block());
^
```
**be/src/common/status.h:532:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/vec/exec/format/parquet/delta_bit_pack_decoder.cpp:127:** +5,
including nesting penalty of 4, nesting level increased to 5
```cpp
RETURN_IF_ERROR(_init_block());
^
```
**be/src/common/status.h:534:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/vec/exec/format/parquet/delta_bit_pack_decoder.cpp:128:** +1,
nesting level increased to 3
```cpp
} else {
^
```
**be/src/vec/exec/format/parquet/delta_bit_pack_decoder.cpp:130:** +4,
including nesting penalty of 3, nesting level increased to 4
```cpp
if (_mini_block_idx < _mini_blocks_per_block) {
^
```
**be/src/vec/exec/format/parquet/delta_bit_pack_decoder.cpp:131:** +5,
including nesting penalty of 4, nesting level increased to 5
```cpp
RETURN_IF_ERROR(_init_mini_block(_delta_bit_widths.data()[_mini_block_idx]));
^
```
**be/src/common/status.h:532:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/vec/exec/format/parquet/delta_bit_pack_decoder.cpp:131:** +6,
including nesting penalty of 5, nesting level increased to 6
```cpp
RETURN_IF_ERROR(_init_mini_block(_delta_bit_widths.data()[_mini_block_idx]));
^
```
**be/src/common/status.h:534:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/vec/exec/format/parquet/delta_bit_pack_decoder.cpp:132:** +1,
nesting level increased to 4
```cpp
} else {
^
```
**be/src/vec/exec/format/parquet/delta_bit_pack_decoder.cpp:133:** +5,
including nesting penalty of 4, nesting level increased to 5
```cpp
RETURN_IF_ERROR(_init_block());
^
```
**be/src/common/status.h:532:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/vec/exec/format/parquet/delta_bit_pack_decoder.cpp:133:** +6,
including nesting penalty of 5, nesting level increased to 6
```cpp
RETURN_IF_ERROR(_init_block());
^
```
**be/src/common/status.h:534:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/vec/exec/format/parquet/delta_bit_pack_decoder.cpp:140:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
for (int j = 0; j < values_decode; ++j) {
^
```
**be/src/vec/exec/format/parquet/delta_bit_pack_decoder.cpp:141:** +3,
including nesting penalty of 2, nesting level increased to 3
```cpp
if (!_bit_reader->GetValue(_delta_bit_width, buffer + i + j)) {
^
```
**be/src/vec/exec/format/parquet/delta_bit_pack_decoder.cpp:145:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
for (int j = 0; j < values_decode; ++j) {
^
```
**be/src/vec/exec/format/parquet/delta_bit_pack_decoder.cpp:157:** +1,
including nesting penalty of 0, nesting level increased to 1
```cpp
if (PREDICT_FALSE(_total_values_remaining == 0)) {
^
```
**be/src/vec/exec/format/parquet/delta_bit_pack_decoder.cpp:158:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
if (!_bit_reader->Advance(_delta_bit_width *
_values_remaining_current_mini_block)) {
^
```
</details>
##########
be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:
##########
@@ -0,0 +1,609 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/parquet/fix_length_plain_decoder.h"
+
+#include <gen_cpp/parquet_types.h>
+#include <stdint.h>
+#include <string.h>
+
+#include <memory>
+#include <vector>
+
+// IWYU pragma: no_include <opentelemetry/common/threadlocal.h>
+#include "common/compiler_util.h" // IWYU pragma: keep
+#include "util/bit_util.h"
+#include "util/slice.h"
+#include "vec/columns/column.h"
+#include "vec/common/string_ref.h"
+#include "vec/core/types.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/exec/format/format_common.h"
+#include "vec/exec/format/parquet/parquet_common.h"
+#include "vec/runtime/vdatetime_value.h"
+
+namespace doris {
+namespace vectorized {
+template <typename T>
+class ColumnDecimal;
+template <typename T>
+class ColumnVector;
+} // namespace vectorized
+} // namespace doris
+
+namespace doris::vectorized {
+
+Status FixLengthPlainDecoder::skip_values(size_t num_values) {
+ _offset += _type_length * num_values;
+ if (UNLIKELY(_offset > _data->size)) {
+ return Status::IOError("Out-of-bounds access in parquet data decoder");
+ }
+ return Status::OK();
+}
+
+Status FixLengthPlainDecoder::decode_values(MutableColumnPtr& doris_column,
DataTypePtr& data_type,
+ ColumnSelectVector& select_vector,
+ bool is_dict_filter) {
+ if (select_vector.has_filter()) {
+ return _decode_values<true>(doris_column, data_type, select_vector,
is_dict_filter);
+ } else {
+ return _decode_values<false>(doris_column, data_type, select_vector,
is_dict_filter);
+ }
+}
+
+template <bool has_filter>
+Status FixLengthPlainDecoder::_decode_values(MutableColumnPtr& doris_column,
DataTypePtr& data_type,
Review Comment:
warning: function '_decode_values' exceeds recommended size/complexity
thresholds [readability-function-size]
```cpp
Status FixLengthPlainDecoder::_decode_values(MutableColumnPtr& doris_column,
DataTypePtr& data_type,
^
```
<details>
<summary>Additional context</summary>
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:68:** 119
lines including whitespace and comments (threshold 80)
```cpp
Status FixLengthPlainDecoder::_decode_values(MutableColumnPtr& doris_column,
DataTypePtr& data_type,
^
```
</details>
##########
be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:
##########
@@ -0,0 +1,609 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/parquet/fix_length_plain_decoder.h"
+
+#include <gen_cpp/parquet_types.h>
+#include <stdint.h>
+#include <string.h>
+
+#include <memory>
+#include <vector>
+
+// IWYU pragma: no_include <opentelemetry/common/threadlocal.h>
+#include "common/compiler_util.h" // IWYU pragma: keep
+#include "util/bit_util.h"
+#include "util/slice.h"
+#include "vec/columns/column.h"
+#include "vec/common/string_ref.h"
+#include "vec/core/types.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/exec/format/format_common.h"
+#include "vec/exec/format/parquet/parquet_common.h"
+#include "vec/runtime/vdatetime_value.h"
+
+namespace doris {
+namespace vectorized {
+template <typename T>
+class ColumnDecimal;
+template <typename T>
+class ColumnVector;
+} // namespace vectorized
+} // namespace doris
+
+namespace doris::vectorized {
+
+Status FixLengthPlainDecoder::skip_values(size_t num_values) {
+ _offset += _type_length * num_values;
+ if (UNLIKELY(_offset > _data->size)) {
+ return Status::IOError("Out-of-bounds access in parquet data decoder");
+ }
+ return Status::OK();
+}
+
+Status FixLengthPlainDecoder::decode_values(MutableColumnPtr& doris_column,
DataTypePtr& data_type,
+ ColumnSelectVector& select_vector,
+ bool is_dict_filter) {
+ if (select_vector.has_filter()) {
+ return _decode_values<true>(doris_column, data_type, select_vector,
is_dict_filter);
+ } else {
+ return _decode_values<false>(doris_column, data_type, select_vector,
is_dict_filter);
+ }
+}
+
+template <bool has_filter>
+Status FixLengthPlainDecoder::_decode_values(MutableColumnPtr& doris_column,
DataTypePtr& data_type,
Review Comment:
warning: function '_decode_values' has cognitive complexity of 90 (threshold
50) [readability-function-cognitive-complexity]
```cpp
Status FixLengthPlainDecoder::_decode_values(MutableColumnPtr& doris_column,
DataTypePtr& data_type,
^
```
<details>
<summary>Additional context</summary>
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:72:** +1,
including nesting penalty of 0, nesting level increased to 1
```cpp
if (UNLIKELY(_offset + _type_length * non_null_size > _data->size)) {
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:76:** +1,
including nesting penalty of 0, nesting level increased to 1
```cpp
switch (logical_type) {
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:94:** +1,
nesting level increased to 2
```cpp
FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:97:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
if (_physical_type == tparquet::Type::INT32) {
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:102:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
if (_physical_type == tparquet::Type::INT32) {
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:108:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
if (_physical_type == tparquet::Type::INT96) {
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:111:** +1,
nesting level increased to 2
```cpp
} else if (_physical_type == tparquet::Type::INT64) {
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:119:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
if (_physical_type == tparquet::Type::INT96) {
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:122:** +1,
nesting level increased to 2
```cpp
} else if (_physical_type == tparquet::Type::INT64) {
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:128:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
if (_physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY) {
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:131:** +1,
nesting level increased to 2
```cpp
} else if (_physical_type == tparquet::Type::INT32) {
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:134:** +1,
nesting level increased to 2
```cpp
} else if (_physical_type == tparquet::Type::INT64) {
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:140:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
if (_physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY) {
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:143:** +1,
nesting level increased to 2
```cpp
} else if (_physical_type == tparquet::Type::INT32) {
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:146:** +1,
nesting level increased to 2
```cpp
} else if (_physical_type == tparquet::Type::INT64) {
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:152:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
if (_physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY) {
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:155:** +1,
nesting level increased to 2
```cpp
} else if (_physical_type == tparquet::Type::INT32) {
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:158:** +1,
nesting level increased to 2
```cpp
} else if (_physical_type == tparquet::Type::INT64) {
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:164:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
if (_physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY) {
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:167:** +1,
nesting level increased to 2
```cpp
} else if (_physical_type == tparquet::Type::INT32) {
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:170:** +1,
nesting level increased to 2
```cpp
} else if (_physical_type == tparquet::Type::INT64) {
^
```
**be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:179:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
if (_physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY) {
^
```
</details>
##########
be/src/vec/exec/format/parquet/delta_bit_pack_decoder.cpp:
##########
@@ -0,0 +1,283 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "delta_bit_pack_decoder.h"
+
+#include <string.h>
Review Comment:
warning: inclusion of deprecated C++ header 'string.h'; consider using
'cstring' instead [modernize-deprecated-headers]
```suggestion
#include <cstring>
```
##########
be/src/vec/exec/format/parquet/vparquet_column_reader.cpp:
##########
@@ -481,108 +476,86 @@ Status ScalarColumnReader::_try_load_dict_page(bool*
loaded, bool* has_dict) {
Status ScalarColumnReader::read_column_data(ColumnPtr& doris_column,
DataTypePtr& type,
Review Comment:
warning: function 'read_column_data' has cognitive complexity of 51
(threshold 50) [readability-function-cognitive-complexity]
```cpp
Status ScalarColumnReader::read_column_data(ColumnPtr& doris_column,
DataTypePtr& type,
^
```
<details>
<summary>Additional context</summary>
**be/src/vec/exec/format/parquet/vparquet_column_reader.cpp:478:** +1,
including nesting penalty of 0, nesting level increased to 1
```cpp
if (_chunk_reader->remaining_num_values() == 0) {
^
```
**be/src/vec/exec/format/parquet/vparquet_column_reader.cpp:479:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
if (!_chunk_reader->has_next_page()) {
^
```
**be/src/vec/exec/format/parquet/vparquet_column_reader.cpp:484:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
RETURN_IF_ERROR(_chunk_reader->next_page());
^
```
**be/src/common/status.h:532:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/vec/exec/format/parquet/vparquet_column_reader.cpp:484:** +3,
including nesting penalty of 2, nesting level increased to 3
```cpp
RETURN_IF_ERROR(_chunk_reader->next_page());
^
```
**be/src/common/status.h:534:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/vec/exec/format/parquet/vparquet_column_reader.cpp:486:** +1,
including nesting penalty of 0, nesting level increased to 1
```cpp
if (_nested_column) {
^
```
**be/src/vec/exec/format/parquet/vparquet_column_reader.cpp:487:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent());
^
```
**be/src/common/status.h:532:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/vec/exec/format/parquet/vparquet_column_reader.cpp:487:** +3,
including nesting penalty of 2, nesting level increased to 3
```cpp
RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent());
^
```
**be/src/common/status.h:534:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/vec/exec/format/parquet/vparquet_column_reader.cpp:496:** +1,
including nesting penalty of 0, nesting level increased to 1
```cpp
if (read_ranges.size() == 0) {
^
```
**be/src/vec/exec/format/parquet/vparquet_column_reader.cpp:499:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
RETURN_IF_ERROR(_chunk_reader->skip_page());
^
```
**be/src/common/status.h:532:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/vec/exec/format/parquet/vparquet_column_reader.cpp:499:** +3,
including nesting penalty of 2, nesting level increased to 3
```cpp
RETURN_IF_ERROR(_chunk_reader->skip_page());
^
```
**be/src/common/status.h:534:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/vec/exec/format/parquet/vparquet_column_reader.cpp:501:** +1,
nesting level increased to 1
```cpp
} else {
^
```
**be/src/vec/exec/format/parquet/vparquet_column_reader.cpp:505:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
if (select_vector.has_filter() && select_vector.filter_ratio() >
0.6) {
^
```
**be/src/vec/exec/format/parquet/vparquet_column_reader.cpp:505:** +1
```cpp
if (select_vector.has_filter() && select_vector.filter_ratio() >
0.6) {
^
```
**be/src/vec/exec/format/parquet/vparquet_column_reader.cpp:511:** +3,
including nesting penalty of 2, nesting level increased to 3
```cpp
if (batch_size >= remaining_num_values &&
^
```
**be/src/vec/exec/format/parquet/vparquet_column_reader.cpp:511:** +1
```cpp
if (batch_size >= remaining_num_values &&
^
```
**be/src/vec/exec/format/parquet/vparquet_column_reader.cpp:516:** +4,
including nesting penalty of 3, nesting level increased to 4
```cpp
RETURN_IF_ERROR(_chunk_reader->skip_page());
^
```
**be/src/common/status.h:532:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/vec/exec/format/parquet/vparquet_column_reader.cpp:516:** +5,
including nesting penalty of 4, nesting level increased to 5
```cpp
RETURN_IF_ERROR(_chunk_reader->skip_page());
^
```
**be/src/common/status.h:534:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/vec/exec/format/parquet/vparquet_column_reader.cpp:518:** +4,
including nesting penalty of 3, nesting level increased to 4
```cpp
if (!_chunk_reader->has_next_page()) {
^
```
**be/src/vec/exec/format/parquet/vparquet_column_reader.cpp:524:** +1
```cpp
batch_size <= remaining_num_values &&
select_vector.can_filter_all(batch_size);
^
```
**be/src/vec/exec/format/parquet/vparquet_column_reader.cpp:525:** +3,
including nesting penalty of 2, nesting level increased to 3
```cpp
if (skip_whole_batch) {
^
```
**be/src/vec/exec/format/parquet/vparquet_column_reader.cpp:530:** +2,
including nesting penalty of 1, nesting level increased to 2
```cpp
RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent());
^
```
**be/src/common/status.h:532:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/vec/exec/format/parquet/vparquet_column_reader.cpp:530:** +3,
including nesting penalty of 2, nesting level increased to 3
```cpp
RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent());
^
```
**be/src/common/status.h:534:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/vec/exec/format/parquet/vparquet_column_reader.cpp:555:** +1,
including nesting penalty of 0, nesting level increased to 1
```cpp
if (_chunk_reader->remaining_num_values() == 0 &&
!_chunk_reader->has_next_page()) {
^
```
</details>
##########
be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:
##########
@@ -0,0 +1,609 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/parquet/fix_length_plain_decoder.h"
+
+#include <gen_cpp/parquet_types.h>
+#include <stdint.h>
+#include <string.h>
+
+#include <memory>
+#include <vector>
+
+// IWYU pragma: no_include <opentelemetry/common/threadlocal.h>
+#include "common/compiler_util.h" // IWYU pragma: keep
+#include "util/bit_util.h"
+#include "util/slice.h"
+#include "vec/columns/column.h"
+#include "vec/common/string_ref.h"
+#include "vec/core/types.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/exec/format/format_common.h"
+#include "vec/exec/format/parquet/parquet_common.h"
+#include "vec/runtime/vdatetime_value.h"
+
+namespace doris {
+namespace vectorized {
Review Comment:
warning: nested namespaces can be concatenated
[modernize-concat-nested-namespaces]
```suggestion
namespace doris::vectorized {
```
be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:44:
```diff
- } // namespace vectorized
- } // namespace doris
+ } // namespace doris
```
##########
be/src/vec/exec/format/parquet/fix_length_dict_decoder.hpp:
##########
@@ -123,17 +334,127 @@ class FixLengthDictDecoder final : public
BaseDictDecoder {
}
return Status::OK();
}
- using ColumnType =
ParquetConvert::PhysicalTypeTraits<PhysicalType>::ColumnType;
- using DataType =
ParquetConvert::PhysicalTypeTraits<PhysicalType>::DataType;
+
+ template <typename DecimalPrimitiveType, typename DecimalPhysicalType,
bool has_filter>
+ Status _decode_primitive_decimal(MutableColumnPtr& doris_column,
DataTypePtr& data_type,
+ ColumnSelectVector& select_vector) {
+ init_decimal_converter<DecimalPrimitiveType>(data_type);
+ DecimalScaleParams& scale_params = _decode_params->decimal_scale;
+#define M(FixedTypeLength, ValueCopyType, ScaleType)
\
+ case FixedTypeLength:
\
+ return _decode_primitive_decimal_internal<DecimalPrimitiveType,
DecimalPhysicalType, \
+ has_filter, FixedTypeLength,
ValueCopyType, \
+ ScaleType>(doris_column,
data_type, \
+ select_vector);
+
+#define APPLY_FOR_DECIMALS(ScaleType) \
+ M(1, int64_t, ScaleType) \
+ M(2, int64_t, ScaleType) \
+ M(3, int64_t, ScaleType) \
+ M(4, int64_t, ScaleType) \
+ M(5, int64_t, ScaleType) \
+ M(6, int64_t, ScaleType) \
+ M(7, int64_t, ScaleType) \
+ M(8, int64_t, ScaleType) \
+ M(9, int128_t, ScaleType) \
+ M(10, int128_t, ScaleType) \
+ M(11, int128_t, ScaleType) \
+ M(12, int128_t, ScaleType) \
+ M(13, int128_t, ScaleType) \
+ M(14, int128_t, ScaleType) \
+ M(15, int128_t, ScaleType) \
+ M(16, int128_t, ScaleType)
+
+ if (scale_params.scale_type == DecimalScaleParams::SCALE_UP) {
+ switch (_type_length) {
+ APPLY_FOR_DECIMALS(DecimalScaleParams::SCALE_UP)
+ default:
+ LOG(FATAL) << "__builtin_unreachable";
+ __builtin_unreachable();
+ }
+ } else if (scale_params.scale_type == DecimalScaleParams::SCALE_DOWN) {
+ switch (_type_length) {
+ APPLY_FOR_DECIMALS(DecimalScaleParams::SCALE_DOWN)
+ default:
+ LOG(FATAL) << "__builtin_unreachable";
+ __builtin_unreachable();
+ }
+ } else {
+ switch (_type_length) {
+ APPLY_FOR_DECIMALS(DecimalScaleParams::NO_SCALE)
+ default:
+ LOG(FATAL) << "__builtin_unreachable";
+ __builtin_unreachable();
+ }
+ }
+ return Status::OK();
+#undef APPLY_FOR_DECIMALS
+#undef M
+ }
+
+ template <typename DecimalPrimitiveType, typename DecimalPhysicalType,
bool has_filter,
+ int fixed_type_length, typename ValueCopyType,
+ DecimalScaleParams::ScaleType ScaleType>
+ Status _decode_primitive_decimal_internal(MutableColumnPtr& doris_column,
+ DataTypePtr& data_type,
+ ColumnSelectVector&
select_vector) {
+ auto& column_data =
+
static_cast<ColumnDecimal<Decimal<DecimalPrimitiveType>>&>(*doris_column)
+ .get_data();
+ size_t data_index = column_data.size();
+ column_data.resize(data_index + select_vector.num_values() -
select_vector.num_filtered());
+ size_t dict_index = 0;
+ DecimalScaleParams& scale_params = _decode_params->decimal_scale;
+
+ ColumnSelectVector::DataReadType read_type;
+ while (size_t run_length =
select_vector.get_next_run<has_filter>(&read_type)) {
+ switch (read_type) {
+ case ColumnSelectVector::CONTENT: {
+ for (size_t i = 0; i < run_length; ++i) {
+ ValueCopyType value =
static_cast<T>(_dict_items[_indexes[dict_index++]]);
+ if constexpr (ScaleType == DecimalScaleParams::SCALE_UP) {
+ value *= scale_params.scale_factor;
+ } else if constexpr (ScaleType ==
DecimalScaleParams::SCALE_DOWN) {
+ value /= scale_params.scale_factor;
+ } else if constexpr (ScaleType ==
DecimalScaleParams::NO_SCALE) {
+ // do nothing
+ } else {
+ LOG(FATAL) << "__builtin_unreachable";
+ __builtin_unreachable();
+ }
+ auto& v =
reinterpret_cast<DecimalPrimitiveType&>(column_data[data_index++]);
+ v = (DecimalPrimitiveType)value;
+ }
+ break;
+ }
+ case ColumnSelectVector::NULL_DATA: {
+ data_index += run_length;
+ break;
+ }
+ case ColumnSelectVector::FILTERED_CONTENT: {
+ dict_index += run_length;
+ break;
+ }
+ case ColumnSelectVector::FILTERED_NULL: {
+ // do nothing
+ break;
+ }
+ }
+ }
+ return Status::OK();
+ }
+
+ tparquet::Type::type _physical_type;
// For dictionary encoding
- std::vector<DataType> _dict_items;
+ std::vector<T> _dict_items;
};
template <>
-class FixLengthDictDecoder<tparquet::Type::FIXED_LEN_BYTE_ARRAY> final :
public BaseDictDecoder {
+class FixLengthDictDecoder<char*> final : public BaseDictDecoder {
public:
- FixLengthDictDecoder() : BaseDictDecoder() {};
+ FixLengthDictDecoder(tparquet::Type::type physical_type)
+ : BaseDictDecoder(), _physical_type(physical_type) {};
Review Comment:
warning: initializer for base class
'std::doris::vectorized::BaseDictDecoder' is redundant
[readability-redundant-member-init]
```suggestion
: , _physical_type(physical_type) {};
```
##########
be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:
##########
@@ -0,0 +1,609 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/parquet/fix_length_plain_decoder.h"
+
+#include <gen_cpp/parquet_types.h>
+#include <stdint.h>
Review Comment:
warning: inclusion of deprecated C++ header 'stdint.h'; consider using
'cstdint' instead [modernize-deprecated-headers]
```suggestion
#include <cstdint>
```
##########
be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:
##########
@@ -0,0 +1,609 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/parquet/fix_length_plain_decoder.h"
+
+#include <gen_cpp/parquet_types.h>
+#include <stdint.h>
+#include <string.h>
Review Comment:
warning: inclusion of deprecated C++ header 'string.h'; consider using
'cstring' instead [modernize-deprecated-headers]
```suggestion
#include <cstring>
```
##########
be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp:
##########
@@ -0,0 +1,609 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/parquet/fix_length_plain_decoder.h"
+
+#include <gen_cpp/parquet_types.h>
+#include <stdint.h>
+#include <string.h>
+
+#include <memory>
+#include <vector>
+
+// IWYU pragma: no_include <opentelemetry/common/threadlocal.h>
+#include "common/compiler_util.h" // IWYU pragma: keep
+#include "util/bit_util.h"
+#include "util/slice.h"
+#include "vec/columns/column.h"
+#include "vec/common/string_ref.h"
+#include "vec/core/types.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/exec/format/format_common.h"
+#include "vec/exec/format/parquet/parquet_common.h"
+#include "vec/runtime/vdatetime_value.h"
+
+namespace doris {
+namespace vectorized {
+template <typename T>
+class ColumnDecimal;
+template <typename T>
+class ColumnVector;
+} // namespace vectorized
+} // namespace doris
+
+namespace doris::vectorized {
+
+Status FixLengthPlainDecoder::skip_values(size_t num_values) {
+ _offset += _type_length * num_values;
+ if (UNLIKELY(_offset > _data->size)) {
+ return Status::IOError("Out-of-bounds access in parquet data decoder");
+ }
+ return Status::OK();
+}
+
+Status FixLengthPlainDecoder::decode_values(MutableColumnPtr& doris_column,
DataTypePtr& data_type,
Review Comment:
warning: method 'decode_values' can be made static
[readability-convert-member-functions-to-static]
```suggestion
static Status FixLengthPlainDecoder::decode_values(MutableColumnPtr&
doris_column, DataTypePtr& data_type,
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
