This is an automated email from the ASF dual-hosted git repository.
zclll pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ab390d9116b [refactor](vec) remove useless be_exec_version (#58672)
ab390d9116b is described below
commit ab390d9116b4312ee7eba7362419fa2db58bfa3d
Author: admiring_xm <[email protected]>
AuthorDate: Fri Dec 5 16:47:14 2025 +0800
[refactor](vec) remove useless be_exec_version (#58672)
In the current version, `be_exec_version >= USE_CONST_SERDE` is always
true, so the corresponding conditional branches in `serialize`,
`deserialize`, and `get_uncompressed_serialized_bytes` of the `datatype`
can be removed.
---
be/src/vec/data_types/data_type_array.cpp | 113 +++-----
be/src/vec/data_types/data_type_bitmap.cpp | 152 ++++-------
be/src/vec/data_types/data_type_decimal.cpp | 112 +++-----
.../data_types/data_type_fixed_length_object.cpp | 163 +++++-------
be/src/vec/data_types/data_type_hll.cpp | 170 +++++-------
be/src/vec/data_types/data_type_jsonb.cpp | 12 +-
be/src/vec/data_types/data_type_jsonb.h | 6 +-
be/src/vec/data_types/data_type_map.cpp | 131 ++++------
be/src/vec/data_types/data_type_nullable.cpp | 182 +++++--------
be/src/vec/data_types/data_type_number_base.cpp | 136 +++-------
be/src/vec/data_types/data_type_quantilestate.cpp | 157 ++++--------
be/src/vec/data_types/data_type_string.cpp | 284 +++++++--------------
be/src/vec/data_types/data_type_struct.cpp | 101 +++-----
be/src/vec/data_types/data_type_varbinary.cpp | 3 -
be/test/olap/wal/wal_manager_test.cpp | 4 +-
be/test/util/test_data/column_variant.bin | Bin 654 -> 0 bytes
be/test/vec/columns/column_variant_test.cpp | 213 ----------------
.../vec/data_types/data_type_datetime_v1_test.cpp | 37 +--
.../vec/data_types/data_type_datetime_v2_test.cpp | 39 +--
be/test/vec/data_types/data_type_decimal_test.cpp | 54 +---
be/test/vec/data_types/data_type_jsonb_test.cpp | 78 ++----
be/test/vec/data_types/data_type_number_test.cpp | 46 +---
be/test/vec/data_types/data_type_string_test.cpp | 78 ++----
23 files changed, 668 insertions(+), 1603 deletions(-)
diff --git a/be/src/vec/data_types/data_type_array.cpp
b/be/src/vec/data_types/data_type_array.cpp
index 287f9c561c4..6b152e857f8 100644
--- a/be/src/vec/data_types/data_type_array.cpp
+++ b/be/src/vec/data_types/data_type_array.cpp
@@ -83,90 +83,51 @@ size_t DataTypeArray::get_number_of_dimensions() const {
// data : data1 | data2 | ...
int64_t DataTypeArray::get_uncompressed_serialized_bytes(const IColumn& column,
int be_exec_version)
const {
- if (be_exec_version >= USE_CONST_SERDE) {
- auto size = sizeof(bool) + sizeof(size_t) + sizeof(size_t);
- bool is_const_column = is_column_const(column);
- auto real_need_copy_num = is_const_column ? 1 : column.size();
- const IColumn* array_column = &column;
- if (is_const_column) {
- const auto& const_column = assert_cast<const ColumnConst&>(column);
- array_column = &(const_column.get_data_column());
- }
- const auto& data_column = assert_cast<const
ColumnArray&>(*array_column);
- size = size + sizeof(ColumnArray::Offset64) * real_need_copy_num;
- return size +
get_nested_type()->get_uncompressed_serialized_bytes(data_column.get_data(),
-
be_exec_version);
- } else {
- auto ptr = column.convert_to_full_column_if_const();
- const auto& data_column = assert_cast<const ColumnArray&>(*ptr.get());
- return sizeof(ColumnArray::Offset64) * (column.size() + 1) +
-
get_nested_type()->get_uncompressed_serialized_bytes(data_column.get_data(),
-
be_exec_version);
+ auto size = sizeof(bool) + sizeof(size_t) + sizeof(size_t);
+ bool is_const_column = is_column_const(column);
+ auto real_need_copy_num = is_const_column ? 1 : column.size();
+ const IColumn* array_column = &column;
+ if (is_const_column) {
+ const auto& const_column = assert_cast<const ColumnConst&>(column);
+ array_column = &(const_column.get_data_column());
}
+ const auto& data_column = assert_cast<const ColumnArray&>(*array_column);
+ size = size + sizeof(ColumnArray::Offset64) * real_need_copy_num;
+ return size +
get_nested_type()->get_uncompressed_serialized_bytes(data_column.get_data(),
+
be_exec_version);
}
char* DataTypeArray::serialize(const IColumn& column, char* buf, int
be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- const auto* array_column = &column;
- size_t real_need_copy_num = 0;
- buf = serialize_const_flag_and_row_num(&array_column, buf,
&real_need_copy_num);
-
- const auto& data_column = assert_cast<const
ColumnArray&>(*array_column);
- // offsets
- memcpy(buf, data_column.get_offsets().data(),
- real_need_copy_num * sizeof(ColumnArray::Offset64));
- buf += real_need_copy_num * sizeof(ColumnArray::Offset64);
- // children
- return get_nested_type()->serialize(data_column.get_data(), buf,
be_exec_version);
- } else {
- auto ptr = column.convert_to_full_column_if_const();
- const auto& data_column = assert_cast<const ColumnArray&>(*ptr.get());
-
- // row num
- unaligned_store<ColumnArray::Offset64>(buf, column.size());
- buf += sizeof(ColumnArray::Offset64);
- // offsets
- memcpy(buf, data_column.get_offsets().data(),
- column.size() * sizeof(ColumnArray::Offset64));
- buf += column.size() * sizeof(ColumnArray::Offset64);
- // children
- return get_nested_type()->serialize(data_column.get_data(), buf,
be_exec_version);
- }
+ const auto* array_column = &column;
+ size_t real_need_copy_num = 0;
+ buf = serialize_const_flag_and_row_num(&array_column, buf,
&real_need_copy_num);
+
+ const auto& data_column = assert_cast<const ColumnArray&>(*array_column);
+ // offsets
+ memcpy(buf, data_column.get_offsets().data(),
+ real_need_copy_num * sizeof(ColumnArray::Offset64));
+ buf += real_need_copy_num * sizeof(ColumnArray::Offset64);
+ // children
+ return get_nested_type()->serialize(data_column.get_data(), buf,
be_exec_version);
}
const char* DataTypeArray::deserialize(const char* buf, MutableColumnPtr*
column,
int be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- auto* origin_column = column->get();
- size_t real_have_saved_num = 0;
- buf = deserialize_const_flag_and_row_num(buf, column,
&real_have_saved_num);
-
- auto* data_column = assert_cast<ColumnArray*>(origin_column);
- auto& offsets = data_column->get_offsets();
-
- // offsets
- offsets.resize(real_have_saved_num);
- memcpy(offsets.data(), buf, sizeof(ColumnArray::Offset64) *
real_have_saved_num);
- buf += sizeof(ColumnArray::Offset64) * real_have_saved_num;
- // children
- auto nested_column = data_column->get_data_ptr()->assume_mutable();
- buf = get_nested_type()->deserialize(buf, &nested_column,
be_exec_version);
- return buf;
- } else {
- auto* data_column = assert_cast<ColumnArray*>(column->get());
- auto& offsets = data_column->get_offsets();
-
- // row num
- auto row_num = unaligned_load<ColumnArray::Offset64>(buf);
- buf += sizeof(ColumnArray::Offset64);
- // offsets
- offsets.resize(row_num);
- memcpy(offsets.data(), buf, sizeof(ColumnArray::Offset64) * row_num);
- buf += sizeof(ColumnArray::Offset64) * row_num;
- // children
- auto nested_column = data_column->get_data_ptr()->assume_mutable();
- return get_nested_type()->deserialize(buf, &nested_column,
be_exec_version);
- }
+ auto* origin_column = column->get();
+ size_t real_have_saved_num = 0;
+ buf = deserialize_const_flag_and_row_num(buf, column,
&real_have_saved_num);
+
+ auto* data_column = assert_cast<ColumnArray*>(origin_column);
+ auto& offsets = data_column->get_offsets();
+
+ // offsets
+ offsets.resize(real_have_saved_num);
+ memcpy(offsets.data(), buf, sizeof(ColumnArray::Offset64) *
real_have_saved_num);
+ buf += sizeof(ColumnArray::Offset64) * real_have_saved_num;
+ // children
+ auto nested_column = data_column->get_data_ptr()->assume_mutable();
+ buf = get_nested_type()->deserialize(buf, &nested_column, be_exec_version);
+ return buf;
}
void DataTypeArray::to_pb_column_meta(PColumnMeta* col_meta) const {
diff --git a/be/src/vec/data_types/data_type_bitmap.cpp
b/be/src/vec/data_types/data_type_bitmap.cpp
index 496f8bedd99..338834f15aa 100644
--- a/be/src/vec/data_types/data_type_bitmap.cpp
+++ b/be/src/vec/data_types/data_type_bitmap.cpp
@@ -36,120 +36,66 @@ namespace doris::vectorized {
// <bitmap array>: bitmap1 | bitmap2 | ...
int64_t DataTypeBitMap::get_uncompressed_serialized_bytes(const IColumn&
column,
int be_exec_version)
const {
- if (be_exec_version >= USE_CONST_SERDE) {
- auto size = sizeof(bool) + sizeof(size_t) + sizeof(size_t);
- bool is_const_column = is_column_const(column);
- auto real_need_copy_num = is_const_column ? 1 : column.size();
-
- const IColumn* bitmap_column = &column;
- if (is_const_column) {
- const auto& const_column = assert_cast<const ColumnConst&>(column);
- bitmap_column = &(const_column.get_data_column());
- }
- const auto& data_column = assert_cast<const
ColumnBitmap&>(*bitmap_column);
- auto allocate_len_size = sizeof(size_t) * real_need_copy_num;
- size_t allocate_content_size = 0;
- for (size_t i = 0; i < real_need_copy_num; ++i) {
- auto& bitmap = data_column.get_element(i);
- allocate_content_size += bitmap.getSizeInBytes();
- }
- return size + allocate_len_size + allocate_content_size;
- } else {
- auto ptr = column.convert_to_full_column_if_const();
- const auto& data_column = assert_cast<const ColumnBitmap&>(*ptr);
-
- auto allocate_len_size = sizeof(size_t) * (column.size() + 1);
- size_t allocate_content_size = 0;
- for (size_t i = 0; i < column.size(); ++i) {
- auto& bitmap = data_column.get_element(i);
- allocate_content_size += bitmap.getSizeInBytes();
- }
-
- return allocate_len_size + allocate_content_size;
+ auto size = sizeof(bool) + sizeof(size_t) + sizeof(size_t);
+ bool is_const_column = is_column_const(column);
+ auto real_need_copy_num = is_const_column ? 1 : column.size();
+
+ const IColumn* bitmap_column = &column;
+ if (is_const_column) {
+ const auto& const_column = assert_cast<const ColumnConst&>(column);
+ bitmap_column = &(const_column.get_data_column());
}
+ const auto& data_column = assert_cast<const ColumnBitmap&>(*bitmap_column);
+ auto allocate_len_size = sizeof(size_t) * real_need_copy_num;
+ size_t allocate_content_size = 0;
+ for (size_t i = 0; i < real_need_copy_num; ++i) {
+ auto& bitmap = data_column.get_element(i);
+ allocate_content_size += bitmap.getSizeInBytes();
+ }
+ return size + allocate_len_size + allocate_content_size;
}
char* DataTypeBitMap::serialize(const IColumn& column, char* buf, int
be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- const auto* bitmap_column = &column;
- size_t real_need_copy_num = 0;
- buf = serialize_const_flag_and_row_num(&bitmap_column, buf,
&real_need_copy_num);
-
- const auto& data_column = assert_cast<const
ColumnBitmap&>(*bitmap_column);
- // serialize the bitmap size array
- auto* meta_ptr = reinterpret_cast<size_t*>(buf);
- for (size_t i = 0; i < real_need_copy_num; ++i) {
- auto& bitmap = data_column.get_element(i);
- unaligned_store<size_t>(&meta_ptr[i], bitmap.getSizeInBytes());
- }
-
- // serialize each bitmap
- char* data_ptr = buf + sizeof(size_t) * real_need_copy_num;
- for (size_t i = 0; i < real_need_copy_num; ++i) {
- auto& bitmap = data_column.get_element(i);
- bitmap.write_to(data_ptr);
- data_ptr += unaligned_load<size_t>(&meta_ptr[i]);
- }
- return data_ptr;
- } else {
- auto ptr = column.convert_to_full_column_if_const();
- const auto& data_column = assert_cast<const ColumnBitmap&>(*ptr);
-
- // serialize the bitmap size array, row num saves at index 0
- auto* meta_ptr = reinterpret_cast<size_t*>(buf);
- meta_ptr[0] = column.size();
- for (size_t i = 0; i < meta_ptr[0]; ++i) {
- auto& bitmap = data_column.get_element(i);
- unaligned_store<size_t>(&meta_ptr[i + 1], bitmap.getSizeInBytes());
- }
+ const auto* bitmap_column = &column;
+ size_t real_need_copy_num = 0;
+ buf = serialize_const_flag_and_row_num(&bitmap_column, buf,
&real_need_copy_num);
+
+ const auto& data_column = assert_cast<const ColumnBitmap&>(*bitmap_column);
+ // serialize the bitmap size array
+ auto* meta_ptr = reinterpret_cast<size_t*>(buf);
+ for (size_t i = 0; i < real_need_copy_num; ++i) {
+ auto& bitmap = data_column.get_element(i);
+ unaligned_store<size_t>(&meta_ptr[i], bitmap.getSizeInBytes());
+ }
- // serialize each bitmap
- char* data_ptr = buf + sizeof(size_t) * (meta_ptr[0] + 1);
- for (size_t i = 0; i < meta_ptr[0]; ++i) {
- auto& bitmap = data_column.get_element(i);
- bitmap.write_to(data_ptr);
- data_ptr += unaligned_load<size_t>(&meta_ptr[i + 1]);
- }
- return data_ptr;
+ // serialize each bitmap
+ char* data_ptr = buf + sizeof(size_t) * real_need_copy_num;
+ for (size_t i = 0; i < real_need_copy_num; ++i) {
+ auto& bitmap = data_column.get_element(i);
+ bitmap.write_to(data_ptr);
+ data_ptr += unaligned_load<size_t>(&meta_ptr[i]);
}
+ return data_ptr;
}
const char* DataTypeBitMap::deserialize(const char* buf, MutableColumnPtr*
column,
int be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- auto* origin_column = column->get();
- size_t real_have_saved_num = 0;
- buf = deserialize_const_flag_and_row_num(buf, column,
&real_have_saved_num);
-
- // deserialize the bitmap size array
- auto& data_column = assert_cast<ColumnBitmap&>(*origin_column);
- auto& data = data_column.get_data();
- // deserialize each bitmap
- data.resize(real_have_saved_num);
- const auto* meta_ptr = reinterpret_cast<const size_t*>(buf);
- const char* data_ptr = buf + sizeof(size_t) * real_have_saved_num;
- for (size_t i = 0; i < real_have_saved_num; ++i) {
- data[i].deserialize(data_ptr);
- data_ptr += unaligned_load<size_t>(&meta_ptr[i]);
- }
- return data_ptr;
- } else {
- auto& data_column = assert_cast<ColumnBitmap&>(*(column->get()));
- auto& data = data_column.get_data();
-
- // deserialize the bitmap size array
- const size_t* meta_ptr = reinterpret_cast<const size_t*>(buf);
-
- // deserialize each bitmap
- data.resize(meta_ptr[0]);
- const char* data_ptr = buf + sizeof(size_t) * (meta_ptr[0] + 1);
- for (size_t i = 0; i < meta_ptr[0]; ++i) {
- data[i].deserialize(data_ptr);
- data_ptr += unaligned_load<size_t>(&meta_ptr[i + 1]);
- }
-
- return data_ptr;
+ auto* origin_column = column->get();
+ size_t real_have_saved_num = 0;
+ buf = deserialize_const_flag_and_row_num(buf, column,
&real_have_saved_num);
+
+ // deserialize the bitmap size array
+ auto& data_column = assert_cast<ColumnBitmap&>(*origin_column);
+ auto& data = data_column.get_data();
+ // deserialize each bitmap
+ data.resize(real_have_saved_num);
+ const auto* meta_ptr = reinterpret_cast<const size_t*>(buf);
+ const char* data_ptr = buf + sizeof(size_t) * real_have_saved_num;
+ for (size_t i = 0; i < real_have_saved_num; ++i) {
+ data[i].deserialize(data_ptr);
+ data_ptr += unaligned_load<size_t>(&meta_ptr[i]);
}
+ return data_ptr;
}
MutableColumnPtr DataTypeBitMap::create_column() const {
diff --git a/be/src/vec/data_types/data_type_decimal.cpp
b/be/src/vec/data_types/data_type_decimal.cpp
index 67867c97672..4f756f239c8 100644
--- a/be/src/vec/data_types/data_type_decimal.cpp
+++ b/be/src/vec/data_types/data_type_decimal.cpp
@@ -137,66 +137,32 @@ std::string DataTypeDecimal<T>::to_string(const
FieldType& value) const {
template <PrimitiveType T>
int64_t DataTypeDecimal<T>::get_uncompressed_serialized_bytes(const IColumn&
column,
int
be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- auto size = sizeof(bool) + sizeof(size_t) + sizeof(size_t);
- auto real_need_copy_num = is_column_const(column) ? 1 : column.size();
- auto mem_size = cast_set<UInt32>(sizeof(FieldType) *
real_need_copy_num);
- if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
- return size + mem_size;
- } else {
- return size + sizeof(size_t) +
- std::max(cast_set<size_t>(mem_size),
-
streamvbyte_max_compressedbytes(upper_int32(mem_size)));
- }
+ auto size = sizeof(bool) + sizeof(size_t) + sizeof(size_t);
+ auto real_need_copy_num = is_column_const(column) ? 1 : column.size();
+ auto mem_size = cast_set<UInt32>(sizeof(FieldType) * real_need_copy_num);
+ if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
+ return size + mem_size;
} else {
- auto size = sizeof(FieldType) * column.size();
- if (size <= SERIALIZED_MEM_SIZE_LIMIT) {
- return sizeof(uint32_t) + size;
- } else {
- return sizeof(uint32_t) + sizeof(size_t) +
- std::max(size,
-
streamvbyte_max_compressedbytes(cast_set<UInt32>(upper_int32(size))));
- }
+ return size + sizeof(size_t) +
+ std::max(cast_set<size_t>(mem_size),
+
streamvbyte_max_compressedbytes(upper_int32(mem_size)));
}
}
template <PrimitiveType T>
char* DataTypeDecimal<T>::serialize(const IColumn& column, char* buf, int
be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- const auto* data_column = &column;
- size_t real_need_copy_num = 0;
- buf = serialize_const_flag_and_row_num(&data_column, buf,
&real_need_copy_num);
+ const auto* data_column = &column;
+ size_t real_need_copy_num = 0;
+ buf = serialize_const_flag_and_row_num(&data_column, buf,
&real_need_copy_num);
- UInt32 mem_size = cast_set<UInt32>(real_need_copy_num *
sizeof(FieldType));
- const auto* origin_data =
- assert_cast<const
ColumnDecimal<T>&>(*data_column).get_data().data();
+ UInt32 mem_size = cast_set<UInt32>(real_need_copy_num * sizeof(FieldType));
+ const auto* origin_data = assert_cast<const
ColumnDecimal<T>&>(*data_column).get_data().data();
- // column data
- if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
- memcpy(buf, origin_data, mem_size);
- return buf + mem_size;
- } else {
- auto encode_size =
- streamvbyte_encode(reinterpret_cast<const
uint32_t*>(origin_data),
- upper_int32(mem_size), (uint8_t*)(buf +
sizeof(size_t)));
- unaligned_store<size_t>(buf, encode_size);
- buf += sizeof(size_t);
- return buf + encode_size;
- }
+ // column data
+ if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
+ memcpy(buf, origin_data, mem_size);
+ return buf + mem_size;
} else {
- // row num
- UInt32 mem_size = cast_set<UInt32>(column.size() * sizeof(FieldType));
- *reinterpret_cast<uint32_t*>(buf) = mem_size;
- buf += sizeof(uint32_t);
- // column data
- auto ptr = column.convert_to_full_column_if_const();
- const auto* origin_data =
- assert_cast<const
ColumnDecimal<T>&>(*ptr.get()).get_data().data();
- if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
- memcpy(buf, origin_data, mem_size);
- return buf + mem_size;
- }
-
auto encode_size =
streamvbyte_encode(reinterpret_cast<const
uint32_t*>(origin_data),
upper_int32(mem_size), (uint8_t*)(buf +
sizeof(size_t)));
@@ -205,47 +171,29 @@ char* DataTypeDecimal<T>::serialize(const IColumn&
column, char* buf, int be_exe
return buf + encode_size;
}
}
+
template <PrimitiveType T>
const char* DataTypeDecimal<T>::deserialize(const char* buf, MutableColumnPtr*
column,
int be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- auto* origin_column = column->get();
- size_t real_have_saved_num = 0;
- buf = deserialize_const_flag_and_row_num(buf, column,
&real_have_saved_num);
+ auto* origin_column = column->get();
+ size_t real_have_saved_num = 0;
+ buf = deserialize_const_flag_and_row_num(buf, column,
&real_have_saved_num);
- // column data
- UInt32 mem_size = cast_set<UInt32>(real_have_saved_num *
sizeof(FieldType));
- auto& container =
assert_cast<ColumnDecimal<T>*>(origin_column)->get_data();
- container.resize(real_have_saved_num);
- if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
- memcpy(container.data(), buf, mem_size);
- buf = buf + mem_size;
- } else {
- auto encode_size = unaligned_load<size_t>(buf);
- buf += sizeof(size_t);
- streamvbyte_decode((const uint8_t*)buf,
(uint32_t*)(container.data()),
- upper_int32(mem_size));
- buf = buf + encode_size;
- }
- return buf;
+ // column data
+ UInt32 mem_size = cast_set<UInt32>(real_have_saved_num *
sizeof(FieldType));
+ auto& container =
assert_cast<ColumnDecimal<T>*>(origin_column)->get_data();
+ container.resize(real_have_saved_num);
+ if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
+ memcpy(container.data(), buf, mem_size);
+ buf = buf + mem_size;
} else {
- // row num
- uint32_t mem_size = *reinterpret_cast<const uint32_t*>(buf);
- buf += sizeof(uint32_t);
- // column data
- auto& container =
assert_cast<ColumnDecimal<T>*>(column->get())->get_data();
- container.resize(mem_size / sizeof(FieldType));
- if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
- memcpy(container.data(), buf, mem_size);
- return buf + mem_size;
- }
-
auto encode_size = unaligned_load<size_t>(buf);
buf += sizeof(size_t);
streamvbyte_decode((const uint8_t*)buf, (uint32_t*)(container.data()),
upper_int32(mem_size));
- return buf + encode_size;
+ buf = buf + encode_size;
}
+ return buf;
}
template <PrimitiveType T>
diff --git a/be/src/vec/data_types/data_type_fixed_length_object.cpp
b/be/src/vec/data_types/data_type_fixed_length_object.cpp
index d6dca09250e..ab80bf72f9f 100644
--- a/be/src/vec/data_types/data_type_fixed_length_object.cpp
+++ b/be/src/vec/data_types/data_type_fixed_length_object.cpp
@@ -33,123 +33,78 @@ namespace doris::vectorized {
char* DataTypeFixedLengthObject::serialize(const IColumn& column, char* buf,
int be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- // const flag
- bool is_const_column = is_column_const(column);
- unaligned_store<bool>(buf, is_const_column);
- buf += sizeof(bool);
-
- // row num
- const auto row_num = column.size();
- unaligned_store<size_t>(buf, row_num);
- buf += sizeof(size_t);
- auto real_need_copy_num = is_const_column ? 1 : row_num;
-
- const IColumn* data_column = &column;
- if (is_const_column) {
- const auto& const_column = assert_cast<const ColumnConst&>(column);
- data_column = &(const_column.get_data_column());
- }
- const auto& src_col = assert_cast<const ColumnType&>(*data_column);
- DCHECK(src_col.item_size() > 0)
- << "[serialize]item size of DataTypeFixedLengthObject should
be greater than 0";
-
- // item size
- unaligned_store<size_t>(buf, src_col.item_size());
- buf += sizeof(size_t);
- // column data
- const auto* origin_data = src_col.get_data().data();
- memcpy(buf, origin_data, real_need_copy_num * src_col.item_size());
- buf += real_need_copy_num * src_col.item_size();
-
- return buf;
- } else {
- // row num
- const UInt32 row_num = cast_set<UInt32>(column.size());
- unaligned_store<uint32_t>(buf, row_num);
- buf += sizeof(uint32_t);
- // column data
- auto ptr = column.convert_to_full_column_if_const();
- const auto& src_col = assert_cast<const ColumnType&>(*ptr.get());
- DCHECK(src_col.item_size() > 0)
- << "[serialize]item size of DataTypeFixedLengthObject should
be greater than 0";
- unaligned_store<size_t>(buf, src_col.item_size());
- buf += sizeof(size_t);
- const auto* origin_data = src_col.get_data().data();
- memcpy(buf, origin_data, row_num * src_col.item_size());
- buf += row_num * src_col.item_size();
-
- return buf;
+ // const flag
+ bool is_const_column = is_column_const(column);
+ unaligned_store<bool>(buf, is_const_column);
+ buf += sizeof(bool);
+
+ // row num
+ const auto row_num = column.size();
+ unaligned_store<size_t>(buf, row_num);
+ buf += sizeof(size_t);
+ auto real_need_copy_num = is_const_column ? 1 : row_num;
+
+ const IColumn* data_column = &column;
+ if (is_const_column) {
+ const auto& const_column = assert_cast<const ColumnConst&>(column);
+ data_column = &(const_column.get_data_column());
}
+ const auto& src_col = assert_cast<const ColumnType&>(*data_column);
+ DCHECK(src_col.item_size() > 0)
+ << "[serialize]item size of DataTypeFixedLengthObject should be
greater than 0";
+
+ // item size
+ unaligned_store<size_t>(buf, src_col.item_size());
+ buf += sizeof(size_t);
+ // column data
+ const auto* origin_data = src_col.get_data().data();
+ memcpy(buf, origin_data, real_need_copy_num * src_col.item_size());
+ buf += real_need_copy_num * src_col.item_size();
+
+ return buf;
}
const char* DataTypeFixedLengthObject::deserialize(const char* buf,
MutableColumnPtr* column,
int be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- //const flag
- bool is_const_column = unaligned_load<bool>(buf);
- buf += sizeof(bool);
- //row num
- size_t row_num = unaligned_load<size_t>(buf);
- buf += sizeof(size_t);
- //item size
- size_t item_size = unaligned_load<size_t>(buf);
- buf += sizeof(size_t);
-
- DCHECK(item_size > 0)
- << "[deserialize]item size of DataTypeFixedLengthObject should
be greater than 0";
-
- auto& dst_col = static_cast<ColumnType&>(*(column->get()));
- dst_col.set_item_size(item_size);
- // column data
- auto real_copy_num = is_const_column ? 1 : row_num;
- dst_col.resize(real_copy_num);
- memcpy(dst_col.get_data().data(), buf, real_copy_num * item_size);
- buf += real_copy_num * item_size;
- if (is_const_column) {
- auto const_column = ColumnConst::create((*column)->get_ptr(),
row_num);
- *column = const_column->get_ptr();
- }
- return buf;
- } else {
- // row num
- uint32_t row_num = unaligned_load<uint32_t>(buf);
- buf += sizeof(uint32_t);
- size_t item_size = unaligned_load<size_t>(buf);
- buf += sizeof(size_t);
-
- DCHECK(item_size > 0)
- << "[deserialize]item size of DataTypeFixedLengthObject should
be greater than 0";
-
- auto& dst_col = static_cast<ColumnType&>(*(column->get()));
- dst_col.set_item_size(item_size);
- // column data
- dst_col.resize(row_num);
- memcpy(dst_col.get_data().data(), buf, row_num * item_size);
- buf += row_num * item_size;
-
- return buf;
+ //const flag
+ bool is_const_column = unaligned_load<bool>(buf);
+ buf += sizeof(bool);
+ //row num
+ size_t row_num = unaligned_load<size_t>(buf);
+ buf += sizeof(size_t);
+ //item size
+ size_t item_size = unaligned_load<size_t>(buf);
+ buf += sizeof(size_t);
+
+ DCHECK(item_size > 0)
+ << "[deserialize]item size of DataTypeFixedLengthObject should be
greater than 0";
+
+ auto& dst_col = static_cast<ColumnType&>(*(column->get()));
+ dst_col.set_item_size(item_size);
+ // column data
+ auto real_copy_num = is_const_column ? 1 : row_num;
+ dst_col.resize(real_copy_num);
+ memcpy(dst_col.get_data().data(), buf, real_copy_num * item_size);
+ buf += real_copy_num * item_size;
+ if (is_const_column) {
+ auto const_column = ColumnConst::create((*column)->get_ptr(), row_num);
+ *column = const_column->get_ptr();
}
+ return buf;
}
// binary: const flag | row num | item size| data
// data : item data1 | item data2...
int64_t DataTypeFixedLengthObject::get_uncompressed_serialized_bytes(const
IColumn& column,
int
be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- auto size = sizeof(bool) + sizeof(size_t) + sizeof(size_t);
- const IColumn* data_column = &column;
- if (is_column_const(column)) {
- const auto& const_column = assert_cast<const ColumnConst&>(column);
- data_column = &(const_column.get_data_column());
- }
- const auto& src_col = assert_cast<const ColumnType&>(*data_column);
- return static_cast<const ColumnType&>(src_col).byte_size() + size;
-
- } else {
- return static_cast<const ColumnType&>(column).byte_size() +
sizeof(uint32_t) +
- sizeof(size_t);
+ auto size = sizeof(bool) + sizeof(size_t) + sizeof(size_t);
+ const IColumn* data_column = &column;
+ if (is_column_const(column)) {
+ const auto& const_column = assert_cast<const ColumnConst&>(column);
+ data_column = &(const_column.get_data_column());
}
+ const auto& src_col = assert_cast<const ColumnType&>(*data_column);
+ return static_cast<const ColumnType&>(src_col).byte_size() + size;
}
MutableColumnPtr DataTypeFixedLengthObject::create_column() const {
diff --git a/be/src/vec/data_types/data_type_hll.cpp
b/be/src/vec/data_types/data_type_hll.cpp
index cf9598d5614..225056dce80 100644
--- a/be/src/vec/data_types/data_type_hll.cpp
+++ b/be/src/vec/data_types/data_type_hll.cpp
@@ -36,56 +36,33 @@ namespace doris::vectorized {
// first: const flag| row num | real_saved_num | hll1 size | hll2 size | ...
// second: hll1 | hll2 | ...
char* DataTypeHLL::serialize(const IColumn& column, char* buf, int
be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- const auto* hll_column = &column;
- size_t real_need_copy_num = 0;
- buf = serialize_const_flag_and_row_num(&hll_column, buf,
&real_need_copy_num);
- // In the code below, if the real_need_copy_num is 0, an empty vector
will be created. A vector with size 0 may return nullptr from data()
- // https://en.cppreference.com/w/cpp/container/vector/data
- // `If size() is 0, data() may or may not return a null pointer.`
- // This would trigger a ubsan error: `null pointer passed as argument
2, which is declared to never be null`
- // Other data types don't have this issue because they use Doris
internal pod array that guarantees data won't be nullptr.
- if (real_need_copy_num == 0) {
- return buf;
- }
-
- const auto& data_column = assert_cast<const ColumnHLL&>(*hll_column);
- std::vector<size_t> hll_size_array(real_need_copy_num);
- auto allocate_len_size = sizeof(size_t) * real_need_copy_num;
- char* buf_start = buf;
- buf += allocate_len_size;
-
- for (size_t i = 0; i < real_need_copy_num; ++i) {
- const auto& hll = data_column.get_element(i);
- size_t actual_size =
hll.serialize(reinterpret_cast<uint8_t*>(buf));
- hll_size_array[i] = actual_size;
- buf += actual_size;
- }
-
- memcpy(buf_start, hll_size_array.data(), allocate_len_size);
- return buf;
- } else {
- auto ptr = column.convert_to_full_column_if_const();
- auto& data_column = assert_cast<const ColumnHLL&>(*ptr);
-
- size_t row_num = column.size();
- std::vector<size_t> hll_size_array(row_num + 1);
- hll_size_array[0] = row_num;
-
- auto allocate_len_size = sizeof(size_t) * (row_num + 1);
- char* buf_start = buf;
- buf += allocate_len_size;
-
- for (size_t i = 0; i < row_num; ++i) {
- auto& hll = data_column.get_element(i);
- size_t actual_size =
hll.serialize(reinterpret_cast<uint8_t*>(buf));
- hll_size_array[i + 1] = actual_size;
- buf += actual_size;
- }
-
- memcpy(buf_start, hll_size_array.data(), allocate_len_size);
+ const auto* hll_column = &column;
+ size_t real_need_copy_num = 0;
+ buf = serialize_const_flag_and_row_num(&hll_column, buf,
&real_need_copy_num);
+ // In the code below, if the real_need_copy_num is 0, an empty vector will
be created. A vector with size 0 may return nullptr from data()
+ // https://en.cppreference.com/w/cpp/container/vector/data
+ // `If size() is 0, data() may or may not return a null pointer.`
+ // This would trigger a ubsan error: `null pointer passed as argument 2,
which is declared to never be null`
+ // Other data types don't have this issue because they use Doris internal
pod array that guarantees data won't be nullptr.
+ if (real_need_copy_num == 0) {
return buf;
}
+
+ const auto& data_column = assert_cast<const ColumnHLL&>(*hll_column);
+ std::vector<size_t> hll_size_array(real_need_copy_num);
+ auto allocate_len_size = sizeof(size_t) * real_need_copy_num;
+ char* buf_start = buf;
+ buf += allocate_len_size;
+
+ for (size_t i = 0; i < real_need_copy_num; ++i) {
+ const auto& hll = data_column.get_element(i);
+ size_t actual_size = hll.serialize(reinterpret_cast<uint8_t*>(buf));
+ hll_size_array[i] = actual_size;
+ buf += actual_size;
+ }
+
+ memcpy(buf_start, hll_size_array.data(), allocate_len_size);
+ return buf;
}
// Two part of binary: <size array> | <hll data array>
@@ -93,45 +70,26 @@ char* DataTypeHLL::serialize(const IColumn& column, char*
buf, int be_exec_versi
// second: hll1 | hll2 | ...
const char* DataTypeHLL::deserialize(const char* buf, MutableColumnPtr* column,
int be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- auto* origin_column = column->get();
- size_t real_have_saved_num = 0;
- buf = deserialize_const_flag_and_row_num(buf, column,
&real_have_saved_num);
-
- auto& data_column = assert_cast<ColumnHLL&>(*origin_column);
- auto& data = data_column.get_data();
- data.resize(real_have_saved_num);
- if (real_have_saved_num == 0) {
- // If real_have_saved_num is 0, we don't need to deserialize any
data.
- return buf;
- }
- std::vector<size_t> hll_size_array(real_have_saved_num);
- memcpy(hll_size_array.data(), buf, sizeof(size_t) *
real_have_saved_num);
- buf += sizeof(size_t) * real_have_saved_num;
-
- for (int i = 0; i < real_have_saved_num; ++i) {
- data[i].deserialize(Slice(buf, hll_size_array[i]));
- buf += hll_size_array[i];
- }
+ auto* origin_column = column->get();
+ size_t real_have_saved_num = 0;
+ buf = deserialize_const_flag_and_row_num(buf, column,
&real_have_saved_num);
+
+ auto& data_column = assert_cast<ColumnHLL&>(*origin_column);
+ auto& data = data_column.get_data();
+ data.resize(real_have_saved_num);
+ if (real_have_saved_num == 0) {
+ // If real_have_saved_num is 0, we don't need to deserialize any data.
return buf;
- } else {
- auto& data_column = assert_cast<ColumnHLL&>(*(column->get()));
- auto& data = data_column.get_data();
-
- size_t row_num = *reinterpret_cast<const size_t*>(buf);
- buf += sizeof(size_t);
- std::vector<size_t> hll_size_array(row_num);
- memcpy(hll_size_array.data(), buf, sizeof(size_t) * row_num);
- buf += sizeof(size_t) * row_num;
-
- data.resize(row_num);
- for (int i = 0; i < row_num; ++i) {
- data[i].deserialize(Slice(buf, hll_size_array[i]));
- buf += hll_size_array[i];
- }
+ }
+ std::vector<size_t> hll_size_array(real_have_saved_num);
+ memcpy(hll_size_array.data(), buf, sizeof(size_t) * real_have_saved_num);
+ buf += sizeof(size_t) * real_have_saved_num;
- return buf;
+ for (int i = 0; i < real_have_saved_num; ++i) {
+ data[i].deserialize(Slice(buf, hll_size_array[i]));
+ buf += hll_size_array[i];
}
+ return buf;
}
// binary: const flag| row num | real_saved_num | size array | data array
@@ -139,36 +97,22 @@ const char* DataTypeHLL::deserialize(const char* buf,
MutableColumnPtr* column,
// <data array>: hll1 | hll1 | ...
int64_t DataTypeHLL::get_uncompressed_serialized_bytes(const IColumn& column,
int be_exec_version)
const {
- if (be_exec_version >= USE_CONST_SERDE) {
- auto size = sizeof(bool) + sizeof(size_t) + sizeof(size_t);
- bool is_const_column = is_column_const(column);
- auto real_need_copy_num = is_const_column ? 1 : column.size();
- const IColumn* hll_column = &column;
- if (is_const_column) {
- const auto& const_column = assert_cast<const ColumnConst&>(column);
- hll_column = &(const_column.get_data_column());
- }
- const auto& data_column = assert_cast<const ColumnHLL&>(*hll_column);
- auto allocate_len_size = sizeof(size_t) * real_need_copy_num;
- size_t allocate_content_size = 0;
- for (size_t i = 0; i < real_need_copy_num; ++i) {
- auto& hll = data_column.get_element(i);
- allocate_content_size += hll.max_serialized_size();
- }
- return size + allocate_len_size + allocate_content_size;
- } else {
- auto ptr = column.convert_to_full_column_if_const();
- const auto& data_column = assert_cast<const ColumnHLL&>(*ptr);
-
- auto allocate_len_size = sizeof(size_t) * (column.size() + 1);
- size_t allocate_content_size = 0;
- for (size_t i = 0; i < column.size(); ++i) {
- auto& hll = data_column.get_element(i);
- allocate_content_size += hll.max_serialized_size();
- }
-
- return allocate_len_size + allocate_content_size;
+ auto size = sizeof(bool) + sizeof(size_t) + sizeof(size_t);
+ bool is_const_column = is_column_const(column);
+ auto real_need_copy_num = is_const_column ? 1 : column.size();
+ const IColumn* hll_column = &column;
+ if (is_const_column) {
+ const auto& const_column = assert_cast<const ColumnConst&>(column);
+ hll_column = &(const_column.get_data_column());
+ }
+ const auto& data_column = assert_cast<const ColumnHLL&>(*hll_column);
+ auto allocate_len_size = sizeof(size_t) * real_need_copy_num;
+ size_t allocate_content_size = 0;
+ for (size_t i = 0; i < real_need_copy_num; ++i) {
+ auto& hll = data_column.get_element(i);
+ allocate_content_size += hll.max_serialized_size();
}
+ return size + allocate_len_size + allocate_content_size;
}
MutableColumnPtr DataTypeHLL::create_column() const {
diff --git a/be/src/vec/data_types/data_type_jsonb.cpp
b/be/src/vec/data_types/data_type_jsonb.cpp
index 3f2645cc4c1..b8edef8e7ea 100644
--- a/be/src/vec/data_types/data_type_jsonb.cpp
+++ b/be/src/vec/data_types/data_type_jsonb.cpp
@@ -70,17 +70,17 @@ bool DataTypeJsonb::equals(const IDataType& rhs) const {
}
int64_t DataTypeJsonb::get_uncompressed_serialized_bytes(const IColumn& column,
- int data_version)
const {
- return data_type_string.get_uncompressed_serialized_bytes(column,
data_version);
+ int be_exec_version)
const {
+ return data_type_string.get_uncompressed_serialized_bytes(column,
be_exec_version);
}
-char* DataTypeJsonb::serialize(const IColumn& column, char* buf, int
data_version) const {
- return data_type_string.serialize(column, buf, data_version);
+char* DataTypeJsonb::serialize(const IColumn& column, char* buf, int
be_exec_version) const {
+ return data_type_string.serialize(column, buf, be_exec_version);
}
const char* DataTypeJsonb::deserialize(const char* buf, MutableColumnPtr*
column,
- int data_version) const {
- return data_type_string.deserialize(buf, column, data_version);
+ int be_exec_version) const {
+ return data_type_string.deserialize(buf, column, be_exec_version);
}
FieldWithDataType DataTypeJsonb::get_field_with_data_type(const IColumn&
column,
diff --git a/be/src/vec/data_types/data_type_jsonb.h
b/be/src/vec/data_types/data_type_jsonb.h
index d887879d117..0d38df2cda8 100644
--- a/be/src/vec/data_types/data_type_jsonb.h
+++ b/be/src/vec/data_types/data_type_jsonb.h
@@ -56,10 +56,10 @@ public:
}
int64_t get_uncompressed_serialized_bytes(const IColumn& column,
- int data_version) const override;
- char* serialize(const IColumn& column, char* buf, int data_version) const
override;
+ int be_exec_version) const
override;
+ char* serialize(const IColumn& column, char* buf, int be_exec_version)
const override;
const char* deserialize(const char* buf, MutableColumnPtr* column,
- int data_version) const override;
+ int be_exec_version) const override;
MutableColumnPtr create_column() const override;
Status check_column(const IColumn& column) const override;
diff --git a/be/src/vec/data_types/data_type_map.cpp
b/be/src/vec/data_types/data_type_map.cpp
index 76fdfbc3f71..c77ff6ede86 100644
--- a/be/src/vec/data_types/data_type_map.cpp
+++ b/be/src/vec/data_types/data_type_map.cpp
@@ -97,99 +97,58 @@ bool DataTypeMap::equals(const IDataType& rhs) const {
// key_col: key1 | key1 ...
// val_col: val1 | val2 ...
int64_t DataTypeMap::get_uncompressed_serialized_bytes(const IColumn& column,
- int data_version) const
{
- if (data_version >= USE_CONST_SERDE) {
- auto size = sizeof(bool) + sizeof(size_t) + sizeof(size_t);
- bool is_const_column = is_column_const(column);
- auto real_need_copy_num = is_const_column ? 1 : column.size();
-
- const IColumn* data_column = &column;
- if (is_const_column) {
- const auto& const_column = assert_cast<const ColumnConst&>(column);
- data_column = &(const_column.get_data_column());
- }
- const auto& map_column = assert_cast<const ColumnMap&>(*data_column);
- size = size + sizeof(ColumnArray::Offset64) * real_need_copy_num;
- size = size +
get_key_type()->get_uncompressed_serialized_bytes(map_column.get_keys(),
-
data_version);
- size = size +
get_value_type()->get_uncompressed_serialized_bytes(map_column.get_values(),
-
data_version);
- return size;
- } else {
- auto ptr = column.convert_to_full_column_if_const();
- const auto& data_column = assert_cast<const ColumnMap&>(*ptr.get());
- return sizeof(ColumnArray::Offset64) * (column.size() + 1) +
-
get_key_type()->get_uncompressed_serialized_bytes(data_column.get_keys(),
- data_version)
+
-
get_value_type()->get_uncompressed_serialized_bytes(data_column.get_values(),
-
data_version);
+ int be_exec_version)
const {
+ auto size = sizeof(bool) + sizeof(size_t) + sizeof(size_t);
+ bool is_const_column = is_column_const(column);
+ auto real_need_copy_num = is_const_column ? 1 : column.size();
+
+ const IColumn* data_column = &column;
+ if (is_const_column) {
+ const auto& const_column = assert_cast<const ColumnConst&>(column);
+ data_column = &(const_column.get_data_column());
}
+ const auto& map_column = assert_cast<const ColumnMap&>(*data_column);
+ size = size + sizeof(ColumnArray::Offset64) * real_need_copy_num;
+ size = size +
get_key_type()->get_uncompressed_serialized_bytes(map_column.get_keys(),
+
be_exec_version);
+ size = size +
get_value_type()->get_uncompressed_serialized_bytes(map_column.get_values(),
+
be_exec_version);
+ return size;
}
// serialize to binary
char* DataTypeMap::serialize(const IColumn& column, char* buf, int
be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- const auto* data_column = &column;
- size_t real_need_copy_num = 0;
- buf = serialize_const_flag_and_row_num(&data_column, buf,
&real_need_copy_num);
-
- const auto& map_column = assert_cast<const ColumnMap&>(*data_column);
- // offsets
- memcpy(buf, map_column.get_offsets().data(),
- real_need_copy_num * sizeof(ColumnArray::Offset64));
- buf += real_need_copy_num * sizeof(ColumnArray::Offset64);
- // key value
- buf = get_key_type()->serialize(map_column.get_keys(), buf,
be_exec_version);
- return get_value_type()->serialize(map_column.get_values(), buf,
be_exec_version);
- } else {
- auto ptr = column.convert_to_full_column_if_const();
- const auto& map_column = assert_cast<const ColumnMap&>(*ptr.get());
-
- // row num
- *reinterpret_cast<ColumnArray::Offset64*>(buf) = column.size();
- buf += sizeof(ColumnArray::Offset64);
- // offsets
- memcpy(buf, map_column.get_offsets().data(), column.size() *
sizeof(ColumnArray::Offset64));
- buf += column.size() * sizeof(ColumnArray::Offset64);
- // key value
- buf = get_key_type()->serialize(map_column.get_keys(), buf,
be_exec_version);
- return get_value_type()->serialize(map_column.get_values(), buf,
be_exec_version);
- }
+ const auto* data_column = &column;
+ size_t real_need_copy_num = 0;
+ buf = serialize_const_flag_and_row_num(&data_column, buf,
&real_need_copy_num);
+
+ const auto& map_column = assert_cast<const ColumnMap&>(*data_column);
+ // offsets
+ memcpy(buf, map_column.get_offsets().data(),
+ real_need_copy_num * sizeof(ColumnArray::Offset64));
+ buf += real_need_copy_num * sizeof(ColumnArray::Offset64);
+ // key value
+ buf = get_key_type()->serialize(map_column.get_keys(), buf,
be_exec_version);
+ return get_value_type()->serialize(map_column.get_values(), buf,
be_exec_version);
}
+
const char* DataTypeMap::deserialize(const char* buf, MutableColumnPtr* column,
int be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- auto* origin_column = column->get();
- size_t real_have_saved_num = 0;
- buf = deserialize_const_flag_and_row_num(buf, column,
&real_have_saved_num);
-
- auto* map_column = assert_cast<ColumnMap*>(origin_column);
- auto& map_offsets = map_column->get_offsets();
- // offsets
- map_offsets.resize(real_have_saved_num);
- memcpy(map_offsets.data(), buf, sizeof(ColumnArray::Offset64) *
real_have_saved_num);
- buf += sizeof(ColumnArray::Offset64) * real_have_saved_num;
- // key value
- auto nested_keys_column = map_column->get_keys_ptr()->assume_mutable();
- auto nested_values_column =
map_column->get_values_ptr()->assume_mutable();
- buf = get_key_type()->deserialize(buf, &nested_keys_column,
be_exec_version);
- buf = get_value_type()->deserialize(buf, &nested_values_column,
be_exec_version);
- return buf;
- } else {
- auto* map_column = assert_cast<ColumnMap*>(column->get());
- auto& map_offsets = map_column->get_offsets();
- // row num
- ColumnArray::Offset64 row_num = *reinterpret_cast<const
ColumnArray::Offset64*>(buf);
- buf += sizeof(ColumnArray::Offset64);
- // offsets
- map_offsets.resize(row_num);
- memcpy(map_offsets.data(), buf, sizeof(ColumnArray::Offset64) *
row_num);
- buf += sizeof(ColumnArray::Offset64) * row_num;
- // key value
- auto nested_keys_column = map_column->get_keys_ptr()->assume_mutable();
- auto nested_values_column =
map_column->get_values_ptr()->assume_mutable();
- buf = get_key_type()->deserialize(buf, &nested_keys_column,
be_exec_version);
- return get_value_type()->deserialize(buf, &nested_values_column,
be_exec_version);
- }
+ auto* origin_column = column->get();
+ size_t real_have_saved_num = 0;
+ buf = deserialize_const_flag_and_row_num(buf, column,
&real_have_saved_num);
+
+ auto* map_column = assert_cast<ColumnMap*>(origin_column);
+ auto& map_offsets = map_column->get_offsets();
+ // offsets
+ map_offsets.resize(real_have_saved_num);
+ memcpy(map_offsets.data(), buf, sizeof(ColumnArray::Offset64) *
real_have_saved_num);
+ buf += sizeof(ColumnArray::Offset64) * real_have_saved_num;
+ // key value
+ auto nested_keys_column = map_column->get_keys_ptr()->assume_mutable();
+ auto nested_values_column = map_column->get_values_ptr()->assume_mutable();
+ buf = get_key_type()->deserialize(buf, &nested_keys_column,
be_exec_version);
+ buf = get_value_type()->deserialize(buf, &nested_values_column,
be_exec_version);
+ return buf;
}
} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/src/vec/data_types/data_type_nullable.cpp
b/be/src/vec/data_types/data_type_nullable.cpp
index 7a9235ffe72..f989405c14a 100644
--- a/be/src/vec/data_types/data_type_nullable.cpp
+++ b/be/src/vec/data_types/data_type_nullable.cpp
@@ -54,143 +54,79 @@ DataTypeNullable::DataTypeNullable(const DataTypePtr&
nested_data_type_)
// <values array>: value1 | value2 | ...>
int64_t DataTypeNullable::get_uncompressed_serialized_bytes(const IColumn&
column,
int
be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- auto size = sizeof(bool) + sizeof(size_t) + sizeof(size_t);
- bool is_const_column = is_column_const(column);
- auto real_need_copy_num = is_const_column ? 1 : column.size();
- const IColumn* data_column = &column;
- if (is_const_column) {
- const auto& const_column = assert_cast<const ColumnConst&>(column);
- data_column = &(const_column.get_data_column());
- }
+ auto size = sizeof(bool) + sizeof(size_t) + sizeof(size_t);
+ bool is_const_column = is_column_const(column);
+ auto real_need_copy_num = is_const_column ? 1 : column.size();
+ const IColumn* data_column = &column;
+ if (is_const_column) {
+ const auto& const_column = assert_cast<const ColumnConst&>(column);
+ data_column = &(const_column.get_data_column());
+ }
- const auto mem_size = real_need_copy_num * sizeof(bool);
- if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
- size += mem_size;
- } else {
- // Throw exception if mem_size is large than UINT32_MAX
- size = size + sizeof(size_t) +
- std::max(mem_size, streamvbyte_max_compressedbytes(
-
cast_set<UInt32>(upper_int32(mem_size))));
- }
- const auto& col = assert_cast<const ColumnNullable&>(*data_column);
- size = size +
nested_data_type->get_uncompressed_serialized_bytes(col.get_nested_column(),
-
be_exec_version);
- return size;
+ const auto mem_size = real_need_copy_num * sizeof(bool);
+ if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
+ size += mem_size;
} else {
- size_t ret = 0;
- if (size_t size = sizeof(bool) * column.size(); size <=
SERIALIZED_MEM_SIZE_LIMIT) {
- ret += size + sizeof(uint32_t);
- } else {
- // Throw exception if mem_size is large than UINT32_MAX
- ret += (sizeof(uint32_t) + sizeof(size_t) +
- std::max(size,
-
streamvbyte_max_compressedbytes(cast_set<UInt32>(upper_int32(size)))));
- }
- ret += nested_data_type->get_uncompressed_serialized_bytes(
- assert_cast<const
ColumnNullable&>(*column.convert_to_full_column_if_const())
- .get_nested_column(),
- be_exec_version);
- return ret;
+ // Throw exception if mem_size is large than UINT32_MAX
+ size = size + sizeof(size_t) +
+ std::max(mem_size,
+
streamvbyte_max_compressedbytes(cast_set<UInt32>(upper_int32(mem_size))));
}
+ const auto& col = assert_cast<const ColumnNullable&>(*data_column);
+ size = size +
nested_data_type->get_uncompressed_serialized_bytes(col.get_nested_column(),
+
be_exec_version);
+ return size;
}
char* DataTypeNullable::serialize(const IColumn& column, char* buf, int
be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- const auto* data_column = &column;
- size_t real_need_copy_num = 0;
- buf = serialize_const_flag_and_row_num(&data_column, buf,
&real_need_copy_num);
-
- // mem_size = real_row_num * sizeof(T)
- const auto mem_size = real_need_copy_num * sizeof(bool);
- const auto& col = assert_cast<const ColumnNullable&>(*data_column);
- // null flags
- if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
- memcpy(buf, col.get_null_map_data().data(), mem_size);
- buf += mem_size;
- } else {
- // Throw exception if mem_size is large than UINT32_MAX
- auto encode_size = streamvbyte_encode(
- reinterpret_cast<const
uint32_t*>(col.get_null_map_data().data()),
- cast_set<UInt32>(upper_int32(mem_size)), (uint8_t*)(buf +
sizeof(size_t)));
- unaligned_store<size_t>(buf, encode_size);
- buf += (sizeof(size_t) + encode_size);
- }
- // data values
- return nested_data_type->serialize(col.get_nested_column(), buf,
be_exec_version);
+ const auto* data_column = &column;
+ size_t real_need_copy_num = 0;
+ buf = serialize_const_flag_and_row_num(&data_column, buf,
&real_need_copy_num);
+
+ // mem_size = real_row_num * sizeof(T)
+ const auto mem_size = real_need_copy_num * sizeof(bool);
+ const auto& col = assert_cast<const ColumnNullable&>(*data_column);
+ // null flags
+ if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
+ memcpy(buf, col.get_null_map_data().data(), mem_size);
+ buf += mem_size;
} else {
- auto ptr = column.convert_to_full_column_if_const();
- const auto& col = assert_cast<const ColumnNullable&>(*ptr.get());
-
- // row num
- auto mem_size = col.size() * sizeof(bool);
- unaligned_store<uint32_t>(buf, static_cast<UInt32>(mem_size));
- buf += sizeof(uint32_t);
- // null flags
- if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
- memcpy(buf, col.get_null_map_data().data(), mem_size);
- buf += mem_size;
- } else {
- // Throw exception if mem_size is large than UINT32_MAX
- auto encode_size = streamvbyte_encode(
- reinterpret_cast<const
uint32_t*>(col.get_null_map_data().data()),
- cast_set<UInt32>(upper_int32(mem_size)), (uint8_t*)(buf +
sizeof(size_t)));
- *reinterpret_cast<size_t*>(buf) = encode_size;
- buf += (sizeof(size_t) + encode_size);
- }
- // data values
- return nested_data_type->serialize(col.get_nested_column(), buf,
be_exec_version);
+ // Throw exception if mem_size is large than UINT32_MAX
+ auto encode_size = streamvbyte_encode(
+ reinterpret_cast<const
uint32_t*>(col.get_null_map_data().data()),
+ cast_set<UInt32>(upper_int32(mem_size)), (uint8_t*)(buf +
sizeof(size_t)));
+ unaligned_store<size_t>(buf, encode_size);
+ buf += (sizeof(size_t) + encode_size);
}
+ // data values
+ return nested_data_type->serialize(col.get_nested_column(), buf,
be_exec_version);
}
const char* DataTypeNullable::deserialize(const char* buf, MutableColumnPtr*
column,
int be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- auto* origin_column = column->get();
- size_t real_have_saved_num = 0;
- buf = deserialize_const_flag_and_row_num(buf, column,
&real_have_saved_num);
-
- auto* col = assert_cast<ColumnNullable*>(origin_column);
- // null flags
- auto mem_size = real_have_saved_num * sizeof(bool);
- col->get_null_map_data().resize(real_have_saved_num);
- if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
- memcpy(col->get_null_map_data().data(), buf, mem_size);
- buf += mem_size;
- } else {
- size_t encode_size = unaligned_load<size_t>(buf);
- buf += sizeof(size_t);
- // Throw exception if mem_size is large than UINT32_MAX
- streamvbyte_decode((const uint8_t*)buf,
(uint32_t*)(col->get_null_map_data().data()),
- cast_set<UInt32>(upper_int32(mem_size)));
- buf += encode_size;
- }
- // column data values
- auto nested = col->get_nested_column_ptr();
- buf = nested_data_type->deserialize(buf, &nested, be_exec_version);
- return buf;
+ auto* origin_column = column->get();
+ size_t real_have_saved_num = 0;
+ buf = deserialize_const_flag_and_row_num(buf, column,
&real_have_saved_num);
+
+ auto* col = assert_cast<ColumnNullable*>(origin_column);
+ // null flags
+ auto mem_size = real_have_saved_num * sizeof(bool);
+ col->get_null_map_data().resize(real_have_saved_num);
+ if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
+ memcpy(col->get_null_map_data().data(), buf, mem_size);
+ buf += mem_size;
} else {
- auto* col = assert_cast<ColumnNullable*>(column->get());
- // row num
- uint32_t mem_size = unaligned_load<uint32_t>(buf);
- buf += sizeof(uint32_t);
- // null flags
- col->get_null_map_data().resize(mem_size / sizeof(bool));
- if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
- memcpy(col->get_null_map_data().data(), buf, mem_size);
- buf += mem_size;
- } else {
- size_t encode_size = *reinterpret_cast<const size_t*>(buf);
- buf += sizeof(size_t);
- // Throw exception if mem_size is large than UINT32_MAX
- streamvbyte_decode((const uint8_t*)buf,
(uint32_t*)(col->get_null_map_data().data()),
- cast_set<UInt32>(upper_int32(mem_size)));
- buf += encode_size;
- }
- // data values
- auto nested = col->get_nested_column_ptr();
- return nested_data_type->deserialize(buf, &nested, be_exec_version);
+ size_t encode_size = unaligned_load<size_t>(buf);
+ buf += sizeof(size_t);
+ // Throw exception if mem_size is large than UINT32_MAX
+ streamvbyte_decode((const uint8_t*)buf,
(uint32_t*)(col->get_null_map_data().data()),
+ cast_set<UInt32>(upper_int32(mem_size)));
+ buf += encode_size;
}
+ // column data values
+ auto nested = col->get_nested_column_ptr();
+ buf = nested_data_type->deserialize(buf, &nested, be_exec_version);
+ return buf;
}
void DataTypeNullable::to_pb_column_meta(PColumnMeta* col_meta) const {
diff --git a/be/src/vec/data_types/data_type_number_base.cpp
b/be/src/vec/data_types/data_type_number_base.cpp
index 5426b59e632..a6e0d5cf03a 100644
--- a/be/src/vec/data_types/data_type_number_base.cpp
+++ b/be/src/vec/data_types/data_type_number_base.cpp
@@ -102,77 +102,38 @@ Field DataTypeNumberBase<T>::get_field(const TExprNode&
node) const {
template <PrimitiveType T>
int64_t DataTypeNumberBase<T>::get_uncompressed_serialized_bytes(const
IColumn& column,
int
be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- auto size = sizeof(bool) + sizeof(size_t) + sizeof(size_t);
- auto real_need_copy_num = is_column_const(column) ? 1 : column.size();
- auto mem_size =
- sizeof(typename PrimitiveTypeTraits<T>::ColumnItemType) *
real_need_copy_num;
- if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
- return size + mem_size;
- } else {
- // Throw exception if mem_size is large than UINT32_MAX
- return size + sizeof(size_t) +
- std::max(mem_size, streamvbyte_max_compressedbytes(
-
cast_set<UInt32>(upper_int32(mem_size))));
- }
+ auto size = sizeof(bool) + sizeof(size_t) + sizeof(size_t);
+ auto real_need_copy_num = is_column_const(column) ? 1 : column.size();
+ auto mem_size = sizeof(typename PrimitiveTypeTraits<T>::ColumnItemType) *
real_need_copy_num;
+ if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
+ return size + mem_size;
} else {
- auto size = sizeof(typename PrimitiveTypeTraits<T>::ColumnItemType) *
column.size();
- if (size <= SERIALIZED_MEM_SIZE_LIMIT) {
- return sizeof(uint32_t) + size;
- } else {
- // Throw exception if mem_size is large than UINT32_MAX
- return sizeof(uint32_t) + sizeof(size_t) +
- std::max(size,
-
streamvbyte_max_compressedbytes(cast_set<UInt32>(upper_int32(size))));
- }
+ // Throw exception if mem_size is large than UINT32_MAX
+ return size + sizeof(size_t) +
+ std::max(mem_size,
+
streamvbyte_max_compressedbytes(cast_set<UInt32>(upper_int32(mem_size))));
}
}
template <PrimitiveType T>
char* DataTypeNumberBase<T>::serialize(const IColumn& column, char* buf,
int be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- const auto* data_column = &column;
- size_t real_need_copy_num = 0;
- buf = serialize_const_flag_and_row_num(&data_column, buf,
&real_need_copy_num);
-
- // mem_size = real_need_copy_num * sizeof(T)
- auto mem_size =
- real_need_copy_num * sizeof(typename
PrimitiveTypeTraits<T>::ColumnItemType);
- const auto* origin_data =
- assert_cast<const typename
PrimitiveTypeTraits<T>::ColumnType&>(*data_column)
- .get_data()
- .data();
-
- // column data
- if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
- memcpy(buf, origin_data, mem_size);
- return buf + mem_size;
- } else {
- // Throw exception if mem_size is large than UINT32_MAX
- auto encode_size = streamvbyte_encode(reinterpret_cast<const
uint32_t*>(origin_data),
-
cast_set<UInt32>(upper_int32(mem_size)),
- (uint8_t*)(buf +
sizeof(size_t)));
- unaligned_store<size_t>(buf, encode_size);
- buf += sizeof(size_t);
- return buf + encode_size;
- }
+ const auto* data_column = &column;
+ size_t real_need_copy_num = 0;
+ buf = serialize_const_flag_and_row_num(&data_column, buf,
&real_need_copy_num);
+
+ // mem_size = real_need_copy_num * sizeof(T)
+ auto mem_size = real_need_copy_num * sizeof(typename
PrimitiveTypeTraits<T>::ColumnItemType);
+ const auto* origin_data =
+ assert_cast<const typename
PrimitiveTypeTraits<T>::ColumnType&>(*data_column)
+ .get_data()
+ .data();
+
+ // column data
+ if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
+ memcpy(buf, origin_data, mem_size);
+ return buf + mem_size;
} else {
- // row num
- const auto mem_size =
- column.size() * sizeof(typename
PrimitiveTypeTraits<T>::ColumnItemType);
- *reinterpret_cast<uint32_t*>(buf) = static_cast<UInt32>(mem_size);
- buf += sizeof(uint32_t);
- // column data
- auto ptr = column.convert_to_full_column_if_const();
- const auto* origin_data =
- assert_cast<const typename
PrimitiveTypeTraits<T>::ColumnType&>(*ptr.get())
- .get_data()
- .data();
- if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
- memcpy(buf, origin_data, mem_size);
- return buf + mem_size;
- }
// Throw exception if mem_size is large than UINT32_MAX
auto encode_size = streamvbyte_encode(reinterpret_cast<const
uint32_t*>(origin_data),
cast_set<UInt32>(upper_int32(mem_size)),
@@ -186,47 +147,26 @@ char* DataTypeNumberBase<T>::serialize(const IColumn&
column, char* buf,
template <PrimitiveType T>
const char* DataTypeNumberBase<T>::deserialize(const char* buf,
MutableColumnPtr* column,
int be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- auto* origin_column = column->get();
- size_t real_have_saved_num = 0;
- buf = deserialize_const_flag_and_row_num(buf, column,
&real_have_saved_num);
-
- // column data
- auto mem_size =
- real_have_saved_num * sizeof(typename
PrimitiveTypeTraits<T>::ColumnItemType);
- auto& container = assert_cast<typename
PrimitiveTypeTraits<T>::ColumnType*>(origin_column)
- ->get_data();
- container.resize(real_have_saved_num);
- if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
- memcpy(container.data(), buf, mem_size);
- buf = buf + mem_size;
- } else {
- size_t encode_size = unaligned_load<size_t>(buf);
- buf += sizeof(size_t);
- streamvbyte_decode((const uint8_t*)buf,
(uint32_t*)(container.data()),
- cast_set<UInt32>(upper_int32(mem_size)));
- buf = buf + encode_size;
- }
- return buf;
+ auto* origin_column = column->get();
+ size_t real_have_saved_num = 0;
+ buf = deserialize_const_flag_and_row_num(buf, column,
&real_have_saved_num);
+
+ // column data
+ auto mem_size = real_have_saved_num * sizeof(typename
PrimitiveTypeTraits<T>::ColumnItemType);
+ auto& container =
+ assert_cast<typename
PrimitiveTypeTraits<T>::ColumnType*>(origin_column)->get_data();
+ container.resize(real_have_saved_num);
+ if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
+ memcpy(container.data(), buf, mem_size);
+ buf = buf + mem_size;
} else {
- // row num
- uint32_t mem_size = *reinterpret_cast<const uint32_t*>(buf);
- buf += sizeof(uint32_t);
- // column data
- auto& container = assert_cast<typename
PrimitiveTypeTraits<T>::ColumnType*>(column->get())
- ->get_data();
- container.resize(mem_size / sizeof(typename
PrimitiveTypeTraits<T>::ColumnItemType));
- if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
- memcpy(container.data(), buf, mem_size);
- return buf + mem_size;
- }
-
size_t encode_size = unaligned_load<size_t>(buf);
buf += sizeof(size_t);
streamvbyte_decode((const uint8_t*)buf, (uint32_t*)(container.data()),
cast_set<UInt32>(upper_int32(mem_size)));
- return buf + encode_size;
+ buf = buf + encode_size;
}
+ return buf;
}
template <PrimitiveType T>
diff --git a/be/src/vec/data_types/data_type_quantilestate.cpp
b/be/src/vec/data_types/data_type_quantilestate.cpp
index 9596dae957e..473bee75f10 100644
--- a/be/src/vec/data_types/data_type_quantilestate.cpp
+++ b/be/src/vec/data_types/data_type_quantilestate.cpp
@@ -33,125 +33,68 @@ namespace doris::vectorized {
// <quantilestate array>: quantilestate1 | quantilestate2 | ...
int64_t DataTypeQuantileState::get_uncompressed_serialized_bytes(const
IColumn& column,
int
be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- auto size = sizeof(bool) + sizeof(size_t) + sizeof(size_t);
- bool is_const_column = is_column_const(column);
- auto real_need_copy_num = is_const_column ? 1 : column.size();
- const IColumn* quantile_column = &column;
- if (is_const_column) {
- const auto& const_column = assert_cast<const ColumnConst&>(column);
- quantile_column = &(const_column.get_data_column());
- }
- const auto& data_column = assert_cast<const
ColumnQuantileState&>(*quantile_column);
- auto allocate_len_size = sizeof(size_t) * real_need_copy_num;
- size_t allocate_content_size = 0;
- for (size_t i = 0; i < real_need_copy_num; ++i) {
- auto& quantile_state = data_column.get_element(i);
- allocate_content_size += quantile_state.get_serialized_size();
- }
- return size + allocate_len_size + allocate_content_size;
- } else {
- auto ptr = column.convert_to_full_column_if_const();
- const auto& data_column = assert_cast<const
ColumnQuantileState&>(*ptr);
-
- auto allocate_len_size = sizeof(size_t) * (column.size() + 1);
- size_t allocate_content_size = 0;
- for (size_t i = 0; i < column.size(); ++i) {
- auto& quantile_state = data_column.get_element(i);
- allocate_content_size += quantile_state.get_serialized_size();
- }
-
- return allocate_len_size + allocate_content_size;
+ auto size = sizeof(bool) + sizeof(size_t) + sizeof(size_t);
+ bool is_const_column = is_column_const(column);
+ auto real_need_copy_num = is_const_column ? 1 : column.size();
+ const IColumn* quantile_column = &column;
+ if (is_const_column) {
+ const auto& const_column = assert_cast<const ColumnConst&>(column);
+ quantile_column = &(const_column.get_data_column());
}
+ const auto& data_column = assert_cast<const
ColumnQuantileState&>(*quantile_column);
+ auto allocate_len_size = sizeof(size_t) * real_need_copy_num;
+ size_t allocate_content_size = 0;
+ for (size_t i = 0; i < real_need_copy_num; ++i) {
+ auto& quantile_state = data_column.get_element(i);
+ allocate_content_size += quantile_state.get_serialized_size();
+ }
+ return size + allocate_len_size + allocate_content_size;
}
char* DataTypeQuantileState::serialize(const IColumn& column, char* buf,
int be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- const auto* quantile_column = &column;
- size_t real_need_copy_num = 0;
- buf = serialize_const_flag_and_row_num(&quantile_column, buf,
&real_need_copy_num);
-
- const auto& data_column = assert_cast<const
ColumnQuantileState&>(*quantile_column);
- // serialize the quantile_state size array, row num saves at index 0
- auto* meta_ptr = (size_t*)buf;
- for (size_t i = 0; i < real_need_copy_num; ++i) {
- auto& quantile_state = data_column.get_element(i);
- unaligned_store<size_t>(&meta_ptr[i],
quantile_state.get_serialized_size());
- }
-
- // serialize each quantile_state
- char* data_ptr = buf + sizeof(size_t) * real_need_copy_num;
- for (size_t i = 0; i < real_need_copy_num; ++i) {
- auto& quantile_state = data_column.get_element(i);
- quantile_state.serialize((uint8_t*)data_ptr);
- data_ptr += unaligned_load<size_t>(&meta_ptr[i]);
- }
- return data_ptr;
- } else {
- auto ptr = column.convert_to_full_column_if_const();
- const auto& data_column = assert_cast<const
ColumnQuantileState&>(*ptr);
-
- // serialize the quantile_state size array, row num saves at index 0
- size_t* meta_ptr = (size_t*)buf;
- meta_ptr[0] = column.size();
- for (size_t i = 0; i < meta_ptr[0]; ++i) {
- auto& quantile_state = data_column.get_element(i);
- unaligned_store<size_t>(&meta_ptr[i + 1],
quantile_state.get_serialized_size());
- }
-
- // serialize each quantile_state
- char* data_ptr = buf + sizeof(size_t) * (meta_ptr[0] + 1);
- for (size_t i = 0; i < meta_ptr[0]; ++i) {
- auto& quantile_state = data_column.get_element(i);
- quantile_state.serialize((uint8_t*)data_ptr);
- data_ptr += unaligned_load<size_t>(&meta_ptr[i + 1]);
- }
+ const auto* quantile_column = &column;
+ size_t real_need_copy_num = 0;
+ buf = serialize_const_flag_and_row_num(&quantile_column, buf,
&real_need_copy_num);
+
+ const auto& data_column = assert_cast<const
ColumnQuantileState&>(*quantile_column);
+ // serialize the quantile_state size array, row num saves at index 0
+ auto* meta_ptr = (size_t*)buf;
+ for (size_t i = 0; i < real_need_copy_num; ++i) {
+ auto& quantile_state = data_column.get_element(i);
+ unaligned_store<size_t>(&meta_ptr[i],
quantile_state.get_serialized_size());
+ }
- return data_ptr;
+ // serialize each quantile_state
+ char* data_ptr = buf + sizeof(size_t) * real_need_copy_num;
+ for (size_t i = 0; i < real_need_copy_num; ++i) {
+ auto& quantile_state = data_column.get_element(i);
+ quantile_state.serialize((uint8_t*)data_ptr);
+ data_ptr += unaligned_load<size_t>(&meta_ptr[i]);
}
+ return data_ptr;
}
const char* DataTypeQuantileState::deserialize(const char* buf,
MutableColumnPtr* column,
int be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- auto* origin_column = column->get();
- size_t real_have_saved_num = 0;
- buf = deserialize_const_flag_and_row_num(buf, column,
&real_have_saved_num);
-
- auto& data_column = assert_cast<ColumnQuantileState&>(*origin_column);
- auto& data = data_column.get_data();
-
- // deserialize each quantile_state
- data.resize(real_have_saved_num);
- const auto* meta_ptr = reinterpret_cast<const size_t*>(buf);
- const char* data_ptr = buf + sizeof(size_t) * real_have_saved_num;
- for (size_t i = 0; i < real_have_saved_num; ++i) {
- const size_t size = unaligned_load<size_t>(&meta_ptr[i]);
- Slice slice(data_ptr, size);
- data[i].deserialize(slice);
- data_ptr += size;
- }
- return data_ptr;
- } else {
- auto& data_column =
assert_cast<ColumnQuantileState&>(*(column->get()));
- auto& data = data_column.get_data();
-
- // deserialize the quantile_state size array
- const size_t* meta_ptr = reinterpret_cast<const size_t*>(buf);
-
- // deserialize each quantile_state
- data.resize(meta_ptr[0]);
- const char* data_ptr = buf + sizeof(size_t) * (meta_ptr[0] + 1);
- for (size_t i = 0; i < meta_ptr[0]; ++i) {
- const size_t size = unaligned_load<size_t>(&meta_ptr[i + 1]);
- Slice slice(data_ptr, size);
- data[i].deserialize(slice);
- data_ptr += size;
- }
-
- return data_ptr;
+ auto* origin_column = column->get();
+ size_t real_have_saved_num = 0;
+ buf = deserialize_const_flag_and_row_num(buf, column,
&real_have_saved_num);
+
+ auto& data_column = assert_cast<ColumnQuantileState&>(*origin_column);
+ auto& data = data_column.get_data();
+
+ // deserialize each quantile_state
+ data.resize(real_have_saved_num);
+ const auto* meta_ptr = reinterpret_cast<const size_t*>(buf);
+ const char* data_ptr = buf + sizeof(size_t) * real_have_saved_num;
+ for (size_t i = 0; i < real_have_saved_num; ++i) {
+ const size_t size = unaligned_load<size_t>(&meta_ptr[i]);
+ Slice slice(data_ptr, size);
+ data[i].deserialize(slice);
+ data_ptr += size;
}
+ return data_ptr;
}
MutableColumnPtr DataTypeQuantileState::create_column() const {
diff --git a/be/src/vec/data_types/data_type_string.cpp
b/be/src/vec/data_types/data_type_string.cpp
index d114bd37832..3cffb9a9f5b 100644
--- a/be/src/vec/data_types/data_type_string.cpp
+++ b/be/src/vec/data_types/data_type_string.cpp
@@ -68,220 +68,120 @@ bool DataTypeString::equals(const IDataType& rhs) const {
// chars : {value_length | <value1> | <value2 ...} or {value_length |
encode_size | <value1> | <value2 ...}
int64_t DataTypeString::get_uncompressed_serialized_bytes(const IColumn&
column,
int be_exec_version)
const {
- if (be_exec_version >= USE_CONST_SERDE) {
- int64_t size = sizeof(bool) + sizeof(size_t) + sizeof(size_t);
- bool is_const_column = is_column_const(column);
- const IColumn* string_column = &column;
- if (is_const_column) {
- const auto& const_column = assert_cast<const ColumnConst&>(column);
- string_column = &(const_column.get_data_column());
- }
- const auto& data_column = assert_cast<const
ColumnString&>(*string_column);
+ int64_t size = sizeof(bool) + sizeof(size_t) + sizeof(size_t);
+ bool is_const_column = is_column_const(column);
+ const IColumn* string_column = &column;
+ if (is_const_column) {
+ const auto& const_column = assert_cast<const ColumnConst&>(column);
+ string_column = &(const_column.get_data_column());
+ }
+ const auto& data_column = assert_cast<const ColumnString&>(*string_column);
- auto real_need_copy_num = is_const_column ? 1 : data_column.size();
- auto offsets_size = real_need_copy_num * sizeof(IColumn::Offset);
- if (offsets_size <= SERIALIZED_MEM_SIZE_LIMIT) {
- size += offsets_size;
- } else {
- // Throw exception if offsets_size is large than UINT32_MAX
- size += sizeof(size_t) +
- std::max(offsets_size, streamvbyte_max_compressedbytes(
-
cast_set<UInt32>(upper_int32(offsets_size))));
- }
- size += sizeof(size_t);
- if (size_t bytes = data_column.get_chars().size(); bytes <=
SERIALIZED_MEM_SIZE_LIMIT) {
- size += bytes;
- } else {
- if (bytes > LZ4_MAX_INPUT_SIZE) {
- throw Exception(ErrorCode::BUFFER_OVERFLOW,
- "LZ4_compressBound meet invalid input size,
input_size={}, "
- "LZ4_MAX_INPUT_SIZE={}",
- bytes, LZ4_MAX_INPUT_SIZE);
- }
- size += sizeof(size_t) +
- std::max(bytes,
(size_t)LZ4_compressBound(cast_set<UInt32>(bytes)));
- }
- return size;
+ auto real_need_copy_num = is_const_column ? 1 : data_column.size();
+ auto offsets_size = real_need_copy_num * sizeof(IColumn::Offset);
+ if (offsets_size <= SERIALIZED_MEM_SIZE_LIMIT) {
+ size += offsets_size;
} else {
- auto ptr = column.convert_to_full_column_if_const();
- const auto& data_column = assert_cast<const ColumnString&>(*ptr.get());
- int64_t size = sizeof(uint32_t) + sizeof(uint64_t);
- if (auto offsets_size = data_column.size() * sizeof(IColumn::Offset);
- offsets_size <= SERIALIZED_MEM_SIZE_LIMIT) {
- size += offsets_size;
- } else {
- // Throw exception if offsets_size is large than UINT32_MAX
- size += sizeof(size_t) +
- std::max(offsets_size, streamvbyte_max_compressedbytes(
-
cast_set<UInt32>(upper_int32(offsets_size))));
- }
-
- if (auto bytes = data_column.get_chars().size(); bytes <=
SERIALIZED_MEM_SIZE_LIMIT) {
- size += bytes;
- } else {
- // Throw exception if bytes is large than UINT32_MAX
- size += sizeof(size_t) +
- std::max(bytes,
(size_t)LZ4_compressBound(cast_set<UInt32>(bytes)));
+ // Throw exception if offsets_size is large than UINT32_MAX
+ size += sizeof(size_t) +
+ std::max(offsets_size, streamvbyte_max_compressedbytes(
+
cast_set<UInt32>(upper_int32(offsets_size))));
+ }
+ size += sizeof(size_t);
+ if (size_t bytes = data_column.get_chars().size(); bytes <=
SERIALIZED_MEM_SIZE_LIMIT) {
+ size += bytes;
+ } else {
+ if (bytes > LZ4_MAX_INPUT_SIZE) {
+ throw Exception(ErrorCode::BUFFER_OVERFLOW,
+ "LZ4_compressBound meet invalid input size,
input_size={}, "
+ "LZ4_MAX_INPUT_SIZE={}",
+ bytes, LZ4_MAX_INPUT_SIZE);
}
- return size;
+ size += sizeof(size_t) +
+ std::max(bytes,
(size_t)LZ4_compressBound(cast_set<UInt32>(bytes)));
}
+ return size;
}
char* DataTypeString::serialize(const IColumn& column, char* buf, int
be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- const auto* data_column = &column;
- size_t real_need_copy_num = 0;
- buf = serialize_const_flag_and_row_num(&data_column, buf,
&real_need_copy_num);
-
- // mem_size = real_row_num * sizeof(IColumn::Offset)
- size_t mem_size = real_need_copy_num * sizeof(IColumn::Offset);
- const auto& string_column = assert_cast<const
ColumnString&>(*data_column);
- // offsets
- if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
- memcpy(buf, string_column.get_offsets().data(), mem_size);
- buf += mem_size;
- } else {
- // Throw exception if mem_size is large than UINT32_MAX
- auto encode_size = streamvbyte_encode(
- reinterpret_cast<const
uint32_t*>(string_column.get_offsets().data()),
- cast_set<uint32_t>(upper_int32(mem_size)), (uint8_t*)(buf
+ sizeof(size_t)));
- unaligned_store<size_t>(buf, encode_size);
- buf += (sizeof(size_t) + encode_size);
- }
-
- // values
- auto value_len = string_column.get_chars().size();
- unaligned_store<size_t>(buf, value_len);
- buf += sizeof(size_t);
- if (value_len <= SERIALIZED_MEM_SIZE_LIMIT) {
- memcpy(buf, string_column.get_chars().data(), value_len);
- buf += value_len;
- } else {
- auto encode_size =
LZ4_compress_fast(string_column.get_chars().raw_data(),
- (buf + sizeof(size_t)),
cast_set<Int32>(value_len),
-
LZ4_compressBound(cast_set<Int32>(value_len)), 1);
- unaligned_store<size_t>(buf, encode_size);
- buf += (sizeof(size_t) + encode_size);
- }
- return buf;
+ const auto* data_column = &column;
+ size_t real_need_copy_num = 0;
+ buf = serialize_const_flag_and_row_num(&data_column, buf,
&real_need_copy_num);
+
+ // mem_size = real_row_num * sizeof(IColumn::Offset)
+ size_t mem_size = real_need_copy_num * sizeof(IColumn::Offset);
+ const auto& string_column = assert_cast<const ColumnString&>(*data_column);
+ // offsets
+ if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
+ memcpy(buf, string_column.get_offsets().data(), mem_size);
+ buf += mem_size;
} else {
- auto ptr = column.convert_to_full_column_if_const();
- const auto& data_column = assert_cast<const ColumnString&>(*ptr.get());
-
- // row num
- size_t mem_size = data_column.size() * sizeof(IColumn::Offset);
- unaligned_store<uint32_t>(buf, static_cast<UInt32>(mem_size));
- buf += sizeof(uint32_t);
- // offsets
- if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
- memcpy(buf, data_column.get_offsets().data(), mem_size);
- buf += mem_size;
- } else {
- // Throw exception if mem_size is large than UINT32_MAX
- auto encode_size = streamvbyte_encode(
- reinterpret_cast<const
uint32_t*>(data_column.get_offsets().data()),
- cast_set<UInt32>(upper_int32(mem_size)), (uint8_t*)(buf +
sizeof(size_t)));
- unaligned_store<size_t>(buf, encode_size);
- buf += (sizeof(size_t) + encode_size);
- }
+ // Throw exception if mem_size is large than UINT32_MAX
+ auto encode_size = streamvbyte_encode(
+ reinterpret_cast<const
uint32_t*>(string_column.get_offsets().data()),
+ cast_set<uint32_t>(upper_int32(mem_size)), (uint8_t*)(buf +
sizeof(size_t)));
+ unaligned_store<size_t>(buf, encode_size);
+ buf += (sizeof(size_t) + encode_size);
+ }
- // values
- uint64_t value_len = data_column.get_chars().size();
- unaligned_store<uint64_t>(buf, value_len);
- buf += sizeof(uint64_t);
- if (value_len <= SERIALIZED_MEM_SIZE_LIMIT) {
- memcpy(buf, data_column.get_chars().data(), value_len);
- buf += value_len;
- return buf;
- }
- auto encode_size =
LZ4_compress_fast(data_column.get_chars().raw_data(),
+ // values
+ auto value_len = string_column.get_chars().size();
+ unaligned_store<size_t>(buf, value_len);
+ buf += sizeof(size_t);
+ if (value_len <= SERIALIZED_MEM_SIZE_LIMIT) {
+ memcpy(buf, string_column.get_chars().data(), value_len);
+ buf += value_len;
+ } else {
+ auto encode_size =
LZ4_compress_fast(string_column.get_chars().raw_data(),
(buf + sizeof(size_t)),
cast_set<Int32>(value_len),
LZ4_compressBound(cast_set<Int32>(value_len)), 1);
unaligned_store<size_t>(buf, encode_size);
buf += (sizeof(size_t) + encode_size);
- return buf;
}
+ return buf;
}
const char* DataTypeString::deserialize(const char* buf, MutableColumnPtr*
column,
int be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- auto* origin_column = column->get();
- size_t real_have_saved_num = 0;
- buf = deserialize_const_flag_and_row_num(buf, column,
&real_have_saved_num);
-
- auto mem_size = real_have_saved_num * sizeof(IColumn::Offset);
- auto* column_string = assert_cast<ColumnString*>(origin_column);
- ColumnString::Chars& data = column_string->get_chars();
- ColumnString::Offsets& offsets = column_string->get_offsets();
- offsets.resize(real_have_saved_num);
-
- // offsets
- if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
- memcpy(offsets.data(), buf, mem_size);
- buf += mem_size;
- } else {
- size_t encode_size = unaligned_load<size_t>(buf);
- buf += sizeof(size_t);
- streamvbyte_decode((const uint8_t*)buf,
(uint32_t*)(offsets.data()),
- cast_set<UInt32>(upper_int32(mem_size)));
- buf += encode_size;
- }
-
- // total length
- size_t value_len = unaligned_load<size_t>(buf);
- buf += sizeof(size_t);
- data.resize(value_len);
-
- // values
- if (value_len <= SERIALIZED_MEM_SIZE_LIMIT) {
- memcpy(data.data(), buf, value_len);
- buf += value_len;
- } else {
- size_t encode_size = unaligned_load<size_t>(buf);
- buf += sizeof(size_t);
- LZ4_decompress_safe(buf, reinterpret_cast<char*>(data.data()),
- cast_set<Int32>(encode_size),
cast_set<Int32>(value_len));
- buf += encode_size;
- }
- return buf;
+ auto* origin_column = column->get();
+ size_t real_have_saved_num = 0;
+ buf = deserialize_const_flag_and_row_num(buf, column,
&real_have_saved_num);
+
+ auto mem_size = real_have_saved_num * sizeof(IColumn::Offset);
+ auto* column_string = assert_cast<ColumnString*>(origin_column);
+ ColumnString::Chars& data = column_string->get_chars();
+ ColumnString::Offsets& offsets = column_string->get_offsets();
+ offsets.resize(real_have_saved_num);
+
+ // offsets
+ if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
+ memcpy(offsets.data(), buf, mem_size);
+ buf += mem_size;
} else {
- auto* column_string = assert_cast<ColumnString*>(column->get());
- ColumnString::Chars& data = column_string->get_chars();
- ColumnString::Offsets& offsets = column_string->get_offsets();
+ size_t encode_size = unaligned_load<size_t>(buf);
+ buf += sizeof(size_t);
+ streamvbyte_decode((const uint8_t*)buf, (uint32_t*)(offsets.data()),
+ cast_set<UInt32>(upper_int32(mem_size)));
+ buf += encode_size;
+ }
- uint32_t mem_size = unaligned_load<uint32_t>(buf);
- buf += sizeof(uint32_t);
- offsets.resize(mem_size / sizeof(IColumn::Offset));
- // offsets
- if (mem_size <= SERIALIZED_MEM_SIZE_LIMIT) {
- memcpy(offsets.data(), buf, mem_size);
- buf += mem_size;
- } else {
- size_t encode_size = unaligned_load<size_t>(buf);
- buf += sizeof(size_t);
- streamvbyte_decode((const uint8_t*)buf,
(uint32_t*)(offsets.data()),
- cast_set<UInt32>(upper_int32(mem_size)));
- buf += encode_size;
- }
- // total length
- uint64_t value_len = unaligned_load<uint64_t>(buf);
- buf += sizeof(uint64_t);
- data.resize(value_len);
+ // total length
+ size_t value_len = unaligned_load<size_t>(buf);
+ buf += sizeof(size_t);
+ data.resize(value_len);
- // values
- if (value_len <= SERIALIZED_MEM_SIZE_LIMIT) {
- memcpy(data.data(), buf, value_len);
- buf += value_len;
- } else {
- size_t encode_size = unaligned_load<size_t>(buf);
- buf += sizeof(size_t);
- LZ4_decompress_safe(buf, reinterpret_cast<char*>(data.data()),
- cast_set<int32_t>(encode_size),
cast_set<Int32>(value_len));
- buf += encode_size;
- }
- return buf;
+ // values
+ if (value_len <= SERIALIZED_MEM_SIZE_LIMIT) {
+ memcpy(data.data(), buf, value_len);
+ buf += value_len;
+ } else {
+ size_t encode_size = unaligned_load<size_t>(buf);
+ buf += sizeof(size_t);
+ LZ4_decompress_safe(buf, reinterpret_cast<char*>(data.data()),
cast_set<Int32>(encode_size),
+ cast_set<Int32>(value_len));
+ buf += encode_size;
}
+ return buf;
}
FieldWithDataType DataTypeString::get_field_with_data_type(const IColumn&
column,
diff --git a/be/src/vec/data_types/data_type_struct.cpp
b/be/src/vec/data_types/data_type_struct.cpp
index 650ff23e58d..d6d3e6a59a9 100644
--- a/be/src/vec/data_types/data_type_struct.cpp
+++ b/be/src/vec/data_types/data_type_struct.cpp
@@ -185,83 +185,48 @@ String DataTypeStruct::get_name_by_position(size_t i)
const {
// childs : child1 | child2 ...
int64_t DataTypeStruct::get_uncompressed_serialized_bytes(const IColumn&
column,
int be_exec_version)
const {
- if (be_exec_version >= USE_CONST_SERDE) {
- auto size = sizeof(bool) + sizeof(size_t) + sizeof(size_t);
- bool is_const_column = is_column_const(column);
- const IColumn* data_column = &column;
- if (is_const_column) {
- const auto& const_column = assert_cast<const ColumnConst&>(column);
- data_column = &(const_column.get_data_column());
- }
- const auto& struct_column = assert_cast<const
ColumnStruct&>(*data_column);
- DCHECK(elems.size() == struct_column.tuple_size());
- int64_t bytes = 0;
- for (size_t i = 0; i < elems.size(); ++i) {
- bytes +=
elems[i]->get_uncompressed_serialized_bytes(struct_column.get_column(i),
-
be_exec_version);
- }
- return size + bytes;
- } else {
- auto ptr = column.convert_to_full_column_if_const();
- const auto& struct_column = assert_cast<const
ColumnStruct&>(*ptr.get());
- DCHECK(elems.size() == struct_column.tuple_size());
-
- int64_t bytes = 0;
- for (size_t i = 0; i < elems.size(); ++i) {
- bytes +=
elems[i]->get_uncompressed_serialized_bytes(struct_column.get_column(i),
-
be_exec_version);
- }
- return bytes;
+ auto size = sizeof(bool) + sizeof(size_t) + sizeof(size_t);
+ bool is_const_column = is_column_const(column);
+ const IColumn* data_column = &column;
+ if (is_const_column) {
+ const auto& const_column = assert_cast<const ColumnConst&>(column);
+ data_column = &(const_column.get_data_column());
+ }
+ const auto& struct_column = assert_cast<const ColumnStruct&>(*data_column);
+ DCHECK(elems.size() == struct_column.tuple_size());
+ int64_t bytes = 0;
+ for (size_t i = 0; i < elems.size(); ++i) {
+ bytes +=
elems[i]->get_uncompressed_serialized_bytes(struct_column.get_column(i),
+ be_exec_version);
}
+ return size + bytes;
}
char* DataTypeStruct::serialize(const IColumn& column, char* buf, int
be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- const auto* data_column = &column;
- [[maybe_unused]] size_t real_need_copy_num = 0;
- buf = serialize_const_flag_and_row_num(&data_column, buf,
&real_need_copy_num);
-
- const auto& struct_column = assert_cast<const
ColumnStruct&>(*data_column);
- DCHECK(elems.size() == struct_column.tuple_size());
- for (size_t i = 0; i < elems.size(); ++i) {
- buf = elems[i]->serialize(struct_column.get_column(i), buf,
be_exec_version);
- }
- return buf;
- } else {
- auto ptr = column.convert_to_full_column_if_const();
- const auto& struct_column = assert_cast<const
ColumnStruct&>(*ptr.get());
- DCHECK(elems.size() == struct_column.tuple_size());
-
- for (size_t i = 0; i < elems.size(); ++i) {
- buf = elems[i]->serialize(struct_column.get_column(i), buf,
be_exec_version);
- }
- return buf;
+ const auto* data_column = &column;
+ [[maybe_unused]] size_t real_need_copy_num = 0;
+ buf = serialize_const_flag_and_row_num(&data_column, buf,
&real_need_copy_num);
+
+ const auto& struct_column = assert_cast<const ColumnStruct&>(*data_column);
+ DCHECK(elems.size() == struct_column.tuple_size());
+ for (size_t i = 0; i < elems.size(); ++i) {
+ buf = elems[i]->serialize(struct_column.get_column(i), buf,
be_exec_version);
}
+ return buf;
}
const char* DataTypeStruct::deserialize(const char* buf, MutableColumnPtr*
column,
int be_exec_version) const {
- if (be_exec_version >= USE_CONST_SERDE) {
- auto* origin_column = column->get();
- [[maybe_unused]] size_t real_have_saved_num = 0;
- buf = deserialize_const_flag_and_row_num(buf, column,
&real_have_saved_num);
-
- auto* struct_column = assert_cast<ColumnStruct*>(origin_column);
- DCHECK(elems.size() == struct_column->tuple_size());
- for (size_t i = 0; i < elems.size(); ++i) {
- auto child_column =
struct_column->get_column_ptr(i)->assume_mutable();
- buf = elems[i]->deserialize(buf, &child_column, be_exec_version);
- }
- return buf;
- } else {
- auto* struct_column = assert_cast<ColumnStruct*>(column->get());
- DCHECK(elems.size() == struct_column->tuple_size());
-
- for (size_t i = 0; i < elems.size(); ++i) {
- auto child_column =
struct_column->get_column_ptr(i)->assume_mutable();
- buf = elems[i]->deserialize(buf, &child_column, be_exec_version);
- }
- return buf;
+ auto* origin_column = column->get();
+ [[maybe_unused]] size_t real_have_saved_num = 0;
+ buf = deserialize_const_flag_and_row_num(buf, column,
&real_have_saved_num);
+
+ auto* struct_column = assert_cast<ColumnStruct*>(origin_column);
+ DCHECK(elems.size() == struct_column->tuple_size());
+ for (size_t i = 0; i < elems.size(); ++i) {
+ auto child_column = struct_column->get_column_ptr(i)->assume_mutable();
+ buf = elems[i]->deserialize(buf, &child_column, be_exec_version);
}
+ return buf;
}
void DataTypeStruct::to_pb_column_meta(PColumnMeta* col_meta) const {
diff --git a/be/src/vec/data_types/data_type_varbinary.cpp
b/be/src/vec/data_types/data_type_varbinary.cpp
index 440b79f112e..41dc037e896 100644
--- a/be/src/vec/data_types/data_type_varbinary.cpp
+++ b/be/src/vec/data_types/data_type_varbinary.cpp
@@ -64,7 +64,6 @@ bool DataTypeVarbinary::equals(const IDataType& rhs) const {
// <data array>: data1 value| data2 value| ... dataN value
int64_t DataTypeVarbinary::get_uncompressed_serialized_bytes(const IColumn&
column,
int
be_exec_version) const {
- DCHECK(be_exec_version >= USE_CONST_SERDE);
auto size = sizeof(bool) + sizeof(size_t) + sizeof(size_t);
bool is_const_column = is_column_const(column);
auto real_need_copy_num = is_const_column ? 1 : column.size();
@@ -85,7 +84,6 @@ int64_t
DataTypeVarbinary::get_uncompressed_serialized_bytes(const IColumn& colu
}
char* DataTypeVarbinary::serialize(const IColumn& column, char* buf, int
be_exec_version) const {
- DCHECK(be_exec_version >= USE_CONST_SERDE);
const auto* varbinary_column = &column;
size_t real_need_copy_num = 0;
buf = serialize_const_flag_and_row_num(&varbinary_column, buf,
&real_need_copy_num);
@@ -110,7 +108,6 @@ char* DataTypeVarbinary::serialize(const IColumn& column,
char* buf, int be_exec
const char* DataTypeVarbinary::deserialize(const char* buf, MutableColumnPtr*
column,
int be_exec_version) const {
- DCHECK(be_exec_version >= USE_CONST_SERDE);
auto* origin_column = column->get();
size_t real_have_saved_num = 0;
buf = deserialize_const_flag_and_row_num(buf, column,
&real_have_saved_num);
diff --git a/be/test/olap/wal/wal_manager_test.cpp
b/be/test/olap/wal/wal_manager_test.cpp
index edf65499502..6cfc8a8e2a6 100644
--- a/be/test/olap/wal/wal_manager_test.cpp
+++ b/be/test/olap/wal/wal_manager_test.cpp
@@ -332,7 +332,7 @@ void
WalManagerTest::generate_scanner(std::shared_ptr<FileScanner>& scanner) {
WARN_IF_ERROR(scanner->init(&_runtime_state, _conjuncts), "fail to prepare
scanner");
}
-TEST_F(WalManagerTest, read_block_normal) {
+TEST_F(WalManagerTest, DISABLED_read_block_normal) {
// read wal file with wal_version=0
_runtime_state._wal_id = _txn_id_1;
std::shared_ptr<FileScanner> scanner = nullptr;
@@ -363,7 +363,7 @@ TEST_F(WalManagerTest, read_block_normal) {
WARN_IF_ERROR(scanner->close(&_runtime_state), "fail to close scanner");
}
-TEST_F(WalManagerTest, read_block_fail_with_not_equal) {
+TEST_F(WalManagerTest, DISABLED_read_block_fail_with_not_equal) {
auto sp = SyncPoint::get_instance();
SyncPoint::CallbackGuard guard1;
sp->set_call_back(
diff --git a/be/test/util/test_data/column_variant.bin
b/be/test/util/test_data/column_variant.bin
deleted file mode 100644
index 5d41c357566..00000000000
Binary files a/be/test/util/test_data/column_variant.bin and /dev/null differ
diff --git a/be/test/vec/columns/column_variant_test.cpp
b/be/test/vec/columns/column_variant_test.cpp
index 4c635d7c71c..d6c6b683683 100644
--- a/be/test/vec/columns/column_variant_test.cpp
+++ b/be/test/vec/columns/column_variant_test.cpp
@@ -3241,219 +3241,6 @@ TEST_F(ColumnVariantTest,
subcolumn_operations_coverage) {
}
}
-// Helper function to create ColumnVariant with various types (int, double,
array, string)
-static auto create_mixed_type_variant_column() {
- auto variant = ColumnVariant::create(1);
-
- // Create test data with different types
- std::vector<std::pair<std::string, doris::vectorized::Field>> test_data = {
- // int type
- {"data.int_field",
doris::vectorized::Field::create_field<TYPE_INT>(42)},
- // double type
- {"data.double_field",
doris::vectorized::Field::create_field<TYPE_DOUBLE>(3.14159)},
- // string type
- {"data.string_field",
-
doris::vectorized::Field::create_field<TYPE_STRING>(String("hello_world"))},
- // array of ints
- {"data.array_int_field",
- [] {
- auto array_field =
doris::vectorized::Field::create_field<TYPE_ARRAY>(Array());
- auto& array = array_field.get<Array>();
-
array.emplace_back(doris::vectorized::Field::create_field<TYPE_INT>(1));
-
array.emplace_back(doris::vectorized::Field::create_field<TYPE_INT>(2));
-
array.emplace_back(doris::vectorized::Field::create_field<TYPE_INT>(3));
- return array_field;
- }()},
- // array of strings
- {"data.array_string_field", [] {
- auto array_field =
doris::vectorized::Field::create_field<TYPE_ARRAY>(Array());
- auto& array = array_field.get<Array>();
- array.emplace_back(
-
doris::vectorized::Field::create_field<TYPE_STRING>(String("apple")));
- array.emplace_back(
-
doris::vectorized::Field::create_field<TYPE_STRING>(String("banana")));
- return array_field;
- }()}};
-
- // Insert 5 rows with the same structure
- for (int i = 0; i < 5; ++i) {
- auto variant_map = VariantUtil::construct_variant_map(test_data);
- variant->try_insert(variant_map);
- }
-
- // Add some additional rows with different structures for sparse column
- std::vector<std::pair<std::string, doris::vectorized::Field>> sparse_data
= {
- {"data.int_field",
doris::vectorized::Field::create_field<TYPE_INT>(100)},
- {"data.double_field",
doris::vectorized::Field::create_field<TYPE_DOUBLE>(2.71828)},
- {"data.nested.sparse_string",
-
doris::vectorized::Field::create_field<TYPE_STRING>(String("sparse_data"))},
- {"data.nested.sparse_int",
doris::vectorized::Field::create_field<TYPE_INT>(999)}};
-
- // Insert 3 more rows with sparse structure
- for (int i = 0; i < 3; ++i) {
- auto sparse_variant_map =
VariantUtil::construct_variant_map(sparse_data);
- variant->try_insert(sparse_variant_map);
- }
-
- return variant;
-}
-
-// // Test to generate ColumnVariant data and serialize to binary file
-// TEST_F(ColumnVariantTest, generate_compatibility_test_data) {
-// // 1. Create ColumnVariant with various types
-// auto variant_column = create_mixed_type_variant_column();
-// variant_column->finalize();
-// EXPECT_TRUE(variant_column->pick_subcolumns_to_sparse_column({},
false).ok());
-// variant_column->finalize();
-// EXPECT_LE(variant_column->get_subcolumns().size(), 2);
-//
-// // 2. Create DataTypeVariant for serialization
-// auto data_type_variant = std::make_shared<DataTypeVariant>();
-//
-// // 3. Calculate serialized size
-// int be_exec_version = 0; // Use current version
-// int64_t serialized_size =
-//
data_type_variant->get_uncompressed_serialized_bytes(*variant_column,
be_exec_version);
-// EXPECT_GT(serialized_size, 0);
-//
-// // 4. Serialize to buffer
-// std::vector<char> buffer(serialized_size);
-// char* buf_end = data_type_variant->serialize(*variant_column,
buffer.data(), be_exec_version);
-// size_t actual_size = buf_end - buffer.data();
-// // EXPECT_EQ(actual_size, serialized_size);
-//
-// // 5. Write to binary file
-// std::string test_data_path =
-// std::string(getenv("ROOT")) +
"/be/test/util/test_data/column_variant.bin";
-// std::ofstream file(test_data_path, std::ios::binary);
-// ASSERT_TRUE(file.is_open()) << "Failed to open file for writing: " <<
test_data_path;
-// file.write(buffer.data(), actual_size);
-// file.close();
-//
-// std::cout << "Successfully generated test data file: " <<
test_data_path << std::endl;
-// std::cout << "Serialized size: " << actual_size << " bytes" <<
std::endl;
-// }
-
-// Test to deserialize from binary file and verify data consistency
-TEST_F(ColumnVariantTest, compatibility_deserialize_and_verify) {
- // 1. Create reference data for comparison
- auto expected_variant = create_mixed_type_variant_column();
- expected_variant->finalize();
-
- // 2. Read binary data from file
- std::string test_data_path =
- std::string(getenv("ROOT")) +
"/be/test/util/test_data/column_variant.bin";
- std::ifstream file(test_data_path, std::ios::binary);
- ASSERT_TRUE(file.is_open()) << "Failed to open test data file: " <<
test_data_path;
-
- // Get file size
- file.seekg(0, std::ios::end);
- size_t file_size = file.tellg();
- file.seekg(0, std::ios::beg);
- EXPECT_GT(file_size, 0);
-
- // Read all data
- std::vector<char> buffer(file_size);
- file.read(buffer.data(), file_size);
- file.close();
-
- // 3. Deserialize from binary data
- auto data_type_variant = std::make_shared<DataTypeVariant>();
- auto deserialized_column = data_type_variant->create_column();
- int be_exec_version = 0;
-
- const char* buf_end =
- data_type_variant->deserialize(buffer.data(),
&deserialized_column, be_exec_version);
- size_t consumed_size = buf_end - buffer.data();
- EXPECT_EQ(consumed_size, file_size);
-
- auto variant_column =
assert_cast<ColumnVariant*>(deserialized_column.get());
- EXPECT_EQ(variant_column->size(), 8);
-
- // 4. Use ColumnVariant::get to retrieve VariantMap and verify data
- for (size_t row = 0; row < variant_column->size(); ++row) {
- Field result_field;
- variant_column->get(row, result_field);
- EXPECT_EQ(result_field.get_type(), TYPE_VARIANT);
-
- const auto& variant_map = result_field.get<VariantMap>();
-
- if (row < 5) {
- // First 5 rows should have the structured data
- EXPECT_TRUE(variant_map.find(PathInData("data.int_field")) !=
variant_map.end());
- EXPECT_TRUE(variant_map.find(PathInData("data.double_field")) !=
variant_map.end());
- EXPECT_TRUE(variant_map.find(PathInData("data.string_field")) !=
variant_map.end());
- EXPECT_TRUE(variant_map.find(PathInData("data.array_int_field"))
!= variant_map.end());
-
EXPECT_TRUE(variant_map.find(PathInData("data.array_string_field")) !=
- variant_map.end());
-
- // Verify specific values
- auto int_iter = variant_map.find(PathInData("data.int_field"));
- if (int_iter != variant_map.end()) {
- EXPECT_EQ(int_iter->second.field.get<Int32>(), 42);
- }
-
- auto double_iter =
variant_map.find(PathInData("data.double_field"));
- if (double_iter != variant_map.end()) {
- EXPECT_DOUBLE_EQ(double_iter->second.field.get<Float64>(),
3.14159);
- }
-
- auto string_iter =
variant_map.find(PathInData("data.string_field"));
- if (string_iter != variant_map.end()) {
- EXPECT_EQ(string_iter->second.field.get<String>(),
"hello_world");
- }
-
- auto array_int_iter =
variant_map.find(PathInData("data.array_int_field"));
- if (array_int_iter != variant_map.end()) {
- const auto& array = array_int_iter->second.field.get<Array>();
- EXPECT_EQ(array.size(), 3);
- EXPECT_EQ(array[0].get<Int32>(), 1);
- EXPECT_EQ(array[1].get<Int32>(), 2);
- EXPECT_EQ(array[2].get<Int32>(), 3);
- }
-
- auto array_string_iter =
variant_map.find(PathInData("data.array_string_field"));
- if (array_string_iter != variant_map.end()) {
- const auto& array =
array_string_iter->second.field.get<Array>();
- EXPECT_EQ(array.size(), 2);
- EXPECT_EQ(array[0].get<String>(), "apple");
- EXPECT_EQ(array[1].get<String>(), "banana");
- }
- } else {
- // Last 3 rows should have sparse data
- EXPECT_TRUE(variant_map.find(PathInData("data.int_field")) !=
variant_map.end());
- EXPECT_TRUE(variant_map.find(PathInData("data.double_field")) !=
variant_map.end());
-
EXPECT_TRUE(variant_map.find(PathInData("data.nested.sparse_string")) !=
- variant_map.end());
- EXPECT_TRUE(variant_map.find(PathInData("data.nested.sparse_int"))
!=
- variant_map.end());
-
- // Verify sparse data values
- auto int_iter = variant_map.find(PathInData("data.int_field"));
- if (int_iter != variant_map.end()) {
- EXPECT_EQ(int_iter->second.field.get<Int32>(), 100);
- }
-
- auto double_iter =
variant_map.find(PathInData("data.double_field"));
- if (double_iter != variant_map.end()) {
- EXPECT_DOUBLE_EQ(double_iter->second.field.get<Float64>(),
2.71828);
- }
-
- auto sparse_string_iter =
variant_map.find(PathInData("data.nested.sparse_string"));
- if (sparse_string_iter != variant_map.end()) {
- EXPECT_EQ(sparse_string_iter->second.field.get<String>(),
"sparse_data");
- }
-
- auto sparse_int_iter =
variant_map.find(PathInData("data.nested.sparse_int"));
- if (sparse_int_iter != variant_map.end()) {
- EXPECT_EQ(sparse_int_iter->second.field.get<Int32>(), 999);
- }
- }
- }
-
- std::cout << "Successfully verified deserialized data integrity!" <<
std::endl;
-}
-
TEST_F(ColumnVariantTest, subcolumn_insert_range_from_test) {
ColumnVariant::Subcolumn subcolumn(0, true /* is_nullable */, false /*
is_root */);
Field int_field = Field::create_field<TYPE_INT>(200000);
diff --git a/be/test/vec/data_types/data_type_datetime_v1_test.cpp
b/be/test/vec/data_types/data_type_datetime_v1_test.cpp
index 47025f0a53d..aa4fed9e00c 100644
--- a/be/test/vec/data_types/data_type_datetime_v1_test.cpp
+++ b/be/test/vec/data_types/data_type_datetime_v1_test.cpp
@@ -360,11 +360,9 @@ TEST_F(DataTypeDateTimeV1Test, ser_deser) {
// binary: const flag| row num | real saved num| data
auto content_uncompressed_size =
dt.get_uncompressed_serialized_bytes(*tmp_col,
be_exec_version);
- if (be_exec_version >= USE_CONST_SERDE) {
- EXPECT_EQ(content_uncompressed_size, 17 + expected_data_size);
- } else {
- EXPECT_EQ(content_uncompressed_size, 4 + expected_data_size);
- }
+
+ EXPECT_EQ(content_uncompressed_size, 17 + expected_data_size);
+
{
std::string column_values;
column_values.resize(content_uncompressed_size);
@@ -383,11 +381,9 @@ TEST_F(DataTypeDateTimeV1Test, ser_deser) {
col_with_type->insert_many_vals(1, count);
expected_data_size = sizeof(typename ColumnType::value_type) * count;
content_uncompressed_size =
dt.get_uncompressed_serialized_bytes(*tmp_col, be_exec_version);
- if (be_exec_version >= USE_CONST_SERDE) {
- EXPECT_EQ(content_uncompressed_size, 17 + expected_data_size);
- } else {
- EXPECT_EQ(content_uncompressed_size, 4 + expected_data_size);
- }
+
+ EXPECT_EQ(content_uncompressed_size, 17 + expected_data_size);
+
{
std::string column_values;
column_values.resize(content_uncompressed_size);
@@ -409,18 +405,13 @@ TEST_F(DataTypeDateTimeV1Test, ser_deser) {
col_with_type->insert_many_vals(1, count);
content_uncompressed_size =
dt.get_uncompressed_serialized_bytes(*tmp_col, be_exec_version);
expected_data_size = sizeof(typename ColumnType::value_type) * count;
- if (be_exec_version >= USE_CONST_SERDE) {
- EXPECT_EQ(content_uncompressed_size,
- 17 + 8 +
- std::max(expected_data_size,
- streamvbyte_max_compressedbytes(
-
cast_set<UInt32>(upper_int32(expected_data_size)))));
- } else {
- EXPECT_EQ(content_uncompressed_size,
- 12 + std::max(expected_data_size,
- streamvbyte_max_compressedbytes(
-
cast_set<UInt32>(upper_int32(expected_data_size)))));
- }
+
+ EXPECT_EQ(content_uncompressed_size,
+ 17 + 8 +
+ std::max(expected_data_size,
+ streamvbyte_max_compressedbytes(
+
cast_set<UInt32>(upper_int32(expected_data_size)))));
+
{
std::string column_values;
column_values.resize(content_uncompressed_size);
@@ -457,10 +448,8 @@ TEST_F(DataTypeDateTimeV1Test, ser_deser) {
}
};
test_func(dt_date, *column_date, USE_CONST_SERDE);
- test_func(dt_date, *column_date, AGGREGATION_2_1_VERSION);
test_func(dt_datetime, *column_datetime, USE_CONST_SERDE);
- test_func(dt_datetime, *column_datetime, AGGREGATION_2_1_VERSION);
}
TEST_F(DataTypeDateTimeV1Test, to_string) {
auto test_func = [](auto& dt, const auto& source_column) {
diff --git a/be/test/vec/data_types/data_type_datetime_v2_test.cpp
b/be/test/vec/data_types/data_type_datetime_v2_test.cpp
index 4d1d0d1b90b..19be034c956 100644
--- a/be/test/vec/data_types/data_type_datetime_v2_test.cpp
+++ b/be/test/vec/data_types/data_type_datetime_v2_test.cpp
@@ -687,11 +687,9 @@ TEST_F(DataTypeDateTimeV2Test, ser_deser) {
// binary: const flag| row num | real saved num| data
auto content_uncompressed_size =
dt.get_uncompressed_serialized_bytes(*tmp_col,
be_exec_version);
- if (be_exec_version >= USE_CONST_SERDE) {
- EXPECT_EQ(content_uncompressed_size, 17 + expected_data_size);
- } else {
- EXPECT_EQ(content_uncompressed_size, 4 + expected_data_size);
- }
+
+ EXPECT_EQ(content_uncompressed_size, 17 + expected_data_size);
+
{
std::string column_values;
column_values.resize(content_uncompressed_size);
@@ -710,11 +708,9 @@ TEST_F(DataTypeDateTimeV2Test, ser_deser) {
col_with_type->insert_many_vals(1, count);
expected_data_size = sizeof(typename ColumnType::value_type) * count;
content_uncompressed_size =
dt.get_uncompressed_serialized_bytes(*tmp_col, be_exec_version);
- if (be_exec_version >= USE_CONST_SERDE) {
- EXPECT_EQ(content_uncompressed_size, 17 + expected_data_size);
- } else {
- EXPECT_EQ(content_uncompressed_size, 4 + expected_data_size);
- }
+
+ EXPECT_EQ(content_uncompressed_size, 17 + expected_data_size);
+
{
std::string column_values;
column_values.resize(content_uncompressed_size);
@@ -736,18 +732,13 @@ TEST_F(DataTypeDateTimeV2Test, ser_deser) {
col_with_type->insert_many_vals(1, count);
content_uncompressed_size =
dt.get_uncompressed_serialized_bytes(*tmp_col, be_exec_version);
expected_data_size = sizeof(typename ColumnType::value_type) * count;
- if (be_exec_version >= USE_CONST_SERDE) {
- EXPECT_EQ(content_uncompressed_size,
- 17 + 8 +
- std::max(expected_data_size,
- streamvbyte_max_compressedbytes(
-
cast_set<UInt32>(upper_int32(expected_data_size)))));
- } else {
- EXPECT_EQ(content_uncompressed_size,
- 12 + std::max(expected_data_size,
- streamvbyte_max_compressedbytes(
-
cast_set<UInt32>(upper_int32(expected_data_size)))));
- }
+
+ EXPECT_EQ(content_uncompressed_size,
+ 17 + 8 +
+ std::max(expected_data_size,
+ streamvbyte_max_compressedbytes(
+
cast_set<UInt32>(upper_int32(expected_data_size)))));
+
{
std::string column_values;
column_values.resize(content_uncompressed_size);
@@ -784,14 +775,10 @@ TEST_F(DataTypeDateTimeV2Test, ser_deser) {
}
};
test_func(dt_date_v2, *column_date_v2, USE_CONST_SERDE);
- test_func(dt_date_v2, *column_date_v2, AGGREGATION_2_1_VERSION);
test_func(dt_datetime_v2_0, *column_datetime_v2_0, USE_CONST_SERDE);
- test_func(dt_datetime_v2_0, *column_datetime_v2_0,
AGGREGATION_2_1_VERSION);
test_func(dt_datetime_v2_5, *column_datetime_v2_5, USE_CONST_SERDE);
- test_func(dt_datetime_v2_5, *column_datetime_v2_5,
AGGREGATION_2_1_VERSION);
test_func(dt_datetime_v2_6, *column_datetime_v2_6, USE_CONST_SERDE);
- test_func(dt_datetime_v2_6, *column_datetime_v2_6,
AGGREGATION_2_1_VERSION);
}
TEST_F(DataTypeDateTimeV2Test, to_string) {
auto test_func = [](auto& dt, const auto& source_column) {
diff --git a/be/test/vec/data_types/data_type_decimal_test.cpp
b/be/test/vec/data_types/data_type_decimal_test.cpp
index 5a7b005bf5b..699231b940d 100644
--- a/be/test/vec/data_types/data_type_decimal_test.cpp
+++ b/be/test/vec/data_types/data_type_decimal_test.cpp
@@ -344,11 +344,9 @@ TEST_F(DataTypeDecimalTest, ser_deser) {
// binary: const flag| row num | real saved num| data
auto content_uncompressed_size =
dt.get_uncompressed_serialized_bytes(*tmp_col,
be_exec_version);
- if (be_exec_version >= USE_CONST_SERDE) {
- EXPECT_EQ(content_uncompressed_size, 17 + expected_data_size);
- } else {
- EXPECT_EQ(content_uncompressed_size, 4 + expected_data_size);
- }
+
+ EXPECT_EQ(content_uncompressed_size, 17 + expected_data_size);
+
{
std::string column_values;
column_values.resize(content_uncompressed_size);
@@ -367,11 +365,9 @@ TEST_F(DataTypeDecimalTest, ser_deser) {
col_with_type->insert_many_defaults(count);
expected_data_size = sizeof(typename ColumnType::value_type) * count;
content_uncompressed_size =
dt.get_uncompressed_serialized_bytes(*tmp_col, be_exec_version);
- if (be_exec_version >= USE_CONST_SERDE) {
- EXPECT_EQ(content_uncompressed_size, 17 + expected_data_size);
- } else {
- EXPECT_EQ(content_uncompressed_size, 4 + expected_data_size);
- }
+
+ EXPECT_EQ(content_uncompressed_size, 17 + expected_data_size);
+
{
std::string column_values;
column_values.resize(content_uncompressed_size);
@@ -393,18 +389,13 @@ TEST_F(DataTypeDecimalTest, ser_deser) {
col_with_type->insert_many_defaults(count);
content_uncompressed_size =
dt.get_uncompressed_serialized_bytes(*tmp_col, be_exec_version);
expected_data_size = sizeof(typename ColumnType::value_type) * count;
- if (be_exec_version >= USE_CONST_SERDE) {
- EXPECT_EQ(content_uncompressed_size,
- 17 + 8 +
- std::max(expected_data_size,
- streamvbyte_max_compressedbytes(
-
cast_set<UInt32>(upper_int32(expected_data_size)))));
- } else {
- EXPECT_EQ(content_uncompressed_size,
- 12 + std::max(expected_data_size,
- streamvbyte_max_compressedbytes(
-
cast_set<UInt32>(upper_int32(expected_data_size)))));
- }
+
+ EXPECT_EQ(content_uncompressed_size,
+ 17 + 8 +
+ std::max(expected_data_size,
+ streamvbyte_max_compressedbytes(
+
cast_set<UInt32>(upper_int32(expected_data_size)))));
+
{
std::string column_values;
column_values.resize(content_uncompressed_size);
@@ -441,39 +432,20 @@ TEST_F(DataTypeDecimalTest, ser_deser) {
}
};
test_func(dt_decimal32_1, *column_decimal32_1, USE_CONST_SERDE);
- test_func(dt_decimal32_1, *column_decimal32_1, AGGREGATION_2_1_VERSION);
test_func(dt_decimal32_2, *column_decimal32_2, USE_CONST_SERDE);
- test_func(dt_decimal32_2, *column_decimal32_2, AGGREGATION_2_1_VERSION);
test_func(dt_decimal32_3, *column_decimal32_3, USE_CONST_SERDE);
- test_func(dt_decimal32_3, *column_decimal32_3, AGGREGATION_2_1_VERSION);
test_func(dt_decimal32_4, *column_decimal32_4, USE_CONST_SERDE);
- test_func(dt_decimal32_4, *column_decimal32_4, AGGREGATION_2_1_VERSION);
test_func(dt_decimal32_5, *column_decimal32_5, USE_CONST_SERDE);
- test_func(dt_decimal32_5, *column_decimal32_5, AGGREGATION_2_1_VERSION);
-
test_func(dt_decimal64_1, *column_decimal64_1, USE_CONST_SERDE);
- test_func(dt_decimal64_1, *column_decimal64_1, AGGREGATION_2_1_VERSION);
test_func(dt_decimal64_2, *column_decimal64_2, USE_CONST_SERDE);
- test_func(dt_decimal64_2, *column_decimal64_2, AGGREGATION_2_1_VERSION);
test_func(dt_decimal64_3, *column_decimal64_3, USE_CONST_SERDE);
- test_func(dt_decimal64_3, *column_decimal64_3, AGGREGATION_2_1_VERSION);
-
test_func(dt_decimal128v2, *column_decimal128_v2, USE_CONST_SERDE);
- test_func(dt_decimal128v2, *column_decimal128_v2, AGGREGATION_2_1_VERSION);
-
test_func(dt_decimal128v3_1, *column_decimal128v3_1, USE_CONST_SERDE);
- test_func(dt_decimal128v3_1, *column_decimal128v3_1,
AGGREGATION_2_1_VERSION);
test_func(dt_decimal128v3_2, *column_decimal128v3_2, USE_CONST_SERDE);
- test_func(dt_decimal128v3_2, *column_decimal128v3_2,
AGGREGATION_2_1_VERSION);
test_func(dt_decimal128v3_3, *column_decimal128v3_3, USE_CONST_SERDE);
- test_func(dt_decimal128v3_3, *column_decimal128v3_3,
AGGREGATION_2_1_VERSION);
-
test_func(dt_decimal256_1, *column_decimal256_1, USE_CONST_SERDE);
- test_func(dt_decimal256_1, *column_decimal256_1, AGGREGATION_2_1_VERSION);
test_func(dt_decimal256_2, *column_decimal256_2, USE_CONST_SERDE);
- test_func(dt_decimal256_2, *column_decimal256_2, AGGREGATION_2_1_VERSION);
test_func(dt_decimal256_3, *column_decimal256_3, USE_CONST_SERDE);
- test_func(dt_decimal256_3, *column_decimal256_3, AGGREGATION_2_1_VERSION);
}
TEST_F(DataTypeDecimalTest, to_pb_column_meta) {
auto test_func = [](auto dt, PGenericType_TypeId expected_type) {
diff --git a/be/test/vec/data_types/data_type_jsonb_test.cpp
b/be/test/vec/data_types/data_type_jsonb_test.cpp
index 40954941282..668ee85322c 100644
--- a/be/test/vec/data_types/data_type_jsonb_test.cpp
+++ b/be/test/vec/data_types/data_type_jsonb_test.cpp
@@ -110,7 +110,6 @@ TEST_F(DataTypeJsonbTest, ser_deser) {
// const flag | row num | read saved num
int64_t prefix_size = sizeof(bool) + sizeof(size_t) + sizeof(size_t);
- int64_t prefix_size2 = sizeof(uint32_t) + sizeof(uint64_t);
{
auto tmp_col = dt.create_column();
@@ -120,19 +119,14 @@ TEST_F(DataTypeJsonbTest, ser_deser) {
auto content_uncompressed_size =
dt.get_uncompressed_serialized_bytes(*tmp_col,
be_exec_version);
- if (be_exec_version >= USE_CONST_SERDE) {
- auto expected_data_size = prefix_size;
- expected_data_size += offsets_size;
- // chars size
- expected_data_size += sizeof(size_t);
- expected_data_size += data_size;
- EXPECT_EQ(content_uncompressed_size, expected_data_size);
- } else {
- auto expected_data_size = prefix_size2;
- expected_data_size += offsets_size;
- expected_data_size += data_size;
- EXPECT_EQ(content_uncompressed_size, expected_data_size);
- }
+
+ auto expected_data_size = prefix_size;
+ expected_data_size += offsets_size;
+ // chars size
+ expected_data_size += sizeof(size_t);
+ expected_data_size += data_size;
+ EXPECT_EQ(content_uncompressed_size, expected_data_size);
+
std::string column_values;
column_values.resize(content_uncompressed_size);
char* buf = column_values.data();
@@ -155,19 +149,14 @@ TEST_F(DataTypeJsonbTest, ser_deser) {
auto content_uncompressed_size =
dt.get_uncompressed_serialized_bytes(*tmp_col,
be_exec_version);
- if (be_exec_version >= USE_CONST_SERDE) {
- auto expected_data_size = prefix_size;
- expected_data_size += offsets_size;
- // chars size
- expected_data_size += sizeof(size_t);
- expected_data_size += data_size;
- EXPECT_EQ(content_uncompressed_size, expected_data_size);
- } else {
- auto expected_data_size = prefix_size2;
- expected_data_size += offsets_size;
- expected_data_size += data_size;
- EXPECT_EQ(content_uncompressed_size, expected_data_size);
- }
+
+ auto expected_data_size = prefix_size;
+ expected_data_size += offsets_size;
+ // chars size
+ expected_data_size += sizeof(size_t);
+ expected_data_size += data_size;
+ EXPECT_EQ(content_uncompressed_size, expected_data_size);
+
std::string column_values;
column_values.resize(content_uncompressed_size);
char* buf = column_values.data();
@@ -195,28 +184,18 @@ TEST_F(DataTypeJsonbTest, ser_deser) {
auto content_uncompressed_size =
dt.get_uncompressed_serialized_bytes(*tmp_col,
be_exec_version);
- if (be_exec_version >= USE_CONST_SERDE) {
- auto expected_data_size = prefix_size;
- expected_data_size +=
- sizeof(size_t) +
- std::max(offsets_size,
streamvbyte_max_compressedbytes(cast_set<UInt32>(
-
upper_int32(offsets_size))));
- expected_data_size += sizeof(size_t);
- expected_data_size +=
- sizeof(size_t) +
- std::max(data_size,
(size_t)LZ4_compressBound(cast_set<UInt32>(data_size)));
- EXPECT_EQ(content_uncompressed_size, expected_data_size);
- } else {
- auto expected_data_size = prefix_size2;
- expected_data_size +=
- sizeof(size_t) +
- std::max(offsets_size,
streamvbyte_max_compressedbytes(cast_set<UInt32>(
-
upper_int32(offsets_size))));
- expected_data_size +=
- sizeof(size_t) +
- std::max(data_size,
(size_t)LZ4_compressBound(cast_set<UInt32>(data_size)));
- EXPECT_EQ(content_uncompressed_size, expected_data_size);
- }
+
+ auto expected_data_size = prefix_size;
+ expected_data_size +=
+ sizeof(size_t) +
+ std::max(offsets_size, streamvbyte_max_compressedbytes(
+
cast_set<UInt32>(upper_int32(offsets_size))));
+ expected_data_size += sizeof(size_t);
+ expected_data_size +=
+ sizeof(size_t) +
+ std::max(data_size,
(size_t)LZ4_compressBound(cast_set<UInt32>(data_size)));
+ EXPECT_EQ(content_uncompressed_size, expected_data_size);
+
std::string column_values;
column_values.resize(content_uncompressed_size);
char* buf = column_values.data();
@@ -252,7 +231,6 @@ TEST_F(DataTypeJsonbTest, ser_deser) {
}
};
test_func(dt_jsonb, *column_jsonb, USE_CONST_SERDE);
- test_func(dt_jsonb, *column_jsonb, AGGREGATION_2_1_VERSION);
}
TEST_F(DataTypeJsonbTest, simple_func_test) {
diff --git a/be/test/vec/data_types/data_type_number_test.cpp
b/be/test/vec/data_types/data_type_number_test.cpp
index 4849cc4cf5e..8f72dbe6c4d 100644
--- a/be/test/vec/data_types/data_type_number_test.cpp
+++ b/be/test/vec/data_types/data_type_number_test.cpp
@@ -247,11 +247,9 @@ TEST_F(DataTypeNumberTest, ser_deser) {
// binary: const flag| row num | real saved num| data
auto content_uncompressed_size =
dt.get_uncompressed_serialized_bytes(*tmp_col,
be_exec_version);
- if (be_exec_version >= USE_CONST_SERDE) {
- EXPECT_EQ(content_uncompressed_size, 17 + expected_data_size);
- } else {
- EXPECT_EQ(content_uncompressed_size, 4 + expected_data_size);
- }
+
+ EXPECT_EQ(content_uncompressed_size, 17 + expected_data_size);
+
{
std::string column_values;
column_values.resize(content_uncompressed_size);
@@ -270,11 +268,9 @@ TEST_F(DataTypeNumberTest, ser_deser) {
col_with_type->insert_many_vals(1, count);
expected_data_size = sizeof(typename ColumnType::value_type) * count;
content_uncompressed_size =
dt.get_uncompressed_serialized_bytes(*tmp_col, be_exec_version);
- if (be_exec_version >= USE_CONST_SERDE) {
- EXPECT_EQ(content_uncompressed_size, 17 + expected_data_size);
- } else {
- EXPECT_EQ(content_uncompressed_size, 4 + expected_data_size);
- }
+
+ EXPECT_EQ(content_uncompressed_size, 17 + expected_data_size);
+
{
std::string column_values;
column_values.resize(content_uncompressed_size);
@@ -296,18 +292,13 @@ TEST_F(DataTypeNumberTest, ser_deser) {
col_with_type->insert_many_vals(1, count);
content_uncompressed_size =
dt.get_uncompressed_serialized_bytes(*tmp_col, be_exec_version);
expected_data_size = sizeof(typename ColumnType::value_type) * count;
- if (be_exec_version >= USE_CONST_SERDE) {
- EXPECT_EQ(content_uncompressed_size,
- 17 + 8 +
- std::max(expected_data_size,
- streamvbyte_max_compressedbytes(
-
cast_set<UInt32>(upper_int32(expected_data_size)))));
- } else {
- EXPECT_EQ(content_uncompressed_size,
- 12 + std::max(expected_data_size,
- streamvbyte_max_compressedbytes(
-
cast_set<UInt32>(upper_int32(expected_data_size)))));
- }
+
+ EXPECT_EQ(content_uncompressed_size,
+ 17 + 8 +
+ std::max(expected_data_size,
+ streamvbyte_max_compressedbytes(
+
cast_set<UInt32>(upper_int32(expected_data_size)))));
+
{
std::string column_values;
column_values.resize(content_uncompressed_size);
@@ -344,22 +335,11 @@ TEST_F(DataTypeNumberTest, ser_deser) {
}
};
test_func(DataTypeInt8(), *column_int8, USE_CONST_SERDE);
- test_func(DataTypeInt8(), *column_int8, AGGREGATION_2_1_VERSION);
-
test_func(DataTypeInt16(), *column_int16, USE_CONST_SERDE);
- test_func(DataTypeInt16(), *column_int16, AGGREGATION_2_1_VERSION);
-
test_func(DataTypeInt32(), *column_int32, USE_CONST_SERDE);
- test_func(DataTypeInt32(), *column_int32, AGGREGATION_2_1_VERSION);
-
test_func(DataTypeInt64(), *column_int64, USE_CONST_SERDE);
- test_func(DataTypeInt64(), *column_int64, AGGREGATION_2_1_VERSION);
-
test_func(DataTypeInt128(), *column_int128, USE_CONST_SERDE);
- test_func(DataTypeInt128(), *column_int128, AGGREGATION_2_1_VERSION);
-
test_func(DataTypeUInt8(), *column_uint8, USE_CONST_SERDE);
- test_func(DataTypeUInt8(), *column_uint8, AGGREGATION_2_1_VERSION);
}
TEST_F(DataTypeNumberTest, to_string) {
diff --git a/be/test/vec/data_types/data_type_string_test.cpp
b/be/test/vec/data_types/data_type_string_test.cpp
index c549f123b00..a56740bedf4 100644
--- a/be/test/vec/data_types/data_type_string_test.cpp
+++ b/be/test/vec/data_types/data_type_string_test.cpp
@@ -113,7 +113,6 @@ TEST_F(DataTypeStringTest, ser_deser) {
// const flag | row num | read saved num
int64_t prefix_size = sizeof(bool) + sizeof(size_t) + sizeof(size_t);
- int64_t prefix_size2 = sizeof(uint32_t) + sizeof(uint64_t);
{
auto tmp_col = dt.create_column();
@@ -123,19 +122,14 @@ TEST_F(DataTypeStringTest, ser_deser) {
auto content_uncompressed_size =
dt.get_uncompressed_serialized_bytes(*tmp_col,
be_exec_version);
- if (be_exec_version >= USE_CONST_SERDE) {
- auto expected_data_size = prefix_size;
- expected_data_size += offsets_size;
- // chars size
- expected_data_size += sizeof(size_t);
- expected_data_size += data_size;
- EXPECT_EQ(content_uncompressed_size, expected_data_size);
- } else {
- auto expected_data_size = prefix_size2;
- expected_data_size += offsets_size;
- expected_data_size += data_size;
- EXPECT_EQ(content_uncompressed_size, expected_data_size);
- }
+
+ auto expected_data_size = prefix_size;
+ expected_data_size += offsets_size;
+ // chars size
+ expected_data_size += sizeof(size_t);
+ expected_data_size += data_size;
+ EXPECT_EQ(content_uncompressed_size, expected_data_size);
+
std::string column_values;
column_values.resize(content_uncompressed_size);
char* buf = column_values.data();
@@ -158,19 +152,14 @@ TEST_F(DataTypeStringTest, ser_deser) {
auto content_uncompressed_size =
dt.get_uncompressed_serialized_bytes(*tmp_col,
be_exec_version);
- if (be_exec_version >= USE_CONST_SERDE) {
- auto expected_data_size = prefix_size;
- expected_data_size += offsets_size;
- // chars size
- expected_data_size += sizeof(size_t);
- expected_data_size += data_size;
- EXPECT_EQ(content_uncompressed_size, expected_data_size);
- } else {
- auto expected_data_size = prefix_size2;
- expected_data_size += offsets_size;
- expected_data_size += data_size;
- EXPECT_EQ(content_uncompressed_size, expected_data_size);
- }
+
+ auto expected_data_size = prefix_size;
+ expected_data_size += offsets_size;
+ // chars size
+ expected_data_size += sizeof(size_t);
+ expected_data_size += data_size;
+ EXPECT_EQ(content_uncompressed_size, expected_data_size);
+
std::string column_values;
column_values.resize(content_uncompressed_size);
char* buf = column_values.data();
@@ -198,28 +187,18 @@ TEST_F(DataTypeStringTest, ser_deser) {
auto content_uncompressed_size =
dt.get_uncompressed_serialized_bytes(*tmp_col,
be_exec_version);
- if (be_exec_version >= USE_CONST_SERDE) {
- auto expected_data_size = prefix_size;
- expected_data_size +=
- sizeof(size_t) +
- std::max(offsets_size,
streamvbyte_max_compressedbytes(cast_set<UInt32>(
-
upper_int32(offsets_size))));
- expected_data_size += sizeof(size_t);
- expected_data_size +=
- sizeof(size_t) +
- std::max(data_size,
(size_t)LZ4_compressBound(cast_set<UInt32>(data_size)));
- EXPECT_EQ(content_uncompressed_size, expected_data_size);
- } else {
- auto expected_data_size = prefix_size2;
- expected_data_size +=
- sizeof(size_t) +
- std::max(offsets_size,
streamvbyte_max_compressedbytes(cast_set<UInt32>(
-
upper_int32(offsets_size))));
- expected_data_size +=
- sizeof(size_t) +
- std::max(data_size,
(size_t)LZ4_compressBound(cast_set<UInt32>(data_size)));
- EXPECT_EQ(content_uncompressed_size, expected_data_size);
- }
+
+ auto expected_data_size = prefix_size;
+ expected_data_size +=
+ sizeof(size_t) +
+ std::max(offsets_size, streamvbyte_max_compressedbytes(
+
cast_set<UInt32>(upper_int32(offsets_size))));
+ expected_data_size += sizeof(size_t);
+ expected_data_size +=
+ sizeof(size_t) +
+ std::max(data_size,
(size_t)LZ4_compressBound(cast_set<UInt32>(data_size)));
+ EXPECT_EQ(content_uncompressed_size, expected_data_size);
+
std::string column_values;
column_values.resize(content_uncompressed_size);
char* buf = column_values.data();
@@ -255,7 +234,6 @@ TEST_F(DataTypeStringTest, ser_deser) {
}
};
test_func(dt_str, *column_str32, USE_CONST_SERDE);
- test_func(dt_str, *column_str32, AGGREGATION_2_1_VERSION);
}
TEST_F(DataTypeStringTest, simple_func_test) {
auto test_func = [](auto& dt) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
