This is an automated email from the ASF dual-hosted git repository.
Mryange pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new c254c020bde [test](be) Improve core BE unit coverage (#64450)
c254c020bde is described below
commit c254c020bde1af3527e807a6116811d2157b5ced
Author: Mryange <[email protected]>
AuthorDate: Tue Jun 23 11:03:00 2026 +0800
[test](be) Improve core BE unit coverage (#64450)
Improve BE unit coverage for low-covered `be/src/core` paths. The change
removes unused core helpers (`ArenaWithFreeLists`, `nested_utils`, and
an unused `LargeIntValue` hash helper), then adds focused tests for
int128 utilities, large integer stream conversion, column filter helper
behavior, comparison specializations, and several data type serde
branches including Nothing, Bitmap, HLL, QuantileState, Variant, Map,
and Struct.
### Release note
None
---
be/src/core/arena.h | 2 +-
be/src/core/arena_with_free_lists.h | 103 ---------------------
be/src/core/data_type/nested_utils.cpp | 66 -------------
be/src/core/value/large_int_value.cpp | 5 -
be/src/core/value/large_int_value.h | 2 -
be/test/core/accurate_comparison_test.cpp | 45 +++++++++
be/test/core/column/column_filter_helper_test.cpp | 52 +++++++++++
be/test/core/data_type/data_type_nothing_test.cpp | 67 +++++++++++++-
.../data_type_serde_bitmap_test.cpp | 11 +++
.../data_type_serde/data_type_serde_hll_test.cpp | 10 ++
.../data_type_serde/data_type_serde_map_test.cpp | 6 ++
.../data_type_serde_quantile_state_test.cpp | 73 ++++++++++++++-
.../data_type_serde_struct_test.cpp | 6 ++
.../data_type_serde_variant_test.cpp | 67 ++++++++++++++
be/test/core/packed_int128_test.cpp | 71 ++++++++++++++
.../core/value/large_int_value_test.cpp} | 43 ++++++---
16 files changed, 436 insertions(+), 193 deletions(-)
diff --git a/be/src/core/arena.h b/be/src/core/arena.h
index 32859b47b40..d423bc81cc6 100644
--- a/be/src/core/arena.h
+++ b/be/src/core/arena.h
@@ -39,7 +39,7 @@ namespace doris {
* - addresses remain valid during lifetime of pool;
* - at destruction of pool, all memory is freed;
* - memory is allocated and freed by large chunks;
- * - freeing parts of data is not possible (but look at ArenaWithFreeLists if
you need);
+ * - freeing parts of data is not possible;
*/
class Arena : private boost::noncopyable {
private:
diff --git a/be/src/core/arena_with_free_lists.h
b/be/src/core/arena_with_free_lists.h
deleted file mode 100644
index d778f760a9a..00000000000
--- a/be/src/core/arena_with_free_lists.h
+++ /dev/null
@@ -1,103 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-// This file is copied from
-//
https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ArenaWithFreeLists.h
-// and modified by Doris
-
-#pragma once
-
-#include <sanitizer/asan_interface.h>
-
-#include "core/arena.h"
-#include "exec/common/bit_helpers.h"
-
-namespace doris {
-
-class ArenaWithFreeLists : private Allocator<false>, private
boost::noncopyable {
-private:
-#if defined(__clang__)
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wzero-length-array"
-#elif defined(__GNUC__)
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wzero-length-bounds"
-#endif
-
- union Block {
- Block* next;
- char data[0];
- };
-
-#if defined(__clang__)
-#pragma clang diagnostic pop
-#elif defined(__GNUC__)
-#pragma GCC diagnostic pop
-#endif
-
- static constexpr size_t max_fixed_block_size = 65536;
-
- static size_t find_free_list_index(const size_t size) {
- return size <= 8 ? 2 : bit_scan_reverse(size - 1);
- }
-
- Arena pool;
-
- Block* free_lists[16] {};
-
-public:
- explicit ArenaWithFreeLists(const size_t initial_size = 4096, const size_t
growth_factor = 2,
- const size_t linear_growth_threshold = 128 *
1024 * 1024)
- : pool {initial_size, growth_factor, linear_growth_threshold} {}
-
- char* alloc(const size_t size) {
- if (size > max_fixed_block_size) {
- return static_cast<char*>(Allocator<false>::alloc(size));
- }
-
- const auto list_idx = find_free_list_index(size);
-
- if (auto& free_block_ptr = free_lists[list_idx]) {
- ASAN_UNPOISON_MEMORY_REGION(free_block_ptr, std::max(size,
sizeof(Block)));
-
- auto* const res = free_block_ptr->data;
- free_block_ptr = free_block_ptr->next;
- return res;
- }
-
- return pool.alloc(1ULL << (list_idx + 1));
- }
-
- void free(char* ptr, const size_t size) {
- if (size > max_fixed_block_size) {
- Allocator<false>::free(ptr, size);
- return;
- }
-
- const auto list_idx = find_free_list_index(size);
-
- auto& free_block_ptr = free_lists[list_idx];
- auto* const old_head = free_block_ptr;
- free_block_ptr = reinterpret_cast<Block*>(ptr);
- free_block_ptr->next = old_head;
-
- ASAN_POISON_MEMORY_REGION(ptr, 1ULL << (list_idx + 1));
- }
-
- size_t allocated_bytes() const { return pool.size(); }
-};
-
-} // namespace doris
\ No newline at end of file
diff --git a/be/src/core/data_type/nested_utils.cpp
b/be/src/core/data_type/nested_utils.cpp
deleted file mode 100644
index d0e98d76155..00000000000
--- a/be/src/core/data_type/nested_utils.cpp
+++ /dev/null
@@ -1,66 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-// This file is copied from
-//
https://github.com/ClickHouse/ClickHouse/blob/master/src/DataTypes/NestedUtils.cpp
-// and modified by Doris
-
-#include "core/data_type/nested_utils.h"
-
-#include "exec/common/string_utils/string_utils.h"
-
-namespace doris {
-
-namespace Nested {
-
-std::string concatenate_name(const std::string& nested_table_name,
- const std::string& nested_field_name) {
- return nested_table_name + "." + nested_field_name;
-}
-
-/** Name can be treated as compound if and only if both parts are simple
identifiers.
- */
-std::pair<std::string, std::string> splitName(const std::string& name) {
- const char* begin = name.data();
- const char* pos = begin;
- const char* end = begin + name.size();
-
- if (pos >= end || !is_valid_identifier_begin(*pos)) return {name, {}};
-
- ++pos;
-
- while (pos < end && is_word_char_ascii(*pos)) ++pos;
-
- if (pos >= end || *pos != '.') return {name, {}};
-
- const char* first_end = pos;
- ++pos;
- const char* second_begin = pos;
-
- if (pos >= end || !is_valid_identifier_begin(*pos)) return {name, {}};
-
- ++pos;
-
- while (pos < end && is_word_char_ascii(*pos)) ++pos;
-
- if (pos != end) return {name, {}};
-
- return {{begin, first_end}, {second_begin, end}};
-}
-
-} // namespace Nested
-
-} // namespace doris
diff --git a/be/src/core/value/large_int_value.cpp
b/be/src/core/value/large_int_value.cpp
index a9b81d84312..4fac5285265 100644
--- a/be/src/core/value/large_int_value.cpp
+++ b/be/src/core/value/large_int_value.cpp
@@ -19,7 +19,6 @@
#include <string>
-#include "util/hash_util.hpp"
#include "util/string_parser.hpp"
namespace doris {
@@ -46,10 +45,6 @@ std::istream& operator>>(std::istream& is, __int128& value) {
return is;
}
-std::size_t hash_value(__int128 const& value) {
- return HashUtil::hash(&value, sizeof(value), 0);
-}
-
} // namespace doris
/* vim: set expandtab ts=4 sw=4 sts=4 tw=100: */
diff --git a/be/src/core/value/large_int_value.h
b/be/src/core/value/large_int_value.h
index fea068b7281..c65879eb95a 100644
--- a/be/src/core/value/large_int_value.h
+++ b/be/src/core/value/large_int_value.h
@@ -45,8 +45,6 @@ std::ostream& operator<<(std::ostream& os, __int128 const&
value);
std::istream& operator>>(std::istream& is, __int128& value);
-std::size_t hash_value(LargeIntValue const& value);
-
} // namespace doris
// Thirdparty printers like gtest needs operator<< to be exported into global
namespace, so that ADL will work.
diff --git a/be/test/core/accurate_comparison_test.cpp
b/be/test/core/accurate_comparison_test.cpp
index 98b0f2ebf96..bbcec6272c9 100644
--- a/be/test/core/accurate_comparison_test.cpp
+++ b/be/test/core/accurate_comparison_test.cpp
@@ -36,4 +36,49 @@ TEST(VAccurateComparison, TestsOP) {
EXPECT_TRUE((GreaterOrEqualsOp<TYPE_INT>::apply(1, 1)));
}
+TEST(VAccurateComparison, StringSpecializations) {
+ StringRef left("abc", 3);
+ StringRef same("abc", 3);
+ StringRef right("abd", 3);
+
+ EXPECT_TRUE((EqualsOp<TYPE_STRING>::apply(left, same)));
+ EXPECT_FALSE((EqualsOp<TYPE_STRING>::apply(left, right)));
+ EXPECT_TRUE((LessOp<TYPE_STRING>::apply(left, right)));
+ EXPECT_TRUE((GreaterOp<TYPE_STRING>::apply(right, left)));
+}
+
+TEST(VAccurateComparison, DateTimeSpecializations) {
+ VecDateTimeValue date_a;
+ VecDateTimeValue date_b;
+ date_a.from_date_int64(20240101);
+ date_b.from_date_int64(20240102);
+ EXPECT_TRUE((EqualsOp<TYPE_DATE>::apply(date_a, date_a)));
+ EXPECT_TRUE((NotEqualsOp<TYPE_DATE>::apply(date_a, date_b)));
+ EXPECT_TRUE((LessOp<TYPE_DATE>::apply(date_a, date_b)));
+ EXPECT_TRUE((GreaterOrEqualsOp<TYPE_DATE>::apply(date_b, date_a)));
+
+ VecDateTimeValue datetime_a;
+ VecDateTimeValue datetime_b;
+ datetime_a.from_date_int64(20240101010203);
+ datetime_b.from_date_int64(20240101010204);
+ EXPECT_TRUE((LessOrEqualsOp<TYPE_DATETIME>::apply(datetime_a,
datetime_b)));
+ EXPECT_TRUE((GreaterOp<TYPE_DATETIME>::apply(datetime_b, datetime_a)));
+
+ DateV2Value<DateV2ValueType> date_v2_a;
+ DateV2Value<DateV2ValueType> date_v2_b;
+ date_v2_a.unchecked_set_time(2024, 1, 1, 0, 0, 0, 0);
+ date_v2_b.unchecked_set_time(2024, 1, 2, 0, 0, 0, 0);
+ EXPECT_TRUE((EqualsOp<TYPE_DATEV2>::apply(date_v2_a, date_v2_a)));
+ EXPECT_TRUE((LessOp<TYPE_DATEV2>::apply(date_v2_a, date_v2_b)));
+ EXPECT_TRUE((GreaterOrEqualsOp<TYPE_DATEV2>::apply(date_v2_b, date_v2_a)));
+
+ DateV2Value<DateTimeV2ValueType> datetime_v2_a;
+ DateV2Value<DateTimeV2ValueType> datetime_v2_b;
+ datetime_v2_a.unchecked_set_time(2024, 1, 1, 1, 2, 3, 0);
+ datetime_v2_b.unchecked_set_time(2024, 1, 1, 1, 2, 4, 0);
+ EXPECT_TRUE((NotEqualsOp<TYPE_DATETIMEV2>::apply(datetime_v2_a,
datetime_v2_b)));
+ EXPECT_TRUE((LessOrEqualsOp<TYPE_DATETIMEV2>::apply(datetime_v2_a,
datetime_v2_b)));
+ EXPECT_TRUE((GreaterOp<TYPE_DATETIMEV2>::apply(datetime_v2_b,
datetime_v2_a)));
+}
+
} // namespace doris
diff --git a/be/test/core/column/column_filter_helper_test.cpp
b/be/test/core/column/column_filter_helper_test.cpp
new file mode 100644
index 00000000000..846110fd47f
--- /dev/null
+++ b/be/test/core/column/column_filter_helper_test.cpp
@@ -0,0 +1,52 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "core/column/column_filter_helper.h"
+
+#include <gtest/gtest.h>
+
+#include "core/assert_cast.h"
+
+namespace doris {
+
+TEST(ColumnFilterHelperTest, ResizeFillAndInsertKeepNullableUInt8NotNull) {
+ auto nullable = ColumnNullable::create(ColumnUInt8::create(),
ColumnUInt8::create());
+
+ ColumnFilterHelper helper(*nullable);
+ helper.reserve(8);
+ helper.resize_fill(3, 1);
+ helper.insert_value(0);
+
+ EXPECT_EQ(helper.size(), 4);
+
+ const auto& value_column = assert_cast<const
ColumnUInt8&>(nullable->get_nested_column());
+ const auto& values = value_column.get_data();
+ ASSERT_EQ(values.size(), 4);
+ EXPECT_EQ(values[0], 1);
+ EXPECT_EQ(values[1], 1);
+ EXPECT_EQ(values[2], 1);
+ EXPECT_EQ(values[3], 0);
+
+ const auto& null_map = nullable->get_null_map_data();
+ ASSERT_EQ(null_map.size(), 4);
+ EXPECT_EQ(null_map[0], 0);
+ EXPECT_EQ(null_map[1], 0);
+ EXPECT_EQ(null_map[2], 0);
+ EXPECT_EQ(null_map[3], 0);
+}
+
+} // namespace doris
diff --git a/be/test/core/data_type/data_type_nothing_test.cpp
b/be/test/core/data_type/data_type_nothing_test.cpp
index daab89cb063..1e15707b5a9 100644
--- a/be/test/core/data_type/data_type_nothing_test.cpp
+++ b/be/test/core/data_type/data_type_nothing_test.cpp
@@ -17,10 +17,16 @@
#include "core/data_type/data_type_nothing.h"
+#include <cctz/time_zone.h>
#include <gtest/gtest.h>
#include "core/column/column_nothing.h"
+#include "core/column/column_vector.h"
#include "core/field.h"
+#include "gen_cpp/types.pb.h"
+#include "util/jsonb_writer.h"
+#include "util/mysql_row_buffer.h"
+#include "util/slice.h"
namespace doris {
@@ -46,4 +52,63 @@ TEST(DataTypeNothingTest, GetFieldWithDataTypeTest) {
EXPECT_EQ(fdt.scale, -1);
}
-} // namespace doris
\ No newline at end of file
+TEST(DataTypeNothingTest, BasicTypeProperties) {
+ DataTypeNothing data_type_nothing;
+
+ EXPECT_EQ(data_type_nothing.get_family_name(), "Nothing");
+ EXPECT_EQ(data_type_nothing.get_primitive_type(),
PrimitiveType::INVALID_TYPE);
+ EXPECT_EQ(data_type_nothing.get_storage_field_type(),
FieldType::OLAP_FIELD_TYPE_NONE);
+ EXPECT_TRUE(data_type_nothing.have_maximum_size_of_value());
+ EXPECT_EQ(data_type_nothing.get_size_of_value_in_memory(), 0);
+
+ auto column = data_type_nothing.create_column();
+ EXPECT_TRUE(data_type_nothing.check_column(*column).ok());
+ EXPECT_FALSE(data_type_nothing.check_column(*ColumnUInt8::create()).ok());
+
+ EXPECT_TRUE(data_type_nothing.equals(DataTypeNothing()));
+ EXPECT_EQ(data_type_nothing.get_uncompressed_serialized_bytes(*column, 0),
0);
+ EXPECT_THROW(data_type_nothing.serialize(*column, nullptr, 0),
doris::Exception);
+ EXPECT_THROW(data_type_nothing.deserialize(nullptr, &column, 0),
doris::Exception);
+
+ TExprNode node;
+ EXPECT_THROW(data_type_nothing.get_field(node), doris::Exception);
+ EXPECT_EQ(data_type_nothing.get_serde()->get_name(), "Nothing");
+}
+
+TEST(DataTypeNothingSerdeTest, UnsupportedFormatsReturnErrorsOrThrow) {
+ DataTypeNothingSerde serde;
+ auto column = ColumnNothing::create(0);
+ DataTypeSerDe::FormatOptions options;
+
+ auto string_column = ColumnString::create();
+ VectorBufferWriter writer(*string_column);
+ Slice slice;
+ std::vector<Slice> slices;
+ uint64_t num_deserialized = 0;
+ PValues values;
+ MysqlRowBinaryBuffer mysql_buffer;
+
+ EXPECT_EQ(serde.get_name(), "Nothing");
+ EXPECT_FALSE(serde.serialize_one_cell_to_json(*column, 0, writer,
options).ok());
+ EXPECT_FALSE(serde.serialize_column_to_json(*column, 0, 0, writer,
options).ok());
+ EXPECT_FALSE(serde.deserialize_one_cell_from_json(*column, slice,
options).ok());
+ EXPECT_FALSE(
+ serde.deserialize_column_from_json_vector(*column, slices,
&num_deserialized, options)
+ .ok());
+ EXPECT_FALSE(serde.write_column_to_pb(*column, values, 0, 0).ok());
+ EXPECT_FALSE(serde.read_column_from_pb(*column, values).ok());
+ EXPECT_FALSE(serde.write_column_to_arrow(*column, nullptr, nullptr, 0, 0,
cctz::utc_time_zone())
+ .ok());
+ EXPECT_FALSE(serde.read_column_from_arrow(*column, nullptr, 0, 0,
cctz::utc_time_zone()).ok());
+ EXPECT_FALSE(serde.write_column_to_mysql_binary(*column, mysql_buffer, 0,
false, options).ok());
+ Arena arena;
+ EXPECT_FALSE(
+ serde.write_column_to_orc("", *column, nullptr, nullptr, 0, 0,
arena, options).ok());
+
+ JsonbWriterT<JsonbOutStream> jsonb_writer;
+ EXPECT_THROW(serde.write_one_cell_to_jsonb(*column, jsonb_writer, arena,
0, 0, options),
+ doris::Exception);
+ EXPECT_THROW(serde.read_one_cell_from_jsonb(*column, nullptr),
doris::Exception);
+}
+
+} // namespace doris
diff --git a/be/test/core/data_type_serde/data_type_serde_bitmap_test.cpp
b/be/test/core/data_type_serde/data_type_serde_bitmap_test.cpp
index d47c9b564a4..86bcda5fcd9 100644
--- a/be/test/core/data_type_serde/data_type_serde_bitmap_test.cpp
+++ b/be/test/core/data_type_serde/data_type_serde_bitmap_test.cpp
@@ -188,4 +188,15 @@ TEST(BitmapSerdeTest, SerdeArrowTest) {
std::cout << "test write/read_column_to_arrow " << std::endl;
}
+TEST(BitmapSerdeTest, UnsupportedArrowReadReturnsError) {
+ auto bitmap_serde = std::make_shared<DataTypeBitMapSerDe>(1);
+ auto column_bitmap = ColumnBitmap::create();
+
+ EXPECT_EQ(bitmap_serde->get_name(), "BitMap");
+ EXPECT_FALSE(
+ bitmap_serde
+ ->read_column_from_arrow(*column_bitmap, nullptr, 0, 0,
cctz::utc_time_zone())
+ .ok());
+}
+
} // namespace doris
diff --git a/be/test/core/data_type_serde/data_type_serde_hll_test.cpp
b/be/test/core/data_type_serde/data_type_serde_hll_test.cpp
index 0ecf85e18a0..c23b8434733 100644
--- a/be/test/core/data_type_serde/data_type_serde_hll_test.cpp
+++ b/be/test/core/data_type_serde/data_type_serde_hll_test.cpp
@@ -197,4 +197,14 @@ TEST(HLLSerdeTest, SerdeArrowTest) {
// CommonDataTypeSerdeTest::compare_two_blocks(block, assert_block);
std::cout << "test write/read_column_to_arrow " << std::endl;
}
+
+TEST(HLLSerdeTest, UnsupportedArrowReadReturnsError) {
+ auto hll_serde = std::make_shared<DataTypeHLLSerDe>(1);
+ auto column_hll = ColumnHLL::create();
+
+ EXPECT_EQ(hll_serde->get_name(), "HLL");
+ EXPECT_FALSE(
+ hll_serde->read_column_from_arrow(*column_hll, nullptr, 0, 0,
cctz::utc_time_zone())
+ .ok());
+}
} // namespace doris
diff --git a/be/test/core/data_type_serde/data_type_serde_map_test.cpp
b/be/test/core/data_type_serde/data_type_serde_map_test.cpp
index cfe6dee93e0..76bc1952588 100644
--- a/be/test/core/data_type_serde/data_type_serde_map_test.cpp
+++ b/be/test/core/data_type_serde/data_type_serde_map_test.cpp
@@ -56,6 +56,12 @@ protected:
static void SetUpTestSuite() {}
};
+TEST_F(DataTypeMapSerDeTest, GetName) {
+ DataTypeMapSerDe serde_map(serde_str_key, serde_str_value);
+
+ EXPECT_EQ(serde_map.get_name(), "Map(String, String)");
+}
+
// Run with UBSan enabled to catch misalignment errors.
TEST_F(DataTypeMapSerDeTest, ArrowMemNotAligned) {
// 1.Prepare the data.
diff --git
a/be/test/core/data_type_serde/data_type_serde_quantile_state_test.cpp
b/be/test/core/data_type_serde/data_type_serde_quantile_state_test.cpp
index e3a9f620708..0b00a1879a4 100644
--- a/be/test/core/data_type_serde/data_type_serde_quantile_state_test.cpp
+++ b/be/test/core/data_type_serde/data_type_serde_quantile_state_test.cpp
@@ -151,4 +151,75 @@ TEST(QuantileStateSerdeTest, serializeColumnToJson) {
quantile_state_res_1.get_value_by_percentile(1));
std::cout << "test serialize/deserialize_column_from_json_vector" <<
std::endl;
}
-} // namespace doris
\ No newline at end of file
+
+TEST(QuantileStateSerdeTest, DeserializeColumnFromJsonVector) {
+ auto quantile_state_serde =
std::make_shared<DataTypeQuantileStateSerDe>(1);
+ QuantileState quantile_state;
+ quantile_state.add_value(123);
+
+ std::string memory_buffer(quantile_state.get_serialized_size(), '0');
+ quantile_state.serialize(reinterpret_cast<uint8_t*>(memory_buffer.data()));
+
+ std::vector<Slice> slices;
+ slices.emplace_back(memory_buffer.data(), memory_buffer.size());
+ uint64_t num_deserialized = 0;
+ auto column_quantile_state = ColumnQuantileState::create();
+ DataTypeSerDe::FormatOptions options;
+
+ auto st = quantile_state_serde->deserialize_column_from_json_vector(
+ *column_quantile_state, slices, &num_deserialized, options);
+
+ EXPECT_TRUE(st.ok()) << st.to_string();
+ EXPECT_EQ(num_deserialized, 1);
+ ASSERT_EQ(column_quantile_state->size(), 1);
+
EXPECT_EQ(column_quantile_state->get_element(0).get_value_by_percentile(1),
123);
+}
+
+TEST(QuantileStateSerdeTest, ArrowMysqlAndToStringPaths) {
+ auto quantile_state_serde =
std::make_shared<DataTypeQuantileStateSerDe>(2);
+ auto column_quantile_state = ColumnQuantileState::create();
+ QuantileState quantile_state;
+ quantile_state.add_value(123);
+ column_quantile_state->insert_value(quantile_state);
+ column_quantile_state->insert_value(QuantileState());
+
+ DataTypeSerDe::FormatOptions options;
+ auto serialized_column = ColumnString::create();
+ VectorBufferWriter writer(*serialized_column);
+ EXPECT_TRUE(quantile_state_serde
+ ->serialize_one_cell_to_json(*column_quantile_state,
0, writer, options)
+ .ok());
+ writer.commit();
+ EXPECT_EQ(serialized_column->get_data_at(0).to_string(), "null");
+
+ auto string_column = ColumnString::create();
+ VectorBufferWriter string_writer(*string_column);
+ quantile_state_serde->to_string(*column_quantile_state, 0, string_writer,
options);
+ string_writer.commit();
+ EXPECT_EQ(string_column->get_data_at(0).size,
quantile_state.get_serialized_size());
+
+ arrow::BinaryBuilder builder;
+ NullMap null_map;
+ null_map.push_back(0);
+ null_map.push_back(1);
+ EXPECT_TRUE(quantile_state_serde
+ ->write_column_to_arrow(*column_quantile_state,
&null_map, &builder, 0, 2,
+ cctz::utc_time_zone())
+ .ok());
+ std::shared_ptr<arrow::Array> array;
+ ASSERT_TRUE(builder.Finish(&array).ok());
+ EXPECT_FALSE(array->IsNull(0));
+ EXPECT_TRUE(array->IsNull(1));
+
+ EXPECT_FALSE(quantile_state_serde
+ ->read_column_from_arrow(*column_quantile_state,
nullptr, 0, 0,
+ cctz::utc_time_zone())
+ .ok());
+
+ MysqlRowBinaryBuffer mysql_buffer;
+ EXPECT_FALSE(quantile_state_serde
+
->write_column_to_mysql_binary(*column_quantile_state, mysql_buffer, 0,
+ false, options)
+ .ok());
+}
+} // namespace doris
diff --git a/be/test/core/data_type_serde/data_type_serde_struct_test.cpp
b/be/test/core/data_type_serde/data_type_serde_struct_test.cpp
index f6d08cd48b0..f6850d82958 100644
--- a/be/test/core/data_type_serde/data_type_serde_struct_test.cpp
+++ b/be/test/core/data_type_serde/data_type_serde_struct_test.cpp
@@ -58,6 +58,12 @@ protected:
static void SetUpTestSuite() {}
};
+TEST_F(DataTypeStructSerDeTest, GetName) {
+ DataTypeStructSerDe serde_struct({serde_int32, serde_str}, {"id", "name"});
+
+ EXPECT_EQ(serde_struct.get_name(), "Struct(id:INT, name:String)");
+}
+
// Run with UBSan enabled to catch misalignment errors.
TEST_F(DataTypeStructSerDeTest, ArrowMemNotAligned) {
// 1.Prepare the data.
diff --git a/be/test/core/data_type_serde/data_type_serde_variant_test.cpp
b/be/test/core/data_type_serde/data_type_serde_variant_test.cpp
new file mode 100644
index 00000000000..968062079d0
--- /dev/null
+++ b/be/test/core/data_type_serde/data_type_serde_variant_test.cpp
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <arrow/array/builder_binary.h>
+#include <arrow/array/builder_primitive.h>
+#include <gtest/gtest.h>
+
+#include "core/column/column_variant.h"
+#include "core/data_type_serde/data_type_variant_serde.h"
+#include "core/string_buffer.hpp"
+#include "gen_cpp/types.pb.h"
+#include "util/mysql_row_buffer.h"
+#include "util/slice.h"
+
+namespace doris {
+
+TEST(VariantSerdeTest, BasicUnsupportedAndArrowPaths) {
+ DataTypeVariantSerDe serde;
+ auto column = ColumnVariant::create(0, false);
+ DataTypeSerDe::FormatOptions options;
+ std::string json = R"({"k": 1})";
+ Slice slice(json.data(), json.size());
+ ASSERT_TRUE(serde.deserialize_one_cell_from_json(*column, slice,
options).ok());
+ column->finalize(ColumnVariant::FinalizeMode::WRITE_MODE);
+
+ EXPECT_EQ(serde.get_name(), "Variant");
+
+ PValues values;
+ EXPECT_FALSE(serde.write_column_to_pb(*column, values, 0,
column->size()).ok());
+ EXPECT_FALSE(serde.read_column_from_pb(*column, values).ok());
+ EXPECT_FALSE(serde.read_column_from_arrow(*column, nullptr, 0, 0,
cctz::utc_time_zone()).ok());
+
+ auto string_column = ColumnString::create();
+ VectorBufferWriter writer(*string_column);
+ serde.to_string(*column, 0, writer, options);
+ writer.commit();
+ EXPECT_FALSE(string_column->get_data_at(0).to_string().empty());
+
+ arrow::StringBuilder string_builder;
+ EXPECT_TRUE(serde.write_column_to_arrow(*column, nullptr, &string_builder,
0, column->size(),
+ cctz::utc_time_zone())
+ .ok());
+ std::shared_ptr<arrow::Array> string_array;
+ ASSERT_TRUE(string_builder.Finish(&string_array).ok());
+ EXPECT_EQ(string_array->length(), column->size());
+
+ arrow::Int32Builder int_builder;
+ EXPECT_FALSE(serde.write_column_to_arrow(*column, nullptr, &int_builder,
0, column->size(),
+ cctz::utc_time_zone())
+ .ok());
+}
+
+} // namespace doris
diff --git a/be/test/core/packed_int128_test.cpp
b/be/test/core/packed_int128_test.cpp
new file mode 100644
index 00000000000..382f0a4a5b7
--- /dev/null
+++ b/be/test/core/packed_int128_test.cpp
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "core/packed_int128.h"
+
+#include <gtest/gtest.h>
+
+#include <array>
+#include <cstring>
+
+namespace doris {
+
+TEST(PackedInt128Test, SignedPackedValueSupportsCopyAndAssignment) {
+ const __int128 positive = (static_cast<__int128>(1) << 100) + 12345;
+ const __int128 negative = -((static_cast<__int128>(1) << 96) + 54321);
+
+ PackedInt128 packed(positive);
+ EXPECT_TRUE(packed.value == positive);
+
+ PackedInt128 copied(packed);
+ EXPECT_TRUE(copied.value == positive);
+
+ copied = negative;
+ EXPECT_TRUE(copied.value == negative);
+
+ PackedInt128 assigned;
+ assigned = copied;
+ EXPECT_TRUE(assigned.value == negative);
+}
+
+TEST(PackedInt128Test, UnsignedPackedValueSupportsCopyAndAssignment) {
+ const uint128_t first = (static_cast<uint128_t>(1) << 100) + 12345;
+ const uint128_t second = (static_cast<uint128_t>(1) << 96) + 54321;
+
+ PackedUInt128 packed(first);
+ EXPECT_TRUE(packed.value == first);
+
+ PackedUInt128 copied(packed);
+ EXPECT_TRUE(copied.value == first);
+
+ copied = second;
+ EXPECT_TRUE(copied.value == second);
+
+ PackedUInt128 assigned;
+ assigned = copied;
+ EXPECT_TRUE(assigned.value == second);
+}
+
+TEST(PackedInt128Test, ReadInt128FromUnalignedAddress) {
+ const int128_t expected = -((static_cast<int128_t>(1) << 100) + 98765);
+ std::array<char, sizeof(int128_t) + 1> buffer {};
+ std::memcpy(buffer.data() + 1, &expected, sizeof(expected));
+
+ EXPECT_TRUE(get_int128_from_unalign(buffer.data() + 1) == expected);
+}
+
+} // namespace doris
diff --git a/be/src/core/data_type/nested_utils.h
b/be/test/core/value/large_int_value_test.cpp
similarity index 50%
rename from be/src/core/data_type/nested_utils.h
rename to be/test/core/value/large_int_value_test.cpp
index 08e5e188b2d..0d400735bd2 100644
--- a/be/src/core/data_type/nested_utils.h
+++ b/be/test/core/value/large_int_value_test.cpp
@@ -14,27 +14,42 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-// This file is copied from
-//
https://github.com/ClickHouse/ClickHouse/blob/master/src/DataTypes/NestedUtils.h
-// and modified by Doris
-#pragma once
+#include "core/value/large_int_value.h"
-#include <string>
-#include <utility>
+#include <gtest/gtest.h>
-#include "core/block/block.h"
+#include <sstream>
namespace doris {
-namespace Nested {
-std::string concatenate_name(const std::string& nested_table_name,
- const std::string& nested_field_name);
+TEST(LargeIntValueTest, StreamOutputUsesDecimalText) {
+ const __int128 value = -((static_cast<__int128>(1) << 100) + 12345);
-std::pair<std::string, std::string> splitName(const std::string& name);
+ std::ostringstream out;
+ out << value;
-/// Replace Array(Tuple(...)) columns to a multiple of Array columns in a form
of `column_name.element_name`.
-Block flatten(const Block& block);
-} // namespace Nested
+ EXPECT_EQ(out.str(), LargeIntValue::to_string(value));
+}
+
+TEST(LargeIntValueTest, StreamInputParsesDecimalText) {
+ const __int128 expected = (static_cast<__int128>(1) << 100) + 54321;
+
+ std::istringstream in(LargeIntValue::to_string(expected));
+ __int128 actual = 0;
+ in >> actual;
+
+ EXPECT_FALSE(in.fail());
+ EXPECT_EQ(actual, expected);
+}
+
+TEST(LargeIntValueTest, StreamInputFailsOnInvalidText) {
+ std::istringstream in("not-a-large-int");
+ __int128 value = 123;
+ in >> value;
+
+ EXPECT_TRUE(in.fail());
+ EXPECT_EQ(value, 0);
+}
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
