This is an automated email from the ASF dual-hosted git repository.
zclllyybb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new bd101604546 [Test](bitmap) Harden BitmapValue serialization test
(#65491)
bd101604546 is described below
commit bd101604546dc0940050f438775a54b0efaae985
Author: linrrarity <[email protected]>
AuthorDate: Mon Jul 13 09:38:08 2026 +0800
[Test](bitmap) Harden BitmapValue serialization test (#65491)
### Release note
- Strengthen BitmapValue serialization tests for deterministic and
non-deterministic formats.
- Keep exact byte assertions for non-SET bitmap encodings.
- Add fixed historical SET binary inputs and verify deserialized bitmap
contents semantically.
- Avoid pinning bitmap_to_base64 output for SET because hash-set
iteration order makes serialized bytes unstable.
---
be/test/core/value/bitmap_value_test.cpp | 124 +++++++++++++++++++--
be/test/exprs/function/function_bitmap_test.cpp | 31 +++++-
.../bitmap_functions/test_bitmap_function.out | 12 ++
.../sql-functions/doc_bitmap_functions_test.groovy | 11 +-
.../bitmap_functions/test_bitmap_function.groovy | 6 +
5 files changed, 163 insertions(+), 21 deletions(-)
diff --git a/be/test/core/value/bitmap_value_test.cpp
b/be/test/core/value/bitmap_value_test.cpp
index 5b25551ae0c..1f89539c366 100644
--- a/be/test/core/value/bitmap_value_test.cpp
+++ b/be/test/core/value/bitmap_value_test.cpp
@@ -21,12 +21,15 @@
#include <gtest/gtest-test-part.h>
#include <roaring/roaring.h>
+#include <algorithm>
#include <cstdint>
#include <string>
+#include <vector>
#include "gtest/gtest.h"
#include "gtest/gtest_pred_impl.h"
#include "util/coding.h"
+#include "util/url_coding.h"
namespace doris {
using roaring::Roaring;
@@ -503,6 +506,20 @@ void check_bitmap_equal(const BitmapValue& left, const
BitmapValue& right) {
}
}
+std::vector<uint64_t> sorted_bitmap_values(const BitmapValue& bitmap) {
+ std::vector<uint64_t> values;
+ for (auto v : bitmap) {
+ values.emplace_back(v);
+ }
+ std::sort(values.begin(), values.end());
+ return values;
+}
+
+void check_bitmap_values(const BitmapValue& bitmap, std::vector<uint64_t>
expected) {
+ std::sort(expected.begin(), expected.end());
+ EXPECT_EQ(sorted_bitmap_values(bitmap), expected);
+}
+
TEST(BitmapValueTest, write_read) {
config::enable_set_in_bitmap_value = true;
BitmapValue bitmap_empty;
@@ -966,14 +983,31 @@ std::string convert_bitmap_to_string(BitmapValue& bitmap)
{
return buf;
}
+std::string decode_base64_to_string(const std::string& base64) {
+ std::string decoded;
+ EXPECT_TRUE(base64_decode(base64, &decoded));
+ return decoded;
+}
+
+BitmapValue deserialize_bitmap_from_string(const std::string& buffer) {
+ BitmapValue bitmap;
+ EXPECT_TRUE(bitmap.deserialize(buffer.data()));
+ return bitmap;
+}
+
TEST(BitmapValueTest, bitmap_serde) {
+ auto old_enable_set = config::enable_set_in_bitmap_value;
+ auto old_serialize_version = config::bitmap_serialize_version;
+ config::enable_set_in_bitmap_value = false;
+ config::bitmap_serialize_version = 1;
+
{ // EMPTY
BitmapValue empty;
std::string buffer = convert_bitmap_to_string(empty);
std::string expect_buffer(1, BitmapTypeCode::EMPTY);
EXPECT_EQ(expect_buffer, buffer);
- BitmapValue out(buffer.data());
+ BitmapValue out = deserialize_bitmap_from_string(buffer);
EXPECT_EQ(0, out.cardinality());
}
{ // SINGLE32
@@ -984,18 +1018,19 @@ TEST(BitmapValueTest, bitmap_serde) {
put_fixed32_le(&expect_buffer, i);
EXPECT_EQ(expect_buffer, buffer);
- BitmapValue out(buffer.data());
+ BitmapValue out = deserialize_bitmap_from_string(buffer);
EXPECT_EQ(1, out.cardinality());
EXPECT_TRUE(out.contains(i));
}
{ // BITMAP32
- BitmapValue bitmap32({0, UINT32_MAX});
+ BitmapValue bitmap32({1, 9999999});
std::string buffer = convert_bitmap_to_string(bitmap32);
+
EXPECT_EQ(decode_base64_to_string("AjowAAACAAAAAAAAAJgAAAAYAAAAGgAAAAEAf5Y="),
buffer);
- BitmapValue out(buffer.data());
+ BitmapValue out = deserialize_bitmap_from_string(buffer);
EXPECT_EQ(2, out.cardinality());
- EXPECT_TRUE(out.contains(0));
- EXPECT_TRUE(out.contains(UINT32_MAX));
+ EXPECT_TRUE(out.contains(1));
+ EXPECT_TRUE(out.contains(9999999));
}
{ // SINGLE64
uint64_t i = static_cast<uint64_t>(UINT32_MAX) + 1;
@@ -1005,19 +1040,88 @@ TEST(BitmapValueTest, bitmap_serde) {
put_fixed64_le(&expect_buffer, i);
EXPECT_EQ(expect_buffer, buffer);
- BitmapValue out(buffer.data());
+ BitmapValue out = deserialize_bitmap_from_string(buffer);
EXPECT_EQ(1, out.cardinality());
EXPECT_TRUE(out.contains(i));
}
{ // BITMAP64
- BitmapValue bitmap64({0, static_cast<uint64_t>(UINT32_MAX) + 1});
+ BitmapValue bitmap64({1, static_cast<uint64_t>(UINT32_MAX) + 1});
std::string buffer = convert_bitmap_to_string(bitmap64);
+ EXPECT_EQ(decode_base64_to_string(
+
"BAIAAAAAOjAAAAEAAAAAAAAAEAAAAAEAAQAAADowAAABAAAAAAAAABAAAAAAAA=="),
+ buffer);
- BitmapValue out(buffer.data());
+ BitmapValue out = deserialize_bitmap_from_string(buffer);
EXPECT_EQ(2, out.cardinality());
- EXPECT_TRUE(out.contains(0));
+ EXPECT_TRUE(out.contains(1));
EXPECT_TRUE(out.contains(static_cast<uint64_t>(UINT32_MAX) + 1));
}
+ { // BITMAP32_V2
+ config::bitmap_serialize_version = 2;
+ std::vector<uint64_t> bits32 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
10,
+ 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
21,
+ 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
32};
+ BitmapValue bitmap32(bits32);
+ std::string buffer = convert_bitmap_to_string(bitmap32);
+ EXPECT_EQ(decode_base64_to_string("DAI7MAAAAQAAIAABAAAAIAA="), buffer);
+
+ BitmapValue out = deserialize_bitmap_from_string(buffer);
+ check_bitmap_values(out, bits32);
+ }
+ { // BITMAP64_V2
+ std::vector<uint64_t> bits64 {
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
+ 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21,
+ 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
static_cast<uint64_t>(UINT32_MAX) + 1};
+ BitmapValue bitmap64(bits64);
+ std::string buffer = convert_bitmap_to_string(bitmap64);
+
EXPECT_EQ(decode_base64_to_string("DQIAAAAAAjswAAABAAAfAAEAAAAfAAEAAAABAQAAAAAAAAA="),
+ buffer);
+
+ BitmapValue out = deserialize_bitmap_from_string(buffer);
+ check_bitmap_values(out, bits64);
+ }
+ { // SET serialization is semantic-only because hash-set iteration order
is not stable.
+ config::enable_set_in_bitmap_value = true;
+ config::bitmap_serialize_version = 1;
+ std::vector<uint64_t> values {UINT64_MAX, 0,
static_cast<uint64_t>(UINT32_MAX) + 1, 7, 1};
+ BitmapValue set_bitmap;
+ for (auto value : values) {
+ set_bitmap.add(value);
+ }
+ EXPECT_EQ(set_bitmap.get_type_code(), BitmapTypeCode::SET);
+
+ std::string buffer = convert_bitmap_to_string(set_bitmap);
+ BitmapValue out = deserialize_bitmap_from_string(buffer);
+ check_bitmap_values(out, values);
+ }
+ { // Deserializing historical SET bytes must not depend on their
serialized payload order.
+ config::enable_set_in_bitmap_value = true;
+ std::vector<uint64_t> values {1, 9999999};
+ std::string set_v1_insert_order =
decode_base64_to_string("BQIBAAAAAAAAAH+WmAAAAAAA");
+ std::string set_v1_reverse_order =
decode_base64_to_string("BQJ/lpgAAAAAAAEAAAAAAAAA");
+ std::string set_v2_reverse_order =
decode_base64_to_string("CgIAAAB/lpgAAAAAAAEAAAAAAAAA");
+
+ BitmapValue set_in_insert_order =
deserialize_bitmap_from_string(set_v1_insert_order);
+ BitmapValue set_in_reverse_order =
deserialize_bitmap_from_string(set_v1_reverse_order);
+ BitmapValue set_v2_in_reverse_order =
deserialize_bitmap_from_string(set_v2_reverse_order);
+
+ check_bitmap_values(set_in_insert_order, values);
+ check_bitmap_values(set_in_reverse_order, values);
+ check_bitmap_values(set_v2_in_reverse_order, values);
+ }
+ { // Historical SET bytes with mixed 32-bit and 64-bit values must remain
readable.
+ config::enable_set_in_bitmap_value = true;
+ std::vector<uint64_t> values {UINT64_MAX, 0,
static_cast<uint64_t>(UINT32_MAX) + 1, 7, 1};
+ std::string set_v1_mixed =
+
decode_base64_to_string("BQX//////////wAAAAAAAAAAAAAAAAEAAAAHAAAAAAAAAAEAAAAAAAAA");
+
+ BitmapValue out = deserialize_bitmap_from_string(set_v1_mixed);
+ check_bitmap_values(out, values);
+ }
+
+ config::enable_set_in_bitmap_value = old_enable_set;
+ config::bitmap_serialize_version = old_serialize_version;
}
// Forked from CRoaring's UT of Roaring64Map
diff --git a/be/test/exprs/function/function_bitmap_test.cpp
b/be/test/exprs/function/function_bitmap_test.cpp
index 6e18e10d2e3..95813289427 100644
--- a/be/test/exprs/function/function_bitmap_test.cpp
+++ b/be/test/exprs/function/function_bitmap_test.cpp
@@ -29,6 +29,7 @@
#include "core/types.h"
#include "core/value/bitmap_value.h"
#include "exprs/function/function_test_util.h"
+#include "util/url_coding.h"
namespace doris {
@@ -98,6 +99,14 @@ namespace doris::config {
DECLARE_Bool(enable_set_in_bitmap_value);
}
+std::string encode_bitmap_to_base64(const BitmapValue& bitmap) {
+ std::string serialized(bitmap.getSizeInBytes(), '\0');
+ bitmap.write_to(serialized.data());
+ std::string encoded;
+ base64_encode(serialized, &encoded);
+ return encoded;
+}
+
TEST(function_bitmap_test, function_bitmap_to_base64) {
config::Register::Field field("bool", "enable_set_in_bitmap_value",
&config::enable_set_in_bitmap_value,
"false", false);
@@ -175,10 +184,8 @@ TEST(function_bitmap_test, function_bitmap_to_base64) {
{
DataSet data_set = {
{{&bitmap32_1}, std::string("AQEAAAA=")},
- {{&bitmap32_2}, std::string("BQIBAAAAAAAAAH+WmAAAAAAA")},
{{&bitmap32_3}, std::string("AjswAAABAAAgAAEAAAAgAA==")},
{{&bitmap64_1}, std::string("AwAAAAABAAAA")},
- {{&bitmap64_2}, std::string("BQIAAAAAAQAAAAEAAAAAAAAA")},
{{&bitmap64_3},
std::string("BAIAAAAAOzAAAAEAAB8AAQAAAB8AAQAAADowAAABAAAAAAAAABAAAAAAAA==")},
{{&empty_bitmap}, std::string("AA==")},
@@ -188,13 +195,20 @@ TEST(function_bitmap_test, function_bitmap_to_base64) {
}
{
- std::string base64("BQQAAAAAAAAAAAEAAAAAAAAAAgAAAAAAAAADAAAAAAAAAA==");
+ // SET serialization depends on hash-set iteration order, so do not
pin a hard-coded
+ // base64 string here. The contract is that the current serialization
can round-trip.
+ DataSet data_set = {{{&bitmap32_2},
encode_bitmap_to_base64(bitmap32_2)},
+ {{&bitmap64_2},
encode_bitmap_to_base64(bitmap64_2)}};
+ static_cast<void>(check_function<DataTypeString, true>(func_name,
input_types, data_set));
+ }
+
+ {
BitmapValue bitmap;
bitmap.add(0);
bitmap.add(1);
bitmap.add(2);
bitmap.add(3);
- DataSet data_set = {{{&bitmap}, base64}};
+ DataSet data_set = {{{&bitmap}, encode_bitmap_to_base64(bitmap)}};
static_cast<void>(check_function<DataTypeString, true>(func_name,
input_types, data_set));
}
@@ -291,6 +305,15 @@ TEST(function_bitmap_test, function_bitmap_from_base64) {
DataSet data_set = {{{base64}, &bitmap}};
static_cast<void>(check_function<DataTypeBitMap, true>(func_name,
input_types, data_set));
}
+ {
+ // Historical SET bytes may store the same elements in different
orders; decoding must
+ // preserve the bitmap contents instead of relying on the serialized
payload order.
+ std::string set_v1_reversed("BQJ/lpgAAAAAAAEAAAAAAAAA");
+ std::string set_v2_reversed("CgIAAAB/lpgAAAAAAAEAAAAAAAAA");
+ BitmapValue bitmap({1, 9999999});
+ DataSet data_set = {{{set_v1_reversed}, &bitmap}, {{set_v2_reversed},
&bitmap}};
+ static_cast<void>(check_function<DataTypeBitMap, true>(func_name,
input_types, data_set));
+ }
{
EXPECT_TRUE(config::set_config("bitmap_serialize_version", "1", false,
true).ok());
diff --git
a/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out
b/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out
index 766b791c4ea..a162f381a4a 100644
---
a/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out
+++
b/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out
@@ -537,6 +537,18 @@ true
-- !sql_bitmap_base64_nereids8 --
1
+-- !sql_bitmap_base64_history_set_v1 --
+1,9999999
+
+-- !sql_bitmap_base64_history_set_v1_reversed --
+1,9999999
+
+-- !sql_bitmap_base64_history_set_v2_reversed --
+1,9999999
+
+-- !sql_bitmap_base64_history_set_v1_mixed --
+0,1,7,4294967296,18446744073709551615
+
-- !sql_bitmap_base64_nereids9 --
0
diff --git
a/regression-test/suites/doc/sql-manual/sql-functions/doc_bitmap_functions_test.groovy
b/regression-test/suites/doc/sql-manual/sql-functions/doc_bitmap_functions_test.groovy
index 06247016e7d..627ee6b8411 100644
---
a/regression-test/suites/doc/sql-manual/sql-functions/doc_bitmap_functions_test.groovy
+++
b/regression-test/suites/doc/sql-manual/sql-functions/doc_bitmap_functions_test.groovy
@@ -31,21 +31,18 @@ suite("doc_bitmap_functions_test") {
''')
def bitmapToBase64Single = sql '''
- SELECT bitmap_to_base64(to_bitmap(1));
+ SELECT
bitmap_to_string(bitmap_from_base64(bitmap_to_base64(to_bitmap(1))));
'''
- def bitmapToBase64SingleExpected = ["BQEBAAAAAAAAAA==", "AQEAAAA="]
-
assertTrue(bitmapToBase64SingleExpected.contains(bitmapToBase64Single[0][0].toString()),
+ assertEquals("1", bitmapToBase64Single[0][0].toString(),
"Unexpected bitmap_to_base64 single result:
${bitmapToBase64Single}")
testFoldConst('''
SELECT bitmap_to_base64(to_bitmap(1));
''')
def bitmapToBase64Multi = sql '''
- SELECT bitmap_to_base64(bitmap_from_string("1,9999999"));
+ SELECT
bitmap_to_string(bitmap_from_base64(bitmap_to_base64(bitmap_from_string("1,9999999"))));
'''
- def bitmapToBase64MultiExpected = ["BQIBAAAAAAAAAH+WmAAAAAAA",
- "AjowAAACAAAAAAAAAJgAAAAYAAAAGgAAAAEAf5Y="]
-
assertTrue(bitmapToBase64MultiExpected.contains(bitmapToBase64Multi[0][0].toString()),
+ assertEquals("1,9999999", bitmapToBase64Multi[0][0].toString(),
"Unexpected bitmap_to_base64 multi result: ${bitmapToBase64Multi}")
testFoldConst('''
SELECT bitmap_to_base64(bitmap_from_string("1,9999999"));
diff --git
a/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy
b/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy
index e09f6e02506..fabfef875d3 100644
---
a/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy
+++
b/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy
@@ -777,6 +777,12 @@ suite("test_bitmap_function") {
qt_sql_bitmap_base64_nereids6 """ select
bitmap_to_string(bitmap_from_base64(bitmap_to_base64(bitmap_from_string("0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32"))))
"""
qt_sql_bitmap_base64_nereids7 """ select
bitmap_to_string(bitmap_from_base64(bitmap_to_base64(bitmap_from_string("0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,4294967296"))))
"""
qt_sql_bitmap_base64_nereids8 """ select
bitmap_to_string(bitmap_from_base64(bitmap_to_base64(to_bitmap(1)))); """
+ // SET serialization order is not stable because it depends on hash-set
iteration order.
+ // These fixed historical bytes verify from_base64 semantics rather than
bitmap_to_base64 bytes.
+ qt_sql_bitmap_base64_history_set_v1 """ select
bitmap_to_string(bitmap_from_base64("BQIBAAAAAAAAAH+WmAAAAAAA")); """
+ qt_sql_bitmap_base64_history_set_v1_reversed """ select
bitmap_to_string(bitmap_from_base64("BQJ/lpgAAAAAAAEAAAAAAAAA")); """
+ qt_sql_bitmap_base64_history_set_v2_reversed """ select
bitmap_to_string(bitmap_from_base64("CgIAAAB/lpgAAAAAAAEAAAAAAAAA")); """
+ qt_sql_bitmap_base64_history_set_v1_mixed """ select
bitmap_to_string(bitmap_from_base64("BQX//////////wAAAAAAAAAAAAAAAAEAAAAHAAAAAAAAAAEAAAAAAAAA"));
"""
// test nullable
sql """ DROP TABLE IF EXISTS test_bitmap_base64 """
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
