This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new ec9d7f18b41 branch-4.2: [fix](be) Preserve valid types for date
defaults #69120 (#69191)
ec9d7f18b41 is described below
commit ec9d7f18b41efe644dec6df58960751e46f0684e
Author: Chenyang Sun <[email protected]>
AuthorDate: Sun Oct 18 11:40:35 2026 +0800
branch-4.2: [fix](be) Preserve valid types for date defaults #69120 (#69191)
### What problem does this PR solve?
Issue Number: None
Related PR: #69120 (master), #56079 (introduced the invalid default
type)
Problem Summary: PR #56079 introduced `VecDateTimeValue::DEFAULT_VALUE`
with the compute-layer `TYPE_DATETIME` enum where the storage-layer
`TimeType` was required. The value is stored in a three-bit field, so it
becomes an invalid type and can serialize to an empty string. This PR
uses valid time types and defines separate `DATE_DEFAULT_VALUE` and
`DATETIME_DEFAULT_VALUE` constants.
This is a backport of #69120 onto branch-4.2.
### Backport notes
The only conflict is
`be/test/exec/runtime_filter/runtime_filter_partition_pruner_test.cpp`:
neither that test nor `RuntimeFilterPartitionPruner` exists on
branch-4.2, so the test-side adaptation for it is dropped. The other 7
files apply unchanged, and the resulting diff is byte-identical to the
change on master.
### Release note
Fix legacy DATE/DATETIME default values so they carry valid time types
and serialize canonically instead of an empty string.
### Check List (For Author)
- Test
- [ ] Regression test
- [x] Unit Test: 6 targeted BE unit tests passed on a branch-4.2 Release
build (see the validation comment below)
- [x] Manual test: full `./build.sh --be` on branch-4.2 with this pick
applied succeeds with no compiler warnings or errors
- [ ] No need to test or manual test. Explain why <!-- At least one must
be included. -->
- Behavior changed:
- [x] Yes. Default DATE/DATETIME values now carry valid time types and
serialize canonically.
- Does this need documentation?
- [x] No.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
be/src/core/column/column_vector.h | 8 +-
be/src/core/value/vdatetime_value.cpp | 4 +-
be/src/core/value/vdatetime_value.h | 19 ++--
be/test/core/column/common_column_test.h | 114 +++++++++++----------
.../data_type/data_type_insert_default_test.cpp | 12 ++-
be/test/core/value/vdatetime_value_test.cpp | 14 +++
.../runtime_filter/runtime_filter_wrapper_test.cpp | 16 +--
7 files changed, 114 insertions(+), 73 deletions(-)
diff --git a/be/src/core/column/column_vector.h
b/be/src/core/column/column_vector.h
index 9debd7e0554..d82c91c3242 100644
--- a/be/src/core/column/column_vector.h
+++ b/be/src/core/column/column_vector.h
@@ -410,7 +410,13 @@ public:
// when run function which need_replace_null_data_to_default, use the
value far from 0 to avoid
// raise errors for null cell.
static value_type default_value() {
- if constexpr (is_date_type(T) || T == PrimitiveType::TYPE_TIMESTAMPTZ)
{
+ if constexpr (T == PrimitiveType::TYPE_DATE) {
+ return PrimitiveTypeTraits<T>::CppType::DATE_DEFAULT_VALUE;
+ } else if constexpr (T == PrimitiveType::TYPE_DATETIME) {
+ return PrimitiveTypeTraits<T>::CppType::DATETIME_DEFAULT_VALUE;
+ } else if constexpr (T == PrimitiveType::TYPE_DATEV2 ||
+ T == PrimitiveType::TYPE_DATETIMEV2 ||
+ T == PrimitiveType::TYPE_TIMESTAMPTZ) {
return PrimitiveTypeTraits<T>::CppType::DEFAULT_VALUE;
} else {
return value_type();
diff --git a/be/src/core/value/vdatetime_value.cpp
b/be/src/core/value/vdatetime_value.cpp
index f02378c8ad5..77a138558e4 100644
--- a/be/src/core/value/vdatetime_value.cpp
+++ b/be/src/core/value/vdatetime_value.cpp
@@ -180,12 +180,12 @@ bool VecDateTimeValue::from_date_int64(int64_t value) {
return check_range_and_set_time(year, month, day, hour, minute, second,
_type);
}
-void VecDateTimeValue::set_zero(int type) {
+void VecDateTimeValue::set_zero(TimeType type) {
memset(this, 0, sizeof(*this));
_type = type;
}
-void VecDateTimeValue::set_type(int type) {
+void VecDateTimeValue::set_type(TimeType type) {
_type = type;
if (type == TIME_DATE) {
_hour = 0;
diff --git a/be/src/core/value/vdatetime_value.h
b/be/src/core/value/vdatetime_value.h
index a94fff3b671..9c7693f61b1 100644
--- a/be/src/core/value/vdatetime_value.h
+++ b/be/src/core/value/vdatetime_value.h
@@ -290,7 +290,8 @@ public:
_year(0) {} // before int128 16 bytes ---> after int64 8 bytes
const static VecDateTimeValue FIRST_DAY;
- const static VecDateTimeValue DEFAULT_VALUE;
+ const static VecDateTimeValue DATE_DEFAULT_VALUE;
+ const static VecDateTimeValue DATETIME_DEFAULT_VALUE;
// The data format of DATE/DATETIME is different in storage layer and
execute layer.
// So we should use different creator to get data from value.
@@ -727,7 +728,7 @@ public:
return (daynr() - rhs.daynr()) * SECOND_PER_HOUR * HOUR_PER_DAY +
time_part_diff(rhs);
}
- void set_type(int type);
+ void set_type(TimeType type);
int type() const { return _type; }
@@ -766,7 +767,7 @@ public:
bool get_date_from_daynr(uint64_t);
// reset 0
- void reset_zero_by_type(int type) { set_zero(type); }
+ void reset_zero_by_type(TimeType type) { set_zero(type); }
private:
// Used to make sure sizeof VecDateTimeValue
@@ -793,7 +794,7 @@ private:
bool disable_lut = false);
// Helper to set max, min, zero
- void set_zero(int type);
+ void set_zero(TimeType type);
void set_max_time(bool neg);
bool from_date_format_str(const char* format, int format_len, const char*
value,
@@ -809,7 +810,7 @@ private:
uint8_t _month;
uint16_t _year;
- VecDateTimeValue(uint8_t neg, uint8_t type, uint8_t hour, uint8_t minute,
uint8_t second,
+ VecDateTimeValue(uint8_t neg, TimeType type, uint8_t hour, uint8_t minute,
uint8_t second,
uint16_t year, uint8_t month, uint8_t day)
: _neg(neg),
_type(type),
@@ -825,9 +826,11 @@
static_assert(std::is_trivially_destructible_v<VecDateTimeValue>,
"VecDateTimeValue must be trivial destructible");
static_assert(std::is_trivially_copyable_v<VecDateTimeValue>,
"VecDateTimeValue must be trivial copyable");
-inline const VecDateTimeValue VecDateTimeValue::FIRST_DAY(false,
TYPE_DATETIME, 0, 0, 0, 1, 1, 1);
-inline const VecDateTimeValue VecDateTimeValue::DEFAULT_VALUE(false,
TYPE_DATETIME, 0, 0, 0, 1970,
- 1, 1);
+inline const VecDateTimeValue VecDateTimeValue::FIRST_DAY(false,
TIME_DATETIME, 0, 0, 0, 1, 1, 1);
+inline const VecDateTimeValue VecDateTimeValue::DATE_DEFAULT_VALUE(false,
TIME_DATE, 0, 0, 0, 1970,
+ 1, 1);
+inline const VecDateTimeValue VecDateTimeValue::DATETIME_DEFAULT_VALUE(false,
TIME_DATETIME, 0, 0,
+ 0,
1970, 1, 1);
template <typename T>
class DateV2Value {
diff --git a/be/test/core/column/common_column_test.h
b/be/test/core/column/common_column_test.h
index 3e41c9402d5..fb03add6a21 100644
--- a/be/test/core/column/common_column_test.h
+++ b/be/test/core/column/common_column_test.h
@@ -2947,8 +2947,13 @@ auto assert_column_vector_insert_default_callback =
[](auto x,
T(PrimitiveTypeTraits<PType>::CppType::DEFAULT_VALUE.to_date_int_val()));
} else if constexpr (PType == PrimitiveType::TYPE_DATE ||
PType == PrimitiveType::TYPE_DATETIME) {
- EXPECT_EQ(col_vec_target->get_element(i),
- T(PrimitiveTypeTraits<PType>::CppType::DEFAULT_VALUE));
+ if constexpr (PType == PrimitiveType::TYPE_DATE) {
+ EXPECT_EQ(col_vec_target->get_element(i),
+
T(PrimitiveTypeTraits<PType>::CppType::DATE_DEFAULT_VALUE));
+ } else {
+ EXPECT_EQ(col_vec_target->get_element(i),
+
T(PrimitiveTypeTraits<PType>::CppType::DATETIME_DEFAULT_VALUE));
+ }
} else if constexpr (PType == PrimitiveType::TYPE_DECIMALV2) {
EXPECT_EQ(col_vec_target->get_element(i), DecimalV2Value());
} else {
@@ -2960,59 +2965,64 @@ auto assert_column_vector_insert_default_callback =
[](auto x,
};
template <PrimitiveType PType>
-auto assert_column_vector_insert_many_defaults_callback =
- [](auto x, const MutableColumnPtr& source_column) {
- using T = decltype(x);
- using ColumnVecType = std::conditional_t<
- std::is_same_v<T, ColumnString>, ColumnString,
- std::conditional_t<std::is_same_v<T, ColumnString64>,
ColumnString64,
- std::conditional_t<IsDecimalNumber<T>,
ColumnDecimal<PType>,
-
ColumnVector<PType>>>>;
- std::vector<size_t> insert_vals_count = {0, 10, 1000};
- auto* col_vec_src =
assert_cast<ColumnVecType*>(source_column.get());
- auto src_size = source_column->size();
+auto assert_column_vector_insert_many_defaults_callback = [](auto x, const
MutableColumnPtr&
+
source_column) {
+ using T = decltype(x);
+ using ColumnVecType = std::conditional_t<
+ std::is_same_v<T, ColumnString>, ColumnString,
+ std::conditional_t<std::is_same_v<T, ColumnString64>,
ColumnString64,
+ std::conditional_t<IsDecimalNumber<T>,
ColumnDecimal<PType>,
+ ColumnVector<PType>>>>;
+ std::vector<size_t> insert_vals_count = {0, 10, 1000};
+ auto* col_vec_src = assert_cast<ColumnVecType*>(source_column.get());
+ auto src_size = source_column->size();
- auto test_func = [&](size_t clone_count) {
- for (auto n : insert_vals_count) {
- size_t actual_clone_count = std::min(clone_count,
src_size);
- auto target_column =
source_column->clone_resized(actual_clone_count);
- auto* col_vec_target =
assert_cast<ColumnVecType*>(target_column.get());
- col_vec_target->insert_many_defaults(n);
- auto target_size = col_vec_target->size();
- EXPECT_EQ(target_size, actual_clone_count + n);
- size_t i = 0;
- for (; i < actual_clone_count; ++i) {
- if constexpr (std::is_same_v<T, ColumnString> ||
- std::is_same_v<T, ColumnString64>) {
- EXPECT_EQ(col_vec_target->get_data_at(i),
col_vec_src->get_data_at(i));
- } else {
- EXPECT_EQ(col_vec_target->get_element(i),
col_vec_src->get_element(i));
- }
- }
- for (; i < target_size; ++i) {
- if constexpr (std::is_same_v<T, ColumnString> ||
- std::is_same_v<T, ColumnString64>) {
-
EXPECT_EQ(col_vec_target->get_data_at(i).to_string(), "");
- } else if constexpr (PType ==
PrimitiveType::TYPE_DATEV2 ||
- PType ==
PrimitiveType::TYPE_DATETIMEV2) {
- EXPECT_EQ(col_vec_target->get_element(i),
-
T(PrimitiveTypeTraits<PType>::CppType::DEFAULT_VALUE
- .to_date_int_val()));
- } else if constexpr (PType == PrimitiveType::TYPE_DATE
||
- PType ==
PrimitiveType::TYPE_DATETIME) {
- EXPECT_EQ(col_vec_target->get_element(i),
-
T(PrimitiveTypeTraits<PType>::CppType::DEFAULT_VALUE));
- } else if constexpr (PType ==
PrimitiveType::TYPE_DECIMALV2) {
- EXPECT_EQ(col_vec_target->get_element(i),
DecimalV2Value());
- } else {
- EXPECT_EQ(col_vec_target->get_element(i), T {});
- }
+ auto test_func = [&](size_t clone_count) {
+ for (auto n : insert_vals_count) {
+ size_t actual_clone_count = std::min(clone_count, src_size);
+ auto target_column =
source_column->clone_resized(actual_clone_count);
+ auto* col_vec_target =
assert_cast<ColumnVecType*>(target_column.get());
+ col_vec_target->insert_many_defaults(n);
+ auto target_size = col_vec_target->size();
+ EXPECT_EQ(target_size, actual_clone_count + n);
+ size_t i = 0;
+ for (; i < actual_clone_count; ++i) {
+ if constexpr (std::is_same_v<T, ColumnString> ||
+ std::is_same_v<T, ColumnString64>) {
+ EXPECT_EQ(col_vec_target->get_data_at(i),
col_vec_src->get_data_at(i));
+ } else {
+ EXPECT_EQ(col_vec_target->get_element(i),
col_vec_src->get_element(i));
+ }
+ }
+ for (; i < target_size; ++i) {
+ if constexpr (std::is_same_v<T, ColumnString> ||
+ std::is_same_v<T, ColumnString64>) {
+ EXPECT_EQ(col_vec_target->get_data_at(i).to_string(), "");
+ } else if constexpr (PType == PrimitiveType::TYPE_DATEV2 ||
+ PType == PrimitiveType::TYPE_DATETIMEV2) {
+ EXPECT_EQ(col_vec_target->get_element(i),
+
T(PrimitiveTypeTraits<PType>::CppType::DEFAULT_VALUE
+ .to_date_int_val()));
+ } else if constexpr (PType == PrimitiveType::TYPE_DATE ||
+ PType == PrimitiveType::TYPE_DATETIME) {
+ if constexpr (PType == PrimitiveType::TYPE_DATE) {
+ EXPECT_EQ(col_vec_target->get_element(i),
+
T(PrimitiveTypeTraits<PType>::CppType::DATE_DEFAULT_VALUE));
+ } else {
+ EXPECT_EQ(col_vec_target->get_element(i),
+
T(PrimitiveTypeTraits<PType>::CppType::DATETIME_DEFAULT_VALUE));
}
+ } else if constexpr (PType == PrimitiveType::TYPE_DECIMALV2) {
+ EXPECT_EQ(col_vec_target->get_element(i),
DecimalV2Value());
+ } else {
+ EXPECT_EQ(col_vec_target->get_element(i), T {});
}
- };
- test_func(0);
- test_func(10);
- };
+ }
+ }
+ };
+ test_func(0);
+ test_func(10);
+};
template <PrimitiveType PType>
auto assert_column_vector_get_bool_callback = [](auto x, const
MutableColumnPtr& source_column) {
diff --git a/be/test/core/data_type/data_type_insert_default_test.cpp
b/be/test/core/data_type/data_type_insert_default_test.cpp
index 1a7928139de..4b49bab21d7 100644
--- a/be/test/core/data_type/data_type_insert_default_test.cpp
+++ b/be/test/core/data_type/data_type_insert_default_test.cpp
@@ -166,14 +166,22 @@ TEST(DataTypeInsertDefaultTest, AllTypeFamilies) {
SCOPED_TRACE("date");
auto actual = get_default_field(make_simple_type(TYPE_DATE));
ASSERT_EQ(actual.get_type(), TYPE_DATE);
- EXPECT_EQ(actual.get<TYPE_DATE>(), VecDateTimeValue::DEFAULT_VALUE);
+ EXPECT_EQ(actual.get<TYPE_DATE>(),
VecDateTimeValue::DATE_DEFAULT_VALUE);
+ EXPECT_EQ(actual.get<TYPE_DATE>().type(), TIME_DATE);
+ char buffer[64];
+ actual.get<TYPE_DATE>().to_string(buffer);
+ EXPECT_STREQ(buffer, "1970-01-01");
}
{
SCOPED_TRACE("datetime");
auto actual = get_default_field(make_simple_type(TYPE_DATETIME));
ASSERT_EQ(actual.get_type(), TYPE_DATETIME);
- EXPECT_EQ(actual.get<TYPE_DATETIME>(),
VecDateTimeValue::DEFAULT_VALUE);
+ EXPECT_EQ(actual.get<TYPE_DATETIME>(),
VecDateTimeValue::DATETIME_DEFAULT_VALUE);
+ EXPECT_EQ(actual.get<TYPE_DATETIME>().type(), TIME_DATETIME);
+ char buffer[64];
+ actual.get<TYPE_DATETIME>().to_string(buffer);
+ EXPECT_STREQ(buffer, "1970-01-01 00:00:00");
}
{
diff --git a/be/test/core/value/vdatetime_value_test.cpp
b/be/test/core/value/vdatetime_value_test.cpp
index 6b3cd7488ae..bc1faa9d611 100644
--- a/be/test/core/value/vdatetime_value_test.cpp
+++ b/be/test/core/value/vdatetime_value_test.cpp
@@ -30,6 +30,20 @@
namespace doris {
+TEST(VDateTimeValueTest, LegacyDefaultsHaveValidTimeTypes) {
+ EXPECT_EQ(VecDateTimeValue::FIRST_DAY.type(), TIME_DATETIME);
+ EXPECT_EQ(VecDateTimeValue::DATE_DEFAULT_VALUE.type(), TIME_DATE);
+ EXPECT_EQ(VecDateTimeValue::DATETIME_DEFAULT_VALUE.type(), TIME_DATETIME);
+
+ char buffer[64];
+ VecDateTimeValue::FIRST_DAY.to_string(buffer);
+ EXPECT_STREQ(buffer, "0001-01-01 00:00:00");
+ VecDateTimeValue::DATE_DEFAULT_VALUE.to_string(buffer);
+ EXPECT_STREQ(buffer, "1970-01-01");
+ VecDateTimeValue::DATETIME_DEFAULT_VALUE.to_string(buffer);
+ EXPECT_STREQ(buffer, "1970-01-01 00:00:00");
+}
+
TEST(VDateTimeValueTest, date_v2_to_uint32_test) {
uint16_t year = 2022;
uint8_t month = 5;
diff --git a/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
b/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
index 3e954f69528..27a9e6507f5 100644
--- a/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
+++ b/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
@@ -231,10 +231,10 @@ TEST_F(RuntimeFilterWrapperTest, TestInAssign) {
APPLY_FOR_PRIMITIVE_BASE_TYPE(TYPE_DATEV2);
APPLY_FOR_PRIMITIVE_BASE_TYPE(TYPE_DATETIMEV2);
APPLY_FOR_PRIMITIVE_BASE_TYPE(TYPE_TIMESTAMP_NS);
- APPLY_FOR_PRIMITIVE_TYPE(TYPE_DATETIME, VecDateTimeValue(0, 3, 0, 0, 0,
2020, 1, 1),
- VecDateTimeValue(0, 3, 0, 0, 0, 2020, 1, 2));
- APPLY_FOR_PRIMITIVE_TYPE(TYPE_DATE, VecDateTimeValue(0, 2, 0, 0, 0, 2020,
1, 1),
- VecDateTimeValue(0, 2, 0, 0, 0, 2020, 1, 2));
+ APPLY_FOR_PRIMITIVE_TYPE(TYPE_DATETIME, VecDateTimeValue(0, TIME_DATETIME,
0, 0, 0, 2020, 1, 1),
+ VecDateTimeValue(0, TIME_DATETIME, 0, 0, 0, 2020,
1, 2));
+ APPLY_FOR_PRIMITIVE_TYPE(TYPE_DATE, VecDateTimeValue(0, TIME_DATE, 0, 0,
0, 2020, 1, 1),
+ VecDateTimeValue(0, TIME_DATE, 0, 0, 0, 2020, 1,
2));
APPLY_FOR_PRIMITIVE_TYPE(TYPE_DECIMALV2, DecimalV2Value(1, 1),
DecimalV2Value(1, 2));
APPLY_FOR_PRIMITIVE_BASE_TYPE(TYPE_DECIMAL32);
APPLY_FOR_PRIMITIVE_BASE_TYPE(TYPE_DECIMAL64);
@@ -311,10 +311,10 @@ TEST_F(RuntimeFilterWrapperTest, TestMinMaxAssign) {
APPLY_FOR_PRIMITIVE_BASE_TYPE(TYPE_DATEV2);
APPLY_FOR_PRIMITIVE_BASE_TYPE(TYPE_DATETIMEV2);
APPLY_FOR_PRIMITIVE_BASE_TYPE(TYPE_TIMESTAMP_NS);
- APPLY_FOR_PRIMITIVE_TYPE(TYPE_DATETIME, VecDateTimeValue(0, 3, 0, 0, 0,
2020, 1, 1),
- VecDateTimeValue(0, 3, 0, 0, 0, 2020, 1, 2));
- APPLY_FOR_PRIMITIVE_TYPE(TYPE_DATE, VecDateTimeValue(0, 2, 0, 0, 0, 2020,
1, 1),
- VecDateTimeValue(0, 2, 0, 0, 0, 2020, 1, 2));
+ APPLY_FOR_PRIMITIVE_TYPE(TYPE_DATETIME, VecDateTimeValue(0, TIME_DATETIME,
0, 0, 0, 2020, 1, 1),
+ VecDateTimeValue(0, TIME_DATETIME, 0, 0, 0, 2020,
1, 2));
+ APPLY_FOR_PRIMITIVE_TYPE(TYPE_DATE, VecDateTimeValue(0, TIME_DATE, 0, 0,
0, 2020, 1, 1),
+ VecDateTimeValue(0, TIME_DATE, 0, 0, 0, 2020, 1,
2));
APPLY_FOR_PRIMITIVE_TYPE(TYPE_DECIMALV2, DecimalV2Value(1, 1),
DecimalV2Value(1, 2));
APPLY_FOR_PRIMITIVE_BASE_TYPE(TYPE_DECIMAL32);
APPLY_FOR_PRIMITIVE_BASE_TYPE(TYPE_DECIMAL64);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
