This is an automated email from the ASF dual-hosted git repository.
panxiaolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1b8120880f2 [Fix](function) fix skip null value for some date
functions (#59616)
1b8120880f2 is described below
commit 1b8120880f274016b74da7600019f7ef767c41da
Author: zclllyybb <[email protected]>
AuthorDate: Thu Jan 8 22:07:43 2026 +0800
[Fix](function) fix skip null value for some date functions (#59616)
fix skip null value for some date functions
---
.../function_date_or_datetime_computation.cpp | 8 +-
.../function_date_or_datetime_computation.h | 148 +++++++--------------
.../function_date_or_datetime_to_string.cpp | 49 +++++--
.../vec/functions/function_needs_to_handle_null.h | 6 +
.../{Map.out => nereids_scalar_fn_map.out} | 8 ++
.../data/nereids_p0/join/test_outer_join.out | 5 +
.../{Map.groovy => nereids_scalar_fn_map.groovy} | 21 +++
.../suites/nereids_p0/join/test_outer_join.groovy | 29 +++-
.../test_date_function_v2.groovy | 4 +-
9 files changed, 159 insertions(+), 119 deletions(-)
diff --git a/be/src/vec/functions/function_date_or_datetime_computation.cpp
b/be/src/vec/functions/function_date_or_datetime_computation.cpp
index 266324f28d6..3495c16d1bb 100644
--- a/be/src/vec/functions/function_date_or_datetime_computation.cpp
+++ b/be/src/vec/functions/function_date_or_datetime_computation.cpp
@@ -119,10 +119,10 @@ using FunctionDatetimeSubQuarters =
using FunctionDatetimeSubYears =
FunctionDateOrDateTimeComputation<SubtractYearsImpl<TYPE_DATETIMEV2>>;
-using FunctionAddTimeDatetime = FunctionAddTime<TYPE_DATETIMEV2, AddTimeImpl>;
-using FunctionAddTimeTime = FunctionAddTime<TYPE_TIMEV2, AddTimeImpl>;
-using FunctionSubTimeDatetime = FunctionAddTime<TYPE_DATETIMEV2, SubTimeImpl>;
-using FunctionSubTimeTime = FunctionAddTime<TYPE_TIMEV2, SubTimeImpl>;
+using FunctionAddTimeDatetime = FunctionNeedsToHandleNull<AddTimeDatetimeImpl,
TYPE_DATETIMEV2>;
+using FunctionAddTimeTime = FunctionNeedsToHandleNull<AddTimeTimeImpl,
TYPE_TIMEV2>;
+using FunctionSubTimeDatetime = FunctionNeedsToHandleNull<SubTimeDatetimeImpl,
TYPE_DATETIMEV2>;
+using FunctionSubTimeTime = FunctionNeedsToHandleNull<SubTimeTimeImpl,
TYPE_TIMEV2>;
#define FUNCTION_TIME_DIFF(NAME, IMPL, TYPE) using NAME##_##TYPE =
FunctionTimeDiff<IMPL<TYPE>>;
diff --git a/be/src/vec/functions/function_date_or_datetime_computation.h
b/be/src/vec/functions/function_date_or_datetime_computation.h
index e2953c9ab9e..2e31144b453 100644
--- a/be/src/vec/functions/function_date_or_datetime_computation.h
+++ b/be/src/vec/functions/function_date_or_datetime_computation.h
@@ -23,6 +23,7 @@
#include <cstddef>
#include <cstdint>
#include <memory>
+#include <string>
#include <string_view>
#include <type_traits>
#include <utility>
@@ -1819,118 +1820,71 @@ public:
}
};
-struct AddTimeImpl {
- static constexpr auto name = "add_time";
- static bool is_negative() { return false; }
-};
-
-struct SubTimeImpl {
- static constexpr auto name = "sub_time";
- static bool is_negative() { return true; }
-};
-
-template <PrimitiveType PType, typename Impl>
-class FunctionAddTime : public IFunction {
+template <PrimitiveType PType, bool IsNegative>
+class AddTimeImplBase {
public:
- static constexpr auto name = Impl::name;
- static constexpr PrimitiveType ReturnType = PType;
- static constexpr PrimitiveType ArgType1 = PType;
- static constexpr PrimitiveType ArgType2 = TYPE_TIMEV2;
- using ColumnType1 = typename PrimitiveTypeTraits<PType>::ColumnType;
- using ColumnType2 = typename PrimitiveTypeTraits<TYPE_TIMEV2>::ColumnType;
+ static constexpr auto name = IsNegative ? "sub_time" : "add_time";
using InputType1 = typename
PrimitiveTypeTraits<PType>::DataType::FieldType;
using InputType2 = typename
PrimitiveTypeTraits<TYPE_TIMEV2>::DataType::FieldType;
- using ReturnNativeType = InputType1;
- using ReturnDataType = typename PrimitiveTypeTraits<PType>::DataType;
+ using ResultColumnType = typename PrimitiveTypeTraits<PType>::ColumnType;
- String get_name() const override { return name; }
- size_t get_number_of_arguments() const override { return 2; }
- DataTypes get_variadic_argument_types_impl() const override {
+ static size_t get_number_of_arguments() { return 2; }
+ static bool is_variadic() { return true; }
+ static DataTypes get_variadic_argument_types_impl() {
return {std::make_shared<typename
PrimitiveTypeTraits<PType>::DataType>(),
std::make_shared<typename
PrimitiveTypeTraits<TYPE_TIMEV2>::DataType>()};
}
- DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments)
const override {
- return std::make_shared<ReturnDataType>();
- }
-
- ReturnNativeType compute(const InputType1& arg1, const InputType2& arg2)
const {
- if constexpr (PType == TYPE_DATETIMEV2) {
- DateV2Value<DateTimeV2ValueType> dtv1 =
- binary_cast<InputType1,
DateV2Value<DateTimeV2ValueType>>(arg1);
- auto tv2 = static_cast<TimeValue::TimeType>(arg2);
- TimeInterval interval(TimeUnit::MICROSECOND, tv2,
Impl::is_negative());
- bool out_range = dtv1.template
date_add_interval<TimeUnit::MICROSECOND>(interval);
- if (UNLIKELY(!out_range)) {
- throw Exception(ErrorCode::INVALID_ARGUMENT,
- "datetime value is out of range in function
{}", name);
- }
- return binary_cast<DateV2Value<DateTimeV2ValueType>,
ReturnNativeType>(dtv1);
- } else if constexpr (PType == TYPE_TIMEV2) {
- auto tv1 = static_cast<TimeValue::TimeType>(arg1);
- auto tv2 = static_cast<TimeValue::TimeType>(arg2);
- double res = TimeValue::limit_with_bound(Impl::is_negative() ? tv1
- tv2 : tv1 + tv2);
- return res;
- } else {
- throw Exception(ErrorCode::FATAL_ERROR, "not support type for
function {}", name);
+ static DataTypePtr get_return_type_impl(const DataTypes& arguments) {
+ if (arguments[0]->is_nullable() || arguments[1]->is_nullable()) {
+ return make_nullable(std::make_shared<typename
PrimitiveTypeTraits<PType>::DataType>());
}
+ return std::make_shared<typename
PrimitiveTypeTraits<PType>::DataType>();
}
- static FunctionPtr create() { return std::make_shared<FunctionAddTime>(); }
-
- Status execute_impl(FunctionContext* context, Block& block, const
ColumnNumbers& arguments,
- uint32_t result, size_t input_rows_count) const
override {
- DCHECK_EQ(arguments.size(), 2);
- const auto& [left_col, left_const] =
- unpack_if_const(block.get_by_position(arguments[0]).column);
- const auto& [right_col, right_const] =
- unpack_if_const(block.get_by_position(arguments[1]).column);
- ColumnPtr nest_col1 = remove_nullable(left_col);
- ColumnPtr nest_col2 = remove_nullable(right_col);
- auto res = ColumnVector<ReturnType>::create(input_rows_count, 0);
-
- if (left_const) {
- execute_constant_vector(assert_cast<const
ColumnType1&>(*nest_col1).get_element(0),
- assert_cast<const
ColumnType2&>(*nest_col2).get_data(),
- res->get_data(), input_rows_count);
- } else if (right_const) {
- execute_vector_constant(assert_cast<const
ColumnType1&>(*nest_col1).get_data(),
- assert_cast<const
ColumnType2&>(*nest_col2).get_element(0),
- res->get_data(), input_rows_count);
- } else {
- execute_vector_vector(assert_cast<const
ColumnType1&>(*nest_col1).get_data(),
- assert_cast<const
ColumnType2&>(*nest_col2).get_data(),
- res->get_data(), input_rows_count);
- }
-
- block.replace_by_position(result, std::move(res));
- return Status::OK();
- }
- void execute_vector_vector(const PaddedPODArray<InputType1>& left_col,
- const PaddedPODArray<InputType2>& right_col,
- PaddedPODArray<ReturnNativeType>& res_data,
- size_t input_rows_count) const {
- for (size_t i = 0; i < input_rows_count; ++i) {
- res_data[i] = compute(left_col[i], right_col[i]);
- }
- }
+ static void execute(const std::vector<ColumnWithConstAndNullMap>&
cols_info,
+ typename ResultColumnType::MutablePtr& res_col,
+ PaddedPODArray<UInt8>& res_null_map_data, size_t
input_rows_count) {
+ const auto& left_data =
+ assert_cast<const
ResultColumnType*>(cols_info[0].nested_col)->get_data();
+ const auto& right_data =
+ assert_cast<const
ColumnVector<TYPE_TIMEV2>*>(cols_info[1].nested_col)->get_data();
- void execute_vector_constant(const PaddedPODArray<InputType1>& left_col,
- const InputType2 right_value,
- PaddedPODArray<ReturnNativeType>& res_data,
- size_t input_rows_count) const {
for (size_t i = 0; i < input_rows_count; ++i) {
- res_data[i] = compute(left_col[i], right_value);
- }
- }
+ if (cols_info[0].is_null_at(i) || cols_info[1].is_null_at(i)) {
+ res_col->insert_default();
+ res_null_map_data[i] = 1;
+ continue;
+ }
- void execute_constant_vector(const InputType1 left_value,
- const PaddedPODArray<InputType2>& right_col,
- PaddedPODArray<ReturnNativeType>& res_data,
- size_t input_rows_count) const {
- for (size_t i = 0; i < input_rows_count; ++i) {
- res_data[i] = compute(left_value, right_col[i]);
+ const auto& arg1 = left_data[index_check_const(i,
cols_info[0].is_const)];
+ const auto& arg2 = right_data[index_check_const(i,
cols_info[1].is_const)];
+
+ if constexpr (PType == TYPE_DATETIMEV2) {
+ DateV2Value<DateTimeV2ValueType> dtv1 =
+ binary_cast<InputType1,
DateV2Value<DateTimeV2ValueType>>(arg1);
+ auto tv2 = static_cast<TimeValue::TimeType>(arg2);
+ TimeInterval interval(TimeUnit::MICROSECOND, tv2, IsNegative);
+ bool out_range = dtv1.template
date_add_interval<TimeUnit::MICROSECOND>(interval);
+ if (!out_range) [[unlikely]] {
+ throw_invalid_strings(name, dtv1.to_string(),
std::to_string(arg2));
+ }
+ res_col->insert_value(
+ binary_cast<DateV2Value<DateTimeV2ValueType>,
InputType1>(dtv1));
+ } else if constexpr (PType == TYPE_TIMEV2) {
+ auto tv1 = static_cast<TimeValue::TimeType>(arg1);
+ auto tv2 = static_cast<TimeValue::TimeType>(arg2);
+ double res = TimeValue::limit_with_bound(IsNegative ? tv1 -
tv2 : tv1 + tv2);
+ res_col->insert_value(res);
+ } else {
+ throw Exception(ErrorCode::FATAL_ERROR, "not support type for
function {}", name);
+ }
}
}
};
+
+using AddTimeDatetimeImpl = AddTimeImplBase<TYPE_DATETIMEV2, false>;
+using AddTimeTimeImpl = AddTimeImplBase<TYPE_TIMEV2, false>;
+using SubTimeDatetimeImpl = AddTimeImplBase<TYPE_DATETIMEV2, true>;
+using SubTimeTimeImpl = AddTimeImplBase<TYPE_TIMEV2, true>;
#include "common/compile_check_avoid_end.h"
} // namespace doris::vectorized
diff --git a/be/src/vec/functions/function_date_or_datetime_to_string.cpp
b/be/src/vec/functions/function_date_or_datetime_to_string.cpp
index 6b068ccf159..8e38aaf0b7f 100644
--- a/be/src/vec/functions/function_date_or_datetime_to_string.cpp
+++ b/be/src/vec/functions/function_date_or_datetime_to_string.cpp
@@ -77,8 +77,6 @@ public:
return {};
}
- ColumnNumbers get_arguments_that_are_always_constant() const override {
return {1}; }
-
// In ICU, Week_array: {"", "Sunday", "Monday", ..., "Saturday"}, size = 8
// Month_array: {"January", "February", ..., "December"}, size = 12
static constexpr size_t DAY_NUM_IN_ICU = 8;
@@ -140,28 +138,48 @@ public:
return IFunction::open(context, scope);
}
+ bool use_default_implementation_for_nulls() const override { return false;
}
+
Status execute_impl(FunctionContext* context, Block& block, const
ColumnNumbers& arguments,
uint32_t result, size_t input_rows_count) const
override {
const ColumnPtr source_col =
block.get_by_position(arguments[0]).column;
+ const NullMap* null_map = nullptr;
+ ColumnPtr actual_col = source_col;
+
+ if (is_column_nullable(*source_col)) {
+ const auto* nullable_col =
check_and_get_column<ColumnNullable>(source_col.get());
+ actual_col = nullable_col->get_nested_column_ptr();
+ null_map = &nullable_col->get_null_map_data();
+ }
+
const auto* sources =
-
check_and_get_column<ColumnVector<Transform::OpArgType>>(source_col.get());
- auto col_res = ColumnString::create();
+
check_and_get_column<ColumnVector<Transform::OpArgType>>(actual_col.get());
+ if (!sources) [[unlikely]] {
+ return Status::FatalError("Illegal column {} of first argument of
function {}",
+
block.get_by_position(arguments[0]).column->get_name(), name);
+ }
- // Support all input of datetime is valind to make sure not null return
- if (sources) {
- vector(context, sources->get_data(), col_res->get_chars(),
col_res->get_offsets());
- block.replace_by_position(result, std::move(col_res));
+ auto col_res = ColumnString::create();
+ vector(context, sources->get_data(), col_res->get_chars(),
col_res->get_offsets(),
+ null_map);
+
+ if (null_map) {
+ const auto* nullable_col =
check_and_get_column<ColumnNullable>(source_col.get());
+ block.replace_by_position(
+ result,
+ ColumnNullable::create(std::move(col_res),
+
nullable_col->get_null_map_column_ptr()->clone_resized(
+ input_rows_count)));
} else {
- return Status::InternalError("Illegal column {} of first argument
of function {}",
-
block.get_by_position(arguments[0]).column->get_name(),
- name);
+ block.replace_by_position(result, std::move(col_res));
}
return Status::OK();
}
private:
static void vector(FunctionContext* context, const
PaddedPODArray<NativeType>& ts,
- ColumnString::Chars& res_data, ColumnString::Offsets&
res_offsets) {
+ ColumnString::Chars& res_data, ColumnString::Offsets&
res_offsets,
+ const NullMap* null_map = nullptr) {
const auto len = ts.size();
res_data.resize(len * Transform::max_size);
res_offsets.resize(len);
@@ -176,7 +194,12 @@ private:
names_ptr = state->month_names;
}
- for (int i = 0; i < len; ++i) {
+ for (size_t i = 0; i < len; ++i) {
+ if (null_map && (*null_map)[i]) {
+ res_offsets[i] = cast_set<UInt32>(offset);
+ continue;
+ }
+
const auto& t = ts[i];
const auto date_time_value = binary_cast<NativeType, DateType>(t);
res_offsets[i] = cast_set<UInt32>(
diff --git a/be/src/vec/functions/function_needs_to_handle_null.h
b/be/src/vec/functions/function_needs_to_handle_null.h
index 0e60da92e68..5ac6584924d 100644
--- a/be/src/vec/functions/function_needs_to_handle_null.h
+++ b/be/src/vec/functions/function_needs_to_handle_null.h
@@ -53,6 +53,12 @@ public:
}
return false;
}
+ DataTypes get_variadic_argument_types_impl() const override {
+ if constexpr (requires { Impl::get_variadic_argument_types_impl(); }) {
+ return Impl::get_variadic_argument_types_impl();
+ }
+ return {};
+ }
bool use_default_implementation_for_nulls() const override { return false;
}
diff --git a/regression-test/data/nereids_function_p0/scalar_function/Map.out
b/regression-test/data/nereids_function_p0/scalar_function/nereids_scalar_fn_map.out
similarity index 99%
rename from regression-test/data/nereids_function_p0/scalar_function/Map.out
rename to
regression-test/data/nereids_function_p0/scalar_function/nereids_scalar_fn_map.out
index c3eb0f6c071..b5e55783718 100644
--- a/regression-test/data/nereids_function_p0/scalar_function/Map.out
+++
b/regression-test/data/nereids_function_p0/scalar_function/nereids_scalar_fn_map.out
@@ -3722,3 +3722,11 @@ false
-- !sql --
{"{"zip":10001}":"{"city":"NY"}", "{"code":10002}":"{"state":"NY"}"}
+-- !element_at_tint_date_dayname --
+\N
+Sunday
+
+-- !element_at_tint_date_dayname_notnull --
+\N
+Saturday
+
diff --git a/regression-test/data/nereids_p0/join/test_outer_join.out
b/regression-test/data/nereids_p0/join/test_outer_join.out
index 9ae6731b347..60c7db8f729 100644
--- a/regression-test/data/nereids_p0/join/test_outer_join.out
+++ b/regression-test/data/nereids_p0/join/test_outer_join.out
@@ -10,3 +10,8 @@
-- !join --
+-- !join --
+\N \N
+1 2023-12-18T23:30:01
+2 2023-12-15T23:30:01
+
diff --git
a/regression-test/suites/nereids_function_p0/scalar_function/Map.groovy
b/regression-test/suites/nereids_function_p0/scalar_function/nereids_scalar_fn_map.groovy
similarity index 97%
rename from
regression-test/suites/nereids_function_p0/scalar_function/Map.groovy
rename to
regression-test/suites/nereids_function_p0/scalar_function/nereids_scalar_fn_map.groovy
index 7c686a08baa..343b298a872 100644
--- a/regression-test/suites/nereids_function_p0/scalar_function/Map.groovy
+++
b/regression-test/suites/nereids_function_p0/scalar_function/nereids_scalar_fn_map.groovy
@@ -35,6 +35,7 @@ suite("nereids_scalar_fn_map") {
order_qt_element_at_str_tint """ select km_str_tint[kstr] from fn_test
"""
order_qt_element_at_date_tint """ select km_date_tint[kdt] from fn_test
"""
order_qt_element_at_dtm_tint """ select km_dtm_tint[kdtm] from fn_test
"""
+
order_qt_element_at_bool_tint_notnull """ select km_bool_tint[kbool]
from fn_test_not_nullable """
order_qt_element_at_tint_tint_notnull """ select km_tint_tint[ktint]
from fn_test_not_nullable """
order_qt_element_at_sint_tint_notnull """ select km_sint_tint[ksint]
from fn_test_not_nullable """
@@ -314,4 +315,24 @@ suite("nereids_scalar_fn_map") {
qt_sql "select map('postal_code', 10001, 'area_code', 10002, 'zip_plus_4',
10003)"
qt_sql "select map('{\"zip\":10001}', '{\"city\":\"NY\"}',
'{\"code\":10002}', '{\"state\":\"NY\"}')"
+ sql """ drop table if exists fn_test_element_at_map_date """
+ sql """
+ create table fn_test_element_at_map_date (
+ id int null,
+ m map<tinyint, date> null
+ ) engine=olap
+ distributed by hash(id) buckets 1
+ properties('replication_num' = '1')
+ """
+ sql """
+ insert into fn_test_element_at_map_date values
+ (1, map(1,null, 2,'2023-12-16')),
+ (2, map(1,'2023-01-01', 2,null));
+ """
+ order_qt_element_at_tint_date_dayname """
+ select dayname(element_at(m, 1)) from fn_test_element_at_map_date
+ """
+ order_qt_element_at_tint_date_dayname_notnull """
+ select dayname(element_at(m, 2)) from fn_test_element_at_map_date
+ """
}
\ No newline at end of file
diff --git a/regression-test/suites/nereids_p0/join/test_outer_join.groovy
b/regression-test/suites/nereids_p0/join/test_outer_join.groovy
index 9264f3fc579..f97c580d84c 100644
--- a/regression-test/suites/nereids_p0/join/test_outer_join.groovy
+++ b/regression-test/suites/nereids_p0/join/test_outer_join.groovy
@@ -84,7 +84,30 @@ suite("test_outer_join", "nereids_p0") {
contains "FULL OUTER JOIN(PARTITIONED)"
}
- sql "DROP TABLE IF EXISTS ${tbl1}"
- sql "DROP TABLE IF EXISTS ${tbl2}"
- sql "DROP TABLE IF EXISTS ${tbl3}"
+ def tbl4 = "test_outer_join4"
+ def tbl5 = "test_outer_join5"
+
+ sql "DROP TABLE IF EXISTS test_outer_join4"
+ sql """
+ CREATE TABLE test_outer_join4 (
+ k INT,
+ d DATEV2
+ )
+ DUPLICATE KEY(k)
+ DISTRIBUTED BY HASH(k) BUCKETS 1 PROPERTIES ("replication_num" = "1");
+ """
+ sql "DROP TABLE IF EXISTS test_outer_join5"
+ sql """
+ CREATE TABLE test_outer_join5 (
+ k INT
+ )
+ DUPLICATE KEY(k)
+ DISTRIBUTED BY HASH(k) BUCKETS 1 PROPERTIES ("replication_num" = "1");
+ """
+ sql """INSERT INTO test_outer_join4 VALUES (1, '2023-12-18'), (2,
'2023-12-15');"""
+ sql """INSERT INTO test_outer_join5 VALUES (1), (2), (3);"""
+
+ order_qt_join """
+ SELECT k, add_time(d, '23:30:01') FROM test_outer_join4 RIGHT JOIN
test_outer_join5 USING (k) ORDER BY k;
+ """
}
diff --git
a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function_v2.groovy
b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function_v2.groovy
index d96a6f7c770..977acfc36a2 100644
---
a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function_v2.groovy
+++
b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function_v2.groovy
@@ -175,7 +175,7 @@ suite("test_date_function_v2") {
test{
sql("select add_time('9999-12-29 00:00:00', '122:35:22.123456');")
- exception "datetime value is out of range in function add_time"
+ exception "Operation add_time of"
}
qt_sql_subtime1("select sub_time('2023-10-14 00:00:00', '22:35:22');")
@@ -193,7 +193,7 @@ suite("test_date_function_v2") {
test{
sql("select sub_time('0000-01-01 00:00:00', '122:35:22.123456');")
- exception "datetime value is out of range in function sub_time"
+ exception "Operation sub_time of"
}
//test computetimearithmetic regular
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
