This is an automated email from the ASF dual-hosted git repository.
zhangstar333 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 09d02c05282 [refactor]remove the redundant nullable value calculation
in function call (#41739)
09d02c05282 is described below
commit 09d02c0528283dfab8dc8199f284b85ca11eef43
Author: vlt <[email protected]>
AuthorDate: Mon Oct 21 15:56:00 2024 +0800
[refactor]remove the redundant nullable value calculation in function call
(#41739)
## Proposed changes
Issue Number: close #xxx
<!--Describe your changes.-->
---------
Co-authored-by: zhiqiang-hhhh <[email protected]>
---
be/src/vec/columns/column_nullable.cpp | 12 ---
be/src/vec/columns/column_nullable.h | 4 -
.../array/function_array_contains_all.cpp | 2 -
.../functions/array/function_array_enumerate.cpp | 1 -
.../vec/functions/array/function_array_range.cpp | 10 ---
be/src/vec/functions/function_bitmap.cpp | 10 ---
be/src/vec/functions/function_bitmap_min_or_max.h | 8 --
be/src/vec/functions/function_conv.cpp | 6 --
.../function_date_or_datetime_to_string.h | 9 --
be/src/vec/functions/function_encryption.cpp | 8 --
be/src/vec/functions/function_jsonb.cpp | 1 -
be/src/vec/functions/function_regexp.cpp | 5 --
be/src/vec/functions/function_timestamp.cpp | 32 --------
be/src/vec/functions/function_totype.h | 2 -
.../data/correctness/test_str_to_date.out | 95 +++++++++++++++++++---
.../sql_functions/math_functions/test_conv.out | 63 ++++++++++++++
.../data/nereids_syntax_p0/test_regexp_replace.out | 50 ++++++++++++
.../conditional_functions/test_json_parse.out | 93 +++++++++++++++++++++
.../suites/correctness/test_str_to_date.groovy | 62 +++++++++++---
.../sql_functions/math_functions/test_conv.groovy | 42 ++++++++--
.../nereids_syntax_p0/test_regexp_replace.groovy | 55 +++++++++++++
.../conditional_functions/test_json_parse.groovy | 69 ++++++++++++++++
22 files changed, 499 insertions(+), 140 deletions(-)
diff --git a/be/src/vec/columns/column_nullable.cpp
b/be/src/vec/columns/column_nullable.cpp
index 54f4bd3c118..8fffe1dfe65 100644
--- a/be/src/vec/columns/column_nullable.cpp
+++ b/be/src/vec/columns/column_nullable.cpp
@@ -640,16 +640,4 @@ ColumnPtr remove_nullable(const ColumnPtr& column) {
return column;
}
-void check_set_nullable(ColumnPtr& argument_column,
ColumnVector<UInt8>::MutablePtr& null_map,
- bool is_single) {
- if (const auto* nullable =
check_and_get_column<ColumnNullable>(*argument_column)) {
- // Danger: Here must dispose the null map data first! Because
- // argument_columns[i]=nullable->get_nested_column_ptr(); will release
the mem
- // of column nullable mem of null map
- VectorizedUtils::update_null_map(null_map->get_data(),
nullable->get_null_map_data(),
- is_single);
- argument_column = nullable->get_nested_column_ptr();
- }
-}
-
} // namespace doris::vectorized
diff --git a/be/src/vec/columns/column_nullable.h
b/be/src/vec/columns/column_nullable.h
index cdf7271a6c3..0eef017bc63 100644
--- a/be/src/vec/columns/column_nullable.h
+++ b/be/src/vec/columns/column_nullable.h
@@ -481,8 +481,4 @@ private:
ColumnPtr make_nullable(const ColumnPtr& column, bool is_nullable = false);
ColumnPtr remove_nullable(const ColumnPtr& column);
-// check if argument column is nullable. If so, extract its concrete column
and set null_map.
-//TODO: use this to replace inner usages.
-// is_single: whether null_map is null map of a ColumnConst
-void check_set_nullable(ColumnPtr&, ColumnVector<UInt8>::MutablePtr& null_map,
bool is_single);
} // namespace doris::vectorized
diff --git a/be/src/vec/functions/array/function_array_contains_all.cpp
b/be/src/vec/functions/array/function_array_contains_all.cpp
index f50ee81679b..de74b648aab 100644
--- a/be/src/vec/functions/array/function_array_contains_all.cpp
+++ b/be/src/vec/functions/array/function_array_contains_all.cpp
@@ -37,8 +37,6 @@ public:
String get_name() const override { return name; }
- bool use_default_implementation_for_nulls() const override { return true; }
-
bool is_variadic() const override { return false; }
size_t get_number_of_arguments() const override { return 2; }
diff --git a/be/src/vec/functions/array/function_array_enumerate.cpp
b/be/src/vec/functions/array/function_array_enumerate.cpp
index 1b8197ac96f..0ce927db897 100644
--- a/be/src/vec/functions/array/function_array_enumerate.cpp
+++ b/be/src/vec/functions/array/function_array_enumerate.cpp
@@ -56,7 +56,6 @@ public:
static constexpr auto name = "array_enumerate";
static FunctionPtr create() { return
std::make_shared<FunctionArrayEnumerate>(); }
String get_name() const override { return name; }
- bool use_default_implementation_for_nulls() const override { return true; }
size_t get_number_of_arguments() const override { return 1; }
DataTypePtr get_return_type_impl(const DataTypes& arguments) const
override {
const DataTypeArray* array_type =
diff --git a/be/src/vec/functions/array/function_array_range.cpp
b/be/src/vec/functions/array/function_array_range.cpp
index 1a5dd08aaac..1b71693f640 100644
--- a/be/src/vec/functions/array/function_array_range.cpp
+++ b/be/src/vec/functions/array/function_array_range.cpp
@@ -65,8 +65,6 @@ public:
bool is_variadic() const override { return true; }
- bool use_default_implementation_for_nulls() const override { return false;
}
-
size_t get_number_of_arguments() const override {
return get_variadic_argument_types_impl().size();
}
@@ -146,14 +144,6 @@ struct RangeImplUtil {
for (int i = 0; i < 3; ++i) {
argument_columns[i] =
block.get_by_position(arguments[i]).column->convert_to_full_column_if_const();
- if (auto* nullable =
check_and_get_column<ColumnNullable>(*argument_columns[i])) {
- // Danger: Here must dispose the null map data first! Because
- // argument_columns[i]=nullable->get_nested_column_ptr(); will
release the mem
- // of column nullable mem of null map
- VectorizedUtils::update_null_map(args_null_map->get_data(),
-
nullable->get_null_map_data());
- argument_columns[i] = nullable->get_nested_column_ptr();
- }
}
auto start_column =
assert_cast<const
ColumnVector<SourceDataType>*>(argument_columns[0].get());
diff --git a/be/src/vec/functions/function_bitmap.cpp
b/be/src/vec/functions/function_bitmap.cpp
index fb0e0471caa..b6072c05cf0 100644
--- a/be/src/vec/functions/function_bitmap.cpp
+++ b/be/src/vec/functions/function_bitmap.cpp
@@ -349,8 +349,6 @@ public:
size_t get_number_of_arguments() const override { return 1; }
- bool use_default_implementation_for_nulls() const override { return true; }
-
Status execute_impl(FunctionContext* context, Block& block, const
ColumnNumbers& arguments,
size_t result, size_t input_rows_count) const override
{
auto res_null_map = ColumnUInt8::create(input_rows_count, 0);
@@ -1148,8 +1146,6 @@ public:
size_t get_number_of_arguments() const override { return 3; }
- bool use_default_implementation_for_nulls() const override { return false;
}
-
Status execute_impl(FunctionContext* context, Block& block, const
ColumnNumbers& arguments,
size_t result, size_t input_rows_count) const override
{
DCHECK_EQ(arguments.size(), 3);
@@ -1168,10 +1164,6 @@ public:
default_preprocess_parameter_columns(argument_columns, col_const, {1,
2}, block, arguments);
- for (int i = 0; i < 3; i++) {
- check_set_nullable(argument_columns[i], res_null_map,
col_const[i]);
- }
-
auto bitmap_column = assert_cast<const
ColumnBitmap*>(argument_columns[0].get());
auto offset_column = assert_cast<const
ColumnVector<Int64>*>(argument_columns[1].get());
auto limit_column = assert_cast<const
ColumnVector<Int64>*>(argument_columns[2].get());
@@ -1207,8 +1199,6 @@ public:
size_t get_number_of_arguments() const override { return 1; }
- bool use_default_implementation_for_nulls() const override { return true; }
-
Status execute_impl(FunctionContext* context, Block& block, const
ColumnNumbers& arguments,
size_t result, size_t input_rows_count) const override
{
auto return_nested_type =
make_nullable(std::make_shared<DataTypeInt64>());
diff --git a/be/src/vec/functions/function_bitmap_min_or_max.h
b/be/src/vec/functions/function_bitmap_min_or_max.h
index 726fdb3b421..85cf9fc3f08 100644
--- a/be/src/vec/functions/function_bitmap_min_or_max.h
+++ b/be/src/vec/functions/function_bitmap_min_or_max.h
@@ -55,14 +55,6 @@ public:
auto result_null_map_column = ColumnUInt8::create(input_rows_count, 0);
ColumnPtr argument_column = block.get_by_position(arguments[0]).column;
- if (auto* nullable =
check_and_get_column<ColumnNullable>(*argument_column)) {
- // Danger: Here must dispose the null map data first! Because
- // argument_columns[i]=nullable->get_nested_column_ptr(); will
release the mem
- // of column nullable mem of null map
-
VectorizedUtils::update_null_map(result_null_map_column->get_data(),
- nullable->get_null_map_data());
- argument_column = nullable->get_nested_column_ptr();
- }
execute_straight(assert_cast<const
ColumnBitmap*>(argument_column.get()),
assert_cast<ColumnInt64*>(result_column.get()),
diff --git a/be/src/vec/functions/function_conv.cpp
b/be/src/vec/functions/function_conv.cpp
index 9db79b89993..3dbfd81e8a2 100644
--- a/be/src/vec/functions/function_conv.cpp
+++ b/be/src/vec/functions/function_conv.cpp
@@ -72,8 +72,6 @@ public:
return get_variadic_argument_types_impl().size();
}
- bool use_default_implementation_for_nulls() const override { return false;
}
-
Status execute_impl(FunctionContext* context, Block& block, const
ColumnNumbers& arguments,
size_t result, size_t input_rows_count) const override
{
auto result_column = ColumnString::create();
@@ -91,10 +89,6 @@ public:
default_preprocess_parameter_columns(argument_columns, col_const, {1,
2}, block, arguments);
- for (int i = 0; i < 3; i++) {
- check_set_nullable(argument_columns[i], result_null_map_column,
col_const[i]);
- }
-
if (col_const[1] && col_const[2]) {
execute_scalar_args(
context,
diff --git a/be/src/vec/functions/function_date_or_datetime_to_string.h
b/be/src/vec/functions/function_date_or_datetime_to_string.h
index 53cb0199721..6ffb67f9c34 100644
--- a/be/src/vec/functions/function_date_or_datetime_to_string.h
+++ b/be/src/vec/functions/function_date_or_datetime_to_string.h
@@ -65,7 +65,6 @@ public:
String get_name() const override { return name; }
size_t get_number_of_arguments() const override { return 1; }
- bool use_default_implementation_for_nulls() const override { return false;
}
DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments)
const override {
RETURN_REAL_TYPE_FOR_DATEV2_FUNCTION(DataTypeString);
@@ -95,14 +94,6 @@ public:
TransformerToStringOneArgument<Transform>::vector(
context, sources->get_data(), col_res->get_chars(),
col_res->get_offsets(),
null_map->get_data());
- if (const auto* nullable_col =
-
check_and_get_column<ColumnNullable>(source_col.get())) {
- NullMap& result_null_map =
assert_cast<ColumnUInt8&>(*null_map).get_data();
- const NullMap& src_null_map =
- assert_cast<const
ColumnUInt8&>(nullable_col->get_null_map_column())
- .get_data();
- VectorizedUtils::update_null_map(result_null_map,
src_null_map);
- }
block.replace_by_position(
result, ColumnNullable::create(std::move(col_res),
std::move(null_map)));
} else {
diff --git a/be/src/vec/functions/function_encryption.cpp
b/be/src/vec/functions/function_encryption.cpp
index d017f761897..9aaefc26a65 100644
--- a/be/src/vec/functions/function_encryption.cpp
+++ b/be/src/vec/functions/function_encryption.cpp
@@ -110,8 +110,6 @@ public:
return get_variadic_argument_types_impl().size();
}
- bool use_default_implementation_for_nulls() const override { return false;
}
-
Status execute_impl(FunctionContext* context, Block& block, const
ColumnNumbers& arguments,
size_t result, size_t input_rows_count) const override
{
return Impl::execute_impl_inner(context, block, arguments, result,
input_rows_count);
@@ -207,9 +205,6 @@ struct EncryptionAndDecryptTwoImpl {
default_preprocess_parameter_columns(argument_columns, col_const, {1,
2}, block, arguments);
- for (int i = 0; i < argument_size; i++) {
- check_set_nullable(argument_columns[i], result_null_map_column,
col_const[i]);
- }
auto& result_data = result_column->get_chars();
auto& result_offset = result_column->get_offsets();
result_offset.resize(input_rows_count);
@@ -327,9 +322,6 @@ struct EncryptionAndDecryptMultiImpl {
arguments);
}
- for (int i = 0; i < argument_size; i++) {
- check_set_nullable(argument_columns[i], result_null_map_column,
col_const[i]);
- }
auto& result_data = result_column->get_chars();
auto& result_offset = result_column->get_offsets();
result_offset.resize(input_rows_count);
diff --git a/be/src/vec/functions/function_jsonb.cpp
b/be/src/vec/functions/function_jsonb.cpp
index 45864a07bdd..3bfcec38ba0 100644
--- a/be/src/vec/functions/function_jsonb.cpp
+++ b/be/src/vec/functions/function_jsonb.cpp
@@ -1350,7 +1350,6 @@ public:
}
bool use_default_implementation_for_nulls() const override { return false;
}
-
Status execute_impl(FunctionContext* context, Block& block, const
ColumnNumbers& arguments,
size_t result, size_t input_rows_count) const override
{
return Impl::execute_impl(context, block, arguments, result,
input_rows_count);
diff --git a/be/src/vec/functions/function_regexp.cpp
b/be/src/vec/functions/function_regexp.cpp
index 525d99b6cc7..d4823523e57 100644
--- a/be/src/vec/functions/function_regexp.cpp
+++ b/be/src/vec/functions/function_regexp.cpp
@@ -391,8 +391,6 @@ public:
String get_name() const override { return name; }
- bool use_default_implementation_for_nulls() const override { return false;
}
-
size_t get_number_of_arguments() const override {
if constexpr (std::is_same_v<Impl, RegexpExtractAllImpl>) {
return 2;
@@ -455,9 +453,6 @@ public:
default_preprocess_parameter_columns(argument_columns, col_const,
{1, 2}, block,
arguments);
}
- for (int i = 0; i < argument_size; i++) {
- check_set_nullable(argument_columns[i], result_null_map,
col_const[i]);
- }
if constexpr (std::is_same_v<Impl, RegexpExtractAllImpl>) {
if (col_const[1]) {
diff --git a/be/src/vec/functions/function_timestamp.cpp
b/be/src/vec/functions/function_timestamp.cpp
index 428f8c2893f..7c2e62e5723 100644
--- a/be/src/vec/functions/function_timestamp.cpp
+++ b/be/src/vec/functions/function_timestamp.cpp
@@ -115,12 +115,9 @@ struct StrToDate {
ColumnPtr argument_columns[2] = {
col_const[0] ? static_cast<const
ColumnConst&>(*col0).convert_to_full_column()
: col0};
- check_set_nullable(argument_columns[0], null_map, col_const[0]);
- //TODO: when we set default implementation for nullable, the
check_set_nullable for arguments is useless. consider to remove it.
std::tie(argument_columns[1], col_const[1]) =
unpack_if_const(block.get_by_position(arguments[1]).column);
- check_set_nullable(argument_columns[1], null_map, col_const[1]);
auto specific_str_column = assert_cast<const
ColumnString*>(argument_columns[0].get());
auto specific_char_column = assert_cast<const
ColumnString*>(argument_columns[1].get());
@@ -265,11 +262,9 @@ struct MakeDateImpl {
ColumnPtr argument_columns[2] = {
col_const[0] ? static_cast<const
ColumnConst&>(*col0).convert_to_full_column()
: col0};
- check_set_nullable(argument_columns[0], null_map, col_const[0]);
std::tie(argument_columns[1], col_const[1]) =
unpack_if_const(block.get_by_position(arguments[1]).column);
- check_set_nullable(argument_columns[1], null_map, col_const[1]);
ColumnPtr res = nullptr;
WhichDataType
which(remove_nullable(block.get_by_position(result).type));
@@ -502,8 +497,6 @@ public:
size_t get_number_of_arguments() const override { return 1; }
- bool use_default_implementation_for_nulls() const override { return true; }
-
DataTypePtr get_return_type_impl(const DataTypes& arguments) const
override {
return make_nullable(std::make_shared<DataTypeDate>());
}
@@ -713,9 +706,6 @@ struct UnixTimeStampStrImpl {
auto& col_result_data = col_result->get_data();
auto& null_map_data = null_map->get_data();
- check_set_nullable(col_left, null_map, source_const);
- check_set_nullable(col_right, null_map, format_const);
-
const auto* col_source = assert_cast<const
ColumnString*>(col_left.get());
const auto* col_format = assert_cast<const
ColumnString*>(col_right.get());
for (int i = 0; i < input_rows_count; i++) {
@@ -772,10 +762,6 @@ public:
return Impl::get_variadic_argument_types();
}
- bool use_default_implementation_for_nulls() const override {
- return !static_cast<bool>(std::is_same_v<Impl, UnixTimeStampStrImpl>);
- }
-
Status execute_impl(FunctionContext* context, Block& block, const
ColumnNumbers& arguments,
size_t result, size_t input_rows_count) const override
{
return Impl::execute_impl(context, block, arguments, result,
input_rows_count);
@@ -849,8 +835,6 @@ public:
String get_name() const override { return name; }
- bool use_default_implementation_for_nulls() const override { return false;
}
-
size_t get_number_of_arguments() const override { return 1; }
bool is_variadic() const override { return true; }
@@ -907,14 +891,6 @@ struct LastDayImpl {
input_rows_count, null_map->get_data(),
data_col->get_data(),
static_cast<ResultColumnType*>(res_column->assume_mutable().get())->get_data());
- if (const auto* nullable_col =
check_and_get_column<ColumnNullable>(
- block.get_by_position(arguments[0]).column.get())) {
- NullMap& result_null_map =
assert_cast<ColumnUInt8&>(*null_map).get_data();
- const NullMap& src_null_map =
- assert_cast<const
ColumnUInt8&>(nullable_col->get_null_map_column())
- .get_data();
- VectorizedUtils::update_null_map(result_null_map,
src_null_map);
- }
block.replace_by_position(result,
ColumnNullable::create(res_column,
std::move(null_map)));
} else {
@@ -1019,14 +995,6 @@ struct MondayImpl {
input_rows_count, null_map->get_data(),
data_col->get_data(),
static_cast<ResultColumnType*>(res_column->assume_mutable().get())->get_data());
- if (const auto* nullable_col =
check_and_get_column<ColumnNullable>(
- block.get_by_position(arguments[0]).column.get())) {
- NullMap& result_null_map =
assert_cast<ColumnUInt8&>(*null_map).get_data();
- const NullMap& src_null_map =
- assert_cast<const
ColumnUInt8&>(nullable_col->get_null_map_column())
- .get_data();
- VectorizedUtils::update_null_map(result_null_map,
src_null_map);
- }
block.replace_by_position(result,
ColumnNullable::create(res_column,
std::move(null_map)));
} else {
diff --git a/be/src/vec/functions/function_totype.h
b/be/src/vec/functions/function_totype.h
index e86f5444299..3309fbcacda 100644
--- a/be/src/vec/functions/function_totype.h
+++ b/be/src/vec/functions/function_totype.h
@@ -322,7 +322,6 @@ public:
for (int i = 0; i < 2; ++i) {
std::tie(argument_columns[i], col_const[i]) =
unpack_if_const(block.get_by_position(arguments[i]).column);
- check_set_nullable(argument_columns[i], null_map, col_const[i]);
}
using ResultDataType = typename Impl<LeftDataType, RightDataType,
ResultDateType,
@@ -401,7 +400,6 @@ public:
for (int i = 0; i < 2; ++i) {
std::tie(argument_columns[i], col_const[i]) =
unpack_if_const(block.get_by_position(arguments[i]).column);
- check_set_nullable(argument_columns[i], null_map, col_const[i]);
}
auto res = Impl::ColumnType::create();
diff --git a/regression-test/data/correctness/test_str_to_date.out
b/regression-test/data/correctness/test_str_to_date.out
index c4326914a7c..dfbb90e234b 100644
--- a/regression-test/data/correctness/test_str_to_date.out
+++ b/regression-test/data/correctness/test_str_to_date.out
@@ -1,27 +1,100 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !select1 --
-2019-12-01 yyyy-MM-dd 2019-12-01T00:00
-20201203 yyyyMMdd 2020-12-03T00:00
-2020-12-03 11:45:14 yyyy-MM-dd HH:mm:ss 2020-12-03T11:45:14
+1 2019-12-01 yyyy-MM-dd 2019-12-01T00:00
+2 20201203 yyyyMMdd 2020-12-03T00:00
+3 2020-12-03 11:45:14 yyyy-MM-dd HH:mm:ss 2020-12-03T11:45:14
+4 \N yyyy-MM-dd \N
+5 2019-12-01 \N \N
+6 \N \N \N
+7 无效日期 yyyy-MM-dd \N
--- !select2 --
+-- !const_test1 --
2019-12-01
--- !select3 --
-2020-12-03
+-- !const_test2 --
+\N
--- !select4 --
-2020-12-03T11:45:14
+-- !const_test3 --
+\N
+
+-- !const_test4 --
+\N
+
+-- !const_test5 --
+\N
-- !short_1 --
2023-01-01
-- !short_2 --
-2023-12-01
+\N
-- !short_3 --
-2023-01-01
+\N
-- !short_4 --
-2020-02-01
+\N
+
+-- !select_from_table1 --
+1 2019-12-01
+2 \N
+3 2020-12-03
+4 \N
+5 2019-12-01
+6 \N
+7 \N
+
+-- !select_from_table2 --
+1 2019-12-01T00:00
+2 2020-12-03T00:00
+3 2020-12-03T11:45:14
+4 \N
+5 \N
+6 \N
+7 \N
+
+-- !check_fe --
+2019-12-01
+
+-- !check_be --
+2019-12-01
+
+-- !check_no_fold --
+2019-12-01
+
+-- !check_fe --
+\N
+
+-- !check_be --
+\N
+
+-- !check_no_fold --
+\N
+
+-- !check_fe --
+\N
+
+-- !check_be --
+\N
+
+-- !check_no_fold --
+\N
+
+-- !check_fe --
+\N
+
+-- !check_be --
+\N
+
+-- !check_no_fold --
+\N
+
+-- !check_fe --
+\N
+
+-- !check_be --
+\N
+
+-- !check_no_fold --
+\N
diff --git
a/regression-test/data/nereids_p0/sql_functions/math_functions/test_conv.out
b/regression-test/data/nereids_p0/sql_functions/math_functions/test_conv.out
index 95e67e726d6..db787d1daf1 100644
--- a/regression-test/data/nereids_p0/sql_functions/math_functions/test_conv.out
+++ b/regression-test/data/nereids_p0/sql_functions/math_functions/test_conv.out
@@ -12,5 +12,68 @@
18446744073709551615
-- !select5 --
+1100100 100
+\N \N
+11001000 200
+\N \N
+
+-- !select6 --
1 1.464868
+\N \N
+10 2.789
+11 3.14159
+
+-- !check_fe --
+\N
+
+-- !check_be --
+\N
+
+-- !check_no_fold --
+\N
+
+-- !check_fe --
+1111
+
+-- !check_be --
+1111
+
+-- !check_no_fold --
+1111
+
+-- !check_fe --
+\N
+
+-- !check_be --
+\N
+
+-- !check_no_fold --
+\N
+
+-- !check_fe --
+\N
+
+-- !check_be --
+\N
+
+-- !check_no_fold --
+\N
+
+-- !check_fe --
+\N
+
+-- !check_be --
+\N
+
+-- !check_no_fold --
+\N
+
+-- !check_fe --
+1111011
+
+-- !check_be --
+1111011
+
+-- !check_no_fold --
+1111011
diff --git a/regression-test/data/nereids_syntax_p0/test_regexp_replace.out
b/regression-test/data/nereids_syntax_p0/test_regexp_replace.out
new file mode 100644
index 00000000000..f06114138f0
--- /dev/null
+++ b/regression-test/data/nereids_syntax_p0/test_regexp_replace.out
@@ -0,0 +1,50 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !basic_replace --
+abcxyz
+
+-- !replace_chinese --
+汉123
+
+-- !check_fe --
+abcxyz
+
+-- !check_be --
+abcxyz
+
+-- !check_no_fold --
+abcxyz
+
+-- !check_fe --
+\N
+
+-- !check_be --
+\N
+
+-- !check_no_fold --
+\N
+
+-- !check_fe --
+\N
+
+-- !check_be --
+\N
+
+-- !check_no_fold --
+\N
+
+-- !check_fe --
+\N
+
+-- !check_be --
+\N
+
+-- !check_no_fold --
+\N
+
+-- !replace_in_table_chinese --
+1 abc123
+2 汉汉汉汉汉456
+3 Phone: 987-654-3210
+4 汉汉汉汉汉汉
+5 \N
+
diff --git
a/regression-test/data/query_p0/sql_functions/conditional_functions/test_json_parse.out
b/regression-test/data/query_p0/sql_functions/conditional_functions/test_json_parse.out
new file mode 100644
index 00000000000..05b650de68a
--- /dev/null
+++
b/regression-test/data/query_p0/sql_functions/conditional_functions/test_json_parse.out
@@ -0,0 +1,93 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !basic_parse --
+{"key":"value"}
+
+-- !invalid_json --
+\N
+
+-- !empty_string --
+\N
+
+-- !null_string --
+null
+
+-- !number_string --
+123
+
+-- !null_input --
+\N
+
+-- !parse_from_table --
+1 {"name":"Alice","age":30}
+2 {"name":"Bob","age":25}
+3 \N
+4 \N
+5 {"list":[1,2,3],"valid":true}
+6 \N
+7 null
+8 123
+9 [1,2,3]
+
+-- !check_fe --
+{"key":"value"}
+
+-- !check_be --
+{"key":"value"}
+
+-- !check_no_fold --
+{"key":"value"}
+
+-- !check_fe --
+\N
+
+-- !check_be --
+\N
+
+-- !check_no_fold --
+\N
+
+-- !check_fe --
+\N
+
+-- !check_be --
+\N
+
+-- !check_no_fold --
+\N
+
+-- !check_fe --
+\N
+
+-- !check_be --
+\N
+
+-- !check_no_fold --
+\N
+
+-- !check_fe --
+null
+
+-- !check_be --
+null
+
+-- !check_no_fold --
+null
+
+-- !check_fe --
+123
+
+-- !check_be --
+123
+
+-- !check_no_fold --
+123
+
+-- !check_fe --
+[1,2,3]
+
+-- !check_be --
+[1,2,3]
+
+-- !check_no_fold --
+[1,2,3]
+
diff --git a/regression-test/suites/correctness/test_str_to_date.groovy
b/regression-test/suites/correctness/test_str_to_date.groovy
index c2fe22f1f72..a45be14902b 100644
--- a/regression-test/suites/correctness/test_str_to_date.groovy
+++ b/regression-test/suites/correctness/test_str_to_date.groovy
@@ -35,21 +35,63 @@ suite("test_str_to_date") {
sql """ INSERT INTO test_str_to_date_db VALUES(1,'2019-12-01',
'yyyy-MM-dd');"""
sql """ INSERT INTO test_str_to_date_db VALUES(2,'20201203',
'yyyyMMdd');"""
sql """ INSERT INTO test_str_to_date_db VALUES(3,'2020-12-03 11:45:14',
'yyyy-MM-dd HH:mm:ss');"""
+ sql """ INSERT INTO test_str_to_date_db VALUES(4,null, 'yyyy-MM-dd');"""
+ sql """ INSERT INTO test_str_to_date_db VALUES(5,'2019-12-01', null);"""
+ sql """ INSERT INTO test_str_to_date_db VALUES(6,null, null);"""
+ sql """ INSERT INTO test_str_to_date_db VALUES(7,'无效日期', 'yyyy-MM-dd');"""
qt_select1 """
- select s1,s2,STR_TO_DATE(s1,s2) from test_str_to_date_db order by id;
+ select id, s1, s2, STR_TO_DATE(s1, s2) from test_str_to_date_db order
by id;
"""
- qt_select2 """
- SELECT STR_TO_DATE('2019-12-01', 'yyyy-MM-dd');
+
+ qt_const_test1 """
+ SELECT STR_TO_DATE('2019-12-01', 'yyyy-MM-dd');
+ """
+ qt_const_test2 """
+ SELECT STR_TO_DATE(null, 'yyyy-MM-dd');
+ """
+ qt_const_test3 """
+ SELECT STR_TO_DATE('2019-12-01', null);
"""
- qt_select3 """
- SELECT STR_TO_DATE('20201203', 'yyyyMMdd');
+ qt_const_test4 """
+ SELECT STR_TO_DATE(null, null);
"""
- qt_select4 """
- SELECT STR_TO_DATE('2020-12-03 11:45:14', 'yyyy-MM-dd HH:mm:ss');
+ qt_const_test5 """
+ SELECT STR_TO_DATE('无效日期', 'yyyy-MM-dd');
"""
+
qt_short_1 " select STR_TO_DATE('2023', '%Y') "
- qt_short_2 " select STR_TO_DATE('2023-12', '%Y-%m') "
- qt_short_3 " select STR_TO_DATE('2023-12', '%Y')"
- qt_short_4 " select STR_TO_DATE('2020%2', '%Y%%%m')"
+ qt_short_2 " select STR_TO_DATE(null, '%Y') "
+ qt_short_3 " select STR_TO_DATE('2023', null) "
+ qt_short_4 " select STR_TO_DATE(null, null) "
+
+ qt_select_from_table1 """
+ SELECT id, STR_TO_DATE(s1, '%Y-%m-%d') as result from
test_str_to_date_db order by id;
+ """
+ qt_select_from_table2 """
+ SELECT id, STR_TO_DATE(s1, s2) as result from test_str_to_date_db
order by id;
+ """
+
+ def re_fe
+ def re_be
+ def re_no_fold
+
+ def check_three_ways = { test_sql ->
+ re_fe = order_sql "select
/*+SET_VAR(enable_fold_constant_by_be=false)*/ ${test_sql}"
+ re_be = order_sql "select
/*+SET_VAR(enable_fold_constant_by_be=true)*/ ${test_sql}"
+ re_no_fold = order_sql "select
/*+SET_VAR(debug_skip_fold_constant=true)*/ ${test_sql}"
+ logger.info("check sql: ${test_sql}")
+ qt_check_fe "select /*+SET_VAR(enable_fold_constant_by_be=false)*/
${test_sql}"
+ qt_check_be "select /*+SET_VAR(enable_fold_constant_by_be=true)*/
${test_sql}"
+ qt_check_no_fold "select /*+SET_VAR(debug_skip_fold_constant=true)*/
${test_sql}"
+ assertEquals(re_fe, re_be)
+ assertEquals(re_fe, re_no_fold)
+ }
+
+ check_three_ways "STR_TO_DATE('2019-12-01', 'yyyy-MM-dd')"
+ check_three_ways "STR_TO_DATE(null, 'yyyy-MM-dd')"
+ check_three_ways "STR_TO_DATE('2019-12-01', null)"
+ check_three_ways "STR_TO_DATE(null, null)"
+ check_three_ways "STR_TO_DATE('无效日期', 'yyyy-MM-dd')"
}
+
diff --git
a/regression-test/suites/nereids_p0/sql_functions/math_functions/test_conv.groovy
b/regression-test/suites/nereids_p0/sql_functions/math_functions/test_conv.groovy
index b46aee62ba3..631df018fbd 100644
---
a/regression-test/suites/nereids_p0/sql_functions/math_functions/test_conv.groovy
+++
b/regression-test/suites/nereids_p0/sql_functions/math_functions/test_conv.groovy
@@ -16,22 +16,46 @@
// under the License.
suite("test_conv") {
- sql "SET enable_nereids_planner=true"
- sql "SET enable_fallback_to_original_planner=false"
qt_select "SELECT CONV(15,10,2)"
qt_select2 "select conv('ffffffffffffff', 24, 2);"
qt_select3 "select conv('-ff', 24, 2);"
- // if beyond the max value of uint64, use max_uint64 as res
qt_select4 "select conv('fffffffffffffffffffffffffffffffff', 24, 10);"
-
- sql """DROP TABLE IF EXISTS `test_tb`; """
- sql """ create table test_tb(int_1 int, float_2 float) PROPERTIES (
+
+ sql """DROP TABLE IF EXISTS `test_tb_with_null`; """
+ sql """ create table test_tb_with_null(int_1 int, float_2 float,
nullable_val varchar(16)) PROPERTIES (
"replication_num" = "1"
);
"""
+
+ sql """ insert into test_tb_with_null values(1, 1.464868, '100'),
+ (2, null, null),
+ (3, 2.789, '200'),
+ (4, 3.14159, null); """
+
+ qt_select5 """ select conv(nullable_val, 10, 2), nullable_val from
test_tb_with_null; """
+
+ qt_select6 """ select conv(float_2,10,2), float_2 from test_tb_with_null;
"""
- sql """ insert into test_tb values(1, 1.464868); """
+ def re_fe
+ def re_be
+ def re_no_fold
- qt_select5 """ select conv(float_2,10,2),float_2 from test_tb; """
-}
+ def check_three_ways = { test_sql ->
+ re_fe = order_sql
"select/*+SET_VAR(enable_fold_constant_by_be=false)*/ ${test_sql}"
+ re_be = order_sql "select/*+SET_VAR(enable_fold_constant_by_be=true)*/
${test_sql}"
+ re_no_fold = order_sql
"select/*+SET_VAR(debug_skip_fold_constant=true)*/ ${test_sql}"
+ logger.info("check on sql: ${test_sql}")
+ qt_check_fe "select/*+SET_VAR(enable_fold_constant_by_be=false)*/
${test_sql}"
+ qt_check_be "select/*+SET_VAR(enable_fold_constant_by_be=true)*/
${test_sql}"
+ qt_check_no_fold "select/*+SET_VAR(debug_skip_fold_constant=true)*/
${test_sql}"
+ assertEquals(re_fe, re_be)
+ assertEquals(re_fe, re_no_fold)
+ }
+ check_three_ways "conv(null, null, null)"
+ check_three_ways "conv(15, 10, 2)"
+ check_three_ways "conv(null, 10, 2)"
+ check_three_ways "conv(15, null, 2)"
+ check_three_ways "conv(15, 10, null)"
+ check_three_ways "conv('123', 10, 2)"
+}
\ No newline at end of file
diff --git
a/regression-test/suites/nereids_syntax_p0/test_regexp_replace.groovy
b/regression-test/suites/nereids_syntax_p0/test_regexp_replace.groovy
new file mode 100644
index 00000000000..33c6c845491
--- /dev/null
+++ b/regression-test/suites/nereids_syntax_p0/test_regexp_replace.groovy
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_regexp_replace") {
+ qt_basic_replace "SELECT regexp_replace('abc123', '123', 'xyz');"
+
+ qt_replace_chinese "SELECT regexp_replace('这是一个测试字符串123', '\\\\p{Han}+',
'汉');"
+
+ def re_fe
+ def re_be
+ def re_no_fold
+
+ def check_three_ways = { test_sql ->
+ re_fe = order_sql
"select/*+SET_VAR(enable_fold_constant_by_be=false)*/ ${test_sql}"
+ re_be = order_sql "select/*+SET_VAR(enable_fold_constant_by_be=true)*/
${test_sql}"
+ re_no_fold = order_sql
"select/*+SET_VAR(debug_skip_fold_constant=true)*/ ${test_sql}"
+ logger.info("check on sql: ${test_sql}")
+ qt_check_fe "select/*+SET_VAR(enable_fold_constant_by_be=false)*/
${test_sql}"
+ qt_check_be "select/*+SET_VAR(enable_fold_constant_by_be=true)*/
${test_sql}"
+ qt_check_no_fold "select/*+SET_VAR(debug_skip_fold_constant=true)*/
${test_sql}"
+ assertEquals(re_fe, re_be)
+ assertEquals(re_fe, re_no_fold)
+ }
+
+ check_three_ways "regexp_replace('abc123', '123', 'xyz')"
+ check_three_ways "regexp_replace(null, 'abc', 'def')"
+ check_three_ways "regexp_replace('abc123', null, 'xyz')"
+ check_three_ways "regexp_replace('abc123', '123', null)"
+
+ sql """DROP TABLE IF EXISTS `test_table_for_regexp`;"""
+ sql """CREATE TABLE test_table_for_regexp (id INT, name VARCHAR(100))
PROPERTIES ("replication_num"="1");"""
+
+ sql """INSERT INTO test_table_for_regexp VALUES
+ (1, 'abc123'),
+ (2, '测试字符串456'),
+ (3, 'Phone: 987-654-3210'),
+ (4, '这是一个测试'),
+ (5, null);"""
+
+ qt_replace_in_table_chinese """SELECT id, regexp_replace(name,
'\\\\p{Han}', '汉') as replaced_name FROM test_table_for_regexp;"""
+}
\ No newline at end of file
diff --git
a/regression-test/suites/query_p0/sql_functions/conditional_functions/test_json_parse.groovy
b/regression-test/suites/query_p0/sql_functions/conditional_functions/test_json_parse.groovy
new file mode 100644
index 00000000000..11674695037
--- /dev/null
+++
b/regression-test/suites/query_p0/sql_functions/conditional_functions/test_json_parse.groovy
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_json_parse") {
+ sql """DROP TABLE IF EXISTS `test_json_parse_table`;"""
+ sql """CREATE TABLE test_json_parse_table (
+ id INT,
+ json_str VARCHAR(500)
+ ) PROPERTIES ("replication_num"="1");"""
+
+ sql """INSERT INTO test_json_parse_table VALUES
+ (1, '{"name": "Alice", "age": 30}'),
+ (2, '{"name": "Bob", "age": 25}'),
+ (3, 'Invalid JSON String'),
+ (4, NULL),
+ (5, '{"list": [1, 2, 3], "valid": true}'),
+ (6, ''),
+ (7, 'null'),
+ (8, '123'),
+ (9, '[1, 2, 3]');"""
+
+ qt_basic_parse "SELECT json_parse_error_to_null('{\"key\": \"value\"}');"
+ qt_invalid_json "SELECT json_parse_error_to_null('Invalid JSON');"
+ qt_empty_string "SELECT json_parse_error_to_null('');"
+ qt_null_string "SELECT json_parse_error_to_null('null');"
+ qt_number_string "SELECT json_parse_error_to_null('123');"
+ qt_null_input "SELECT json_parse_error_to_null(NULL);"
+
+ qt_parse_from_table "SELECT id, json_parse_error_to_null(json_str) FROM
test_json_parse_table ORDER BY id;"
+
+ def re_fe
+ def re_be
+ def re_no_fold
+
+ def check_three_ways = { test_sql ->
+ re_fe = order_sql "SELECT
/*+SET_VAR(enable_fold_constant_by_be=false)*/ ${test_sql};"
+ re_be = order_sql "SELECT
/*+SET_VAR(enable_fold_constant_by_be=true)*/ ${test_sql};"
+ re_no_fold = order_sql "SELECT
/*+SET_VAR(debug_skip_fold_constant=true)*/ ${test_sql};"
+ logger.info("check on sql: ${test_sql}")
+ qt_check_fe "SELECT /*+SET_VAR(enable_fold_constant_by_be=false)*/
${test_sql};"
+ qt_check_be "SELECT /*+SET_VAR(enable_fold_constant_by_be=true)*/
${test_sql};"
+ qt_check_no_fold "SELECT /*+SET_VAR(debug_skip_fold_constant=true)*/
${test_sql};"
+ assertEquals(re_fe, re_be)
+ assertEquals(re_fe, re_no_fold)
+ }
+
+ check_three_ways "json_parse_error_to_null('{\"key\": \"value\"}')"
+ check_three_ways "json_parse_error_to_null('Invalid JSON')"
+ check_three_ways "json_parse_error_to_null(NULL)"
+ check_three_ways "json_parse_error_to_null('')"
+ check_three_ways "json_parse_error_to_null('null')"
+ check_three_ways "json_parse_error_to_null('123')"
+ check_three_ways "json_parse_error_to_null('[1, 2, 3]')"
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
