This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new edd9015de37 [branch-2.1](function) fix error result in auto partition
name (#41130) (#43977)
edd9015de37 is described below
commit edd9015de37fc6ab87ee97ede48a35e385fe6380
Author: Uniqueyou <[email protected]>
AuthorDate: Fri Nov 15 19:11:42 2024 +0800
[branch-2.1](function) fix error result in auto partition name (#41130)
(#43977)
pick https://github.com/apache/doris/pull/41130
https://github.com/apache/doris/pull/41372
---------
Co-authored-by: zhaochangle <[email protected]>
---
be/src/vec/functions/function_string.h | 63 ++++++++++++++--------
.../functions/scalar/AutoPartitionName.java | 14 +++--
.../data/nereids_function_p0/scalar_function/A.out | 16 ++++++
.../nereids_function_p0/scalar_function/A.groovy | 12 ++++-
4 files changed, 76 insertions(+), 29 deletions(-)
diff --git a/be/src/vec/functions/function_string.h
b/be/src/vec/functions/function_string.h
index 256e5943990..777f461cfa1 100644
--- a/be/src/vec/functions/function_string.h
+++ b/be/src/vec/functions/function_string.h
@@ -532,7 +532,7 @@ public:
String get_name() const override { return name; }
size_t get_number_of_arguments() const override { return 0; }
bool is_variadic() const override { return true; }
-
+ bool use_default_implementation_for_nulls() const override { return false;
}
DataTypePtr get_return_type_impl(const DataTypes& arguments) const
override {
return std::make_shared<DataTypeString>();
}
@@ -540,23 +540,36 @@ public:
Status execute_impl(FunctionContext* context, Block& block, const
ColumnNumbers& arguments,
size_t result, size_t input_rows_count) const override
{
size_t argument_size = arguments.size();
- if (argument_size < 2) {
- return Status::InvalidArgument(
- "auto_partition_name must contains at least two
arguments");
- }
+ auto const_null_map = ColumnUInt8::create(input_rows_count, 0);
+ auto null_map = ColumnUInt8::create(input_rows_count, 0);
std::vector<const ColumnString::Chars*> chars_list(argument_size);
std::vector<const ColumnString::Offsets*> offsets_list(argument_size);
std::vector<bool> is_const_args(argument_size);
+ std::vector<const ColumnUInt8::Container*> null_list(argument_size);
+ std::vector<ColumnPtr> argument_null_columns(argument_size);
+ std::vector<ColumnPtr> argument_columns(argument_size);
for (int i = 0; i < argument_size; ++i) {
+ argument_columns[i] =
+
block.get_by_position(arguments[i]).column->convert_to_full_column_if_const();
+ if (const auto* nullable =
+ check_and_get_column<const
ColumnNullable>(*argument_columns[i])) {
+ null_list[i] = &nullable->get_null_map_data();
+ argument_null_columns[i] = nullable->get_null_map_column_ptr();
+ argument_columns[i] = nullable->get_nested_column_ptr();
+ } else {
+ null_list[i] = &const_null_map->get_data();
+ }
+
const auto& [col, is_const] =
unpack_if_const(block.get_by_position(arguments[i]).column);
- const auto* col_str = assert_cast<const ColumnString*>(col.get());
+ const auto* col_str = assert_cast<const
ColumnString*>(argument_columns[i].get());
chars_list[i] = &col_str->get_chars();
offsets_list[i] = &col_str->get_offsets();
is_const_args[i] = is_const;
}
+
auto res = ColumnString::create();
auto& res_data = res->get_chars();
auto& res_offset = res->get_offsets();
@@ -564,11 +577,11 @@ public:
const char* partition_type = chars_list[0]->raw_data();
// partition type is list|range
- if (std::strncmp(partition_type, "list", 4) == 0) {
- return _auto_partition_type_of_list(chars_list, offsets_list,
is_const_args, res_data,
- res_offset, input_rows_count,
argument_size, block,
- result, res);
- } else {
+ if (!std::strncmp(partition_type, "list", 4)) {
+ return _auto_partition_type_of_list(chars_list, offsets_list,
is_const_args, null_list,
+ res_data, res_offset,
input_rows_count,
+ argument_size, block, result,
res);
+ } else if (!std::strncmp(partition_type, "range", 5)) {
return _auto_partition_type_of_range(chars_list, offsets_list,
is_const_args, res_data,
res_offset, input_rows_count,
argument_size, block,
result, res);
@@ -615,8 +628,9 @@ private:
}
Status _auto_partition_type_of_list(std::vector<const
ColumnString::Chars*>& chars_list,
std::vector<const
ColumnString::Offsets*>& offsets_list,
- std::vector<bool>& is_const_args,
auto& res_data,
- auto& res_offset, size_t
input_rows_count,
+ std::vector<bool>& is_const_args,
+ const std::vector<const
ColumnUInt8::Container*>& null_list,
+ auto& res_data, auto& res_offset,
size_t input_rows_count,
size_t argument_size, Block& block,
size_t result,
auto& res) const {
int curr_len = 0;
@@ -627,16 +641,21 @@ private:
for (int col = 1; col < argument_size; col++) {
const auto& current_offsets = *offsets_list[col];
const auto& current_chars = *chars_list[col];
+ const auto& current_nullmap = *null_list[col];
- auto idx = index_check_const(row, is_const_args[col]);
- int size = current_offsets[idx] - current_offsets[idx - 1];
- const char* raw_chars =
- reinterpret_cast<const
char*>(¤t_chars[current_offsets[idx - 1]]);
-
- // convert string to u16string in order to convert to unicode
strings
- const std::string raw_str(raw_chars, size);
- auto u16string = _string_to_u16string(raw_str);
- res_p += _string_to_unicode(u16string) +
std::to_string(u16string.size());
+ if (current_nullmap[row]) {
+ res_p += 'X';
+ } else {
+ auto idx = index_check_const(row, is_const_args[col]);
+
+ int size = current_offsets[idx] - current_offsets[idx - 1];
+ const char* raw_chars =
+ reinterpret_cast<const
char*>(¤t_chars[current_offsets[idx - 1]]);
+ // convert string to u16string in order to convert to
unicode strings
+ const std::string raw_str(raw_chars, size);
+ auto u16string = _string_to_u16string(raw_str);
+ res_p += _string_to_unicode(u16string) +
std::to_string(u16string.size());
+ }
}
// check the name of length
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/AutoPartitionName.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/AutoPartitionName.java
index 716cb0a6ee5..0ec040ea2f3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/AutoPartitionName.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/AutoPartitionName.java
@@ -20,10 +20,9 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
-import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
-import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.StringType;
import org.apache.doris.nereids.types.VarcharType;
@@ -40,7 +39,7 @@ import java.util.List;
* GenerateFunction.
*/
public class AutoPartitionName extends ScalarFunction
- implements UnaryExpression, ExplicitlyCastableSignature,
PropagateNullable {
+ implements ExplicitlyCastableSignature, AlwaysNotNullable {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).varArgs(VarcharType.SYSTEM_DEFAULT),
@@ -69,11 +68,11 @@ public class AutoPartitionName extends ScalarFunction
throw new AnalysisException("function auto_partition_name must
contains at least two arguments");
}
if (!child(0).isLiteral()) {
- throw new AnalysisException("auto_partition_name must accept
literal for 1nd argument");
+ throw new AnalysisException("auto_partition_name must accept
literal for 1st argument");
}
final String partition_type = ((VarcharLiteral)
getArgument(0)).getStringValue().toLowerCase();
if (!Lists.newArrayList("range", "list").contains(partition_type)) {
- throw new AnalysisException("function auto_partition_name must
accept range|list for 1nd argument");
+ throw new AnalysisException("function auto_partition_name must
accept range|list for 1st argument");
} else if (Lists.newArrayList("range").contains(partition_type)) {
if (!child(1).isLiteral()) {
throw new AnalysisException("auto_partition_name must accept
literal for 2nd argument");
@@ -93,6 +92,11 @@ public class AutoPartitionName extends ScalarFunction
}
}
+ @Override
+ public void checkLegalityBeforeTypeCoercion() {
+ checkLegalityAfterRewrite();
+ }
+
@Override
public List<FunctionSignature> getSignatures() {
return SIGNATURES;
diff --git a/regression-test/data/nereids_function_p0/scalar_function/A.out
b/regression-test/data/nereids_function_p0/scalar_function/A.out
index a08da4e70f4..a525070f5bc 100644
--- a/regression-test/data/nereids_function_p0/scalar_function/A.out
+++ b/regression-test/data/nereids_function_p0/scalar_function/A.out
@@ -484,6 +484,16 @@ pchar116varchar119
pchar126varchar129
pchar136varchar139
+-- !sql_auto_partition_name_list_column --
+pchar116Xvarchar119
+pchar126Xvarchar129
+pchar136Xvarchar139
+
+-- !sql_auto_partition_name_list_column --
+pXstring17XX
+pXstring27XX
+pXstring37XX
+
-- !sql_auto_partition_name_list_literal_empty --
p0
@@ -496,6 +506,12 @@ p_2dhello6
-- !sql_auto_partition_name_list_literal_mixed --
p4023ffe5257e7cworld111112e2e2e2e20
+-- !sql_auto_partition_name_list_literal_mixed --
+plist4X11X
+
+-- !sql_auto_partition_name_list_literal_null --
+pXXX
+
-- !sql_auto_partition_name_range_literal_notnull --
p20221212000000
diff --git
a/regression-test/suites/nereids_function_p0/scalar_function/A.groovy
b/regression-test/suites/nereids_function_p0/scalar_function/A.groovy
index 76387e343a7..b4df5d9e9c4 100644
--- a/regression-test/suites/nereids_function_p0/scalar_function/A.groovy
+++ b/regression-test/suites/nereids_function_p0/scalar_function/A.groovy
@@ -83,10 +83,14 @@ suite("nereids_scalar_fn_A") {
qt_sql_auto_partition_name_list_column_type_mixed "select
auto_partition_name('list', kchrs1, kbool) from fn_test_not_nullable where id <
3 order by kchrs1"
qt_sql_auto_partition_name_list_column_literal_mixed "select
auto_partition_name('list', kstr, 'hello') from fn_test_not_nullable where id <
3 order by kstr"
qt_sql_auto_partition_name_list_column "select
auto_partition_name('list', kchrs1, kvchrs1) from fn_test_not_nullable where id
< 3 order by kchrs1"
+ qt_sql_auto_partition_name_list_column "select
auto_partition_name('list', kchrs1, null, kvchrs1) from fn_test where id < 3
order by kchrs1"
+ qt_sql_auto_partition_name_list_column "select
auto_partition_name('list', null, kstr, null, null) from fn_test where id < 3
order by kstr"
qt_sql_auto_partition_name_list_literal_empty "select
auto_partition_name('list', '')"
qt_sql_auto_partition_name_list_literal_mixed "select
auto_partition_name('list', '你好', true, false)"
qt_sql_auto_partition_name_list_literal_mixed "select
auto_partition_name('list', '-hello')"
qt_sql_auto_partition_name_list_literal_mixed "select
auto_partition_name('list', '@#￥%~|world11111....')"
+ qt_sql_auto_partition_name_list_literal_mixed "select
auto_partition_name('list', 'list', null, true, null)"
+ qt_sql_auto_partition_name_list_literal_null "select
auto_partition_name('list', null, null, null);"
qt_sql_auto_partition_name_range_literal_notnull "select
auto_partition_name('range', 'day', '2022-12-12 19:20:30')"
qt_sql_auto_partition_name_range_literal_notnull "select
auto_partition_name('range', 'month', '2022-12-12 19:20:30')"
qt_sql_auto_partition_name_range_literal_notnull "select
auto_partition_name('range', 'year', '2022-12-12 19:20:30')"
@@ -110,7 +114,7 @@ suite("nereids_scalar_fn_A") {
}
test{
sql """select auto_partition_name(kdt, 'day', kdt) from
fn_test_not_nullable order by kdt"""
- exception "auto_partition_name must accept literal for 1nd
argument"
+ exception "auto_partition_name must accept literal for 1st
argument"
}
test{
sql """select auto_partition_name('range', kdt, kdt) from
fn_test_not_nullable order by kdt"""
@@ -142,7 +146,11 @@ suite("nereids_scalar_fn_A") {
}
test{
sql """select auto_partition_name('ranges', 'year', 'hello');"""
- exception "function auto_partition_name must accept range|list
for 1nd argument"
+ exception "function auto_partition_name must accept range|list
for 1st argument"
+ }
+ test{
+ sql """select auto_partition_name('lists', 'year', 'hello');"""
+ exception "function auto_partition_name must accept range|list
for 1st argument"
}
test{
sql """select auto_partition_name('range', 'years', 'hello');"""
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
