This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new c4bee9adb30 branch-3.0: [fix](fold) fixed an issue with be computing
constants #43410 (#45691)
c4bee9adb30 is described below
commit c4bee9adb308bca0fb4b2f1f5edef8e88e20acf6
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Fri Dec 20 17:54:59 2024 +0800
branch-3.0: [fix](fold) fixed an issue with be computing constants #43410
(#45691)
Cherry-picked from #43410
Co-authored-by: lw112 <[email protected]>
---
.../nereids/rules/expression/rules/FoldConstantRuleOnBE.java | 12 ++++++++++--
.../apache/doris/nereids/trees/expressions/LiteralTest.java | 8 ++++++--
.../expression/fold_constant/fold_constant_by_be.out | 6 ++++++
.../expression/fold_constant/fold_constant_by_be.groovy | 11 +++++++----
4 files changed, 29 insertions(+), 8 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
index 64f7c2784f7..2fc5412846b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
@@ -487,8 +487,16 @@ public class FoldConstantRuleOnBE implements
ExpressionPatternRuleFactory {
} else if (type.isStringLikeType()) {
int num = resultContent.getStringValueCount();
for (int i = 0; i < num; ++i) {
- Literal literal = new
StringLiteral(resultContent.getStringValue(i));
- res.add(literal);
+ // get the raw byte data to avoid character encoding
conversion problems
+ ByteString bytesValues = resultContent.getBytesValue(i);
+ // use UTF-8 encoding to ensure proper handling of binary data
+ String stringValue = bytesValues.toStringUtf8();
+ // handle special NULL value cases
+ if ("\\N".equalsIgnoreCase(stringValue) &&
resultContent.hasHasNull()) {
+ res.add(new NullLiteral(type));
+ } else {
+ res.add(new StringLiteral(stringValue));
+ }
}
} else if (type.isArrayType()) {
ArrayType arrayType = (ArrayType) type;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java
index fcb64ff0bfa..9c7e2e5b151 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java
@@ -233,7 +233,9 @@ class LiteralTest {
PValues.Builder resultContentBuilder = PValues.newBuilder();
for (int i = 0; i < elementsArray.length; i = i + 2) {
childBuilder1.addInt32Value(elementsArray[i]);
- childBuilder2.addStringValue("str" + (i + 1));
+ String strValue = "str" + (i + 1);
+ childBuilder2.addStringValue(strValue);
+
childBuilder2.addBytesValue(com.google.protobuf.ByteString.copyFromUtf8(strValue));
}
childBuilder1.setType(childTypeBuilder1.build());
childBuilder2.setType(childTypeBuilder2.build());
@@ -280,7 +282,9 @@ class LiteralTest {
PValues.Builder resultContentBuilder = PValues.newBuilder();
for (int i = 0; i < elementsArray.length; i = i + 2) {
childBuilder1.addInt32Value(elementsArray[i]);
- childBuilder2.addStringValue("str" + (i + 1));
+ String strValue = "str" + (i + 1);
+ childBuilder2.addStringValue(strValue);
+
childBuilder2.addBytesValue(com.google.protobuf.ByteString.copyFromUtf8(strValue));
}
childBuilder1.setType(childTypeBuilder1.build());
childBuilder2.setType(childTypeBuilder2.build());
diff --git
a/regression-test/data/nereids_p0/expression/fold_constant/fold_constant_by_be.out
b/regression-test/data/nereids_p0/expression/fold_constant/fold_constant_by_be.out
index c7c506292a5..8d9d704684e 100644
---
a/regression-test/data/nereids_p0/expression/fold_constant/fold_constant_by_be.out
+++
b/regression-test/data/nereids_p0/expression/fold_constant/fold_constant_by_be.out
@@ -1,4 +1,10 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+C2BD89103557CCBF7ED97B51860225A0
+
+-- !sql --
+C2BD89103557CCBF7ED97B51860225A0
+
-- !sql_1 --
80000
diff --git
a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_by_be.groovy
b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_by_be.groovy
index 09a80209c04..f3b1b0cdcd5 100644
---
a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_by_be.groovy
+++
b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_by_be.groovy
@@ -22,6 +22,9 @@ suite("fold_constant_by_be") {
sql 'set enable_fallback_to_original_planner=false'
sql 'set enable_fold_constant_by_be=true'
+ qt_sql """ select hex(from_base64('wr2JEDVXzL9+2XtRhgIloA==')); """
+ qt_sql """ select hex(s) from (select
from_base64('wr2JEDVXzL9+2XtRhgIloA==') as s) t; """
+
test {
sql '''
select if(
@@ -32,8 +35,8 @@ suite("fold_constant_by_be") {
result([['9999-07-31']])
}
- sql """
- CREATE TABLE IF NOT EXISTS str_tb (k1 VARCHAR(10) NULL, v1 STRING
NULL)
+ sql """
+ CREATE TABLE IF NOT EXISTS str_tb (k1 VARCHAR(10) NULL, v1 STRING NULL)
UNIQUE KEY(k1) DISTRIBUTED BY HASH(k1) BUCKETS 5
properties("replication_num" = "1");
"""
@@ -53,7 +56,7 @@ suite("fold_constant_by_be") {
sql 'set query_timeout=12;'
qt_sql "select sleep(sign(1)*5);"
-
+
explain {
sql("verbose select substring('123456', 1, 3)")
contains "varchar(3)"
@@ -71,7 +74,7 @@ suite("fold_constant_by_be") {
col_varchar_1000__undef_signed varchar(1000) null ,
col_varchar_1000__undef_signed_not_null varchar(1000) not
null ,
col_varchar_1001__undef_signed varchar(1001) null ,
- col_varchar_1001__undef_signed_not_null varchar(1001) not
null
+ col_varchar_1001__undef_signed_not_null varchar(1001) not
null
) engine=olap
DUPLICATE KEY(pk, col_char_255__undef_signed,
col_char_100__undef_signed)
distributed by hash(pk) buckets 10
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
