This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 28b305e3eaf [bugfix](fold) Skip constant folding in from_base64
function to fix binary data handling (#50453)
28b305e3eaf is described below
commit 28b305e3eaf3a3ae60161db7716aacd8fd30e4c9
Author: lw112 <[email protected]>
AuthorDate: Tue Jun 3 10:56:48 2025 +0800
[bugfix](fold) Skip constant folding in from_base64 function to fix binary
data handling (#50453)
### What problem does this PR solve?
revert: #43410
1. `In PR 43410, I used getBytesValue, but bytes value had no data, so
an error was reported when trying to get it, causing constant folding to
fail. So the query result was correct in the end, but this fix was
wrong. Now I undo my previous fix.`
2. `The current pr skips the from_base64 function in constant folding
processing.`
---
.../rules/expression/rules/FoldConstantRuleOnBE.java | 18 ++++++++----------
.../doris/nereids/trees/expressions/LiteralTest.java | 8 ++------
2 files changed, 10 insertions(+), 16 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
index 90b044e2b81..8a507808915 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
@@ -39,6 +39,7 @@ import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Match;
import org.apache.doris.nereids.trees.expressions.functions.BoundFunction;
import
org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.FromBase64;
import org.apache.doris.nereids.trees.expressions.functions.scalar.NonNullable;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Sleep;
@@ -231,6 +232,11 @@ public class FoldConstantRuleOnBE implements
ExpressionPatternRuleFactory {
return true;
}
+ // Skip from_base64 function to avoid incorrect binary data processing
during constant folding
+ if (expr instanceof FromBase64) {
+ return true;
+ }
+
// TableGeneratingFunction need pass PlanTranslatorContext value
if (expr instanceof TableGeneratingFunction) {
return true;
@@ -490,16 +496,8 @@ public class FoldConstantRuleOnBE implements
ExpressionPatternRuleFactory {
} else if (type.isStringLikeType()) {
int num = resultContent.getStringValueCount();
for (int i = 0; i < num; ++i) {
- // get the raw byte data to avoid character encoding
conversion problems
- ByteString bytesValues = resultContent.getBytesValue(i);
- // use UTF-8 encoding to ensure proper handling of binary data
- String stringValue = bytesValues.toStringUtf8();
- // handle special NULL value cases
- if ("\\N".equalsIgnoreCase(stringValue) &&
resultContent.hasHasNull()) {
- res.add(new NullLiteral(type));
- } else {
- res.add(new StringLiteral(stringValue));
- }
+ Literal literal = new
StringLiteral(resultContent.getStringValue(i));
+ res.add(literal);
}
} else if (type.isArrayType()) {
ArrayType arrayType = (ArrayType) type;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java
index 9c7e2e5b151..fcb64ff0bfa 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java
@@ -233,9 +233,7 @@ class LiteralTest {
PValues.Builder resultContentBuilder = PValues.newBuilder();
for (int i = 0; i < elementsArray.length; i = i + 2) {
childBuilder1.addInt32Value(elementsArray[i]);
- String strValue = "str" + (i + 1);
- childBuilder2.addStringValue(strValue);
-
childBuilder2.addBytesValue(com.google.protobuf.ByteString.copyFromUtf8(strValue));
+ childBuilder2.addStringValue("str" + (i + 1));
}
childBuilder1.setType(childTypeBuilder1.build());
childBuilder2.setType(childTypeBuilder2.build());
@@ -282,9 +280,7 @@ class LiteralTest {
PValues.Builder resultContentBuilder = PValues.newBuilder();
for (int i = 0; i < elementsArray.length; i = i + 2) {
childBuilder1.addInt32Value(elementsArray[i]);
- String strValue = "str" + (i + 1);
- childBuilder2.addStringValue(strValue);
-
childBuilder2.addBytesValue(com.google.protobuf.ByteString.copyFromUtf8(strValue));
+ childBuilder2.addStringValue("str" + (i + 1));
}
childBuilder1.setType(childTypeBuilder1.build());
childBuilder2.setType(childTypeBuilder2.build());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
