This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new dc966005238 [fix](nereids)Use utf-8 when convert string like literal
to double. (#50085)
dc966005238 is described below
commit dc9660052381f3f8f31647b17157ddb339ed6f9a
Author: James <[email protected]>
AuthorDate: Fri Apr 18 09:55:02 2025 +0800
[fix](nereids)Use utf-8 when convert string like literal to double. (#50085)
### What problem does this PR solve?
Use utf-8 when convert string like literal to double.
StringLike columns in Doris are all stored with utf-8 encoding. So we
need to use utf-8 encoding to read the column statistics min/max value.
Otherwise, Java will use the system default encoding. In this case,
doris may read wrong statistics min/max value.
---
.../doris/nereids/trees/expressions/literal/StringLikeLiteral.java | 3 ++-
.../nereids/trees/expressions/literal/StringLikeLiteralTest.java | 7 +++++++
2 files changed, 9 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java
index 546e0ab64c9..c82b7aebe55 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java
@@ -22,6 +22,7 @@ import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.types.DataType;
import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
import java.util.Objects;
/**
@@ -49,7 +50,7 @@ public abstract class StringLikeLiteral extends Literal
implements ComparableLit
* get double value
*/
public static double getDouble(String str) {
- byte[] bytes = str.getBytes();
+ byte[] bytes = str.getBytes(StandardCharsets.UTF_8);
long v = 0;
int pos = 0;
int len = Math.min(bytes.length, 7);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteralTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteralTest.java
index c1e9bc0e839..d9724f6b324 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteralTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteralTest.java
@@ -31,4 +31,11 @@ public class StringLikeLiteralTest {
double d2 = StringLikeLiteral.getDouble(maxStr);
Assertions.assertTrue(d1 < d2);
}
+
+ @Test
+ public void testUtf8() {
+ System.setProperty("file.encoding", "ANSI_X3.4-1968");
+ double d1 = StringLikeLiteral.getDouble("一般风险准备");
+ Assertions.assertEquals(d1, 6.4379158486625512E16);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
