This is an automated email from the ASF dual-hosted git repository.
morrySnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 366fd37b283 [fix](function) Preserve invalid URL decoded bytes (#67841)
366fd37b283 is described below
commit 366fd37b283e55916bbe1a951e703a78a2f7fa3d
Author: morrySnow <[email protected]>
AuthorDate: Mon Sep 14 10:52:26 2026 +0800
[fix](function) Preserve invalid URL decoded bytes (#67841)
## Problem
`URL_DECODE` returned different bytes depending on whether the
expression was folded by FE. For example:
```sql
SELECT HEX(URL_DECODE('%C0%AF'));
```
The folded path returned `EFBFBDEFBFBD`, while evaluation in BE returned
`C0AF`.
## Cause
The FE executable function delegates to Java `URLDecoder`, whose UTF-8
conversion silently replaces malformed byte sequences with U+FFFD. The
BE URL decoder has byte-level semantics and preserves each decoded `%HH`
byte. A Java string literal therefore cannot represent the BE result
losslessly when the decoded bytes are invalid UTF-8.
## Fix
- Reconstruct the byte stream produced by URL decoding and validate it
with a strict UTF-8 decoder before FE constant evaluation.
- If the decoded bytes are invalid UTF-8, abort FE evaluation so the
original function remains in the plan and BE preserves the bytes.
- Keep valid UTF-8 folding unchanged, including a legitimately encoded
U+FFFD value.
- Preserve the existing handling of malformed percent escapes.
## Tests
- Added executable-function unit coverage for `%80`, `%C0%AF`,
`%E0%80%80`, `%ED%A0%80`, `%FF`, valid multibyte text, and encoded
U+FFFD.
- Added regression coverage comparing constant and dynamic execution
paths by their hexadecimal result.
- `StringArithmeticTest`: 7 tests passed.
- Full FE build and checkstyle passed.
- Targeted regression suite: 1/1 passed.
---
.../functions/executable/StringArithmetic.java | 51 ++++++++++++++++++++++
.../functions/executable/StringArithmeticTest.java | 25 +++++++++++
.../scalar_function/url_decode_invalid_utf8.out | 9 ++++
.../scalar_function/url_decode_invalid_utf8.groovy | 39 +++++++++++++++++
4 files changed, 124 insertions(+)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java
index c499c31682a..f670ab0a2d8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java
@@ -46,10 +46,14 @@ import org.apache.doris.nereids.types.ArrayType;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
+import java.io.ByteArrayOutputStream;
import java.io.UnsupportedEncodingException;
import java.math.BigInteger;
import java.net.URLDecoder;
import java.net.URLEncoder;
+import java.nio.ByteBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.CodingErrorAction;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
@@ -1075,6 +1079,11 @@ public class StringArithmetic {
*/
@ExecFunction(name = "url_decode")
public static Expression urlDecode(StringLikeLiteral first) {
+ if (!isValidUtf8AfterUrlDecode(first.getValue())) {
+ // String literals cannot preserve an invalid UTF-8 byte sequence.
Let BE evaluate it
+ // instead of folding the replacement characters produced by
java.net.URLDecoder.
+ throw new IllegalArgumentException("URL-decoded value is not valid
UTF-8");
+ }
try {
return castStringLikeLiteral(first,
URLDecoder.decode(first.getValue(), StandardCharsets.UTF_8.name()));
} catch (UnsupportedEncodingException e) {
@@ -1082,6 +1091,48 @@ public class StringArithmetic {
}
}
+ private static boolean isValidUtf8AfterUrlDecode(String value) {
+ ByteArrayOutputStream decodedBytes = new
ByteArrayOutputStream(value.length());
+ int index = 0;
+ while (index < value.length()) {
+ char current = value.charAt(index);
+ if (current == '%') {
+ if (index + 2 >= value.length()) {
+ // Preserve URLDecoder's existing malformed-escape
handling.
+ return true;
+ }
+ int high = Character.digit(value.charAt(index + 1), 16);
+ int low = Character.digit(value.charAt(index + 2), 16);
+ if (high < 0 || low < 0) {
+ // Preserve URLDecoder's existing malformed-escape
handling.
+ return true;
+ }
+ decodedBytes.write((high << 4) + low);
+ index += 3;
+ } else if (current == '+') {
+ decodedBytes.write(' ');
+ index++;
+ } else {
+ int start = index;
+ while (index < value.length() && value.charAt(index) != '%' &&
value.charAt(index) != '+') {
+ index++;
+ }
+ byte[] originalBytes = value.substring(start,
index).getBytes(StandardCharsets.UTF_8);
+ decodedBytes.write(originalBytes, 0, originalBytes.length);
+ }
+ }
+
+ try {
+ StandardCharsets.UTF_8.newDecoder()
+ .onMalformedInput(CodingErrorAction.REPORT)
+ .onUnmappableCharacter(CodingErrorAction.REPORT)
+ .decode(ByteBuffer.wrap(decodedBytes.toByteArray()));
+ return true;
+ } catch (CharacterCodingException e) {
+ return false;
+ }
+ }
+
/**
* Executable arithmetic functions urlencode
*/
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java
index 985b0029b82..5381bc8e062 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java
@@ -17,9 +17,14 @@
package org.apache.doris.nereids.trees.expressions.functions.executable;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.ExpressionEvaluator;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.UrlDecode;
import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral;
import org.apache.doris.nereids.trees.expressions.literal.FloatLiteral;
import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral;
import org.junit.jupiter.api.Assertions;
@@ -74,4 +79,24 @@ class StringArithmeticTest {
Assertions.assertEquals(2, result.getValue());
}
+
+ @Test
+ void testUrlDecodeDoesNotFoldInvalidUtf8() {
+ String[] invalidUtf8Values = {"%80", "%C0%AF", "%E0%80%80",
"%ED%A0%80", "%FF"};
+ for (String value : invalidUtf8Values) {
+ UrlDecode urlDecode = new UrlDecode(new StringLiteral(value));
+ Assertions.assertSame(urlDecode,
ExpressionEvaluator.INSTANCE.eval(urlDecode), value);
+ }
+ }
+
+ @Test
+ void testUrlDecodeStillFoldsValidUtf8() {
+ assertUrlDecodeValue("%E4%B8%AD+text", "中 text");
+ assertUrlDecodeValue("%EF%BF%BD", "�");
+ }
+
+ private void assertUrlDecodeValue(String encoded, String expected) {
+ Expression result = ExpressionEvaluator.INSTANCE.eval(new
UrlDecode(new StringLiteral(encoded)));
+ Assertions.assertEquals(expected, ((StringLikeLiteral)
result).getValue());
+ }
}
diff --git
a/regression-test/data/nereids_function_p0/scalar_function/url_decode_invalid_utf8.out
b/regression-test/data/nereids_function_p0/scalar_function/url_decode_invalid_utf8.out
new file mode 100644
index 00000000000..3a1c28a3f41
--- /dev/null
+++
b/regression-test/data/nereids_function_p0/scalar_function/url_decode_invalid_utf8.out
@@ -0,0 +1,9 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !invalid_utf8_constant --
+80 C0AF E08080 EDA080 FF
+
+-- !invalid_utf8_runtime --
+80 C0AF E08080 EDA080 FF
+
+-- !valid_utf8_constant --
+中 text EFBFBD
diff --git
a/regression-test/suites/nereids_function_p0/scalar_function/url_decode_invalid_utf8.groovy
b/regression-test/suites/nereids_function_p0/scalar_function/url_decode_invalid_utf8.groovy
new file mode 100644
index 00000000000..6c9de7d9b03
--- /dev/null
+++
b/regression-test/suites/nereids_function_p0/scalar_function/url_decode_invalid_utf8.groovy
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("url_decode_invalid_utf8") {
+ qt_invalid_utf8_constant """
+ SELECT HEX(URL_DECODE('%80')),
+ HEX(URL_DECODE('%C0%AF')),
+ HEX(URL_DECODE('%E0%80%80')),
+ HEX(URL_DECODE('%ED%A0%80')),
+ HEX(URL_DECODE('%FF'))
+ """
+
+ qt_invalid_utf8_runtime """
+ SELECT HEX(URL_DECODE(IF(number = 0, '%80', ''))),
+ HEX(URL_DECODE(IF(number = 0, '%C0%AF', ''))),
+ HEX(URL_DECODE(IF(number = 0, '%E0%80%80', ''))),
+ HEX(URL_DECODE(IF(number = 0, '%ED%A0%80', ''))),
+ HEX(URL_DECODE(IF(number = 0, '%FF', '')))
+ FROM numbers("number" = "1")
+ """
+
+ qt_valid_utf8_constant """
+ SELECT URL_DECODE('%E4%B8%AD+text'), HEX(URL_DECODE('%EF%BF%BD'))
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
