This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new ca07a0396db branch-4.2: [fix](be) Limit parse_url QUERY key lookup to
the query component #68011 (#70124)
ca07a0396db is described below
commit ca07a0396db46b4003b46fbdfd1036e14356cb3c
Author: morrySnow <[email protected]>
AuthorDate: Wed Nov 4 08:36:14 2026 +0800
branch-4.2: [fix](be) Limit parse_url QUERY key lookup to the query
component #68011 (#70124)
picked from master #68011
Co-authored-by: TengJianPing <[email protected]>
---
be/src/util/url_parser.cpp | 132 ++++++++++++---------
be/src/util/url_parser.h | 6 +
be/test/exprs/function/function_string_test.cpp | 28 ++++-
be/test/exprs/function/function_url_test.cpp | 68 +++++++++++
.../functions/executable/StringArithmetic.java | 11 ++
.../functions/executable/StringArithmeticTest.java | 38 ++++++
.../data/function_p0/test_parse_url_key.out | 43 +++++++
.../suites/function_p0/test_parse_url_key.groovy | 82 +++++++++++++
.../fold_constant_string_arithmatic.groovy | 14 +++
9 files changed, 360 insertions(+), 62 deletions(-)
diff --git a/be/src/util/url_parser.cpp b/be/src/util/url_parser.cpp
index 00ed91f52f9..292ed950876 100644
--- a/be/src/util/url_parser.cpp
+++ b/be/src/util/url_parser.cpp
@@ -21,7 +21,9 @@
#include <stdint.h>
#include <algorithm>
+#include <limits>
#include <string>
+#include <utility>
#include "core/string_ref.h"
#include "util/string_search.hpp"
@@ -50,6 +52,29 @@ const StringSearch UrlParser::_s_colon_search(&_s_colon);
const StringSearch UrlParser::_s_question_search(&_s_question);
const StringSearch UrlParser::_s_hash_search(&_s_hash);
+bool UrlParser::find_query_component(const StringRef& url, StringRef* query) {
+ // The query component starts right after the first '?'.
+ int32_t query_pos = _s_question_search.search(&url);
+ if (query_pos < 0) {
+ // Query component is missing.
+ return false;
+ }
+
+ // The first '#' bounds the query component from the right, so this single
search also
+ // provides the position of the fragment.
+ int32_t fragment_pos = _s_hash_search.search(&url);
+ if (fragment_pos >= 0 && fragment_pos < query_pos) {
+ // The '#' comes before the '?', so the '?' and everything behind it
belongs to the
+ // fragment and url has no query component.
+ return false;
+ }
+
+ int32_t query_start = query_pos + cast_set<int32_t>(_s_question.size);
+ int32_t query_end = fragment_pos >= 0 ? fragment_pos :
cast_set<int32_t>(url.size);
+ *query = url.substring(query_start, query_end - query_start);
+ return true;
+}
+
bool UrlParser::parse_url(const StringRef& url, UrlPart part, StringRef*
result) {
result->data = nullptr;
result->size = 0;
@@ -139,18 +164,15 @@ bool UrlParser::parse_url(const StringRef& url, UrlPart
part, StringRef* result)
}
case QUERY: {
- // Find first '?'.
- int32_t start_pos = _s_question_search.search(&protocol_end);
-
- if (start_pos < 0) {
+ // The query component starts at the first '?' and ends before the '#'
that starts the
+ // fragment, so a url whose first '#' comes before its first '?' has
no query.
+ StringRef query;
+ if (!find_query_component(protocol_end, &query)) {
// Indicate no query was found.
return false;
}
- StringRef query_start = protocol_end.substring(start_pos +
_s_question.size);
- // End string _s_at next '#'.
- int32_t end_pos = _s_hash_search.search(&query_start);
- *result = query_start.substring(0, end_pos);
+ *result = query;
break;
}
@@ -227,57 +249,58 @@ bool UrlParser::parse_url_key(const StringRef& url,
UrlPart part, const StringRe
// Remove leading and trailing spaces.
StringRef trimmed_url = url.trim();
- // Search for the key in the url, ignoring malformed URLs for now.
+ // The key can only be found in the query component, which starts at the
first '?' and ends
+ // before the '#' that starts the fragment (if any).
+ StringRef query;
+ if (!find_query_component(trimmed_url, &query)) {
+ // Query component is missing, the whole url is the path plus the
fragment.
+ return false;
+ }
+
+ // Search for the key inside the query component, ignoring malformed URLs
for now.
StringSearch key_search(&key);
+ // Offset of the next search inside the query component. The query
component starts right
+ // after the '?', so a key at offset 0 is a query key as well.
+ int32_t offset = 0;
- while (trimmed_url.size > 0) {
- // Search for the key in the current substring.
- int32_t key_pos = key_search.search(&trimmed_url);
- bool match = true;
+ while (std::cmp_less(offset, query.size)) {
+ // Search for the key in the remaining part of the query component.
+ StringRef rest = query.substring(offset);
+ int32_t key_pos = key_search.search(&rest);
if (key_pos < 0) {
- return false;
- }
-
- // Key pos must be != 0 because it must be preceded by a '?' or a '&'.
- // Check that the char before key_pos is either '?' or '&'.
- if (key_pos == 0 ||
- (trimmed_url.data[key_pos - 1] != '?' && trimmed_url.data[key_pos
- 1] != '&')) {
- match = false;
+ // No (more) key in the query component.
+ break;
}
- // Advance substring beyond matching key.
- trimmed_url = trimmed_url.substring(key_pos + key.size);
-
- if (!match) {
+ offset += key_pos;
+ // The key must start the query component or be preceded by a '&'.
+ if (offset != 0 && query.data[offset - 1] != '&') {
+ // The matched text is not a key, step over it and keep searching.
+ offset += cast_set<int32_t>(key.size);
continue;
}
- if (trimmed_url.size <= 0) {
- break;
- }
+ // Positioned to the char right after the key.
+ int32_t value_pos = offset + cast_set<int32_t>(key.size);
- // Next character must be '=', otherwise the match cannot be a key in
the query part.
- if (trimmed_url.data[0] != '=') {
+ // The key must be followed by a '=' and a value, otherwise the match
cannot be a key.
+ if (value_pos >= cast_set<int32_t>(query.size) ||
query.data[value_pos] != '=') {
+ // Step over the matched text and keep searching.
+ offset += cast_set<int32_t>(key.size);
continue;
}
- int32_t pos = 1;
+ ++value_pos;
- // Find ending position of key's value by matching '#' or '&'.
- while (pos < trimmed_url.size) {
- switch (trimmed_url.data[pos]) {
- case '#':
- case '&':
- *result = trimmed_url.substring(1, pos - 1);
- return true;
- }
-
- ++pos;
- }
-
- // Ending position is end of string.
- *result = trimmed_url.substring(1);
+ // Find the ending position of the key's value by matching '&'.
+ StringRef value_rest = query.substring(value_pos);
+ size_t value_end_rel_pos = value_rest.find_first_of('&');
+ int32_t value_end_pos = value_end_rel_pos ==
std::numeric_limits<size_t>::max()
+ ? cast_set<int32_t>(query.size)
+ : value_pos +
cast_set<int32_t>(value_end_rel_pos);
+ // A duplicated key keeps the behaviour of returning the first value.
+ *result = query.substring(value_pos, value_end_pos - value_pos);
return true;
}
@@ -360,23 +383,15 @@ StringRef UrlParser::extract_url(StringRef url, StringRef
name) {
StringRef result("", 0);
// Remove leading and trailing spaces.
StringRef trimmed_url = url.trim();
- // find '?'
- int32_t question_pos = _s_question_search.search(&trimmed_url);
- if (question_pos < 0) {
+ // The parameters can only be found in the query component, which starts
at the first '?'
+ // and ends before the '#' that starts the fragment (if any).
+ StringRef sub_url;
+ if (!find_query_component(trimmed_url, &sub_url)) {
// this url no parameters.
// Example: https://doris.apache.org/
return result;
}
- // find '#'
- int32_t hash_pos = _s_hash_search.search(&trimmed_url);
- StringRef sub_url;
- if (hash_pos < 0) {
- sub_url = trimmed_url.substring(question_pos + 1, trimmed_url.size -
question_pos - 1);
- } else {
- sub_url = trimmed_url.substring(question_pos + 1, hash_pos -
question_pos - 1);
- }
-
// find '&' and '=', and extract target parameter
// Example: k1=aa&k2=bb&k3=cc&test=dd
int64_t and_pod;
@@ -391,8 +406,7 @@ StringRef UrlParser::extract_url(StringRef url, StringRef
name) {
key_url = sub_url.substring(0, and_pod);
sub_url = sub_url.substring(and_pod + 1, len - and_pod - 1);
} else {
- auto end_pos = sub_url.find_first_of('#');
- key_url = end_pos == -1 ? sub_url : sub_url.substring(0, end_pos);
+ key_url = sub_url;
sub_url = result;
}
len = sub_url.size;
diff --git a/be/src/util/url_parser.h b/be/src/util/url_parser.h
index df1182c6651..790a5c536cb 100644
--- a/be/src/util/url_parser.h
+++ b/be/src/util/url_parser.h
@@ -66,6 +66,12 @@ public:
static StringRef extract_url(StringRef url, StringRef name);
private:
+ // Locates the query component of url and stores it in query. The query
component starts
+ // right after the first '?' and ends before the '#' that starts the
fragment (if any).
+ // Returns false when url has no query component, which is also the case
when the first
+ // '#' comes before the first '?' because the '?' then belongs to the
fragment.
+ static bool find_query_component(const StringRef& url, StringRef* query);
+
// Constants representing parts of a URL.
static const StringRef _s_url_authority;
static const StringRef _s_url_file;
diff --git a/be/test/exprs/function/function_string_test.cpp
b/be/test/exprs/function/function_string_test.cpp
index afb9bbbaabf..7805e08eb16 100644
--- a/be/test/exprs/function/function_string_test.cpp
+++ b/be/test/exprs/function/function_string_test.cpp
@@ -2696,7 +2696,14 @@ TEST(function_string_test,
function_extract_url_parameter_test) {
{{VARCHAR("http://doris.apache.org?k1=aa&k2=bb&test=dd#999/";),
VARCHAR("k3")},
{VARCHAR("")}},
{{VARCHAR("http://doris.apache.org?k1=aa&k2=bb&test=dd#999/";),
VARCHAR("test")},
- {VARCHAR("dd")}}};
+ {VARCHAR("dd")}},
+ // The first '#' comes before the first '?', so the '?' belongs to
the fragment and
+ // the url has no parameters.
+ {{VARCHAR("http://doris.apache.org#f?k1=aa";), VARCHAR("k1")},
{VARCHAR("")}},
+ {{VARCHAR("http://doris.apache.org#f?k1=aa";), VARCHAR("aa")},
{VARCHAR("")}},
+ // The parameters end before the fragment.
+ {{VARCHAR("http://doris.apache.org?k1=aa#f?k2=bb";),
VARCHAR("k1")}, {VARCHAR("aa")}},
+ {{VARCHAR("http://doris.apache.org?k1=aa#f?k2=bb";),
VARCHAR("k2")}, {VARCHAR("")}}};
check_function_all_arg_comb<DataTypeString, true>(func_name, input_types,
data_set);
}
@@ -2746,7 +2753,14 @@ TEST(function_string_test, function_parse_url_test) {
"https://www.facebook.com/aa/bb?returnpage=https://www.facebook.com/";),
std::string("HosT")},
std::string("www.facebook.com")},
- {{std::string("http://www.baidu.com";), std::string("FILE")},
{std::string("")}}};
+ {{std::string("http://www.baidu.com";), std::string("FILE")},
{std::string("")}},
+ // The first '#' comes before the first '?', so the '?'
belongs to the fragment
+ // and the url has no query component.
+ {{std::string("http://h/p#f?k=v";), std::string("QUERY")},
{Null()}},
+ {{std::string("http://h/p#f/?#k=v";), std::string("QUERY")},
{Null()}},
+ // The query component ends before the fragment.
+ {{std::string("http://h/p?k=1#f&k=2";), std::string("QUERY")},
{std::string("k=1")}},
+ {{std::string("http://h/p?";), std::string("QUERY")},
{std::string("")}}};
check_function_all_arg_comb<DataTypeString, true>(func_name,
input_types, data_set);
}
@@ -2765,7 +2779,15 @@ TEST(function_string_test, function_parse_url_test) {
{Null()}},
{{std::string("http://fb.com/path/p1.p?q=1#f";),
std::string("HOST"),
std::string("q")},
- {Null()}}};
+ {Null()}},
+ // The only '?' is inside the fragment, so the url has no
query component.
+ {{std::string("http://h/p#f?k=v";), std::string("QUERY"),
std::string("k")},
+ {Null()}},
+ // A duplicated key returns the first value.
+ {{std::string("http://h/p?k=1&k=2#f";), std::string("QUERY"),
std::string("k")},
+ {std::string("1")}},
+ {{std::string("http://h/p?k=1&k=2&k=3";), std::string("QUERY"),
std::string("k")},
+ {std::string("1")}}};
check_function_all_arg_comb<DataTypeString, true>(func_name,
input_types, data_set);
}
diff --git a/be/test/exprs/function/function_url_test.cpp
b/be/test/exprs/function/function_url_test.cpp
index 36f8b7c3457..7fc20fb104c 100644
--- a/be/test/exprs/function/function_url_test.cpp
+++ b/be/test/exprs/function/function_url_test.cpp
@@ -93,4 +93,72 @@ TEST(FunctionUrlTEST, ProtocolTest) {
static_cast<void>(check_function<DataTypeString, true>(func_name,
input_types, data_set));
}
+TEST(FunctionUrlTEST, ParseUrlQueryKeyTest) {
+ std::string func_name = "parse_url";
+ InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR,
PrimitiveType::TYPE_VARCHAR,
+ PrimitiveType::TYPE_VARCHAR};
+
+ DataSet data_set = {
+ // The only '?' is inside the fragment, so the url has no query
component.
+ {{STRING("http://h/p#f?k=v";), STRING("QUERY"), STRING("k")},
Null()},
+ // The '#' comes before the '?', so it is a fragment instead of a
query.
+ {{STRING("http://h/p#f/?#k=v";), STRING("QUERY"), STRING("k")},
Null()},
+ {{STRING("http://h/p#?k=v";), STRING("QUERY"), STRING("k")},
Null()},
+ // The url has no query component at all.
+ {{STRING("http://h/p&k=v";), STRING("QUERY"), STRING("k")}, Null()},
+ // The key only exists in the path.
+ {{STRING("http://h/p&k=v?x=1";), STRING("QUERY"), STRING("k")},
Null()},
+ // The query component of this url is 'x=1'.
+ {{STRING("http://h/p&k=v?x=1";), STRING("QUERY"), STRING("x")},
STRING("1")},
+ // The key only exists in the fragment.
+ {{STRING("http://h/p?x=1#f&k=v";), STRING("QUERY"), STRING("k")},
Null()},
+ // The key exists in the path, in the query and in the fragment.
+ {{STRING("http://h/p&k=v?k=1#f&k=2";), STRING("QUERY"),
STRING("k")}, STRING("1")},
+ {{STRING("http://h/p?a=1&k=2";), STRING("QUERY"), STRING("k")},
STRING("2")},
+ // A duplicated key keeps the behaviour of returning the first
value.
+ {{STRING("http://h/p?k=1&k=2#f";), STRING("QUERY"), STRING("k")},
STRING("1")},
+ {{STRING("http://h/p?k=1&k=2&k=3";), STRING("QUERY"), STRING("k")},
STRING("1")},
+ // Only the query component is searched, so the duplicated key in
the fragment is
+ // not part of the result.
+ {{STRING("http://h/p?k=1#k=2&k=3";), STRING("QUERY"), STRING("k")},
STRING("1")},
+ // A key without any '=' is not a valid query parameter.
+ {{STRING("http://h/p?k";), STRING("QUERY"), STRING("k")}, Null()},
+ {{STRING("http://h/p?";), STRING("QUERY"), STRING("k")}, Null()},
+ // The key is the first query parameter.
+ {{STRING("http://h/p?k=1";), STRING("QUERY"), STRING("k")},
STRING("1")},
+ {{STRING(" http://h/p?k=1 "), STRING("QUERY"), STRING("k")},
STRING("1")},
+ {{STRING(" http://h/p?sk=0&k=1";), STRING("QUERY"), STRING("k")},
STRING("1")},
+ {{STRING(" http://h/p?k&k=1";), STRING("QUERY"), STRING("k")},
STRING("1")},
+ {{STRING(" http://h/p?k=&k=1";), STRING("QUERY"), STRING("k")},
STRING("")},
+ {{STRING("http://h/p?k=1";), STRING("HOST"), STRING("k")}, Null()},
+ };
+
+ static_cast<void>(check_function<DataTypeString, true>(func_name,
input_types, data_set));
+}
+
+TEST(FunctionUrlTEST, ParseUrlQueryTest) {
+ std::string func_name = "parse_url";
+ InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR,
PrimitiveType::TYPE_VARCHAR};
+
+ DataSet data_set = {
+ // The only '?' is inside the fragment, so the url has no query
component. The
+ // keyed form of parse_url must report the same.
+ {{STRING("http://h/p#f?k=v";), STRING("QUERY")}, Null()},
+ {{STRING("http://h/p#f/?#k=v";), STRING("QUERY")}, Null()},
+ {{STRING("http://h/p#?k=v";), STRING("QUERY")}, Null()},
+ // The url has no query component at all.
+ {{STRING("http://h/p&k=v";), STRING("QUERY")}, Null()},
+ {{STRING("http://h/p";), STRING("QUERY")}, Null()},
+ // The query component starts at the first '?' and ends before the
fragment.
+ {{STRING("http://h/p?k=1";), STRING("QUERY")}, STRING("k=1")},
+ {{STRING("http://h/p?k=1#f&k=2";), STRING("QUERY")}, STRING("k=1")},
+ {{STRING("http://h/p?a=1&k=2";), STRING("QUERY")},
STRING("a=1&k=2")},
+ // An empty query component is not NULL.
+ {{STRING("http://h/p?";), STRING("QUERY")}, STRING("")},
+ {{STRING(" http://h/p?k=1 "), STRING("QUERY")}, STRING("k=1")},
+ };
+
+ static_cast<void>(check_function<DataTypeString, true>(func_name,
input_types, data_set));
+}
+
} // namespace doris
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java
index db0742117fa..3a422fda284 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java
@@ -1033,6 +1033,12 @@ public class StringArithmetic {
if (startPos < 0) {
return null;
}
+ int fragmentPos = protocolEnd.indexOf('#');
+ if (fragmentPos >= 0 && fragmentPos < startPos) {
+ // The '#' comes before the '?', so the '?' and everything behind
it belongs to the
+ // fragment and the url has no query component.
+ return null;
+ }
String queryStart = protocolEnd.substring(startPos + 1);
return substringEnd(queryStart, queryStart.indexOf('#'));
}
@@ -1181,6 +1187,11 @@ public class StringArithmetic {
return castStringLikeLiteral(first, "");
}
int hashPos = trimmedUrl.indexOf('#');
+ if (hashPos >= 0 && hashPos < questionPos) {
+ // The '#' comes before the '?', so the '?' and everything behind
it belongs to the
+ // fragment and the url has no query parameters.
+ return castStringLikeLiteral(first, "");
+ }
String subUrl = hashPos < 0
? trimmedUrl.substring(questionPos + 1)
: trimmedUrl.substring(questionPos + 1, hashPos);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java
index 5381bc8e062..fda0a6f31b6 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java
@@ -23,6 +23,7 @@ import
org.apache.doris.nereids.trees.expressions.functions.scalar.UrlDecode;
import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral;
import org.apache.doris.nereids.trees.expressions.literal.FloatLiteral;
import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral;
@@ -99,4 +100,41 @@ class StringArithmeticTest {
Expression result = ExpressionEvaluator.INSTANCE.eval(new
UrlDecode(new StringLiteral(encoded)));
Assertions.assertEquals(expected, ((StringLikeLiteral)
result).getValue());
}
+
+ @Test
+ void testParseUrlQueryStopsAtFragment() {
+ // The only '?' is inside the fragment, so the url has no query
component.
+ assertParseUrlQueryIsNull("http://h/p#f?k=v";);
+ assertParseUrlQueryIsNull("http://h/p#f/?#k=v";);
+ // The query component starts at the first '?' and ends before the
fragment.
+ assertParseUrlQuery("http://h/p?k=1#f&k=2";, "k=1");
+ assertParseUrlQuery("http://h/p?a=1&k=2";, "a=1&k=2");
+ }
+
+ @Test
+ void testExtractUrlParameterStopsAtFragment() {
+ // The only '?' is inside the fragment, so the url has no parameters.
+ assertExtractUrlParameter("http://h/p#f?k=v";, "k", "");
+ // The parameters end before the fragment.
+ assertExtractUrlParameter("http://h/p?k=1#f&k=2";, "k", "1");
+ assertExtractUrlParameter("http://h/p?k1=aa&k2=bb#f";, "k2", "bb");
+ }
+
+ private void assertParseUrlQuery(String url, String expected) {
+ Expression result = StringArithmetic.parseurl(
+ new StringLiteral(url), new StringLiteral("QUERY"));
+ Assertions.assertEquals(expected, ((StringLikeLiteral)
result).getValue());
+ }
+
+ private void assertParseUrlQueryIsNull(String url) {
+ Expression result = StringArithmetic.parseurl(
+ new StringLiteral(url), new StringLiteral("QUERY"));
+ Assertions.assertTrue(result instanceof NullLiteral, url);
+ }
+
+ private void assertExtractUrlParameter(String url, String parameter,
String expected) {
+ Expression result = StringArithmetic.extractUrlParameter(
+ new StringLiteral(url), new StringLiteral(parameter));
+ Assertions.assertEquals(expected, ((StringLikeLiteral)
result).getValue());
+ }
}
diff --git a/regression-test/data/function_p0/test_parse_url_key.out
b/regression-test/data/function_p0/test_parse_url_key.out
new file mode 100644
index 00000000000..69e7ac35fb0
--- /dev/null
+++ b/regression-test/data/function_p0/test_parse_url_key.out
@@ -0,0 +1,43 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+1 \N
+2 x=1
+3 \N
+4 x=1
+5 k=1
+6 \N
+7 \N
+8 a=1&k=2
+9 k=1&k=2
+
+-- !sql --
+1 \N
+2 \N
+3 \N
+4 \N
+5 1
+6 \N
+7 \N
+8 2
+9 1
+
+-- !sql --
+1 \N
+2 1
+3 \N
+4 1
+5 \N
+6 \N
+7 \N
+8 \N
+9 \N
+
+-- !sql --
+9 1
+
+-- !sql --
+\N \N k=1 1
+
+-- !sql --
+ 2
+
diff --git a/regression-test/suites/function_p0/test_parse_url_key.groovy
b/regression-test/suites/function_p0/test_parse_url_key.groovy
new file mode 100644
index 00000000000..139a8dd0d1f
--- /dev/null
+++ b/regression-test/suites/function_p0/test_parse_url_key.groovy
@@ -0,0 +1,82 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_parse_url_key") {
+
+ sql """
+ drop table if exists test_parse_url_key;
+ """
+
+ sql """
+ CREATE TABLE `test_parse_url_key` (
+ `id` int NULL,
+ `url` text NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`)
+ DISTRIBUTED BY RANDOM BUCKETS AUTO
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ sql """
+ insert into test_parse_url_key values
+ (1, 'http://h/p#f?k=v'),
+ (2, 'http://h/p&k=v?x=1'),
+ (3, 'http://h/p#f/?#k=v'),
+ (4, 'http://h/p?x=1#f&k=v'),
+ (5, 'http://h/p&k=v?k=1#f&k=2'),
+ (6, 'http://h/p#f&k=v?k=2'),
+ (7, 'http://h/p'),
+ (8, 'http://h/p?a=1&k=2'),
+ (9, 'http://h/p?k=1&k=2#f')
+ """
+
+ // The query component is located between the first '?' and the following
'#'. Keys
+ // appearing in the path or in the fragment must not be returned, and a
url whose '#'
+ // comes before its '?' has no query component at all.
+ qt_sql """
+ select id, parse_url(url, 'QUERY') as query from test_parse_url_key
order by id
+ """
+
+ qt_sql """
+ select id, parse_url(url, 'QUERY', 'k') as query_k from
test_parse_url_key order by id
+ """
+
+ qt_sql """
+ select id, parse_url(url, 'QUERY', 'x') as query_x from
test_parse_url_key order by id
+ """
+
+ // A duplicated key returns its first value.
+ qt_sql """
+ select id, parse_url(url, 'QUERY', 'k') as first_k from
test_parse_url_key where id = 9
+ """
+
+ // The constant folding of the same urls must agree with the runtime
evaluation.
+ qt_sql """
+ select parse_url('http://h/p#f?k=v', 'QUERY'),
+ parse_url('http://h/p#f?k=v', 'QUERY', 'k'),
+ parse_url('http://h/p?k=1#f&k=2', 'QUERY'),
+ parse_url('http://h/p?k=1&k=2#f', 'QUERY', 'k')
+ """
+
+ // extract_url_parameter has to bound the parameters by the query
component as well.
+ qt_sql """
+ select extract_url_parameter('http://h/p#f?k=v', 'k'),
+ extract_url_parameter('http://h/p?a=1&k=2#f', 'k')
+ """
+}
diff --git
a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_string_arithmatic.groovy
b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_string_arithmatic.groovy
index 2d4ccdbddff..f7f590c28ec 100644
---
a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_string_arithmatic.groovy
+++
b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_string_arithmatic.groovy
@@ -631,6 +631,13 @@ suite("fold_constant_string_arithmatic") {
testFoldConst("select parse_url('http://www.example.com/path?query=こんにちは',
'QUERY')")
testFoldConst("select
parse_url(\"http://www.example.com/path?query=a\b\'\", 'QUERY')")
testFoldConst("select
parse_url(\"http://www.example.com/path.query=a\b\'\", 'QUERY')")
+ // The query component is located between the first '?' and the fragment,
so a url whose
+ // '#' comes before its '?' has no query component.
+ testFoldConst("select parse_url('http://h/p#f?k=v', 'QUERY')")
+ testFoldConst("select parse_url('http://h/p#f/?#k=v', 'QUERY')")
+ // The query component ends before the fragment.
+ testFoldConst("select parse_url('http://h/p?k=1#f&k=2', 'QUERY')")
+ testFoldConst("select parse_url('http://h/p?k=1&k=2#f', 'QUERY')")
testFoldConst("select PARSE_URL('http://example.com', 'PROTOCOL')")
testFoldConst("select PARSE_URL('http://example.com', 'protocol')")
testFoldConst("select PARSE_URL('http://example.com', 'Protocol')")
@@ -1502,6 +1509,13 @@ suite("fold_constant_string_arithmatic") {
testFoldConst("select
extract_url_parameter('http://user:[email protected]?a=b', null)")
testFoldConst("select extract_url_parameter(null, 'a')")
testFoldConst("select
extract_url_parameter('http://user:[email protected]?a=b', 'a&b')")
+ // The parameters are located between the first '?' and the fragment, so a
url whose '#'
+ // comes before its '?' has no parameters.
+ testFoldConst("select extract_url_parameter('http://h/p#f?k=v', 'k')")
+ testFoldConst("select extract_url_parameter('http://h/p#f?k=v', 'v')")
+ // The parameters end before the fragment.
+ testFoldConst("select extract_url_parameter('http://h/p?a=1&k=2#f', 'k')")
+ testFoldConst("select extract_url_parameter('http://h/p?a=1#f&k=2', 'k')")
testFoldConst("select
extract_url_parameter('http://user:[email protected]?a=b&c=d', 'c')")
testFoldConst("select
extract_url_parameter('http://user:[email protected]?a=b&c=d', 'C')")
testFoldConst("select
extract_url_parameter('http://user:[email protected]?a=b&c=d', 'd')")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
