This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 55b477ec6bb branch-4.2: [fix](function) Align parse_url boundaries and
nullable execution #68490 (#70131)
55b477ec6bb is described below
commit 55b477ec6bb5dbc7781816333d469c9f6e688f8a
Author: morrySnow <[email protected]>
AuthorDate: Thu Nov 5 09:30:23 2026 +0800
branch-4.2: [fix](function) Align parse_url boundaries and nullable
execution #68490 (#70131)
picked from master #68490
Co-authored-by: Mryange <[email protected]>
---
be/src/exprs/function/function_string_url.cpp | 148 ++++++++-------------
be/src/util/url_parser.cpp | 136 +++++++++++--------
be/src/util/url_parser.h | 4 +
be/test/exprs/function/function_url_test.cpp | 47 +++++++
.../functions/executable/StringArithmetic.java | 86 ++++++++----
.../functions/executable/StringArithmeticTest.java | 41 ++++++
6 files changed, 288 insertions(+), 174 deletions(-)
diff --git a/be/src/exprs/function/function_string_url.cpp
b/be/src/exprs/function/function_string_url.cpp
index 3592356f98d..dcdaa6fccf7 100644
--- a/be/src/exprs/function/function_string_url.cpp
+++ b/be/src/exprs/function/function_string_url.cpp
@@ -18,13 +18,13 @@
#include <cstddef>
#include <string>
#include <string_view>
-#include <vector>
#include "common/status.h"
#include "core/assert_cast.h"
#include "core/block/block.h"
#include "core/block/column_numbers.h"
#include "core/column/column_const.h"
+#include "core/column/column_execute_util.h"
#include "core/column/column_nullable.h"
#include "core/column/column_string.h"
#include "core/column/column_vector.h"
@@ -32,7 +32,6 @@
#include "core/data_type/data_type_string.h"
#include "core/string_ref.h"
#include "exec/common/stringop_substring.h"
-#include "exec/common/template_helpers.hpp"
#include "exprs/function/function.h"
#include "exprs/function/function_helpers.h"
#include "exprs/function/simple_function_factory.h"
@@ -93,6 +92,7 @@ public:
String get_name() const override { return name; }
size_t get_number_of_arguments() const override { return 0; }
bool is_variadic() const override { return true; }
+ bool use_default_implementation_for_nulls() const override { return false;
}
DataTypePtr get_return_type_impl(const DataTypes& arguments) const
override {
return make_nullable(std::make_shared<DataTypeString>());
@@ -111,101 +111,67 @@ public:
size_t argument_size = arguments.size();
const bool has_key = argument_size == 3;
- std::vector<ColumnPtr> argument_columns(argument_size);
- std::vector<UInt8> col_const(argument_size);
- for (size_t i = 0; i < argument_size; ++i) {
- std::tie(argument_columns[i], col_const[i]) =
-
unpack_if_const(block.get_by_position(arguments[i]).column);
- }
-
- const auto* url_col = assert_cast<const
ColumnString*>(argument_columns[0].get());
- const auto* part_col = assert_cast<const
ColumnString*>(argument_columns[1].get());
- const bool part_const = col_const[1];
- std::vector<UrlParser::UrlPart> url_parts;
- const int part_nums = part_const ? 1 : input_rows_count;
-
- url_parts.resize(part_nums);
- for (int i = 0; i < part_nums; i++) {
- StringRef part = part_col->get_data_at(i);
- UrlParser::UrlPart url_part = UrlParser::get_url_part(part);
- if (url_part == UrlParser::INVALID) {
- return Status::RuntimeError("Invalid URL part: {}\n{}",
- std::string(part.data, part.size),
- "(Valid URL parts are 'PROTOCOL',
'HOST', "
- "'PATH', 'REF', 'AUTHORITY', "
- "'FILE', 'USERINFO', 'PORT' and
'QUERY')");
- }
- url_parts[i] = url_part;
- }
-
+ const auto url_col =
+
ColumnView<TYPE_STRING>::create(block.get_by_position(arguments[0]).column);
+ const auto part_col =
+
ColumnView<TYPE_STRING>::create(block.get_by_position(arguments[1]).column);
if (has_key) {
- const bool url_const = col_const[0];
- const bool key_const = col_const[2];
- const auto* key_col = assert_cast<const
ColumnString*>(argument_columns[2].get());
- RETURN_IF_ERROR(std::visit(
- [&](auto url_const, auto part_const, auto key_const) {
- return vector_parse_key<url_const, part_const,
key_const>(
- url_col, url_parts, key_col, input_rows_count,
null_map_data,
- res_chars, res_offsets);
- },
- make_bool_variant(url_const),
make_bool_variant(part_const),
- make_bool_variant(key_const)));
- } else {
- const bool url_const = col_const[0];
- RETURN_IF_ERROR(std::visit(
- [&](auto url_const, auto part_const) {
- return vector_parse<url_const, part_const>(url_col,
url_parts,
-
input_rows_count, null_map_data,
- res_chars,
res_offsets);
- },
- make_bool_variant(url_const),
make_bool_variant(part_const)));
- }
- block.get_by_position(result).column =
- ColumnNullable::create(std::move(res), std::move(null_map));
- return Status::OK();
- }
- template <bool url_const, bool part_const>
- static Status vector_parse(const ColumnString* url_col,
- std::vector<UrlParser::UrlPart>& url_parts,
const int size,
- ColumnUInt8::Container& null_map_data,
- ColumnString::Chars& res_chars,
ColumnString::Offsets& res_offsets) {
- for (size_t i = 0; i < size; ++i) {
- UrlParser::UrlPart& url_part =
url_parts[index_check_const<part_const>(i)];
- StringRef url_val =
url_col->get_data_at(index_check_const<url_const>(i));
- StringRef parse_res;
- if (UrlParser::parse_url(url_val, url_part, &parse_res)) {
- if (parse_res.empty()) [[unlikely]] {
- StringOP::push_empty_string(i, res_chars, res_offsets);
+ const auto key_col =
+
ColumnView<TYPE_STRING>::create(block.get_by_position(arguments[2]).column);
+ for (size_t i = 0; i < input_rows_count; ++i) {
+ if (url_col.is_null_at(i) || part_col.is_null_at(i) ||
key_col.is_null_at(i)) {
+ StringOP::push_null_string(i, res_chars, res_offsets,
null_map_data);
continue;
}
- StringOP::push_value_string(std::string_view(parse_res.data,
parse_res.size), i,
- res_chars, res_offsets);
- } else {
- StringOP::push_null_string(i, res_chars, res_offsets,
null_map_data);
+ const auto part = part_col.value_at(i);
+ const auto url_part = UrlParser::get_url_part(part);
+ if (url_part == UrlParser::INVALID) {
+ return Status::RuntimeError("Invalid URL part: {}\n{}",
+ std::string(part.data,
part.size),
+ "(Valid URL parts are
'PROTOCOL', 'HOST', "
+ "'PATH', 'REF', 'AUTHORITY', "
+ "'FILE', 'USERINFO', 'PORT'
and 'QUERY')");
+ }
+ StringRef parse_res;
+ if (UrlParser::parse_url_key(url_col.value_at(i), url_part,
key_col.value_at(i),
+ &parse_res)) {
+
StringOP::push_value_string(std::string_view(parse_res.data, parse_res.size), i,
+ res_chars, res_offsets);
+ } else {
+ StringOP::push_null_string(i, res_chars, res_offsets,
null_map_data);
+ }
}
- }
- return Status::OK();
- }
- template <bool url_const, bool part_const, bool key_const>
- static Status vector_parse_key(const ColumnString* url_col,
- std::vector<UrlParser::UrlPart>& url_parts,
- const ColumnString* key_col, const int size,
- ColumnUInt8::Container& null_map_data,
- ColumnString::Chars& res_chars,
- ColumnString::Offsets& res_offsets) {
- for (size_t i = 0; i < size; ++i) {
- UrlParser::UrlPart& url_part =
url_parts[index_check_const<part_const>(i)];
- StringRef url_val =
url_col->get_data_at(index_check_const<url_const>(i));
- StringRef url_key =
key_col->get_data_at(index_check_const<key_const>(i));
- StringRef parse_res;
- if (UrlParser::parse_url_key(url_val, url_part, url_key,
&parse_res)) {
- StringOP::push_value_string(std::string_view(parse_res.data,
parse_res.size), i,
- res_chars, res_offsets);
- } else {
- StringOP::push_null_string(i, res_chars, res_offsets,
null_map_data);
- continue;
+ } else {
+ for (size_t i = 0; i < input_rows_count; ++i) {
+ if (url_col.is_null_at(i) || part_col.is_null_at(i)) {
+ StringOP::push_null_string(i, res_chars, res_offsets,
null_map_data);
+ continue;
+ }
+ const auto part = part_col.value_at(i);
+ const auto url_part = UrlParser::get_url_part(part);
+ if (url_part == UrlParser::INVALID) {
+ return Status::RuntimeError("Invalid URL part: {}\n{}",
+ std::string(part.data,
part.size),
+ "(Valid URL parts are
'PROTOCOL', 'HOST', "
+ "'PATH', 'REF', 'AUTHORITY', "
+ "'FILE', 'USERINFO', 'PORT'
and 'QUERY')");
+ }
+ StringRef parse_res;
+ if (UrlParser::parse_url(url_col.value_at(i), url_part,
&parse_res)) {
+ if (parse_res.empty()) {
+ StringOP::push_empty_string(i, res_chars, res_offsets);
+ } else {
+ StringOP::push_value_string(
+ std::string_view(parse_res.data,
parse_res.size), i, res_chars,
+ res_offsets);
+ }
+ } else {
+ StringOP::push_null_string(i, res_chars, res_offsets,
null_map_data);
+ }
}
}
+ block.get_by_position(result).column =
+ ColumnNullable::create(std::move(res), std::move(null_map));
return Status::OK();
}
};
diff --git a/be/src/util/url_parser.cpp b/be/src/util/url_parser.cpp
index 292ed950876..e168a999910 100644
--- a/be/src/util/url_parser.cpp
+++ b/be/src/util/url_parser.cpp
@@ -75,6 +75,21 @@ bool UrlParser::find_query_component(const StringRef& url,
StringRef* query) {
return true;
}
+StringRef UrlParser::find_authority(const StringRef& protocol_end) {
+ // The authority component runs from the end of '://' up to the first '/',
'?' or '#',
+ // whichever comes first.
+ int32_t end_pos = _s_slash_search.search(&protocol_end);
+ int32_t question_pos = _s_question_search.search(&protocol_end);
+ if (question_pos >= 0 && (end_pos < 0 || question_pos < end_pos)) {
+ end_pos = question_pos;
+ }
+ int32_t hash_pos = _s_hash_search.search(&protocol_end);
+ if (hash_pos >= 0 && (end_pos < 0 || hash_pos < end_pos)) {
+ end_pos = hash_pos;
+ }
+ return protocol_end.substring(0, end_pos);
+}
+
bool UrlParser::parse_url(const StringRef& url, UrlPart part, StringRef*
result) {
result->data = nullptr;
result->size = 0;
@@ -92,18 +107,21 @@ bool UrlParser::parse_url(const StringRef& url, UrlPart
part, StringRef* result)
switch (part) {
case AUTHORITY: {
- // Find first '/'.
- int32_t end_pos = _s_slash_search.search(&protocol_end);
- *result = protocol_end.substring(0, end_pos);
+ *result = find_authority(protocol_end);
break;
}
case FILE:
case PATH: {
- // Find first '/'.
- int32_t start_pos = _s_slash_search.search(&protocol_end);
-
- if (start_pos < 0) {
+ int32_t slash_pos = _s_slash_search.search(&protocol_end);
+ int32_t question_pos = _s_question_search.search(&protocol_end);
+ int32_t hash_pos = _s_hash_search.search(&protocol_end);
+ int32_t start_pos = slash_pos;
+ if (part == FILE && (start_pos < 0 || (question_pos >= 0 &&
question_pos < start_pos))) {
+ start_pos = question_pos;
+ }
+ if (start_pos < 0 || (part == PATH && question_pos >= 0 &&
question_pos < start_pos) ||
+ (hash_pos >= 0 && hash_pos < start_pos)) {
// Return empty string. This is what Hive does.
return true;
}
@@ -117,10 +135,9 @@ bool UrlParser::parse_url(const StringRef& url, UrlPart
part, StringRef* result)
} else {
// End string _s_at next '?' or '#'.
end_pos = _s_question_search.search(&path_start);
-
- if (end_pos < 0) {
- // No '?' was found, look for '#'.
- end_pos = _s_hash_search.search(&path_start);
+ int32_t path_hash_pos = _s_hash_search.search(&path_start);
+ if (end_pos < 0 || (path_hash_pos >= 0 && path_hash_pos <
end_pos)) {
+ end_pos = path_hash_pos;
}
}
@@ -129,31 +146,29 @@ bool UrlParser::parse_url(const StringRef& url, UrlPart
part, StringRef* result)
}
case HOST: {
- // Find '@'.
- int32_t start_pos = _s_at_search.search(&protocol_end);
-
- if (start_pos < 0) {
- // No '@' was found, i.e., no user:pass info was given, start
after _s_protocol.
- start_pos = 0;
- } else {
- // Skip '@'.
- start_pos += _s_at.size;
- }
-
- StringRef host_start = protocol_end.substring(start_pos);
- // Find first '?'.
- int32_t query_start_pos = _s_question_search.search(&host_start);
- if (query_start_pos > 0) {
- host_start = host_start.substring(0, query_start_pos);
+ StringRef authority = find_authority(protocol_end);
+ int32_t userinfo_end = -1;
+ for (int32_t i = 0; std::cmp_less(i, authority.size); ++i) {
+ if (authority.data[i] == '@') {
+ userinfo_end = i;
+ }
}
- // Find ':' to strip out port.
- int32_t end_pos = _s_colon_search.search(&host_start);
- if (end_pos < 0) {
- // No port was given. search for '/' to determine ending position.
- end_pos = _s_slash_search.search(&host_start);
+ StringRef host_start = authority.substring(userinfo_end + 1);
+ auto end_pos = cast_set<int32_t>(host_start.size);
+ if (!host_start.empty() && host_start.data[0] == '[') {
+ for (int32_t i = 1; std::cmp_less(i, host_start.size); ++i) {
+ if (host_start.data[i] == ']') {
+ end_pos = i + 1;
+ break;
+ }
+ }
+ } else {
+ int32_t colon_pos = _s_colon_search.search(&host_start);
+ if (colon_pos >= 0) {
+ end_pos = colon_pos;
+ }
}
-
*result = host_start.substring(0, end_pos);
break;
}
@@ -190,45 +205,54 @@ bool UrlParser::parse_url(const StringRef& url, UrlPart
part, StringRef* result)
}
case USERINFO: {
- // Find '@'.
- int32_t end_pos = _s_at_search.search(&protocol_end);
+ StringRef authority = find_authority(protocol_end);
+ int32_t end_pos = -1;
+ for (int32_t i = 0; std::cmp_less(i, authority.size); ++i) {
+ if (authority.data[i] == '@') {
+ end_pos = i;
+ }
+ }
if (end_pos < 0) {
// Indicate no user and pass were given.
return false;
}
- *result = protocol_end.substring(0, end_pos);
+ *result = authority.substring(0, end_pos);
break;
}
case PORT: {
- // Find '@'.
- int32_t start_pos = _s_at_search.search(&protocol_end);
+ StringRef authority = find_authority(protocol_end);
+ int32_t userinfo_end = -1;
+ for (int32_t i = 0; std::cmp_less(i, authority.size); ++i) {
+ if (authority.data[i] == '@') {
+ userinfo_end = i;
+ }
+ }
- if (start_pos < 0) {
- // No '@' was found, i.e., no user:pass info was given, start
after _s_protocol.
- start_pos = 0;
+ StringRef host_start = authority.substring(userinfo_end + 1);
+ int32_t port_start = -1;
+ if (!host_start.empty() && host_start.data[0] == '[') {
+ for (int32_t i = 1; std::cmp_less(i, host_start.size); ++i) {
+ if (host_start.data[i] == ']') {
+ if (std::cmp_less(i + 1, host_start.size) &&
host_start.data[i + 1] == ':') {
+ port_start = i + 2;
+ }
+ break;
+ }
+ }
} else {
- // Skip '@'.
- start_pos += _s_at.size;
+ int32_t colon_pos = _s_colon_search.search(&host_start);
+ if (colon_pos >= 0) {
+ port_start = colon_pos + 1;
+ }
}
-
- StringRef host_start = protocol_end.substring(start_pos);
- // Find ':' to strip out port.
- int32_t end_pos = _s_colon_search.search(&host_start);
- //no port found
- if (end_pos < 0) {
+ if (port_start < 0) {
return false;
}
- StringRef port_start_str = host_start.substring(end_pos +
_s_colon.size);
- int32_t port_end_pos = _s_slash_search.search(&port_start_str);
- //if '/' not found, try to find '?'
- if (port_end_pos < 0) {
- port_end_pos = _s_question_search.search(&port_start_str);
- }
- *result = port_start_str.substring(0, port_end_pos);
+ *result = host_start.substring(port_start);
break;
}
diff --git a/be/src/util/url_parser.h b/be/src/util/url_parser.h
index 790a5c536cb..41c876daeaf 100644
--- a/be/src/util/url_parser.h
+++ b/be/src/util/url_parser.h
@@ -72,6 +72,10 @@ private:
// '#' comes before the first '?' because the '?' then belongs to the
fragment.
static bool find_query_component(const StringRef& url, StringRef* query);
+ // Returns the authority component of url, which has already had its
protocol stripped.
+ // The authority ends at the first '/', '?' or '#'.
+ static StringRef find_authority(const StringRef& protocol_end);
+
// Constants representing parts of a URL.
static const StringRef _s_url_authority;
static const StringRef _s_url_file;
diff --git a/be/test/exprs/function/function_url_test.cpp
b/be/test/exprs/function/function_url_test.cpp
index 7fc20fb104c..263346170c7 100644
--- a/be/test/exprs/function/function_url_test.cpp
+++ b/be/test/exprs/function/function_url_test.cpp
@@ -99,6 +99,10 @@ TEST(FunctionUrlTEST, ParseUrlQueryKeyTest) {
PrimitiveType::TYPE_VARCHAR};
DataSet data_set = {
+ // Every nullable argument must short-circuit independently in the
keyed form.
+ {{Null(), STRING("QUERY"), STRING("k")}, Null()},
+ {{STRING("http://h/p?k=1";), Null(), STRING("k")}, Null()},
+ {{STRING("http://h/p?k=1";), STRING("QUERY"), Null()}, Null()},
// The only '?' is inside the fragment, so the url has no query
component.
{{STRING("http://h/p#f?k=v";), STRING("QUERY"), STRING("k")},
Null()},
// The '#' comes before the '?', so it is a fragment instead of a
query.
@@ -161,4 +165,47 @@ TEST(FunctionUrlTEST, ParseUrlQueryTest) {
static_cast<void>(check_function<DataTypeString, true>(func_name,
input_types, data_set));
}
+TEST(FunctionUrlTEST, ParseUrlAuthorityTest) {
+ std::string func_name = "parse_url";
+ InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR,
PrimitiveType::TYPE_VARCHAR};
+
+ DataSet data_set = {
+ // A nullable dynamic part must be handled row by row. A NULL part
must not be
+ // materialized as an invalid empty part while another row is
being parsed.
+ {{Null(), STRING("HOST")}, Null()},
+ {{STRING("http://example.com/path?x=1";), STRING("HOST")},
STRING("example.com")},
+ {{STRING("http://example.com/path?x=1";), Null()}, Null()},
+ // A ':' in the path is not a port separator, and an '@' in the
path is not a
+ // userinfo separator.
+ {{STRING("http://example.com/a:b";), STRING("HOST")},
STRING("example.com")},
+ {{STRING("http://example.com/a:b";), STRING("PORT")}, Null()},
+ {{STRING("http://example.com/a:b";), STRING("AUTHORITY")},
STRING("example.com")},
+ {{STRING("http://example.com/a@b:c";), STRING("HOST")},
STRING("example.com")},
+ {{STRING("http://example.com/a@b:c";), STRING("USERINFO")}, Null()},
+ // A ':' in the query or the fragment is not a port separator
either.
+ {{STRING("http://example.com/p?r=http:8080";), STRING("PORT")},
Null()},
+ {{STRING("http://example.com#f:1";), STRING("HOST")},
STRING("example.com")},
+ {{STRING("http://example.com#f:1";), STRING("PORT")}, Null()},
+ {{STRING("http://example.com?x=1";), STRING("AUTHORITY")},
STRING("example.com")},
+ {{STRING("http://[2001:db8::1]:8080/a?x=1#r";), STRING("HOST")},
+ STRING("[2001:db8::1]")},
+ {{STRING("http://[2001:db8::1]:8080/a?x=1#r";), STRING("PORT")},
STRING("8080")},
+ {{STRING("http://h/p#frag?x=1";), STRING("PATH")}, STRING("/p")},
+ {{STRING("http://h/p#frag?x=1";), STRING("QUERY")}, Null()},
+ // FILE includes a query even when the URL has no path, while PATH
remains empty.
+ {{STRING("http://h?k=/v#frag";), STRING("FILE")}, STRING("?k=/v")},
+ {{STRING("http://h?k=/v#frag";), STRING("PATH")}, STRING("")},
+ // A real port and a real userinfo are still returned.
+ {{STRING("http://user:[email protected]:80/a:b";), STRING("HOST")},
+ STRING("example.com")},
+ {{STRING("http://user:[email protected]:80/a:b";), STRING("PORT")},
STRING("80")},
+ {{STRING("http://user:[email protected]:80/a:b";),
STRING("USERINFO")},
+ STRING("user:pass")},
+ {{STRING("http://user:[email protected]:80/a:b";),
STRING("AUTHORITY")},
+ STRING("user:[email protected]:80")},
+ };
+
+ static_cast<void>(check_function<DataTypeString, true>(func_name,
input_types, data_set));
+}
+
} // namespace doris
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java
index 3a422fda284..98a79583168 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java
@@ -987,17 +987,39 @@ public class StringArithmetic {
return Math.min(firstIndex, secondIndex);
}
+ private static int firstIndexOf(String value, char first, char second,
char third) {
+ return firstIndexOf(value, firstIndexOf(value, first, second), third);
+ }
+
+ private static int firstIndexOf(String value, int firstIndex, char second)
{
+ int secondIndex = value.indexOf(second);
+ if (firstIndex < 0) {
+ return secondIndex;
+ }
+ if (secondIndex < 0) {
+ return firstIndex;
+ }
+ return Math.min(firstIndex, secondIndex);
+ }
+
private static String substringEnd(String value, int end) {
return end < 0 ? value : value.substring(0, end);
}
private static String parseUrlAuthority(String protocolEnd) {
- return substringEnd(protocolEnd, protocolEnd.indexOf('/'));
+ // The authority component runs from the end of "://" up to the first
'/', '?' or '#',
+ // whichever comes first.
+ int endPos = firstIndexOf(protocolEnd, '?', '#');
+ int slashPos = protocolEnd.indexOf('/');
+ if (slashPos >= 0 && (endPos < 0 || slashPos < endPos)) {
+ endPos = slashPos;
+ }
+ return substringEnd(protocolEnd, endPos);
}
private static String parseUrlPath(String protocolEnd) {
- int startPos = protocolEnd.indexOf('/');
- if (startPos < 0) {
+ int startPos = firstIndexOf(protocolEnd, '/', '?', '#');
+ if (startPos < 0 || protocolEnd.charAt(startPos) != '/') {
return "";
}
String pathStart = protocolEnd.substring(startPos);
@@ -1006,7 +1028,12 @@ public class StringArithmetic {
private static String parseUrlFile(String protocolEnd) {
int startPos = protocolEnd.indexOf('/');
- if (startPos < 0) {
+ int queryPos = protocolEnd.indexOf('?');
+ int fragmentPos = protocolEnd.indexOf('#');
+ if (startPos < 0 || (queryPos >= 0 && queryPos < startPos)) {
+ startPos = queryPos;
+ }
+ if (startPos < 0 || (fragmentPos >= 0 && fragmentPos < startPos)) {
return "";
}
String pathStart = protocolEnd.substring(startPos);
@@ -1014,18 +1041,15 @@ public class StringArithmetic {
}
private static String parseUrlHost(String protocolEnd) {
- int startPos = protocolEnd.indexOf('@');
+ String authority = parseUrlAuthority(protocolEnd);
+ int startPos = authority.lastIndexOf('@');
startPos = startPos < 0 ? 0 : startPos + 1;
- String hostStart = protocolEnd.substring(startPos);
- int queryStartPos = hostStart.indexOf('?');
- if (queryStartPos > 0) {
- hostStart = hostStart.substring(0, queryStartPos);
- }
- int endPos = hostStart.indexOf(':');
- if (endPos < 0) {
- endPos = hostStart.indexOf('/');
+ String hostStart = authority.substring(startPos);
+ if (hostStart.startsWith("[")) {
+ int closeBracket = hostStart.indexOf(']');
+ return closeBracket < 0 ? hostStart : hostStart.substring(0,
closeBracket + 1);
}
- return substringEnd(hostStart, endPos);
+ return substringEnd(hostStart, hostStart.indexOf(':'));
}
private static String parseUrlQuery(String protocolEnd) {
@@ -1052,27 +1076,35 @@ public class StringArithmetic {
}
private static String parseUrlUserInfo(String protocolEnd) {
- int endPos = protocolEnd.indexOf('@');
+ String authority = parseUrlAuthority(protocolEnd);
+ int endPos = authority.lastIndexOf('@');
if (endPos < 0) {
return null;
}
- return protocolEnd.substring(0, endPos);
+ return authority.substring(0, endPos);
}
private static String parseUrlPort(String protocolEnd) {
- int startPos = protocolEnd.indexOf('@');
+ String authority = parseUrlAuthority(protocolEnd);
+ int startPos = authority.lastIndexOf('@');
startPos = startPos < 0 ? 0 : startPos + 1;
- String hostStart = protocolEnd.substring(startPos);
- int endPos = hostStart.indexOf(':');
- if (endPos < 0) {
- return null;
- }
- String portStart = hostStart.substring(endPos + 1);
- int portEndPos = portStart.indexOf('/');
- if (portEndPos < 0) {
- portEndPos = portStart.indexOf('?');
+ String hostStart = authority.substring(startPos);
+ int portStart;
+ if (hostStart.startsWith("[")) {
+ int closeBracket = hostStart.indexOf(']');
+ if (closeBracket < 0 || closeBracket + 1 >= hostStart.length()
+ || hostStart.charAt(closeBracket + 1) != ':') {
+ return null;
+ }
+ portStart = closeBracket + 2;
+ } else {
+ int colonPos = hostStart.indexOf(':');
+ if (colonPos < 0) {
+ return null;
+ }
+ portStart = colonPos + 1;
}
- return substringEnd(portStart, portEndPos);
+ return hostStart.substring(portStart);
}
/**
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java
index fda0a6f31b6..22f0292f7c9 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java
@@ -120,6 +120,47 @@ class StringArithmeticTest {
assertExtractUrlParameter("http://h/p?k1=aa&k2=bb#f";, "k2", "bb");
}
+ @Test
+ void testParseUrlStopsAtTheAuthority() {
+ // A ':' in the path is not a port separator, and an '@' in the path
is not a
+ // userinfo separator.
+ assertParseUrl("http://example.com/a:b";, "HOST", "example.com");
+ assertParseUrlIsNull("http://example.com/a:b";, "PORT");
+ assertParseUrl("http://example.com/a:b";, "AUTHORITY", "example.com");
+ assertParseUrl("http://example.com/a@b:c";, "HOST", "example.com");
+ assertParseUrlIsNull("http://example.com/a@b:c";, "USERINFO");
+ // A ':' in the query or the fragment is not a port separator either.
+ assertParseUrlIsNull("http://example.com/p?r=http:8080";, "PORT");
+ assertParseUrl("http://example.com#f:1";, "HOST", "example.com");
+ assertParseUrlIsNull("http://example.com#f:1";, "PORT");
+ assertParseUrl("http://example.com?x=1";, "AUTHORITY", "example.com");
+ assertParseUrl("http://[2001:db8::1]:8080/a?x=1#r";, "HOST",
"[2001:db8::1]");
+ assertParseUrl("http://[2001:db8::1]:8080/a?x=1#r";, "PORT", "8080");
+ assertParseUrl("http://h/p#frag?x=1";, "PATH", "/p");
+ assertParseUrlIsNull("http://h/p#frag?x=1";, "QUERY");
+ // FILE includes a query even when the URL has no path, while PATH
remains empty.
+ assertParseUrl("http://h?k=/v#frag";, "FILE", "?k=/v");
+ assertParseUrl("http://h?k=/v#frag";, "PATH", "");
+ // A real port and a real userinfo are still returned.
+ assertParseUrl("http://user:[email protected]:80/a:b";, "HOST",
"example.com");
+ assertParseUrl("http://user:[email protected]:80/a:b";, "PORT", "80");
+ assertParseUrl("http://user:[email protected]:80/a:b";, "USERINFO",
"user:pass");
+ assertParseUrl("http://user:[email protected]:80/a:b";, "AUTHORITY",
+ "user:[email protected]:80");
+ }
+
+ private void assertParseUrl(String url, String part, String expected) {
+ Expression result = StringArithmetic.parseurl(
+ new StringLiteral(url), new StringLiteral(part));
+ Assertions.assertEquals(expected, ((StringLikeLiteral)
result).getValue());
+ }
+
+ private void assertParseUrlIsNull(String url, String part) {
+ Expression result = StringArithmetic.parseurl(
+ new StringLiteral(url), new StringLiteral(part));
+ Assertions.assertTrue(result instanceof NullLiteral, url + " " + part);
+ }
+
private void assertParseUrlQuery(String url, String expected) {
Expression result = StringArithmetic.parseurl(
new StringLiteral(url), new StringLiteral("QUERY"));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
