This is an automated email from the ASF dual-hosted git repository.
Mryange pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 91e9e8671a6 [fix](function) Preserve NULL values in parse_data_size
(#67910)
91e9e8671a6 is described below
commit 91e9e8671a63c69da2e348577ba6e3b377f622c1
Author: Mryange <[email protected]>
AuthorDate: Mon Sep 14 11:05:32 2026 +0800
[fix](function) Preserve NULL values in parse_data_size (#67910)
`parse_data_size()` fails on string columns containing both valid values
and NULL. Root cause: the default nullable wrapper passes NULL payloads
to the parser as empty strings. Use `ColumnView` to skip NULL rows and
preserve nullability. Invalid non-NULL inputs still report errors.
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/src/exprs/function/function_string.cpp | 55 +++++++++++++++-------
be/test/exprs/function/function_string_test.cpp | 24 ++++++++++
.../test_parse_data_size_nullable.out | 24 ++++++++++
.../test_parse_data_size_nullable.groovy | 50 ++++++++++++++++++++
4 files changed, 135 insertions(+), 18 deletions(-)
diff --git a/be/src/exprs/function/function_string.cpp
b/be/src/exprs/function/function_string.cpp
index 422f0b17560..d62573112bb 100644
--- a/be/src/exprs/function/function_string.cpp
+++ b/be/src/exprs/function/function_string.cpp
@@ -38,6 +38,7 @@
#include "common/logging.h"
#include "common/status.h"
#include "core/column/column.h"
+#include "core/column/column_execute_util.h"
#include "core/column/column_string.h"
#include "core/data_type/data_type_nullable.h"
#include "core/pod_array_fwd.h"
@@ -78,10 +79,6 @@ struct StringASCII {
}
};
-struct NameParseDataSize {
- static constexpr auto name = "parse_data_size";
-};
-
static const std::map<std::string_view, Int128> UNITS = {
{"B", static_cast<Int128>(1)}, {"kB", static_cast<Int128>(1) <<
10},
{"MB", static_cast<Int128>(1) << 20}, {"GB", static_cast<Int128>(1) <<
30},
@@ -89,24 +86,47 @@ static const std::map<std::string_view, Int128> UNITS = {
{"EB", static_cast<Int128>(1) << 60}, {"ZB", static_cast<Int128>(1) <<
70},
{"YB", static_cast<Int128>(1) << 80}};
-struct ParseDataSize {
- using ReturnType = DataTypeInt128;
- static constexpr auto PrimitiveTypeImpl = PrimitiveType::TYPE_STRING;
- using Type = String;
- using ReturnColumnType = ColumnInt128;
+class FunctionStringParseDataSize : public IFunction {
+public:
+ static constexpr auto name = "parse_data_size";
+ static FunctionPtr create() { return
std::make_shared<FunctionStringParseDataSize>(); }
+ String get_name() const override { return name; }
+ size_t get_number_of_arguments() const override { return 1; }
+ bool use_default_implementation_for_nulls() const override { return false;
}
- static Status vector(const ColumnString::Chars& data, const
ColumnString::Offsets& offsets,
- PaddedPODArray<Int128>& res) {
- auto size = offsets.size();
- res.resize(size);
- for (int i = 0; i < size; ++i) {
- const char* raw_str = reinterpret_cast<const
char*>(&data[offsets[i - 1]]);
- int str_size = offsets[i] - offsets[i - 1];
- res[i] = parse_data_size(std::string_view(raw_str, str_size));
+ DataTypePtr get_return_type_impl(const DataTypes& arguments) const
override {
+ auto type = std::make_shared<DataTypeInt128>();
+ return arguments[0]->is_nullable() ? make_nullable(type) : type;
+ }
+
+ Status execute_impl(FunctionContext* context, Block& block, const
ColumnNumbers& arguments,
+ uint32_t result, size_t input_rows_count) const
override {
+ const auto input =
+
ColumnView<TYPE_STRING>::create(block.get_by_position(arguments[0]).column);
+ auto res = ColumnInt128::create(input_rows_count, 0);
+ auto& values = res->get_data();
+ ColumnUInt8::MutablePtr null_map;
+ if (input.null_map != nullptr) {
+ null_map = ColumnUInt8::create(input_rows_count, 0);
+ }
+ for (size_t i = 0; i < input_rows_count; ++i) {
+ if (input.is_null_at(i)) {
+ null_map->get_data()[i] = 1;
+ continue;
+ }
+ const auto value = input.value_at(i);
+ values[i] = parse_data_size(std::string_view(value.data,
value.size));
+ }
+ if (null_map) {
+ block.replace_by_position(result,
+ ColumnNullable::create(std::move(res),
std::move(null_map)));
+ } else {
+ block.replace_by_position(result, std::move(res));
}
return Status::OK();
}
+private:
static Int128 parse_data_size(const std::string_view& dataSize) {
int digit_length = 0;
for (char c : dataSize) {
@@ -1349,7 +1369,6 @@ template <typename LeftDataType, typename RightDataType>
using StringFindInSetImpl = StringFunctionImpl<LeftDataType, RightDataType,
FindInSetOp>;
// ready for regist function
-using FunctionStringParseDataSize = FunctionUnaryToType<ParseDataSize,
NameParseDataSize>;
using FunctionStringASCII = FunctionUnaryToType<StringASCII, NameStringASCII>;
using FunctionStringLength = FunctionUnaryToType<StringLengthImpl,
NameStringLength>;
using FunctionCrc32 = FunctionUnaryToType<Crc32Impl, NameCrc32>;
diff --git a/be/test/exprs/function/function_string_test.cpp
b/be/test/exprs/function/function_string_test.cpp
index c9c818d2d59..f20fd3f7233 100644
--- a/be/test/exprs/function/function_string_test.cpp
+++ b/be/test/exprs/function/function_string_test.cpp
@@ -81,6 +81,30 @@ DataSet make_md5_varbinary_dataset(const
std::vector<std::string>& inputs) {
} // namespace
+TEST(function_string_test, parse_data_size_nullable) {
+ const InputTypeSet input_types = {PrimitiveType::TYPE_STRING};
+ const DataSet data_set = {{{Null()}, Null()},
+ {{std::string("1MB")}, LARGEINT(1048576)},
+ {{Null()}, Null()},
+ {{std::string("2.5MB")}, LARGEINT(2621440)},
+ {{std::string("0B")}, LARGEINT(0)},
+ {{Null()}, Null()}};
+ check_function_all_arg_comb<DataTypeInt128, true>("parse_data_size",
input_types, data_set);
+ check_function_all_arg_comb<DataTypeInt128, true>("parse_data_size",
input_types,
+ {{{Null()}, Null()},
{{Null()}, Null()}});
+
+ const InputTypeSet not_null_types = {Notnull {PrimitiveType::TYPE_STRING}};
+ const DataSet not_null_data = {{{std::string("1MB")}, LARGEINT(1048576)},
+ {{std::string("0B")}, LARGEINT(0)}};
+ ASSERT_TRUE(
+ check_function<DataTypeInt128>("parse_data_size", not_null_types,
not_null_data).ok());
+ const InputTypeSet const_not_null_types = {ConstedNotnull
{PrimitiveType::TYPE_STRING}};
+ for (const auto& row : not_null_data) {
+ ASSERT_TRUE(check_function<DataTypeInt128>("parse_data_size",
const_not_null_types, {row})
+ .ok());
+ }
+}
+
TEST(function_string_test, function_auto_partition_name_case_insensitive_test)
{
const InputTypeSet list_input_types = {Consted
{PrimitiveType::TYPE_VARCHAR},
Consted
{PrimitiveType::TYPE_VARCHAR}};
diff --git
a/regression-test/data/query_p0/sql_functions/string_functions/test_parse_data_size_nullable.out
b/regression-test/data/query_p0/sql_functions/string_functions/test_parse_data_size_nullable.out
new file mode 100644
index 00000000000..8eccd452d89
--- /dev/null
+++
b/regression-test/data/query_p0/sql_functions/string_functions/test_parse_data_size_nullable.out
@@ -0,0 +1,24 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !mixed --
+1 \N
+2 1048576
+3 \N
+4 2621440
+5 0
+6 \N
+
+-- !all_null --
+1 \N
+3 \N
+6 \N
+
+-- !non_null --
+2 1048576
+4 2621440
+5 0
+
+-- !constants --
+\N 1048576
+
+-- !empty --
+
diff --git
a/regression-test/suites/query_p0/sql_functions/string_functions/test_parse_data_size_nullable.groovy
b/regression-test/suites/query_p0/sql_functions/string_functions/test_parse_data_size_nullable.groovy
new file mode 100644
index 00000000000..0223136fe13
--- /dev/null
+++
b/regression-test/suites/query_p0/sql_functions/string_functions/test_parse_data_size_nullable.groovy
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_parse_data_size_nullable") {
+ sql "DROP TABLE IF EXISTS test_parse_data_size_nullable"
+ sql """
+ CREATE TABLE test_parse_data_size_nullable (
+ id INT NOT NULL,
+ value STRING NULL
+ ) DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_num" = "1")
+ """
+ sql """INSERT INTO test_parse_data_size_nullable VALUES
+ (1, NULL), (2, '1MB'), (3, NULL), (4, '2.5MB'), (5, '0B'), (6, NULL)"""
+
+ qt_mixed "SELECT id, parse_data_size(value) FROM
test_parse_data_size_nullable ORDER BY id"
+ qt_all_null """SELECT id, parse_data_size(value) FROM
test_parse_data_size_nullable
+ WHERE value IS NULL ORDER BY id"""
+ qt_non_null """SELECT id, parse_data_size(value) FROM
test_parse_data_size_nullable
+ WHERE value IS NOT NULL ORDER BY id"""
+ qt_constants "SELECT parse_data_size(NULL), parse_data_size('1MB')"
+ qt_empty "SELECT parse_data_size(value) FROM test_parse_data_size_nullable
WHERE id < 0"
+
+ sql "INSERT INTO test_parse_data_size_nullable VALUES (7, '')"
+ test {
+ sql "SELECT parse_data_size(value) FROM test_parse_data_size_nullable
ORDER BY id"
+ exception 'Invalid Input argument "" of function parse_data_size'
+ }
+ sql "INSERT INTO test_parse_data_size_nullable VALUES (8, 'invalid')"
+ test {
+ sql """SELECT parse_data_size(value) FROM test_parse_data_size_nullable
+ WHERE id IN (1, 8) ORDER BY id"""
+ exception 'Invalid Input argument "invalid" of function
parse_data_size'
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
