This is an automated email from the ASF dual-hosted git repository.
Mryange pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new a042859e923 [fix](function) Preserve trailing zero bytes in string
distances (#66236)
a042859e923 is described below
commit a042859e9239af14bc206e59d66547329a9d1430
Author: Mryange <[email protected]>
AuthorDate: Tue Aug 4 10:49:40 2026 +0800
[fix](function) Preserve trailing zero bytes in string distances (#66236)
### What problem does this PR solve?
Problem Summary: String distance functions trimmed trailing zero bytes
from every input during execution. CHAR padding is already removed at
the storage read boundary, so this work was redundant for CHAR values
and incorrectly discarded legitimate trailing zero bytes from STRING and
VARCHAR values. Use the complete ColumnString values for Hamming,
Levenshtein, and Damerau-Levenshtein distance calculations.
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
.../exprs/function/function_hamming_distance.cpp | 33 ++++++++++------------
be/src/exprs/function/function_levenshtein.cpp | 12 ++++----
2 files changed, 20 insertions(+), 25 deletions(-)
diff --git a/be/src/exprs/function/function_hamming_distance.cpp
b/be/src/exprs/function/function_hamming_distance.cpp
index 230b02c32d6..ddbbd00b610 100644
--- a/be/src/exprs/function/function_hamming_distance.cpp
+++ b/be/src/exprs/function/function_hamming_distance.cpp
@@ -81,12 +81,11 @@ public:
if (!has_nullable) {
if (left_const) {
-
RETURN_IF_ERROR(scalar_vector(left_str_col->get_data_at(0).trim_tail_padding_zero(),
- *right_str_col, res_data));
+ RETURN_IF_ERROR(
+ scalar_vector(left_str_col->get_data_at(0),
*right_str_col, res_data));
} else if (right_const) {
- RETURN_IF_ERROR(vector_scalar(
- *left_str_col,
right_str_col->get_data_at(0).trim_tail_padding_zero(),
- res_data));
+ RETURN_IF_ERROR(
+ vector_scalar(*left_str_col,
right_str_col->get_data_at(0), res_data));
} else {
RETURN_IF_ERROR(vector_vector(*left_str_col, *right_str_col,
res_data));
}
@@ -104,7 +103,7 @@ public:
return Status::OK();
}
- const auto left =
left_str_col->get_data_at(0).trim_tail_padding_zero();
+ const auto left = left_str_col->get_data_at(0);
RETURN_IF_ERROR(scalar_vector_nullable(left, *right_str_col,
right_null_map, res_data,
null_map));
} else if (right_const) {
@@ -115,9 +114,8 @@ public:
return Status::OK();
}
- RETURN_IF_ERROR(vector_scalar_nullable(
- *left_str_col,
right_str_col->get_data_at(0).trim_tail_padding_zero(),
- left_null_map, res_data, null_map));
+ RETURN_IF_ERROR(vector_scalar_nullable(*left_str_col,
right_str_col->get_data_at(0),
+ left_null_map, res_data,
null_map));
} else {
for (size_t i = 0; i < input_rows_count; ++i) {
const bool left_is_null = left_null_map && (*left_null_map)[i];
@@ -128,9 +126,8 @@ public:
continue;
}
- RETURN_IF_ERROR(hamming_distance(
- left_str_col->get_data_at(i).trim_tail_padding_zero(),
-
right_str_col->get_data_at(i).trim_tail_padding_zero(), res_data[i], i));
+ RETURN_IF_ERROR(hamming_distance(left_str_col->get_data_at(i),
+
right_str_col->get_data_at(i), res_data[i], i));
}
}
@@ -149,8 +146,8 @@ private:
std::vector<size_t> left_offsets;
std::vector<size_t> right_offsets;
for (size_t i = 0; i < size; ++i) {
- const auto left = lcol.get_data_at(i).trim_tail_padding_zero();
- const auto right = rcol.get_data_at(i).trim_tail_padding_zero();
+ const auto left = lcol.get_data_at(i);
+ const auto right = rcol.get_data_at(i);
RETURN_IF_ERROR(hamming_distance_with_offsets(
left, left_offsets, false,
simd::VStringFunctions::is_ascii(left), right,
right_offsets, false,
simd::VStringFunctions::is_ascii(right), res[i], i));
@@ -167,7 +164,7 @@ private:
simd::VStringFunctions::get_utf8_char_offsets(rdata, right_offsets);
std::vector<size_t> left_offsets;
for (size_t i = 0; i < size; ++i) {
- const auto left = lcol.get_data_at(i).trim_tail_padding_zero();
+ const auto left = lcol.get_data_at(i);
RETURN_IF_ERROR(hamming_distance_with_offsets(
left, left_offsets, false,
simd::VStringFunctions::is_ascii(left), rdata,
right_offsets, true, right_ascii, res[i], i));
@@ -184,7 +181,7 @@ private:
simd::VStringFunctions::get_utf8_char_offsets(ldata, left_offsets);
std::vector<size_t> right_offsets;
for (size_t i = 0; i < size; ++i) {
- const auto right = rcol.get_data_at(i).trim_tail_padding_zero();
+ const auto right = rcol.get_data_at(i);
RETURN_IF_ERROR(hamming_distance_with_offsets(
ldata, left_offsets, true, left_ascii, right,
right_offsets, false,
simd::VStringFunctions::is_ascii(right), res[i], i));
@@ -208,7 +205,7 @@ private:
continue;
}
- const auto left = lcol.get_data_at(i).trim_tail_padding_zero();
+ const auto left = lcol.get_data_at(i);
RETURN_IF_ERROR(hamming_distance_with_offsets(
left, left_offsets, false,
simd::VStringFunctions::is_ascii(left), rdata,
right_offsets, true, right_ascii, res[i], i));
@@ -232,7 +229,7 @@ private:
continue;
}
- const auto right = rcol.get_data_at(i).trim_tail_padding_zero();
+ const auto right = rcol.get_data_at(i);
RETURN_IF_ERROR(hamming_distance_with_offsets(
ldata, left_offsets, true, left_ascii, right,
right_offsets, false,
simd::VStringFunctions::is_ascii(right), res[i], i));
diff --git a/be/src/exprs/function/function_levenshtein.cpp
b/be/src/exprs/function/function_levenshtein.cpp
index b4ee35e9976..06f20eca113 100644
--- a/be/src/exprs/function/function_levenshtein.cpp
+++ b/be/src/exprs/function/function_levenshtein.cpp
@@ -50,8 +50,7 @@ static StringRef string_ref_at(const ColumnString::Chars&
data,
const ColumnString::Offsets& offsets, size_t i)
{
DCHECK_LT(i, offsets.size());
const auto previous_offset = i == 0 ? 0 : offsets[i - 1];
- return StringRef(data.data() + previous_offset, offsets[i] -
previous_offset)
- .trim_tail_padding_zero();
+ return StringRef(data.data() + previous_offset, offsets[i] -
previous_offset);
}
static void get_utf8_char_offsets(const StringRef& ref, Utf8Offsets& offsets) {
@@ -390,15 +389,14 @@ private:
ResultPaddedPODArray& res) {
const size_t size = offsets.size();
res.resize(size);
- const auto constant_ref = constant.trim_tail_padding_zero();
- const bool constant_ascii =
simd::VStringFunctions::is_ascii(constant_ref);
+ const bool constant_ascii = simd::VStringFunctions::is_ascii(constant);
Utf8Offsets constant_offsets;
- get_utf8_char_offsets(constant_ref, constant_offsets);
+ get_utf8_char_offsets(constant, constant_offsets);
Utf8Offsets value_offsets;
for (size_t i = 0; i < size; ++i) {
RETURN_IF_ERROR(distance_with_const_offsets(string_ref_at(data,
offsets, i),
- value_offsets,
constant_ref,
- constant_offsets,
constant_ascii, res[i]));
+ value_offsets,
constant, constant_offsets,
+ constant_ascii,
res[i]));
}
return Status::OK();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
