This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new f9862636c3b branch-4.1: [bug](analytic) fix partition by column when
it overflow haven't convert to column64 #65240 (#65297)
f9862636c3b is described below
commit f9862636c3b80c2d23aed0d0e411f7b297b60dc7
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Mon Jul 13 14:10:54 2026 +0800
branch-4.1: [bug](analytic) fix partition by column when it overflow
haven't convert to column64 #65240 (#65297)
Cherry-picked from #65240
Co-authored-by: zhangstar333 <[email protected]>
---
be/src/core/column/column_string.cpp | 56 +++++++++++++---------
be/src/exec/operator/analytic_sink_operator.cpp | 8 ++++
be/test/core/column/column_string_test.cpp | 20 ++++++++
.../window_functions/test_column_boundary.out | 3 +-
4 files changed, 62 insertions(+), 25 deletions(-)
diff --git a/be/src/core/column/column_string.cpp
b/be/src/core/column/column_string.cpp
index b222030c293..bb921b93d1a 100644
--- a/be/src/core/column/column_string.cpp
+++ b/be/src/core/column/column_string.cpp
@@ -96,33 +96,43 @@ void ColumnStr<T>::insert_range_from_ignore_overflow(const
doris::IColumn& src,
return;
}
- const auto& src_concrete = assert_cast<const ColumnStr<T>&>(src);
- if (start + length > src_concrete.offsets.size()) {
- throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR,
- "Parameter out of bound in "
-
"IColumnStr<T>::insert_range_from_ignore_overflow method.");
- }
+ auto do_insert = [&](const auto& src_concrete) {
+ const auto& src_offsets = src_concrete.get_offsets();
+ const auto& src_chars = src_concrete.get_chars();
+ if (start + length > src_offsets.size()) {
+ throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR,
+ "Parameter out of bound in "
+
"IColumnStr<T>::insert_range_from_ignore_overflow method.");
+ }
- auto nested_offset = src_concrete.offset_at(start);
- auto nested_length = src_concrete.offsets[start + length - 1] -
nested_offset;
+ auto nested_offset = src_offsets[static_cast<ssize_t>(start) - 1];
+ auto nested_length = src_offsets[start + length - 1] - nested_offset;
- size_t old_chars_size = chars.size();
- chars.resize(old_chars_size + nested_length);
- memcpy(&chars[old_chars_size], &src_concrete.chars[nested_offset],
nested_length);
+ size_t old_chars_size = chars.size();
+ chars.resize(old_chars_size + nested_length);
+ memcpy(&chars[old_chars_size], &src_chars[nested_offset],
nested_length);
- if (start == 0 && offsets.empty()) {
- offsets.assign(src_concrete.offsets.begin(),
src_concrete.offsets.begin() + length);
- } else {
- size_t old_size = offsets.size();
- auto prev_max_offset = offsets.back(); /// -1th index is Ok, see
PaddedPODArray
- offsets.resize(old_size + length);
-
- for (size_t i = 0; i < length; ++i) {
- // unsinged integer overflow is well defined in C++,
- // so we don't need to check the overflow here.
- offsets[old_size + i] =
- src_concrete.offsets[start + i] - nested_offset +
prev_max_offset;
+ using OffsetsType = std::decay_t<decltype(src_offsets)>;
+ if (std::is_same_v<T, typename OffsetsType::value_type> && start == 0
&& offsets.empty()) {
+ offsets.assign(src_offsets.begin(), src_offsets.begin() + length);
+ } else {
+ size_t old_size = offsets.size();
+ auto prev_max_offset = offsets.back(); /// -1th index is Ok, see
PaddedPODArray
+ offsets.resize(old_size + length);
+
+ for (size_t i = 0; i < length; ++i) {
+ // unsinged integer overflow is well defined in C++,
+ // so we don't need to check the overflow here.
+ offsets[old_size + i] =
+ static_cast<T>(src_offsets[start + i] - nested_offset)
+ prev_max_offset;
+ }
}
+ };
+
+ if (src.is_column_string64()) {
+ do_insert(assert_cast<const ColumnStr<uint64_t>&>(src));
+ } else {
+ do_insert(assert_cast<const ColumnStr<uint32_t>&>(src));
}
#ifndef NDEBUG
diff --git a/be/src/exec/operator/analytic_sink_operator.cpp
b/be/src/exec/operator/analytic_sink_operator.cpp
index 0360d6168c3..133b4ec9e60 100644
--- a/be/src/exec/operator/analytic_sink_operator.cpp
+++ b/be/src/exec/operator/analytic_sink_operator.cpp
@@ -772,6 +772,12 @@ Status
AnalyticSinkOperatorX::_add_input_block(doris::RuntimeState* state, Block
local_state._input_block_first_row_positions.emplace_back(local_state._input_total_rows);
size_t block_rows = input_block->rows();
local_state._input_total_rows += block_rows;
+ auto convert_column_if_overflow = [](MutableColumnPtr& column) {
+ auto converted_column = column->convert_column_if_overflow();
+ if (converted_column.get() != column.get()) {
+ column = converted_column->assert_mutable();
+ }
+ };
// record origin columns, maybe be after this, could cast some column but
no need to output
auto column_to_keep = input_block->columns();
@@ -792,6 +798,7 @@ Status
AnalyticSinkOperatorX::_add_input_block(doris::RuntimeState* state, Block
RETURN_IF_ERROR(
_insert_range_column(input_block,
local_state._partition_by_eq_expr_ctxs[i],
local_state._partition_by_columns[i].get(), block_rows));
+ convert_column_if_overflow(local_state._partition_by_columns[i]);
}
}
{
@@ -800,6 +807,7 @@ Status
AnalyticSinkOperatorX::_add_input_block(doris::RuntimeState* state, Block
RETURN_IF_ERROR(_insert_range_column(input_block,
local_state._order_by_eq_expr_ctxs[i],
local_state._order_by_columns[i].get(),
block_rows));
+ convert_column_if_overflow(local_state._order_by_columns[i]);
}
}
{
diff --git a/be/test/core/column/column_string_test.cpp
b/be/test/core/column/column_string_test.cpp
index cf807f88615..3581be59893 100644
--- a/be/test/core/column/column_string_test.cpp
+++ b/be/test/core/column/column_string_test.cpp
@@ -185,6 +185,26 @@ TEST_F(ColumnStringTest, is_variable_length) {
ColumnString64::MutablePtr col64 = ColumnString64::create();
EXPECT_TRUE(col64->is_variable_length());
}
+
+TEST(ColumnStringStandaloneTest,
insert_range_from_ignore_overflow_to_string64_from_string32) {
+ auto src = ColumnString::create();
+ src->insert_data("a", 1);
+ src->insert_data("bc", 2);
+ src->insert_data("def", 3);
+
+ auto dst = ColumnString64::create();
+ dst->insert_range_from_ignore_overflow(*src, 0, src->size());
+ ASSERT_EQ(dst->size(), 3);
+ EXPECT_EQ(dst->get_data_at(0).to_string(), "a");
+ EXPECT_EQ(dst->get_data_at(1).to_string(), "bc");
+ EXPECT_EQ(dst->get_data_at(2).to_string(), "def");
+
+ dst->insert_range_from_ignore_overflow(*src, 1, 2);
+ ASSERT_EQ(dst->size(), 5);
+ EXPECT_EQ(dst->get_data_at(3).to_string(), "bc");
+ EXPECT_EQ(dst->get_data_at(4).to_string(), "def");
+}
+
TEST_F(ColumnStringTest, sanity_check) {
auto test_func = [](auto& col) {
auto& chars = col->get_chars();
diff --git
a/regression-test/data/query_p0/sql_functions/window_functions/test_column_boundary.out
b/regression-test/data/query_p0/sql_functions/window_functions/test_column_boundary.out
index 7a238f99935..33fb7867e55 100644
---
a/regression-test/data/query_p0/sql_functions/window_functions/test_column_boundary.out
+++
b/regression-test/data/query_p0/sql_functions/window_functions/test_column_boundary.out
@@ -3,5 +3,4 @@
200000000
-- !sql_2 --
-38135901019538738
-
+40000000000
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
