This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 5c46a390173 branch-4.0: [fix](crc32c) fix bug of crc32c (#60322)
(#60330)
5c46a390173 is described below
commit 5c46a390173abf582f5299905b8a64f2538ac3dc
Author: TengJianPing <[email protected]>
AuthorDate: Thu Jan 29 14:05:34 2026 +0800
branch-4.0: [fix](crc32c) fix bug of crc32c (#60322) (#60330)
Cherry-picked from #60322
Problem Summary:
None
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change. - [ ] No code files have been
changed. - [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/src/vec/columns/column_decimal.cpp | 5 +-
be/src/vec/columns/column_nullable.cpp | 3 +-
be/src/vec/columns/column_vector.cpp | 5 +-
.../data/datatype_p0/complex_types/test_array.out | 23 ++++
.../datatype_p0/complex_types/test_array.groovy | 117 +++++++++++++++++++++
5 files changed, 145 insertions(+), 8 deletions(-)
diff --git a/be/src/vec/columns/column_decimal.cpp
b/be/src/vec/columns/column_decimal.cpp
index f4d37242c41..34d600d07f7 100644
--- a/be/src/vec/columns/column_decimal.cpp
+++ b/be/src/vec/columns/column_decimal.cpp
@@ -193,15 +193,14 @@ void ColumnDecimal<T>::update_crc32c_batch(uint32_t*
__restrict hashes,
template <PrimitiveType T>
void ColumnDecimal<T>::update_crc32c_single(size_t start, size_t end,
uint32_t& hash,
const uint8_t* __restrict
null_map) const {
- auto s = size();
if (null_map) {
- for (size_t i = 0; i < s; ++i) {
+ for (size_t i = start; i < end; ++i) {
if (null_map[i] == 0) {
hash = HashUtil::crc32c_fixed(data[i], hash);
}
}
} else {
- for (size_t i = 0; i < s; ++i) {
+ for (size_t i = start; i < end; ++i) {
hash = HashUtil::crc32c_fixed(data[i], hash);
}
}
diff --git a/be/src/vec/columns/column_nullable.cpp
b/be/src/vec/columns/column_nullable.cpp
index 51784ab6d11..4ad84bd2a3e 100644
--- a/be/src/vec/columns/column_nullable.cpp
+++ b/be/src/vec/columns/column_nullable.cpp
@@ -140,8 +140,7 @@ void ColumnNullable::update_crc32c_single(size_t start,
size_t end, uint32_t& ha
const auto* __restrict real_null_data =
assert_cast<const
ColumnUInt8&>(get_null_map_column()).get_data().data();
constexpr int NULL_VALUE = 0;
- auto s = size();
- for (int i = 0; i < s; ++i) {
+ for (size_t i = start; i < end; ++i) {
if (real_null_data[i] != 0) {
hash = HashUtil::crc32c_fixed(NULL_VALUE, hash);
}
diff --git a/be/src/vec/columns/column_vector.cpp
b/be/src/vec/columns/column_vector.cpp
index 3d5a51bea43..65f85a8895a 100644
--- a/be/src/vec/columns/column_vector.cpp
+++ b/be/src/vec/columns/column_vector.cpp
@@ -222,15 +222,14 @@ void ColumnVector<T>::update_crc32c_batch(uint32_t*
__restrict hashes,
template <PrimitiveType T>
void ColumnVector<T>::update_crc32c_single(size_t start, size_t end, uint32_t&
hash,
const uint8_t* __restrict null_map)
const {
- auto s = size();
if (null_map) {
- for (size_t i = 0; i < s; ++i) {
+ for (size_t i = start; i < end; ++i) {
if (null_map[i] == 0) {
hash = _crc32c_hash(hash, i);
}
}
} else {
- for (size_t i = 0; i < s; ++i) {
+ for (size_t i = start; i < end; ++i) {
hash = _crc32c_hash(hash, i);
}
}
diff --git a/regression-test/data/datatype_p0/complex_types/test_array.out
b/regression-test/data/datatype_p0/complex_types/test_array.out
new file mode 100644
index 00000000000..16958938501
--- /dev/null
+++ b/regression-test/data/datatype_p0/complex_types/test_array.out
@@ -0,0 +1,23 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !window_function_partition_by_array_int --
+Badminton Racket [3] 3 3
+Basketball [1, 3] 1 4
+Basketball [1, 3] 3 4
+Laptop [1, 2, 3] 1 6
+Laptop [1, 2, 3] 2 6
+Laptop [1, 2, 3] 3 6
+Mechanical Keyboard [1, 2] 1 3
+Mechanical Keyboard [1, 2] 2 3
+Shirt [4] 4 4
+
+-- !window_function_partition_by_array_decimal --
+Badminton Racket [3.00] 3 3
+Basketball [1.00, 3.00] 1 4
+Basketball [1.00, 3.00] 3 4
+Laptop [1.00, 2.00, 3.00] 1 6
+Laptop [1.00, 2.00, 3.00] 2 6
+Laptop [1.00, 2.00, 3.00] 3 6
+Mechanical Keyboard [1.00, 2.00] 1 3
+Mechanical Keyboard [1.00, 2.00] 2 3
+Shirt [4.00] 4 4
+
diff --git a/regression-test/suites/datatype_p0/complex_types/test_array.groovy
b/regression-test/suites/datatype_p0/complex_types/test_array.groovy
new file mode 100644
index 00000000000..d94d0e06583
--- /dev/null
+++ b/regression-test/suites/datatype_p0/complex_types/test_array.groovy
@@ -0,0 +1,117 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_array") {
+ multi_sql """
+ set batch_size=3;
+ set enable_local_shuffle=false;
+ set parallel_pipeline_task_num = 2;
+ set enable_new_shuffle_hash_method=true;
+ """
+
+ sql "DROP TABLE IF EXISTS `test_array_int_table`"
+ multi_sql """
+ CREATE TABLE test_array_int_table(
+ id INT,
+ name VARCHAR(50),
+ tags ARRAY<VARCHAR(50)>,
+ price DECIMAL(10,2),
+ category_ids ARRAY<INT>,
+ cid INT
+ )
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 2
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+
+ INSERT INTO test_array_int_table VALUES
+ (1, 'Laptop', ['Electronics', 'Office', 'High-End', 'Laptop'],
5999.99, [1, 2, 3], 1),
+ (2, 'Mechanical Keyboard', ['Electronics', 'Accessories'], 399.99, [1,
2], 1),
+ (3, 'Basketball', ['Sports', 'Outdoor'], 199.99, [1,3], 1),
+ (4, 'Badminton Racket', ['Sports', 'Equipment'], 299.99, [3], 3),
+ (5, 'Shirt', ['Clothing', 'Office', 'Shirt'], 259.00, [4], 4);
+
+ INSERT INTO test_array_int_table VALUES
+ (1, 'Laptop', ['Electronics', 'Office', 'High-End', 'Laptop'],
5999.99, [1, 2, 3], 2),
+ (2, 'Mechanical Keyboard', ['Electronics', 'Accessories'], 399.99,
[1, 2], 2),
+ (3, 'Basketball', ['Sports', 'Outdoor'], 199.99, [1,3], 3);
+
+ INSERT INTO test_array_int_table VALUES
+ (1, 'Laptop', ['Electronics', 'Office', 'High-End', 'Laptop'],
5999.99, [1, 2, 3], 3);
+ """
+
+ qt_window_function_partition_by_array_int """
+ select
+ name,
+ category_ids,
+ cid,
+ sum(cid) over(partition by category_ids)
+ from
+ test_array_int_table
+ group by
+ name, category_ids, cid
+ order by
+ name, category_ids, cid;
+ """
+ // test decimal
+ sql "DROP TABLE IF EXISTS `test_array_decimal_table`"
+ multi_sql """
+ CREATE TABLE test_array_decimal_table(
+ id INT,
+ name VARCHAR(50),
+ tags ARRAY<VARCHAR(50)>,
+ price DECIMAL(10,2),
+ category_ids ARRAY<decimalv3(10,2)>,
+ cid INT
+ )
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 2
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+
+ INSERT INTO test_array_decimal_table VALUES
+ (1, 'Laptop', ['Electronics', 'Office', 'High-End', 'Laptop'],
5999.99, [1, 2, 3], 1),
+ (2, 'Mechanical Keyboard', ['Electronics', 'Accessories'], 399.99, [1,
2], 1),
+ (3, 'Basketball', ['Sports', 'Outdoor'], 199.99, [1,3], 1),
+ (4, 'Badminton Racket', ['Sports', 'Equipment'], 299.99, [3], 3),
+ (5, 'Shirt', ['Clothing', 'Office', 'Shirt'], 259.00, [4], 4);
+
+ INSERT INTO test_array_decimal_table VALUES
+ (1, 'Laptop', ['Electronics', 'Office', 'High-End', 'Laptop'],
5999.99, [1, 2, 3], 2),
+ (2, 'Mechanical Keyboard', ['Electronics', 'Accessories'], 399.99,
[1, 2], 2),
+ (3, 'Basketball', ['Sports', 'Outdoor'], 199.99, [1,3], 3);
+
+ INSERT INTO test_array_decimal_table VALUES
+ (1, 'Laptop', ['Electronics', 'Office', 'High-End', 'Laptop'],
5999.99, [1, 2, 3], 3);
+ """
+
+ qt_window_function_partition_by_array_decimal """
+ select
+ name,
+ category_ids,
+ cid,
+ sum(cid) over(partition by category_ids)
+ from
+ test_array_decimal_table
+ group by
+ name, category_ids, cid
+ order by
+ name, category_ids, cid;
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
