This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1ff4dc8f85fc1e1a9b53ef9a59a412d30abd06eb Author: Pxl <[email protected]> AuthorDate: Fri May 10 11:20:45 2024 +0800 [Bug](runtime-filter) fix coredump won change_null_to_true when argument column is not null… (#34602) fix coredump won change_null_to_true when argument column is not nullable --- be/src/vec/utils/util.hpp | 58 +++++++ .../query_p0/join/rqg/rqg471923/rqg1333347798.out | 16 ++ .../join/rqg/rqg471923/rqg1333347798.groovy | 178 +++++++++++++++++++++ 3 files changed, 252 insertions(+) diff --git a/be/src/vec/utils/util.hpp b/be/src/vec/utils/util.hpp index 30609799e7f..f5c0aee990b 100644 --- a/be/src/vec/utils/util.hpp +++ b/be/src/vec/utils/util.hpp @@ -169,6 +169,64 @@ inline std::string remove_suffix(const std::string& name, const std::string& suf return name.substr(0, name.length() - suffix.length()); }; +inline ColumnPtr create_always_true_column(size_t size, bool is_nullable) { + auto res_data_column = ColumnUInt8::create(size, 1); + if (is_nullable) { + auto null_map = ColumnVector<UInt8>::create(size, 0); + return ColumnNullable::create(std::move(res_data_column), std::move(null_map)); + } + return res_data_column; +} + +// change null element to true element +inline void change_null_to_true(ColumnPtr column, ColumnPtr argument = nullptr) { + size_t rows = column->size(); + if (is_column_const(*column)) { + change_null_to_true(assert_cast<const ColumnConst*>(column.get())->get_data_column_ptr()); + } else if (column->has_null()) { + auto* nullable = + const_cast<ColumnNullable*>(assert_cast<const ColumnNullable*>(column.get())); + auto* __restrict data = assert_cast<ColumnUInt8*>(nullable->get_nested_column_ptr().get()) + ->get_data() + .data(); + auto* __restrict null_map = const_cast<uint8_t*>(nullable->get_null_map_data().data()); + for (size_t i = 0; i < rows; ++i) { + data[i] |= null_map[i]; + } + memset(null_map, 0, rows); + } else if (argument != nullptr && argument->has_null()) { + const auto* __restrict null_map = + assert_cast<const ColumnNullable*>(argument.get())->get_null_map_data().data(); + auto* __restrict data = + const_cast<ColumnUInt8*>(assert_cast<const ColumnUInt8*>(column.get())) + ->get_data() + .data(); + for (size_t i = 0; i < rows; ++i) { + data[i] |= null_map[i]; + } + } +} + +inline size_t calculate_false_number(ColumnPtr column) { + size_t rows = column->size(); + if (is_column_const(*column)) { + return calculate_false_number( + assert_cast<const ColumnConst*>(column.get())->get_data_column_ptr()) * + rows; + } else if (column->is_nullable()) { + const auto* nullable = assert_cast<const ColumnNullable*>(column.get()); + const auto* data = assert_cast<const ColumnUInt8*>(nullable->get_nested_column_ptr().get()) + ->get_data() + .data(); + const auto* __restrict null_map = nullable->get_null_map_data().data(); + return simd::count_zero_num(reinterpret_cast<const int8_t* __restrict>(data), null_map, + rows); + } else { + const auto* data = assert_cast<const ColumnUInt8*>(column.get())->get_data().data(); + return simd::count_zero_num(reinterpret_cast<const int8_t* __restrict>(data), rows); + } +} + } // namespace doris::vectorized namespace apache::thrift { diff --git a/regression-test/data/query_p0/join/rqg/rqg471923/rqg1333347798.out b/regression-test/data/query_p0/join/rqg/rqg471923/rqg1333347798.out new file mode 100644 index 00000000000..ad1c5c38198 --- /dev/null +++ b/regression-test/data/query_p0/join/rqg/rqg471923/rqg1333347798.out @@ -0,0 +1,16 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !test -- +\N 8 +3 3 +12 6 +13 2 +14 4 +16 2 +17 2 +18 3 +19 4 +20 9 +21 15 +22 4 +23 1 + diff --git a/regression-test/suites/query_p0/join/rqg/rqg471923/rqg1333347798.groovy b/regression-test/suites/query_p0/join/rqg/rqg471923/rqg1333347798.groovy new file mode 100644 index 00000000000..e1036c452ec --- /dev/null +++ b/regression-test/suites/query_p0/join/rqg/rqg471923/rqg1333347798.groovy @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("rqg471923") { + sql( + """ + DROP TABLE IF EXISTS `table_200_undef_partitions2_keys3_properties4_distributed_by59`; + """) + sql(""" + create table table_200_undef_partitions2_keys3_properties4_distributed_by59 ( +col_date_undef_signed_not_null date not null , +col_bigint_undef_signed_not_null_index_inverted bigint not null , +col_bigint_undef_signed_not_null bigint not null , +col_int_undef_signed int null , +col_int_undef_signed_index_inverted int null , +col_int_undef_signed_not_null int not null , +col_int_undef_signed_not_null_index_inverted int not null , +col_bigint_undef_signed bigint null , +col_bigint_undef_signed_index_inverted bigint null , +col_date_undef_signed date null , +col_date_undef_signed_index_inverted date null , +col_date_undef_signed_not_null_index_inverted date not null , +col_varchar_10__undef_signed varchar(10) null , +col_varchar_10__undef_signed_index_inverted varchar(10) null , +col_varchar_10__undef_signed_not_null varchar(10) not null , +col_varchar_10__undef_signed_not_null_index_inverted varchar(10) not null , +col_varchar_1024__undef_signed varchar(1024) null , +col_varchar_1024__undef_signed_index_inverted varchar(1024) null , +col_varchar_1024__undef_signed_not_null varchar(1024) not null , +col_varchar_1024__undef_signed_not_null_index_inverted varchar(1024) not null , +pk int, +INDEX col_int_undef_signed_index_inverted_idx (`col_int_undef_signed_index_inverted`) USING INVERTED, +INDEX col_int_undef_signed_not_null_index_inverted_idx (`col_int_undef_signed_not_null_index_inverted`) USING INVERTED, +INDEX col_bigint_undef_signed_index_inverted_idx (`col_bigint_undef_signed_index_inverted`) USING INVERTED, +INDEX col_bigint_undef_signed_not_null_index_inverted_idx (`col_bigint_undef_signed_not_null_index_inverted`) USING INVERTED, +INDEX col_date_undef_signed_index_inverted_idx (`col_date_undef_signed_index_inverted`) USING INVERTED, +INDEX col_date_undef_signed_not_null_index_inverted_idx (`col_date_undef_signed_not_null_index_inverted`) USING INVERTED, +INDEX col_varchar_10__undef_signed_index_inverted_idx (`col_varchar_10__undef_signed_index_inverted`) USING INVERTED, +INDEX col_varchar_10__undef_signed_not_null_index_inverted_idx (`col_varchar_10__undef_signed_not_null_index_inverted`) USING INVERTED, +INDEX col_varchar_1024__undef_signed_index_inverted_idx (`col_varchar_1024__undef_signed_index_inverted`) USING INVERTED, +INDEX col_varchar_1024__undef_signed_not_null_index_inverted_idx (`col_varchar_1024__undef_signed_not_null_index_inverted`) USING INVERTED +) engine=olap +UNIQUE KEY(col_date_undef_signed_not_null, col_bigint_undef_signed_not_null_index_inverted, col_bigint_undef_signed_not_null) +PARTITION BY RANGE(col_date_undef_signed_not_null) ( + FROM ('2023-12-08') TO ('2024-01-01') INTERVAL 1 DAY, + FROM ('2024-01-01') TO ('2024-12-01') INTERVAL 1 MONTH, + FROM ('2025-01-01') TO ('2027-01-01') INTERVAL 1 YEAR, + FROM ('2027-01-01') TO ('2032-01-01') INTERVAL 2 YEAR + ) +distributed by hash(col_bigint_undef_signed_not_null_index_inverted) +properties("enable_unique_key_merge_on_write" = "true", "replication_num" = "1"); +""") +sql(""" +insert into table_200_undef_partitions2_keys3_properties4_distributed_by59(pk,col_int_undef_signed,col_int_undef_signed_index_inverted,col_int_undef_signed_not_null,col_int_undef_signed_not_null_index_inverted,col_bigint_undef_signed,col_bigint_undef_signed_index_inverted,col_bigint_undef_signed_not_null,col_bigint_undef_signed_not_null_index_inverted,col_date_undef_signed,col_date_undef_signed_index_inverted,col_date_undef_signed_not_null,col_date_undef_signed_not_null_index_inverted,co [...] + """) + + + sql(""" + DROP TABLE IF EXISTS `table_200_undef_partitions2_keys3_properties4_distributed_by54`; + """) + sql(""" +create table table_200_undef_partitions2_keys3_properties4_distributed_by54 ( +col_date_undef_signed_not_null_index_inverted date not null , +col_bigint_undef_signed_not_null_index_inverted bigint not null , +col_bigint_undef_signed_not_null bigint not null , +col_int_undef_signed int null , +col_int_undef_signed_index_inverted int null , +col_int_undef_signed_not_null int not null , +col_int_undef_signed_not_null_index_inverted int not null , +col_bigint_undef_signed bigint null , +col_bigint_undef_signed_index_inverted bigint null , +col_date_undef_signed date null , +col_date_undef_signed_index_inverted date null , +col_date_undef_signed_not_null date not null , +col_varchar_10__undef_signed varchar(10) null , +col_varchar_10__undef_signed_index_inverted varchar(10) null , +col_varchar_10__undef_signed_not_null varchar(10) not null , +col_varchar_10__undef_signed_not_null_index_inverted varchar(10) not null , +col_varchar_1024__undef_signed varchar(1024) null , +col_varchar_1024__undef_signed_index_inverted varchar(1024) null , +col_varchar_1024__undef_signed_not_null varchar(1024) not null , +col_varchar_1024__undef_signed_not_null_index_inverted varchar(1024) not null , +pk int, +INDEX col_int_undef_signed_index_inverted_idx (`col_int_undef_signed_index_inverted`) USING INVERTED, +INDEX col_int_undef_signed_not_null_index_inverted_idx (`col_int_undef_signed_not_null_index_inverted`) USING INVERTED, +INDEX col_bigint_undef_signed_index_inverted_idx (`col_bigint_undef_signed_index_inverted`) USING INVERTED, +INDEX col_bigint_undef_signed_not_null_index_inverted_idx (`col_bigint_undef_signed_not_null_index_inverted`) USING INVERTED, +INDEX col_date_undef_signed_index_inverted_idx (`col_date_undef_signed_index_inverted`) USING INVERTED, +INDEX col_date_undef_signed_not_null_index_inverted_idx (`col_date_undef_signed_not_null_index_inverted`) USING INVERTED, +INDEX col_varchar_10__undef_signed_index_inverted_idx (`col_varchar_10__undef_signed_index_inverted`) USING INVERTED, +INDEX col_varchar_10__undef_signed_not_null_index_inverted_idx (`col_varchar_10__undef_signed_not_null_index_inverted`) USING INVERTED, +INDEX col_varchar_1024__undef_signed_index_inverted_idx (`col_varchar_1024__undef_signed_index_inverted`) USING INVERTED, +INDEX col_varchar_1024__undef_signed_not_null_index_inverted_idx (`col_varchar_1024__undef_signed_not_null_index_inverted`) USING INVERTED +) engine=olap +UNIQUE KEY(col_date_undef_signed_not_null_index_inverted, col_bigint_undef_signed_not_null_index_inverted, col_bigint_undef_signed_not_null) +PARTITION BY RANGE(col_date_undef_signed_not_null_index_inverted) ( + FROM ('2023-12-09') TO ('2024-03-09') INTERVAL 1 DAY, + FROM ('2025-02-16') TO ('2025-03-09') INTERVAL 1 DAY, + FROM ('2025-06-18') TO ('2025-06-20') INTERVAL 1 DAY, + FROM ('2026-01-01') TO ('2026-03-09') INTERVAL 1 DAY, + FROM ('2027-01-01') TO ('2027-02-09') INTERVAL 1 DAY + ) +distributed by hash(col_bigint_undef_signed_not_null) +properties("enable_unique_key_merge_on_write" = "true", "replication_num" = "1"); +""") +sql(""" +insert into table_200_undef_partitions2_keys3_properties4_distributed_by54(pk,col_int_undef_signed,col_int_undef_signed_index_inverted,col_int_undef_signed_not_null,col_int_undef_signed_not_null_index_inverted,col_bigint_undef_signed,col_bigint_undef_signed_index_inverted,col_bigint_undef_signed_not_null,col_bigint_undef_signed_not_null_index_inverted,col_date_undef_signed,col_date_undef_signed_index_inverted,col_date_undef_signed_not_null,col_date_undef_signed_not_null_index_inverted,co [...] + """) + + sql(""" + DROP TABLE IF EXISTS `table_50_undef_partitions2_keys3_properties4_distributed_by511`; + """) + sql(""" +create table table_50_undef_partitions2_keys3_properties4_distributed_by511 ( +col_date_undef_signed_not_null_index_inverted date not null , +col_bigint_undef_signed_not_null_index_inverted bigint not null , +col_bigint_undef_signed_not_null bigint not null , +col_int_undef_signed int null , +col_int_undef_signed_index_inverted int null , +col_int_undef_signed_not_null int not null , +col_int_undef_signed_not_null_index_inverted int not null , +col_bigint_undef_signed bigint null , +col_bigint_undef_signed_index_inverted bigint null , +col_date_undef_signed date null , +col_date_undef_signed_index_inverted date null , +col_date_undef_signed_not_null date not null , +col_varchar_10__undef_signed varchar(10) null , +col_varchar_10__undef_signed_index_inverted varchar(10) null , +col_varchar_10__undef_signed_not_null varchar(10) not null , +col_varchar_10__undef_signed_not_null_index_inverted varchar(10) not null , +col_varchar_1024__undef_signed varchar(1024) null , +col_varchar_1024__undef_signed_index_inverted varchar(1024) null , +col_varchar_1024__undef_signed_not_null varchar(1024) not null , +col_varchar_1024__undef_signed_not_null_index_inverted varchar(1024) not null , +pk int, +INDEX col_int_undef_signed_index_inverted_idx (`col_int_undef_signed_index_inverted`) USING INVERTED, +INDEX col_int_undef_signed_not_null_index_inverted_idx (`col_int_undef_signed_not_null_index_inverted`) USING INVERTED, +INDEX col_bigint_undef_signed_index_inverted_idx (`col_bigint_undef_signed_index_inverted`) USING INVERTED, +INDEX col_bigint_undef_signed_not_null_index_inverted_idx (`col_bigint_undef_signed_not_null_index_inverted`) USING INVERTED, +INDEX col_date_undef_signed_index_inverted_idx (`col_date_undef_signed_index_inverted`) USING INVERTED, +INDEX col_date_undef_signed_not_null_index_inverted_idx (`col_date_undef_signed_not_null_index_inverted`) USING INVERTED, +INDEX col_varchar_10__undef_signed_index_inverted_idx (`col_varchar_10__undef_signed_index_inverted`) USING INVERTED, +INDEX col_varchar_10__undef_signed_not_null_index_inverted_idx (`col_varchar_10__undef_signed_not_null_index_inverted`) USING INVERTED, +INDEX col_varchar_1024__undef_signed_index_inverted_idx (`col_varchar_1024__undef_signed_index_inverted`) USING INVERTED, +INDEX col_varchar_1024__undef_signed_not_null_index_inverted_idx (`col_varchar_1024__undef_signed_not_null_index_inverted`) USING INVERTED +) engine=olap +UNIQUE KEY(col_date_undef_signed_not_null_index_inverted, col_bigint_undef_signed_not_null_index_inverted, col_bigint_undef_signed_not_null) +PARTITION BY RANGE(col_date_undef_signed_not_null_index_inverted) ( + FROM ('2023-12-08') TO ('2024-01-01') INTERVAL 1 DAY, + FROM ('2024-01-01') TO ('2024-12-01') INTERVAL 1 MONTH, + FROM ('2025-01-01') TO ('2027-01-01') INTERVAL 1 YEAR, + FROM ('2027-01-01') TO ('2032-01-01') INTERVAL 2 YEAR + ) +distributed by hash(col_bigint_undef_signed_not_null_index_inverted) +properties("enable_unique_key_merge_on_write" = "true", "replication_num" = "1"); +""") +sql(""" +insert into table_50_undef_partitions2_keys3_properties4_distributed_by511(pk,col_int_undef_signed,col_int_undef_signed_index_inverted,col_int_undef_signed_not_null,col_int_undef_signed_not_null_index_inverted,col_bigint_undef_signed,col_bigint_undef_signed_index_inverted,col_bigint_undef_signed_not_null,col_bigint_undef_signed_not_null_index_inverted,col_date_undef_signed,col_date_undef_signed_index_inverted,col_date_undef_signed_not_null,col_date_undef_signed_not_null_index_inverted,co [...] + """) + + qt_test(""" + SELECT DAY ( DATE_ADD ( table1 . col_date_undef_signed , INTERVAL 3 DAY ) ) AS field1, COUNT( distinct table1 . col_varchar_10__undef_signed ) AS field2 FROM table_200_undef_partitions2_keys3_properties4_distributed_by59 AS table1 RIGHT JOIN table_200_undef_partitions2_keys3_properties4_distributed_by54 AS table2 ON ( table2 . col_date_undef_signed_not_null_index_inverted = table1 . col_date_undef_signed_index_inverted ) INNER JOIN table_50_undef_partitions2_keys3_properties4_distrib [...] + """) +} \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
