taiyang-li commented on code in PR #6857: URL: https://github.com/apache/incubator-gluten/pull/6857#discussion_r1767819022
########## cpp-ch/local-engine/Functions/SparkFunctionArraysOverlap.cpp: ########## @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <Columns/ColumnString.h> +#include <Columns/ColumnNullable.h> +#include <Functions/IFunction.h> +#include <Functions/FunctionFactory.h> +#include <DataTypes/DataTypeString.h> +#include <DataTypes/DataTypesNumber.h> + +using namespace DB; + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} +} + +namespace local_engine +{ +class SparkFunctionArraysOverlap : public IFunction +{ +public: + static constexpr auto name = "sparkArraysOverlap"; + static FunctionPtr create(ContextPtr) { return std::make_shared<SparkFunctionArraysOverlap>(); } + SparkFunctionArraysOverlap() = default; + ~SparkFunctionArraysOverlap() override = default; + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } + size_t getNumberOfArguments() const override { return 2; } + String getName() const override { return name; } + bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForConstants() const override { return false; } Review Comment: why is it false? if it is false, we need to handle the case that all input arguments are constant, which could be avoided by setting it to true. ########## cpp-ch/local-engine/Functions/SparkFunctionArraysOverlap.cpp: ########## @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <Columns/ColumnString.h> +#include <Columns/ColumnNullable.h> +#include <Functions/IFunction.h> +#include <Functions/FunctionFactory.h> +#include <DataTypes/DataTypeString.h> +#include <DataTypes/DataTypesNumber.h> + +using namespace DB; + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} +} + +namespace local_engine +{ +class SparkFunctionArraysOverlap : public IFunction +{ +public: + static constexpr auto name = "sparkArraysOverlap"; + static FunctionPtr create(ContextPtr) { return std::make_shared<SparkFunctionArraysOverlap>(); } + SparkFunctionArraysOverlap() = default; + ~SparkFunctionArraysOverlap() override = default; + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } + size_t getNumberOfArguments() const override { return 2; } + String getName() const override { return name; } + bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForConstants() const override { return false; } + + DB::DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName &) const override + { + auto data_type = std::make_shared<DataTypeUInt8>(); + return makeNullable(data_type); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + if (arguments.size() != 2) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} must have 2 arguments", getName()); + + auto res = ColumnUInt8::create(input_rows_count, 0); + auto null_map = ColumnUInt8::create(input_rows_count, 0); + PaddedPODArray<UInt8> & res_data = res->getData(); + PaddedPODArray<UInt8> & null_map_data = null_map->getData(); + if (input_rows_count == 0) + return ColumnNullable::create(std::move(res), std::move(null_map)); + + const ColumnArray * array_col_1 = nullptr, * array_col_2 = nullptr; + const ColumnConst * const_col_1 = checkAndGetColumn<ColumnConst>(arguments[0].column.get()); + const ColumnConst * const_col_2 = checkAndGetColumn<ColumnConst>(arguments[1].column.get()); + if ((const_col_1 && const_col_1->onlyNull()) || (const_col_2 && const_col_2->onlyNull())) + { + null_map_data[0] = 1; + return ColumnNullable::create(std::move(res), std::move(null_map)); + } + if (const_col_1) + array_col_1 = checkAndGetColumn<ColumnArray>(const_col_1->getDataColumnPtr().get()); + else + { + const auto * null_col_1 = checkAndGetColumn<ColumnNullable>(arguments[0].column.get()); + array_col_1 = checkAndGetColumn<ColumnArray>(null_col_1->getNestedColumnPtr().get()); + } + if (const_col_2) + array_col_2 = checkAndGetColumn<ColumnArray>(const_col_2->getDataColumnPtr().get()); + else + { + const auto * null_col_2 = checkAndGetColumn<ColumnNullable>(arguments[1].column.get()); + array_col_2 = checkAndGetColumn<ColumnArray>(null_col_2->getNestedColumnPtr().get()); + } + if (!array_col_1 || !array_col_2) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function {} 1st/2nd argument must be array type", getName()); + + const ColumnArray::Offsets & array_offsets_1 = array_col_1->getOffsets(); + const ColumnArray::Offsets & array_offsets_2 = array_col_2->getOffsets(); + + size_t current_offset_1 = 0, current_offset_2 = 0; + size_t array_pos_1 = 0, array_pos_2 = 0; + for (size_t i = 0; i < array_col_1->size(); ++i) + { + if (arguments[0].column->isNullAt(i) || arguments[1].column->isNullAt(i)) + { + null_map_data[i] = 1; + continue; + } + size_t array_size_1 = array_offsets_1[i] - current_offset_1; + size_t array_size_2 = array_offsets_2[i] - current_offset_2; + auto executeCompare = [&](const IColumn & col1, const IColumn & col2, const ColumnUInt8 * null_map1, const ColumnUInt8 * null_map2) -> void + { + for (size_t j = 0; j < array_size_1 && !res_data[i]; ++j) Review Comment: 两个for循环性能会比较差，可以用这个数据结构： using Set = HashSetWithStackMemory<StringRef, StringRefHash, 4>; ########## cpp-ch/local-engine/Functions/SparkFunctionArraysOverlap.cpp: ########## @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <Columns/ColumnString.h> +#include <Columns/ColumnNullable.h> +#include <Functions/IFunction.h> +#include <Functions/FunctionFactory.h> +#include <DataTypes/DataTypeString.h> +#include <DataTypes/DataTypesNumber.h> + +using namespace DB; + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} +} + +namespace local_engine +{ +class SparkFunctionArraysOverlap : public IFunction +{ +public: + static constexpr auto name = "sparkArraysOverlap"; + static FunctionPtr create(ContextPtr) { return std::make_shared<SparkFunctionArraysOverlap>(); } + SparkFunctionArraysOverlap() = default; + ~SparkFunctionArraysOverlap() override = default; + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } + size_t getNumberOfArguments() const override { return 2; } + String getName() const override { return name; } + bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForConstants() const override { return false; } + + DB::DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName &) const override + { + auto data_type = std::make_shared<DataTypeUInt8>(); + return makeNullable(data_type); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + if (arguments.size() != 2) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} must have 2 arguments", getName()); + + auto res = ColumnUInt8::create(input_rows_count, 0); + auto null_map = ColumnUInt8::create(input_rows_count, 0); + PaddedPODArray<UInt8> & res_data = res->getData(); + PaddedPODArray<UInt8> & null_map_data = null_map->getData(); + if (input_rows_count == 0) + return ColumnNullable::create(std::move(res), std::move(null_map)); + + const ColumnArray * array_col_1 = nullptr, * array_col_2 = nullptr; + const ColumnConst * const_col_1 = checkAndGetColumn<ColumnConst>(arguments[0].column.get()); + const ColumnConst * const_col_2 = checkAndGetColumn<ColumnConst>(arguments[1].column.get()); + if ((const_col_1 && const_col_1->onlyNull()) || (const_col_2 && const_col_2->onlyNull())) + { + null_map_data[0] = 1; + return ColumnNullable::create(std::move(res), std::move(null_map)); + } + if (const_col_1) + array_col_1 = checkAndGetColumn<ColumnArray>(const_col_1->getDataColumnPtr().get()); Review Comment: const里包的有可能是nullable array或array, 两种情况都需要考虑 ########## cpp-ch/local-engine/Functions/SparkFunctionArraysOverlap.cpp: ########## @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <Columns/ColumnString.h> +#include <Columns/ColumnNullable.h> +#include <Functions/IFunction.h> +#include <Functions/FunctionFactory.h> +#include <DataTypes/DataTypeString.h> +#include <DataTypes/DataTypesNumber.h> + +using namespace DB; + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} +} + +namespace local_engine +{ +class SparkFunctionArraysOverlap : public IFunction +{ +public: + static constexpr auto name = "sparkArraysOverlap"; + static FunctionPtr create(ContextPtr) { return std::make_shared<SparkFunctionArraysOverlap>(); } + SparkFunctionArraysOverlap() = default; + ~SparkFunctionArraysOverlap() override = default; + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } + size_t getNumberOfArguments() const override { return 2; } + String getName() const override { return name; } + bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForConstants() const override { return false; } + + DB::DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName &) const override + { + auto data_type = std::make_shared<DataTypeUInt8>(); + return makeNullable(data_type); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + if (arguments.size() != 2) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} must have 2 arguments", getName()); + + auto res = ColumnUInt8::create(input_rows_count, 0); + auto null_map = ColumnUInt8::create(input_rows_count, 0); + PaddedPODArray<UInt8> & res_data = res->getData(); + PaddedPODArray<UInt8> & null_map_data = null_map->getData(); + if (input_rows_count == 0) + return ColumnNullable::create(std::move(res), std::move(null_map)); + + const ColumnArray * array_col_1 = nullptr, * array_col_2 = nullptr; + const ColumnConst * const_col_1 = checkAndGetColumn<ColumnConst>(arguments[0].column.get()); + const ColumnConst * const_col_2 = checkAndGetColumn<ColumnConst>(arguments[1].column.get()); + if ((const_col_1 && const_col_1->onlyNull()) || (const_col_2 && const_col_2->onlyNull())) + { + null_map_data[0] = 1; + return ColumnNullable::create(std::move(res), std::move(null_map)); + } + if (const_col_1) + array_col_1 = checkAndGetColumn<ColumnArray>(const_col_1->getDataColumnPtr().get()); + else + { + const auto * null_col_1 = checkAndGetColumn<ColumnNullable>(arguments[0].column.get()); Review Comment: arguments[0]也许是nullable array或array, 两种情况都需要考虑。arguments[1]也一样 ########## cpp-ch/local-engine/Functions/SparkFunctionArraysOverlap.cpp: ########## @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <Columns/ColumnString.h> +#include <Columns/ColumnNullable.h> +#include <Functions/IFunction.h> +#include <Functions/FunctionFactory.h> +#include <DataTypes/DataTypeString.h> +#include <DataTypes/DataTypesNumber.h> + +using namespace DB; + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} +} + +namespace local_engine +{ +class SparkFunctionArraysOverlap : public IFunction +{ +public: + static constexpr auto name = "sparkArraysOverlap"; + static FunctionPtr create(ContextPtr) { return std::make_shared<SparkFunctionArraysOverlap>(); } + SparkFunctionArraysOverlap() = default; + ~SparkFunctionArraysOverlap() override = default; + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } + size_t getNumberOfArguments() const override { return 2; } + String getName() const override { return name; } + bool useDefaultImplementationForNulls() const override { return false; } Review Comment: 这里设置成false的话，就必须自己处理arr1[i] = null或arr2[i] = null的问题了，增加了很多不必要的麻烦 ########## cpp-ch/local-engine/Functions/SparkFunctionArraysOverlap.cpp: ########## @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <Columns/ColumnString.h> +#include <Columns/ColumnNullable.h> +#include <Functions/IFunction.h> +#include <Functions/FunctionFactory.h> +#include <DataTypes/DataTypeString.h> +#include <DataTypes/DataTypesNumber.h> + +using namespace DB; + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} +} + +namespace local_engine +{ +class SparkFunctionArraysOverlap : public IFunction +{ +public: + static constexpr auto name = "sparkArraysOverlap"; + static FunctionPtr create(ContextPtr) { return std::make_shared<SparkFunctionArraysOverlap>(); } + SparkFunctionArraysOverlap() = default; + ~SparkFunctionArraysOverlap() override = default; + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } + size_t getNumberOfArguments() const override { return 2; } + String getName() const override { return name; } + bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForConstants() const override { return false; } + + DB::DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName &) const override + { + auto data_type = std::make_shared<DataTypeUInt8>(); + return makeNullable(data_type); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + if (arguments.size() != 2) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} must have 2 arguments", getName()); + + auto res = ColumnUInt8::create(input_rows_count, 0); + auto null_map = ColumnUInt8::create(input_rows_count, 0); + PaddedPODArray<UInt8> & res_data = res->getData(); + PaddedPODArray<UInt8> & null_map_data = null_map->getData(); + if (input_rows_count == 0) + return ColumnNullable::create(std::move(res), std::move(null_map)); + + const ColumnArray * array_col_1 = nullptr, * array_col_2 = nullptr; + const ColumnConst * const_col_1 = checkAndGetColumn<ColumnConst>(arguments[0].column.get()); + const ColumnConst * const_col_2 = checkAndGetColumn<ColumnConst>(arguments[1].column.get()); + if ((const_col_1 && const_col_1->onlyNull()) || (const_col_2 && const_col_2->onlyNull())) + { + null_map_data[0] = 1; + return ColumnNullable::create(std::move(res), std::move(null_map)); + } + if (const_col_1) + array_col_1 = checkAndGetColumn<ColumnArray>(const_col_1->getDataColumnPtr().get()); + else + { + const auto * null_col_1 = checkAndGetColumn<ColumnNullable>(arguments[0].column.get()); + array_col_1 = checkAndGetColumn<ColumnArray>(null_col_1->getNestedColumnPtr().get()); + } + if (const_col_2) + array_col_2 = checkAndGetColumn<ColumnArray>(const_col_2->getDataColumnPtr().get()); + else + { + const auto * null_col_2 = checkAndGetColumn<ColumnNullable>(arguments[1].column.get()); + array_col_2 = checkAndGetColumn<ColumnArray>(null_col_2->getNestedColumnPtr().get()); + } + if (!array_col_1 || !array_col_2) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function {} 1st/2nd argument must be array type", getName()); + + const ColumnArray::Offsets & array_offsets_1 = array_col_1->getOffsets(); + const ColumnArray::Offsets & array_offsets_2 = array_col_2->getOffsets(); + + size_t current_offset_1 = 0, current_offset_2 = 0; + size_t array_pos_1 = 0, array_pos_2 = 0; + for (size_t i = 0; i < array_col_1->size(); ++i) + { + if (arguments[0].column->isNullAt(i) || arguments[1].column->isNullAt(i)) Review Comment: for循环中高频调用虚函数 isNullAt会影响性能。既然拿到了null_map_data这里为什么不用？ ########## cpp-ch/local-engine/Functions/SparkFunctionArraysOverlap.cpp: ########## @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <Columns/ColumnString.h> +#include <Columns/ColumnNullable.h> +#include <Functions/IFunction.h> +#include <Functions/FunctionFactory.h> +#include <DataTypes/DataTypeString.h> +#include <DataTypes/DataTypesNumber.h> + +using namespace DB; + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} +} + +namespace local_engine +{ +class SparkFunctionArraysOverlap : public IFunction +{ +public: + static constexpr auto name = "sparkArraysOverlap"; + static FunctionPtr create(ContextPtr) { return std::make_shared<SparkFunctionArraysOverlap>(); } + SparkFunctionArraysOverlap() = default; + ~SparkFunctionArraysOverlap() override = default; + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } + size_t getNumberOfArguments() const override { return 2; } + String getName() const override { return name; } + bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForConstants() const override { return false; } + + DB::DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName &) const override + { + auto data_type = std::make_shared<DataTypeUInt8>(); + return makeNullable(data_type); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override Review Comment: 尽量不要把所有的逻辑放在一个大函数里。参考src/Functions/regexpExtract.cpp的实现，按照两个参数是否为constant分成若干种情况，每种情况对应一个函数。 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
