KevinyhZou commented on code in PR #6857: URL: https://github.com/apache/incubator-gluten/pull/6857#discussion_r1767939298
########## cpp-ch/local-engine/Functions/SparkFunctionArraysOverlap.cpp: ########## @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <Columns/ColumnString.h> +#include <Columns/ColumnNullable.h> +#include <Functions/IFunction.h> +#include <Functions/FunctionFactory.h> +#include <DataTypes/DataTypeString.h> +#include <DataTypes/DataTypesNumber.h> + +using namespace DB; + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} +} + +namespace local_engine +{ +class SparkFunctionArraysOverlap : public IFunction +{ +public: + static constexpr auto name = "sparkArraysOverlap"; + static FunctionPtr create(ContextPtr) { return std::make_shared<SparkFunctionArraysOverlap>(); } + SparkFunctionArraysOverlap() = default; + ~SparkFunctionArraysOverlap() override = default; + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } + size_t getNumberOfArguments() const override { return 2; } + String getName() const override { return name; } + bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForConstants() const override { return false; } + + DB::DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName &) const override + { + auto data_type = std::make_shared<DataTypeUInt8>(); + return makeNullable(data_type); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + if (arguments.size() != 2) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} must have 2 arguments", getName()); + + auto res = ColumnUInt8::create(input_rows_count, 0); + auto null_map = ColumnUInt8::create(input_rows_count, 0); + PaddedPODArray<UInt8> & res_data = res->getData(); + PaddedPODArray<UInt8> & null_map_data = null_map->getData(); + if (input_rows_count == 0) + return ColumnNullable::create(std::move(res), std::move(null_map)); + + const ColumnArray * array_col_1 = nullptr, * array_col_2 = nullptr; + const ColumnConst * const_col_1 = checkAndGetColumn<ColumnConst>(arguments[0].column.get()); + const ColumnConst * const_col_2 = checkAndGetColumn<ColumnConst>(arguments[1].column.get()); + if ((const_col_1 && const_col_1->onlyNull()) || (const_col_2 && const_col_2->onlyNull())) + { + null_map_data[0] = 1; + return ColumnNullable::create(std::move(res), std::move(null_map)); + } + if (const_col_1) + array_col_1 = checkAndGetColumn<ColumnArray>(const_col_1->getDataColumnPtr().get()); + else + { + const auto * null_col_1 = checkAndGetColumn<ColumnNullable>(arguments[0].column.get()); + array_col_1 = checkAndGetColumn<ColumnArray>(null_col_1->getNestedColumnPtr().get()); + } + if (const_col_2) + array_col_2 = checkAndGetColumn<ColumnArray>(const_col_2->getDataColumnPtr().get()); + else + { + const auto * null_col_2 = checkAndGetColumn<ColumnNullable>(arguments[1].column.get()); + array_col_2 = checkAndGetColumn<ColumnArray>(null_col_2->getNestedColumnPtr().get()); + } + if (!array_col_1 || !array_col_2) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function {} 1st/2nd argument must be array type", getName()); + + const ColumnArray::Offsets & array_offsets_1 = array_col_1->getOffsets(); + const ColumnArray::Offsets & array_offsets_2 = array_col_2->getOffsets(); + + size_t current_offset_1 = 0, current_offset_2 = 0; + size_t array_pos_1 = 0, array_pos_2 = 0; + for (size_t i = 0; i < array_col_1->size(); ++i) + { + if (arguments[0].column->isNullAt(i) || arguments[1].column->isNullAt(i)) + { + null_map_data[i] = 1; + continue; + } + size_t array_size_1 = array_offsets_1[i] - current_offset_1; + size_t array_size_2 = array_offsets_2[i] - current_offset_2; + auto executeCompare = [&](const IColumn & col1, const IColumn & col2, const ColumnUInt8 * null_map1, const ColumnUInt8 * null_map2) -> void + { + for (size_t j = 0; j < array_size_1 && !res_data[i]; ++j) Review Comment: HashSetWithStackMemory 可以用来存放FixedLength的数据（INT, FLOAT），通过调用getDataAt(i)。但是如果用来存放string 这种不是FixedLength的情况，就会导致抛异常。不是一个通用的方法 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
