lidavidm commented on a change in pull request #10410: URL: https://github.com/apache/arrow/pull/10410#discussion_r643067951
########## File path: cpp/src/arrow/compute/kernels/scalar_if_else_test.cc ########## @@ -0,0 +1,266 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <arrow/array.h> +#include <arrow/compute/api_scalar.h> +#include <arrow/compute/kernels/test_util.h> +#include <arrow/testing/gtest_util.h> +#include <gtest/gtest.h> + +namespace arrow { +namespace compute { + +void CheckIfElseOutput(const Datum& cond, const Datum& left, const Datum& right, + const Datum& expected) { + ASSERT_OK_AND_ASSIGN(Datum datum_out, IfElse(cond, left, right)); + if (datum_out.is_array()) { + std::shared_ptr<Array> result = datum_out.make_array(); + ASSERT_OK(result->ValidateFull()); + std::shared_ptr<Array> expected_ = expected.make_array(); + AssertArraysEqual(*expected_, *result, /*verbose=*/true); + } else { // expecting scalar + const std::shared_ptr<Scalar>& result = datum_out.scalar(); + const std::shared_ptr<Scalar>& expected_ = expected.scalar(); + AssertScalarsEqual(*expected_, *result, /*verbose=*/true); + } +} + +class TestIfElseKernel : public ::testing::Test {}; + +template <typename Type> +class TestIfElsePrimitive : public ::testing::Test {}; + +using PrimitiveTypes = ::testing::Types<Int8Type, UInt8Type, Int16Type, UInt16Type, + Int32Type, UInt32Type, Int64Type, UInt64Type, + FloatType, DoubleType, Date32Type, Date64Type>; + +TYPED_TEST_SUITE(TestIfElsePrimitive, PrimitiveTypes); + +TYPED_TEST(TestIfElsePrimitive, IfElseFixedSizeRand) { + using ArrayType = typename TypeTraits<TypeParam>::ArrayType; + auto type = TypeTraits<TypeParam>::type_singleton(); + + random::RandomArrayGenerator rand(/*seed=*/0); + int64_t len = 1000; + auto cond = std::static_pointer_cast<BooleanArray>( + rand.ArrayOf(boolean(), len, /*null_probability=*/0.01)); + auto left = std::static_pointer_cast<ArrayType>( + rand.ArrayOf(type, len, /*null_probability=*/0.01)); + auto right = std::static_pointer_cast<ArrayType>( + rand.ArrayOf(type, len, /*null_probability=*/0.01)); + + typename TypeTraits<TypeParam>::BuilderType builder; + + for (int64_t i = 0; i < len; ++i) { + if (!cond->IsValid(i) || (cond->Value(i) && !left->IsValid(i)) || + (!cond->Value(i) && !right->IsValid(i))) { + ASSERT_OK(builder.AppendNull()); + continue; + } + + if (cond->Value(i)) { + ASSERT_OK(builder.Append(left->Value(i))); + } else { + ASSERT_OK(builder.Append(right->Value(i))); + } + } + ASSERT_OK_AND_ASSIGN(auto expected_data, builder.Finish()); + + CheckIfElseOutput(cond, left, right, expected_data); +} + +template <typename Type> +struct DatumWrapper { + using CType = typename TypeTraits<Type>::CType; + using ArrayType = typename TypeTraits<Type>::ArrayType; + using ScalarType = typename TypeTraits<Type>::ScalarType; + + util::Variant<std::shared_ptr<ScalarType>, std::shared_ptr<ArrayType>> datum; + bool is_scalar; + + explicit DatumWrapper(const Datum& datum_) : is_scalar(datum_.is_scalar()) { + if (is_scalar) { + datum = std::move(std::static_pointer_cast<ScalarType>(datum_.scalar())); + } else { + datum = std::move(std::static_pointer_cast<ArrayType>(datum_.make_array())); + } + } + + bool IsValid(int64_t i) const { + return is_scalar ? util::get<std::shared_ptr<ScalarType>>(datum)->is_valid + : util::get<std::shared_ptr<ArrayType>>(datum)->IsValid(i); + } + + CType Value(int64_t i) const { + return is_scalar ? util::get<std::shared_ptr<ScalarType>>(datum)->value + : util::get<std::shared_ptr<ArrayType>>(datum)->Value(i); + } +}; + +template <typename Type> +void GenerateExpected(const Datum& cond, const Datum& left, const Datum& right, + Datum* out) { + int64_t len = cond.is_array() ? cond.length() + : left.is_array() ? left.length() + : right.is_array() ? right.length() : 1; + + DatumWrapper<BooleanType> cond_(cond); + DatumWrapper<Type> left_(left); + DatumWrapper<Type> right_(right); + + int64_t i = 0; + + // if all scalars + if (cond.is_scalar() && left.is_scalar() && right.is_scalar()) { + if (!cond_.IsValid(i) || (cond_.Value(i) && !left_.IsValid(i)) || + (!cond_.Value(i) && !right_.IsValid(i))) { + *out = MakeNullScalar(left.type()); + return; + } + + if (cond_.Value(i)) { + *out = left; + return; + } else { + *out = right; + return; + } + } + + typename TypeTraits<Type>::BuilderType builder; + + for (; i < len; ++i) { + if (!cond_.IsValid(i) || (cond_.Value(i) && !left_.IsValid(i)) || + (!cond_.Value(i) && !right_.IsValid(i))) { + ASSERT_OK(builder.AppendNull()); + continue; + } + + if (cond_.Value(i)) { + ASSERT_OK(builder.Append(left_.Value(i))); + } else { + ASSERT_OK(builder.Append(right_.Value(i))); + } + } + ASSERT_OK_AND_ASSIGN(auto expected_data, builder.Finish()); + + *out = expected_data; +} + +TYPED_TEST(TestIfElsePrimitive, IfElseFixedSizeGen) { + auto type = TypeTraits<TypeParam>::type_singleton(); + + std::vector<Datum> cond_datums{ArrayFromJSON(boolean(), "[true, true, true, false]"), + ArrayFromJSON(boolean(), "[true, null, true, false]"), + MakeScalar(boolean(), true).ValueOrDie(), + MakeNullScalar(boolean())}; + + std::vector<Datum> left_datums{ + ArrayFromJSON(type, "[1, 2, 3, 4]"), ArrayFromJSON(type, "[1, 2, null, 4]"), + MakeScalar(type, 100).ValueOrDie(), MakeNullScalar(type)}; + + std::vector<Datum> right_datums{ + ArrayFromJSON(type, "[5, 6, 7, 8]"), ArrayFromJSON(type, "[5, 6, 7, null]"), + MakeScalar(type, 111).ValueOrDie(), MakeNullScalar(type)}; + + for (auto&& cond : cond_datums) { + for (auto&& left : left_datums) { + for (auto&& right : right_datums) { + Datum exp; + GenerateExpected<TypeParam>(cond, left, right, &exp); + CheckIfElseOutput(cond, left, right, exp); + } + } + } +} + +TEST_F(TestIfElseKernel, IfElseBooleanGen) { + auto type = boolean(); + + std::vector<Datum> cond_datums{ArrayFromJSON(boolean(), "[true, true, true, false]"), + ArrayFromJSON(boolean(), "[true, true, null, false]"), + MakeScalar(boolean(), true).ValueOrDie(), + MakeNullScalar(boolean())}; + + std::vector<Datum> left_datums{ArrayFromJSON(type, "[false, false, false, false]"), + ArrayFromJSON(type, "[false, false, null, false]"), + MakeScalar(type, false).ValueOrDie(), + MakeNullScalar(type)}; + + std::vector<Datum> right_datums{ArrayFromJSON(type, "[true, true, true, true]"), + ArrayFromJSON(type, "[true, true, true, null]"), + MakeScalar(type, true).ValueOrDie(), + MakeNullScalar(type)}; + + for (auto&& cond : cond_datums) { + for (auto&& left : left_datums) { + for (auto&& right : right_datums) { + Datum exp; + GenerateExpected<BooleanType>(cond, left, right, &exp); + CheckIfElseOutput(cond, left, right, exp); + } + } + } +} + +TYPED_TEST(TestIfElsePrimitive, IfElseBooleanRand) { + auto type = boolean(); + random::RandomArrayGenerator rand(/*seed=*/0); + int64_t len = 1000; + auto cond = std::static_pointer_cast<BooleanArray>( + rand.ArrayOf(boolean(), len, /*null_probability=*/0.01)); + auto left = std::static_pointer_cast<BooleanArray>( + rand.ArrayOf(type, len, /*null_probability=*/0.01)); + auto right = std::static_pointer_cast<BooleanArray>( + rand.ArrayOf(type, len, /*null_probability=*/0.01)); + + BooleanBuilder builder; + for (int64_t i = 0; i < len; ++i) { + if (!cond->IsValid(i) || (cond->Value(i) && !left->IsValid(i)) || + (!cond->Value(i) && !right->IsValid(i))) { + ASSERT_OK(builder.AppendNull()); + continue; + } + + if (cond->Value(i)) { + ASSERT_OK(builder.Append(left->Value(i))); + } else { + ASSERT_OK(builder.Append(right->Value(i))); + } + } + ASSERT_OK_AND_ASSIGN(auto expected_data, builder.Finish()); + + CheckIfElseOutput(cond, left, right, expected_data); +} + +TEST_F(TestIfElseKernel, IfElseNull) { + CheckIfElseOutput(ArrayFromJSON(boolean(), "[null, null, null, null]"), + ArrayFromJSON(null(), "[null, null, null, null]"), + ArrayFromJSON(null(), "[null, null, null, null]"), + ArrayFromJSON(null(), "[null, null, null, null]")); +} + +TEST_F(TestIfElseKernel, IfElseWithOffset) { Review comment: You could also do this in the random tests by testing in a loop, say, every offset from 0 to 8 at the end. ########## File path: cpp/src/arrow/compute/kernels/scalar_if_else_test.cc ########## @@ -0,0 +1,266 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <arrow/array.h> +#include <arrow/compute/api_scalar.h> +#include <arrow/compute/kernels/test_util.h> +#include <arrow/testing/gtest_util.h> +#include <gtest/gtest.h> + +namespace arrow { +namespace compute { + +void CheckIfElseOutput(const Datum& cond, const Datum& left, const Datum& right, + const Datum& expected) { + ASSERT_OK_AND_ASSIGN(Datum datum_out, IfElse(cond, left, right)); + if (datum_out.is_array()) { + std::shared_ptr<Array> result = datum_out.make_array(); + ASSERT_OK(result->ValidateFull()); + std::shared_ptr<Array> expected_ = expected.make_array(); + AssertArraysEqual(*expected_, *result, /*verbose=*/true); + } else { // expecting scalar + const std::shared_ptr<Scalar>& result = datum_out.scalar(); + const std::shared_ptr<Scalar>& expected_ = expected.scalar(); + AssertScalarsEqual(*expected_, *result, /*verbose=*/true); + } +} + +class TestIfElseKernel : public ::testing::Test {}; + +template <typename Type> +class TestIfElsePrimitive : public ::testing::Test {}; + +using PrimitiveTypes = ::testing::Types<Int8Type, UInt8Type, Int16Type, UInt16Type, + Int32Type, UInt32Type, Int64Type, UInt64Type, + FloatType, DoubleType, Date32Type, Date64Type>; + +TYPED_TEST_SUITE(TestIfElsePrimitive, PrimitiveTypes); + +TYPED_TEST(TestIfElsePrimitive, IfElseFixedSizeRand) { + using ArrayType = typename TypeTraits<TypeParam>::ArrayType; + auto type = TypeTraits<TypeParam>::type_singleton(); + + random::RandomArrayGenerator rand(/*seed=*/0); + int64_t len = 1000; + auto cond = std::static_pointer_cast<BooleanArray>( + rand.ArrayOf(boolean(), len, /*null_probability=*/0.01)); + auto left = std::static_pointer_cast<ArrayType>( + rand.ArrayOf(type, len, /*null_probability=*/0.01)); + auto right = std::static_pointer_cast<ArrayType>( + rand.ArrayOf(type, len, /*null_probability=*/0.01)); + + typename TypeTraits<TypeParam>::BuilderType builder; + + for (int64_t i = 0; i < len; ++i) { + if (!cond->IsValid(i) || (cond->Value(i) && !left->IsValid(i)) || + (!cond->Value(i) && !right->IsValid(i))) { + ASSERT_OK(builder.AppendNull()); + continue; + } + + if (cond->Value(i)) { + ASSERT_OK(builder.Append(left->Value(i))); + } else { + ASSERT_OK(builder.Append(right->Value(i))); + } + } + ASSERT_OK_AND_ASSIGN(auto expected_data, builder.Finish()); + + CheckIfElseOutput(cond, left, right, expected_data); +} + +template <typename Type> +struct DatumWrapper { + using CType = typename TypeTraits<Type>::CType; + using ArrayType = typename TypeTraits<Type>::ArrayType; + using ScalarType = typename TypeTraits<Type>::ScalarType; + + util::Variant<std::shared_ptr<ScalarType>, std::shared_ptr<ArrayType>> datum; + bool is_scalar; + + explicit DatumWrapper(const Datum& datum_) : is_scalar(datum_.is_scalar()) { + if (is_scalar) { + datum = std::move(std::static_pointer_cast<ScalarType>(datum_.scalar())); + } else { + datum = std::move(std::static_pointer_cast<ArrayType>(datum_.make_array())); + } + } + + bool IsValid(int64_t i) const { + return is_scalar ? util::get<std::shared_ptr<ScalarType>>(datum)->is_valid + : util::get<std::shared_ptr<ArrayType>>(datum)->IsValid(i); + } + + CType Value(int64_t i) const { + return is_scalar ? util::get<std::shared_ptr<ScalarType>>(datum)->value + : util::get<std::shared_ptr<ArrayType>>(datum)->Value(i); + } +}; + +template <typename Type> +void GenerateExpected(const Datum& cond, const Datum& left, const Datum& right, Review comment: nit: why not just return the scalar? ########## File path: cpp/src/arrow/compute/kernels/scalar_if_else.cc ########## @@ -0,0 +1,577 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <arrow/compute/api.h> +#include <arrow/compute/kernels/codegen_internal.h> +#include <arrow/compute/util_internal.h> +#include <arrow/util/bit_block_counter.h> +#include <arrow/util/bitmap.h> +#include <arrow/util/bitmap_ops.h> + +namespace arrow { +using internal::BitBlockCount; +using internal::BitBlockCounter; +using internal::Bitmap; + +namespace compute { + +namespace { + +util::optional<uint64_t> GetConstantValidityWord(const Datum& data) { + if (data.is_scalar()) { + return data.scalar()->is_valid ? ~uint64_t(0) : uint64_t(0); + } + + if (data.array()->null_count == data.array()->length) return uint64_t(0); + + if (!data.array()->MayHaveNulls()) return ~uint64_t(0); + + // no constant validity word available + return {}; +} + +inline Bitmap GetBitmap(const Datum& datum, int i) { + if (datum.is_scalar()) return {}; + const ArrayData& a = *datum.array(); + return Bitmap{a.buffers[i], a.offset, a.length}; +} + +// if the condition is null then output is null otherwise we take validity from the +// selected argument +// ie. cond.valid & (cond.data & left.valid | ~cond.data & right.valid) +Status PromoteNullsVisitor(KernelContext* ctx, const Datum& cond_d, const Datum& left_d, + const Datum& right_d, ArrayData* output) { + auto cond_const = GetConstantValidityWord(cond_d); + auto left_const = GetConstantValidityWord(left_d); + auto right_const = GetConstantValidityWord(right_d); + + enum { COND_CONST = 1, LEFT_CONST = 2, RIGHT_CONST = 4 }; + auto flag = COND_CONST * cond_const.has_value() | LEFT_CONST * left_const.has_value() | + RIGHT_CONST * right_const.has_value(); + + const ArrayData& cond = *cond_d.array(); + // cond.data will always be available + Bitmap cond_data{cond.buffers[1], cond.offset, cond.length}; + Bitmap cond_valid{cond.buffers[0], cond.offset, cond.length}; + Bitmap left_valid = GetBitmap(left_d, 0); + Bitmap right_valid = GetBitmap(right_d, 0); + // sometimes Bitmaps will be ignored, in which case we replace access to them with + // duplicated (probably elided) access to cond_data + const Bitmap& _ = cond_data; + + // lambda function that will be used inside the visitor + uint64_t* out_validity = nullptr; + int64_t i = 0; + auto apply = [&](uint64_t c_valid, uint64_t c_data, uint64_t l_valid, + uint64_t r_valid) { + out_validity[i] = c_valid & ((c_data & l_valid) | (~c_data & r_valid)); + i++; + }; + + // cond.valid & (cond.data & left.valid | ~cond.data & right.valid) + // In the following cases, we dont need to allocate out_valid bitmap + switch (flag) { + case COND_CONST | LEFT_CONST | RIGHT_CONST: + // if cond & left & right all ones, then output is all valid --> out_valid = nullptr + if ((*cond_const & *left_const & *right_const) == UINT64_MAX) { + return Status::OK(); + } Review comment: Can we put `// fallthrough` here to make it explicit that this is intentional? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
