pitrou commented on a change in pull request #11882:
URL: https://github.com/apache/arrow/pull/11882#discussion_r793690969
##########
File path: cpp/src/arrow/compute/kernels/scalar_compare.cc
##########
@@ -156,39 +212,49 @@ struct Maximum {
}
};
+// Check if timestamp timezones are comparable (either all are empty or none
is).
+Status CheckCompareTimestamps(const ExecBatch& batch) {
+ if (batch.num_values() > 0) {
+ int invalid_states = 0;
+ for (int i = 0; i < batch.num_values(); i++) {
+ const auto& tsi = checked_cast<const TimestampType&>(*batch[i].type());
+ invalid_states += int(tsi.timezone().empty());
+ }
+ if (invalid_states * (invalid_states - batch.num_values()) != 0) {
+ return Status::TypeError(
+ "Cannot compare timestamp with timezone to timestamp without
timezone");
+ }
Review comment:
Ok, the logic here is still gratuitously sophisticated. How about
(untested):
```suggestion
bool have_no_timezone =
checked_cast<const
TimestampType&>(*batch[0].type()).timezone().empty();
for (int i = 1; i < batch.num_values(); i++) {
if (checked_cast<const
TimestampType&>(*batch[i].type()).timezone().empty() !=
have_no_timezone) {
return Status::TypeError(
"Cannot compare timestamp with timezone to timestamp without
timezone");
}
}
```
##########
File path: cpp/src/arrow/compute/kernels/scalar_compare.cc
##########
@@ -746,6 +805,199 @@ std::shared_ptr<ScalarFunction>
MakeScalarMinMax(std::string name,
return func;
}
+template <template <BetweenOptions::Inclusive> class Op>
+Status MakeBetweenArrayExec(Type::type type_id, KernelContext* ctx,
+ const ExecBatch& batch, Datum* out) {
+ using BetweenState = OptionsWrapper<BetweenOptions>;
+ const auto& state = static_cast<const BetweenState&>(*ctx->state());
+ if (type_id == Type::FIXED_SIZE_BINARY) {
+ switch (state.options.inclusive) {
+ case BetweenOptions::Inclusive::BOTH:
+ return ScalarTernaryEqualTypes<BooleanType, FixedSizeBinaryType,
+
Op<BetweenOptions::Inclusive::BOTH>>::Exec(ctx,
+
batch,
+
out);
+ case BetweenOptions::Inclusive::LEFT:
+ return ScalarTernaryEqualTypes<BooleanType, FixedSizeBinaryType,
+
Op<BetweenOptions::Inclusive::LEFT>>::Exec(ctx,
+
batch,
+
out);
+ case BetweenOptions::Inclusive::RIGHT:
+ return ScalarTernaryEqualTypes<BooleanType, FixedSizeBinaryType,
+
Op<BetweenOptions::Inclusive::RIGHT>>::Exec(ctx,
+
batch,
+
out);
+ case BetweenOptions::Inclusive::NEITHER:
+ return ScalarTernaryEqualTypes<
+ BooleanType, FixedSizeBinaryType,
+ Op<BetweenOptions::Inclusive::NEITHER>>::Exec(ctx, batch, out);
+ default:
+ return Status::NotImplemented("between inclusiveness not implemented:
",
+ state.options.ToString());
+ }
+ } else if (type_id == Type::DECIMAL128 || type_id == Type::DECIMAL256) {
Review comment:
(untested, but should probably work)
```suggestion
} else if (is_decimal(type_id)) {
```
##########
File path: cpp/src/arrow/compute/api_scalar.h
##########
@@ -1350,5 +1363,26 @@ ARROW_EXPORT Result<Datum> AssumeTimezone(const Datum&
values,
AssumeTimezoneOptions options,
ExecContext* ctx = NULLPTR);
+/// \brief Between compares each element in `values`
+/// with `left` as a lower bound and 'right' as an upper bound
Review comment:
```suggestion
/// \brief Compute whether each element in `values` is between `left` and
`right`
```
##########
File path: cpp/src/arrow/util/bit_block_counter.h
##########
@@ -422,6 +426,96 @@ class ARROW_EXPORT OptionalBinaryBitBlockCounter {
}
};
+/// \brief A class that computes popcounts on the result of bitwise operations
+/// between three bitmaps, 64 bits at a time. A 64-bit word is loaded from each
+/// bitmap, then the popcount is computed on e.g. the bitwise-and of the three
+/// words.
+class ARROW_EXPORT TernaryBitBlockCounter {
Review comment:
Apparently you forgot to remove it :-)
##########
File path: cpp/src/arrow/compute/kernels/scalar_compare.cc
##########
@@ -772,6 +1024,16 @@ const FunctionDoc less_equal_doc{
("A null on either side emits a null comparison result."),
{"x", "y"}};
+const FunctionDoc between_doc{
+ "Check if values are in the given range, val between a and b",
Review comment:
```suggestion
"Check if values are in the given range",
```
##########
File path: cpp/src/arrow/ipc/json_simple.cc
##########
@@ -983,6 +983,16 @@ Status ScalarFromJSON(const std::shared_ptr<DataType>&
type,
return Status::OK();
}
+Status ScalarFromJSON(const std::shared_ptr<DataType>& type,
Review comment:
These two overloads are not used anywhere, are they?
(they are not even exposed in `json_simple.h`)
##########
File path: python/pyarrow/tests/test_compute.py
##########
@@ -1360,6 +1361,27 @@ def test_filter_null_type():
assert len(table.filter(mask).column(0)) == 5
[email protected]("ty", ["inclusive"])
+def test_between_array_array_scalar(ty):
+ BetweenOptions = partial(pc.BetweenOptions)
+
+ val = pa.array([1, 1, 4, 3, 2, 5])
+ arr1 = pa.array([1, 2, 3, 2, None, 2])
+ scalar2 = pa.scalar(4)
+
+ inclusive_and_expected = {
+ "both": [True, False, True, True, None, False],
+ "left": [True, False, False, True, None, False],
+ "right": [False, False, True, True, None, False],
+ "neither": [False, False, False, True, None, False],
+ }
+
+ for inclusive, expected in inclusive_and_expected.items():
+ options = BetweenOptions(inclusive=inclusive)
+ result = pc.between(val, arr1, scalar2, options=options)
Review comment:
You might just as well pass the option directly:
```suggestion
result = pc.between(val, arr1, scalar2, inclusive=inclusive)
```
##########
File path: cpp/src/arrow/compute/kernels/scalar_compare.cc
##########
@@ -746,6 +805,199 @@ std::shared_ptr<ScalarFunction>
MakeScalarMinMax(std::string name,
return func;
}
+template <template <BetweenOptions::Inclusive> class Op>
+Status MakeBetweenArrayExec(Type::type type_id, KernelContext* ctx,
+ const ExecBatch& batch, Datum* out) {
+ using BetweenState = OptionsWrapper<BetweenOptions>;
+ const auto& state = static_cast<const BetweenState&>(*ctx->state());
+ if (type_id == Type::FIXED_SIZE_BINARY) {
+ switch (state.options.inclusive) {
+ case BetweenOptions::Inclusive::BOTH:
+ return ScalarTernaryEqualTypes<BooleanType, FixedSizeBinaryType,
+
Op<BetweenOptions::Inclusive::BOTH>>::Exec(ctx,
+
batch,
+
out);
+ case BetweenOptions::Inclusive::LEFT:
+ return ScalarTernaryEqualTypes<BooleanType, FixedSizeBinaryType,
+
Op<BetweenOptions::Inclusive::LEFT>>::Exec(ctx,
+
batch,
+
out);
+ case BetweenOptions::Inclusive::RIGHT:
+ return ScalarTernaryEqualTypes<BooleanType, FixedSizeBinaryType,
+
Op<BetweenOptions::Inclusive::RIGHT>>::Exec(ctx,
+
batch,
+
out);
+ case BetweenOptions::Inclusive::NEITHER:
+ return ScalarTernaryEqualTypes<
+ BooleanType, FixedSizeBinaryType,
+ Op<BetweenOptions::Inclusive::NEITHER>>::Exec(ctx, batch, out);
+ default:
+ return Status::NotImplemented("between inclusiveness not implemented:
",
+ state.options.ToString());
+ }
+ } else if (type_id == Type::DECIMAL128 || type_id == Type::DECIMAL256) {
+ switch (state.options.inclusive) {
+ case BetweenOptions::Inclusive::BOTH:
+ return GenerateDecimal<ScalarTernaryEqualTypes, BooleanType,
+
Op<BetweenOptions::Inclusive::BOTH>>(type_id)(ctx, batch,
+
out);
+ case BetweenOptions::Inclusive::LEFT:
+ return GenerateDecimal<ScalarTernaryEqualTypes, BooleanType,
+
Op<BetweenOptions::Inclusive::LEFT>>(type_id)(ctx, batch,
+
out);
+ case BetweenOptions::Inclusive::RIGHT:
+ return GenerateDecimal<ScalarTernaryEqualTypes, BooleanType,
+
Op<BetweenOptions::Inclusive::RIGHT>>(type_id)(ctx, batch,
+
out);
+ case BetweenOptions::Inclusive::NEITHER:
+ return GenerateDecimal<ScalarTernaryEqualTypes, BooleanType,
+
Op<BetweenOptions::Inclusive::NEITHER>>(type_id)(
+ ctx, batch, out);
+ default:
+ return Status::NotImplemented("between inclusiveness not implemented:
",
+ state.options.ToString());
+ }
+ } else if (type_id == Type::BINARY || type_id == Type::STRING ||
+ type_id == Type::LARGE_BINARY || type_id == Type::LARGE_STRING) {
Review comment:
(untested, but should probably work)
```suggestion
} else if (is_base_binary_like(type_id)) {
```
##########
File path: python/pyarrow/tests/test_compute.py
##########
@@ -1360,6 +1361,27 @@ def test_filter_null_type():
assert len(table.filter(mask).column(0)) == 5
[email protected]("ty", ["inclusive"])
+def test_between_array_array_scalar(ty):
+ BetweenOptions = partial(pc.BetweenOptions)
Review comment:
This is harmless, but does it actually serve a purpose?
##########
File path: cpp/src/arrow/compute/kernels/scalar_compare_test.cc
##########
@@ -1865,5 +1928,654 @@ TEST(TestMaxElementWiseMinElementWise, CommonTemporal) {
ResultWith(ScalarFromJSON(date64(), "86400000")));
}
+static void ValidateBetween(BetweenOptions options, const Datum& val, const
Datum& lhs,
+ const Datum& rhs, const Datum& expected) {
+ ASSERT_OK_AND_ASSIGN(Datum result, Between(val, lhs, rhs, options, nullptr));
+ if ((val.is_scalar()) && (lhs.is_scalar()) && (rhs.is_scalar())) {
+ AssertScalarsEqual(*expected.scalar(), *result.scalar(), /*verbose=*/true);
+ } else {
+ AssertArraysEqual(*expected.make_array(), *result.make_array(),
+ /*verbose=*/true);
+ }
+}
+
+void ValidateBetween(const Datum& val, const Datum& lhs, const Datum& rhs) {
+ CompareOperator lhs_val;
+ CompareOperator val_rhs;
+ for (auto inclusive :
+ {BetweenOptions::Inclusive::BOTH, BetweenOptions::Inclusive::LEFT,
+ BetweenOptions::Inclusive::RIGHT, BetweenOptions::Inclusive::NEITHER})
{
+ auto options = BetweenOptions(inclusive);
+ if (inclusive == BetweenOptions::Inclusive::NEITHER) {
+ lhs_val = LESS;
+ val_rhs = LESS;
+ } else if (inclusive == BetweenOptions::Inclusive::LEFT) {
+ lhs_val = LESS_EQUAL;
+ val_rhs = LESS;
+ } else if (inclusive == BetweenOptions::Inclusive::RIGHT) {
+ lhs_val = LESS;
+ val_rhs = LESS_EQUAL;
+ } else {
+ lhs_val = LESS_EQUAL;
+ val_rhs = LESS_EQUAL;
+ }
+
+ ASSERT_OK_AND_ASSIGN(
+ Datum resultl, CallFunction(CompareOperatorToFunctionName(lhs_val),
{lhs, val}));
+ ASSERT_OK_AND_ASSIGN(
+ Datum resultr, CallFunction(CompareOperatorToFunctionName(val_rhs),
{val, rhs}));
+ ASSERT_OK_AND_ASSIGN(Datum expected, CallFunction("and", {resultl,
resultr}));
+
+ ValidateBetween(options, val, lhs, rhs, expected);
+ }
+}
+
+class TestNumericBetweenKernel : public ::testing::Test {};
+
+std::shared_ptr<DataType> GetType(std::shared_ptr<DataType> type) {
+ auto type_string = type->ToString();
+ if (type_string == "duration[s]") {
+ return int64();
+ } else if (type_string == "duration[ms]") {
+ return int64();
+ } else if (type_string == "duration[ns]") {
+ return int64();
+ } else if (type_string == "uint8") {
+ return uint8();
+ } else if (type_string == "uint16") {
+ return uint16();
+ } else if (type_string == "uint32") {
+ return uint32();
+ } else if (type_string == "uint64") {
+ return uint64();
+ } else if (type_string == "int8") {
+ return int8();
+ } else if (type_string == "int16") {
+ return int16();
+ } else if (type_string == "int32") {
+ return int32();
+ } else if (type_string == "int64") {
+ return int64();
+ } else if (type_string == "float") {
+ return float32();
+ } else if (type_string == "double") {
+ return float64();
+ } else {
+ return int64();
+ }
+}
+
+TEST(TestNumericBetweenKernel, 3Scalars) {
+ for (const auto& types : {DurationTypes(), NumericTypes()}) {
+ for (const std::shared_ptr<DataType>& ty : types) {
+ ARROW_SCOPED_TRACE("type = ", ty->ToString());
+ auto tt = GetType(ty);
+ auto zero = Datum(ScalarFromJSON(tt, "0"));
+ auto two = Datum(ScalarFromJSON(tt, "2"));
+ auto four = Datum(ScalarFromJSON(tt, "4"));
+ auto null = Datum(ScalarFromJSON(tt, "null"));
+ ValidateBetween(zero, two, four);
+ ValidateBetween(two, zero, four);
+ ValidateBetween(two, two, four);
+ ValidateBetween(four, two, four);
+ ValidateBetween(null, two, four);
+ ValidateBetween(two, null, four);
+ ValidateBetween(two, zero, null);
+ }
+ }
+}
+
+TEST(TestNumericBetweenKernel, 1Array2Scalars) {
+ for (const auto& types : {DurationTypes(), NumericTypes()}) {
+ for (const std::shared_ptr<DataType>& ty : types) {
+ ARROW_SCOPED_TRACE("type = ", ty->ToString());
+ auto tt = GetType(ty);
+ auto zero = Datum(ScalarFromJSON(tt, "0"));
+ auto four = Datum(ScalarFromJSON(tt, "4"));
+ auto null = Datum(ScalarFromJSON(tt, "null"));
+ ValidateBetween(Datum(ArrayFromJSON(tt, "[]")), zero, four);
+ ValidateBetween(Datum(ArrayFromJSON(tt, "[null]")), zero, four);
+ ValidateBetween(Datum(ArrayFromJSON(tt, "[0,1,2,3,4,5]")), zero, four);
+ ValidateBetween(Datum(ArrayFromJSON(tt, "[null,0,1,1]")), zero, four);
+ ValidateBetween(Datum(ArrayFromJSON(tt, "[5,4,3,2,1,0]")), null, four);
+ ValidateBetween(Datum(ArrayFromJSON(tt, "[5,4,3,2,1,0]")), zero, null);
+ ValidateBetween(zero, Datum(ArrayFromJSON(tt, "[]")), four);
+ ValidateBetween(zero, Datum(ArrayFromJSON(tt, "[null]")), four);
+ ValidateBetween(zero, Datum(ArrayFromJSON(tt, "[0,1,2,3,4,5]")), four);
+ ValidateBetween(zero, Datum(ArrayFromJSON(tt, "[null,0,1,1]")), four);
+ ValidateBetween(null, Datum(ArrayFromJSON(tt, "[5,4,3,2,1,0]")), four);
+ ValidateBetween(zero, Datum(ArrayFromJSON(tt, "[5,4,3,2,1,0]")), null);
+ ValidateBetween(zero, four, Datum(ArrayFromJSON(tt, "[]")));
+ ValidateBetween(zero, four, Datum(ArrayFromJSON(tt, "[null]")));
+ ValidateBetween(zero, four, Datum(ArrayFromJSON(tt, "[0,1,2,3,4,5]")));
+ ValidateBetween(zero, four, Datum(ArrayFromJSON(tt, "[null,0,1,1]")));
+ ValidateBetween(null, four, Datum(ArrayFromJSON(tt, "[5,4,3,2,1,0]")));
+ ValidateBetween(zero, null, Datum(ArrayFromJSON(tt, "[5,4,3,2,1,0]")));
+ }
+ }
+}
+
+TEST(TestNumericBetweenKernel, 2Arrays1Scalar) {
+ for (const auto& types : {DurationTypes(), NumericTypes()}) {
+ for (const std::shared_ptr<DataType>& ty : types) {
+ ARROW_SCOPED_TRACE("type = ", ty->ToString());
+ auto tt = GetType(ty);
+ auto one = Datum(ScalarFromJSON(tt, "1"));
+ ValidateBetween(one, Datum(ArrayFromJSON(tt, "[]")),
+ Datum(ArrayFromJSON(tt, "[]")));
+ ValidateBetween(one, Datum(ArrayFromJSON(tt, "[null]")),
+ Datum(ArrayFromJSON(tt, "[null]")));
+ ValidateBetween(one, Datum(ArrayFromJSON(tt, "[0,0,1,3,3]")),
+ Datum(ArrayFromJSON(tt, "[10,10,2,5,5]")));
+ ValidateBetween(one, Datum(ArrayFromJSON(tt, "[0,0,1,null,3,3]")),
+ Datum(ArrayFromJSON(tt, "[0,10,2,2,5,5]")));
+ ValidateBetween(Datum(ArrayFromJSON(tt, "[]")), one,
+ Datum(ArrayFromJSON(tt, "[]")));
+ ValidateBetween(Datum(ArrayFromJSON(tt, "[null]")), one,
+ Datum(ArrayFromJSON(tt, "[null]")));
+ ValidateBetween(Datum(ArrayFromJSON(tt, "[0,0,1,3,3]")), one,
+ Datum(ArrayFromJSON(tt, "[10,10,2,5,5]")));
+ ValidateBetween(Datum(ArrayFromJSON(tt, "[0,0,1,null,3,3]")), one,
+ Datum(ArrayFromJSON(tt, "[0,10,2,2,5,5]")));
+ ValidateBetween(Datum(ArrayFromJSON(tt, "[]")), Datum(ArrayFromJSON(tt,
"[]")),
+ one);
+ ValidateBetween(Datum(ArrayFromJSON(tt, "[null]")),
+ Datum(ArrayFromJSON(tt, "[null]")), one);
+ ValidateBetween(Datum(ArrayFromJSON(tt, "[0,0,1,3,3]")),
+ Datum(ArrayFromJSON(tt, "[10,10,2,5,5]")), one);
+ ValidateBetween(Datum(ArrayFromJSON(tt, "[0,0,1,null,3,3]")),
+ Datum(ArrayFromJSON(tt, "[0,10,2,2,5,5]")), one);
+ }
+ }
+}
+
+TEST(TestNumericBetweenKernel, 3Arrays) {
+ for (const auto& types : {DurationTypes(), NumericTypes()}) {
+ for (const std::shared_ptr<DataType>& ty : types) {
+ ARROW_SCOPED_TRACE("type = ", ty->ToString());
+ ValidateBetween(Datum(ArrayFromJSON(ty, "[]")), Datum(ArrayFromJSON(ty,
"[]")),
+ Datum(ArrayFromJSON(ty, "[]")));
+ ValidateBetween(Datum(ArrayFromJSON(ty, "[null]")),
+ Datum(ArrayFromJSON(ty, "[null]")),
+ Datum(ArrayFromJSON(ty, "[null]")));
+ ValidateBetween(Datum(ArrayFromJSON(ty, "[1,1,2,2,2]")),
+ Datum(ArrayFromJSON(ty, "[0,0,1,3,3]")),
+ Datum(ArrayFromJSON(ty, "[10,10,2,5,5]")));
+ ValidateBetween(Datum(ArrayFromJSON(ty, "[0,1,2,2,2,2]")),
+ Datum(ArrayFromJSON(ty, "[0,0,1,null,3,3]")),
+ Datum(ArrayFromJSON(ty, "[0,10,2,2,5,5]")));
+ }
+ }
+}
+
+TEST(TestNumericBetweenKernel, Random) {
+ for (const auto& types : {DurationTypes(), NumericTypes()}) {
+ for (const std::shared_ptr<DataType>& ty : types) {
+ auto rand = random::RandomArrayGenerator(0x5416447);
+ const int64_t length = 100;
+ for (auto null_probability : {0.0, 0.01, 0.1, 0.25, 0.5, 1.0}) {
+ ARROW_SCOPED_TRACE("type = ", ty->ToString());
+ auto tt = GetType(ty);
+ auto metadata =
+ key_value_metadata({"null_probability"},
{std::to_string(null_probability)});
+ auto field = ::arrow::field("[0,100]", std::move(ty),
std::move(metadata));
+ auto data1 = rand.ArrayOf(*field, length);
+ auto data2 = rand.ArrayOf(*field, length);
+ auto data3 = rand.ArrayOf(*field, length);
+
+ // Create view of data as the type (e.g. float64)
+ auto array1 = Datum(*data1->View(tt));
+ auto array2 = Datum(*data2->View(tt));
+ auto array3 = Datum(*data3->View(tt));
+ auto scalar1 = Datum(ScalarFromJSON(tt, "10"));
+ auto scalar2 = Datum(ScalarFromJSON(tt, "30"));
+ auto scalar3 = Datum(ScalarFromJSON(tt, "50"));
+ ValidateBetween(scalar1, scalar2, scalar3);
+ ValidateBetween(array1, scalar2, scalar3);
+ ValidateBetween(array1, array2, scalar3);
+ ValidateBetween(array1, array2, array3);
+ ValidateBetween(array1, scalar2, scalar3);
+ ValidateBetween(scalar1, array2, array3);
+ ValidateBetween(scalar1, array2, scalar3);
+ ValidateBetween(scalar1, scalar2, array3);
+ ValidateBetween(array1, scalar2, array3);
+ }
+ }
+ }
+}
+
+class TestStringBetweenKernel : public ::testing::Test {};
+
+TEST(TestStringBetweenKernel, Random) {
+ using ScalarType = typename TypeTraits<StringType>::ScalarType;
+
+ auto rand = random::RandomArrayGenerator(0x5416447);
+ for (size_t i = 3; i < 10; i++) {
+ for (auto null_probability : {0.0, 0.01, 0.1, 0.25, 0.5, 1.0}) {
+ const int64_t length = static_cast<int64_t>(1ULL << i);
+ auto array1 = Datum(rand.String(length, 0, 16, null_probability));
+ auto array2 = Datum(rand.String(length, 0, 16, null_probability));
+ auto array3 = Datum(rand.String(length, 0, 16, null_probability));
+ auto scalar1 = Datum(std::make_shared<ScalarType>("fupi"));
+ auto scalar2 = Datum(std::make_shared<ScalarType>("tupu"));
+ auto scalar3 = Datum(std::make_shared<ScalarType>("zito"));
+ ValidateBetween(scalar1, scalar2, scalar3);
+ ValidateBetween(array1, scalar2, scalar3);
+ ValidateBetween(scalar1, array2, scalar3);
+ ValidateBetween(scalar1, scalar2, array3);
+ ValidateBetween(scalar1, array2, array3);
+ ValidateBetween(array1, scalar2, array3);
+ ValidateBetween(array1, array2, scalar3);
+ ValidateBetween(array1, array2, array3);
+ }
+ }
+}
+
+TEST(TestStringBetweenKernel, 1Array2Scalars) {
+ using ScalarType = typename TypeTraits<StringType>::ScalarType;
+ auto l = Datum(std::make_shared<ScalarType>("abc"));
+ auto r = Datum(std::make_shared<ScalarType>("zzz"));
+
ValidateBetween(Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
"[]")), l,
+ r);
+ ValidateBetween(
+ Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
"[null]")), l, r);
+ ValidateBetween(Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["aaa", "aaaa", "ccc", "z"])")),
+ l, r);
+ ValidateBetween(Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["abc", "baa", "fff", "zzz"])")),
+ l, r);
+ ValidateBetween(Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["abd", null, null, "zzx"])")),
+ l, r);
+ ValidateBetween(l,
Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(), "[]")),
+ r);
+ ValidateBetween(l,
+ Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
"[null]")), r);
+ ValidateBetween(l,
Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["aaa", "aaaa", "ccc", "z"])")),
+ r);
+ ValidateBetween(l,
Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["abc", "baa", "fff", "zzz"])")),
+ r);
+ ValidateBetween(l,
Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["abd", null, null, "zzx"])")),
+ r);
+ ValidateBetween(l, r,
+
Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(), "[]")));
+ ValidateBetween(l, r,
+ Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
"[null]")));
+ ValidateBetween(l, r,
Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["aaa", "aaaa", "ccc", "z"])")));
+ ValidateBetween(l, r,
Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["abc", "baa", "fff", "zzz"])")));
+ ValidateBetween(l, r,
Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["abd", null, null, "zzx"])")));
+}
+
+TEST(TestStringBetweenKernel, 2Arrays1Scalar) {
+ using ScalarType = typename TypeTraits<StringType>::ScalarType;
+ auto r = Datum(std::make_shared<ScalarType>("zzz"));
+ ValidateBetween(r,
Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(), "[]")),
+
Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(), "[]")));
+ ValidateBetween(r,
Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["aaa", "aaaa", "ccc", "z"])")),
+ Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["abc", "baa", "fff", "zzz"])")));
+ ValidateBetween(r,
Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["abc", "baa", "fff", "zzz"])")),
+ Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["abd", null, null, "zzx"])")));
+
ValidateBetween(Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
"[]")), r,
+
Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(), "[]")));
+ ValidateBetween(Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["aaa", "aaaa", "ccc", "z"])")), r,
+ Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["abc", "baa", "fff", "zzz"])")));
+ ValidateBetween(Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["abc", "baa", "fff", "zzz"])")), r,
+ Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["abd", null, null, "zzx"])")));
+
ValidateBetween(Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
"[]")),
+
Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(), "[]")),
+ r);
+ ValidateBetween(Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["aaa", "aaaa", "ccc", "z"])")),
+ Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["abc", "baa", "fff", "zzz"])")),
+ r);
+ ValidateBetween(Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["abc", "baa", "fff", "zzz"])")),
+ Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["abd", null, null, "zzx"])")),
+ r);
+}
+
+TEST(TestStringBetweenKernel, 3Arrays) {
+ ValidateBetween(Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["david","hello","world"])")),
+ Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["adam","hi","whirl"])")),
+ Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+
R"(["robert","goeiemoreen","whirlwind"])")));
+ ValidateBetween(
+ Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
R"(["x","a","f"])")),
+ Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
R"(["w","a","e"])")),
+ Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
R"(["z","a","g"])")));
+ ValidateBetween(Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["block","bit","binary"])")),
+ Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["bit","nibble","ternary"])")),
+ Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["word","d","xyz"])")));
+ ValidateBetween(Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["よしもと","の","ち"])")),
+ Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["は","へ","あ"])")),
+ Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["な","を","ち"])")));
+ ValidateBetween(Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["A","ア","王"])")),
+ Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["た","あ","歩"])")),
+ Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
+ R"(["李","田",null])")));
+ ValidateBetween(
+ Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
R"(["Б",null,"Я"])")),
+ Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
R"(["А","Ж","Щ"])")),
+ Datum(ArrayFromJSON(TypeTraits<StringType>::type_singleton(),
R"(["Д","Л","Ф"])")));
+}
+
+TEST(TestTimestampsBetweenKernel, 1Array2Scalars) {
+ const std::string scalar1_json = R"("1980-02-02")";
+ const std::string scalar2_json = R"("1970-01-01")";
+ const std::string array_json = R"(["1970-01-02","1980-02-02","1970-02-28"])";
+ // Same units should be fine
+ ValidateBetween(Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND),
array_json)),
+ Datum(ScalarFromJSON(timestamp(TimeUnit::SECOND),
scalar1_json)),
+ Datum(ScalarFromJSON(timestamp(TimeUnit::SECOND),
scalar2_json)));
+ ValidateBetween(Datum(ScalarFromJSON(timestamp(TimeUnit::SECOND),
scalar1_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND),
array_json)),
+ Datum(ScalarFromJSON(timestamp(TimeUnit::SECOND),
scalar2_json)));
+ ValidateBetween(Datum(ScalarFromJSON(timestamp(TimeUnit::SECOND),
scalar1_json)),
+ Datum(ScalarFromJSON(timestamp(TimeUnit::SECOND),
scalar2_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND),
array_json)));
+ // Different timezones should be fine
+ ValidateBetween(
+ Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND, "Africa/Cairo"),
array_json)),
+ Datum(ScalarFromJSON(timestamp(TimeUnit::SECOND, "America/Chicago"),
scalar1_json)),
+ Datum(ScalarFromJSON(timestamp(TimeUnit::SECOND, "Asia/Beijing"),
scalar2_json)));
+ ValidateBetween(
+ Datum(ScalarFromJSON(timestamp(TimeUnit::NANO, "Europe/Berlin"),
scalar1_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::NANO, "America/Phoenix"),
array_json)),
+ Datum(ScalarFromJSON(timestamp(TimeUnit::NANO, "Africa/Nairobi"),
scalar2_json)));
+ ValidateBetween(
+ Datum(ScalarFromJSON(timestamp(TimeUnit::NANO, "Europe/Berlin"),
scalar1_json)),
+ Datum(ScalarFromJSON(timestamp(TimeUnit::NANO, "Asia/Tokyo"),
scalar2_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::NANO, "Africa/Nairobi"),
array_json)));
+ // Different units should be fine
+ ValidateBetween(Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND),
array_json)),
+ Datum(ScalarFromJSON(timestamp(TimeUnit::MILLI),
scalar1_json)),
+ Datum(ScalarFromJSON(timestamp(TimeUnit::SECOND),
scalar2_json)));
+ ValidateBetween(Datum(ScalarFromJSON(timestamp(TimeUnit::SECOND),
scalar1_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::NANO), array_json)),
+ Datum(ScalarFromJSON(timestamp(TimeUnit::SECOND),
scalar2_json)));
+ ValidateBetween(Datum(ScalarFromJSON(timestamp(TimeUnit::SECOND),
scalar1_json)),
+ Datum(ScalarFromJSON(timestamp(TimeUnit::NANO),
scalar2_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND),
array_json)));
+ // But comparing naive to zoned is not OK
+ for (auto inclusive :
+ {BetweenOptions::Inclusive::BOTH, BetweenOptions::Inclusive::LEFT,
+ BetweenOptions::Inclusive::RIGHT, BetweenOptions::Inclusive::NEITHER})
{
+ auto options = BetweenOptions(inclusive);
+ EXPECT_RAISES_WITH_MESSAGE_THAT(
+ TypeError,
+ ::testing::HasSubstr(
+ "Cannot compare timestamp with timezone to timestamp without
timezone"),
+ Between(ArrayFromJSON(timestamp(TimeUnit::SECOND), array_json),
+ ScalarFromJSON(timestamp(TimeUnit::SECOND), scalar1_json),
+ ScalarFromJSON(timestamp(TimeUnit::SECOND, "Asia/Tokyo"),
scalar2_json),
+ options, nullptr));
+ EXPECT_RAISES_WITH_MESSAGE_THAT(
+ TypeError,
+ ::testing::HasSubstr(
+ "Cannot compare timestamp with timezone to timestamp without
timezone"),
+ Between(ScalarFromJSON(timestamp(TimeUnit::SECOND), scalar1_json),
+ ArrayFromJSON(timestamp(TimeUnit::SECOND, "America/New_York"),
array_json),
+ ScalarFromJSON(timestamp(TimeUnit::SECOND, "Europe/Berlin"),
scalar2_json),
+ options, nullptr));
+ EXPECT_RAISES_WITH_MESSAGE_THAT(
+ TypeError,
+ ::testing::HasSubstr(
+ "Cannot compare timestamp with timezone to timestamp without
timezone"),
+ Between(ScalarFromJSON(timestamp(TimeUnit::SECOND, "Africa/Nairobi"),
scalar1_json),
+ ArrayFromJSON(timestamp(TimeUnit::SECOND), array_json),
+ ScalarFromJSON(timestamp(TimeUnit::SECOND), scalar2_json),
options, nullptr));
+ }
+}
+
+TEST(TestTimestampsBetweenKernel, 2Arrays1Scalar) {
+ const std::string scalar_json = R"("1980-02-02")";
+ const std::string array1_json =
R"(["1970-01-01","1980-02-01","1970-02-28"])";
+ const std::string array2_json =
R"(["1970-01-02","1980-02-02","1970-02-28"])";
+ // Same units should be fine
+ ValidateBetween(Datum(ScalarFromJSON(timestamp(TimeUnit::SECOND),
scalar_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND),
array1_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND),
array2_json)));
+ ValidateBetween(Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND),
array1_json)),
+ Datum(ScalarFromJSON(timestamp(TimeUnit::SECOND),
scalar_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND),
array2_json)));
+ ValidateBetween(Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND),
array1_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND),
array2_json)),
+ Datum(ScalarFromJSON(timestamp(TimeUnit::SECOND),
scalar_json)));
+ // Different timezones should be fine
+ ValidateBetween(
+ Datum(ScalarFromJSON(timestamp(TimeUnit::SECOND, "Africa/Cairo"),
scalar_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND, "America/Chicago"),
array1_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND, "Asia/Beijing"),
array2_json)));
+ ValidateBetween(
+ Datum(ArrayFromJSON(timestamp(TimeUnit::NANO, "Europe/Berlin"),
array1_json)),
+ Datum(ScalarFromJSON(timestamp(TimeUnit::NANO, "America/Phoenix"),
scalar_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::NANO, "Africa/Nairobi"),
array2_json)));
+ ValidateBetween(
+ Datum(ArrayFromJSON(timestamp(TimeUnit::NANO, "Europe/Berlin"),
array1_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::NANO, "Asia/Tokyo"),
array2_json)),
+ Datum(ScalarFromJSON(timestamp(TimeUnit::NANO, "Africa/Nairobi"),
scalar_json)));
+ // Different units should be fine
+ ValidateBetween(Datum(ScalarFromJSON(timestamp(TimeUnit::SECOND),
scalar_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::MILLI),
array1_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND),
array2_json)));
+ ValidateBetween(Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND),
array1_json)),
+ Datum(ScalarFromJSON(timestamp(TimeUnit::NANO),
scalar_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND),
array2_json)));
+ ValidateBetween(Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND),
array1_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::NANO), array2_json)),
+ Datum(ScalarFromJSON(timestamp(TimeUnit::SECOND),
scalar_json)));
+ // But comparing naive to zoned is not OK
+ for (auto inclusive :
+ {BetweenOptions::Inclusive::BOTH, BetweenOptions::Inclusive::LEFT,
+ BetweenOptions::Inclusive::RIGHT, BetweenOptions::Inclusive::NEITHER})
{
+ auto options = BetweenOptions(inclusive);
+ EXPECT_RAISES_WITH_MESSAGE_THAT(
+ TypeError,
+ ::testing::HasSubstr(
+ "Cannot compare timestamp with timezone to timestamp without
timezone"),
+ Between(ScalarFromJSON(timestamp(TimeUnit::SECOND), scalar_json),
+ ArrayFromJSON(timestamp(TimeUnit::SECOND), array1_json),
+ ArrayFromJSON(timestamp(TimeUnit::SECOND, "Asia/Tokyo"),
array2_json),
+ options, nullptr));
+ EXPECT_RAISES_WITH_MESSAGE_THAT(
+ TypeError,
+ ::testing::HasSubstr(
+ "Cannot compare timestamp with timezone to timestamp without
timezone"),
+ Between(ArrayFromJSON(timestamp(TimeUnit::SECOND), array1_json),
+ ScalarFromJSON(timestamp(TimeUnit::SECOND,
"America/New_York"), scalar_json),
+ ArrayFromJSON(timestamp(TimeUnit::SECOND, "Europe/Berlin"),
array2_json),
+ options, nullptr));
+ EXPECT_RAISES_WITH_MESSAGE_THAT(
+ TypeError,
+ ::testing::HasSubstr(
+ "Cannot compare timestamp with timezone to timestamp without
timezone"),
+ Between(ArrayFromJSON(timestamp(TimeUnit::SECOND, "Africa/Nairobi"),
array1_json),
+ ScalarFromJSON(timestamp(TimeUnit::SECOND), scalar_json),
+ ArrayFromJSON(timestamp(TimeUnit::SECOND), array2_json),
options, nullptr));
+ }
+}
+
+TEST(TestTimestampsBetweenKernel, 3Arrays) {
+ const std::string arr_json = R"(["1970-01-01","1980-02-02","1970-02-28"])";
+ const std::string lhs_json = R"(["1970-01-01","1980-02-01","1970-02-28"])";
+ const std::string rhs_json = R"(["1970-01-02","1980-02-02","1970-02-28"])";
+ // Same units should be fine
+ ValidateBetween(Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND), arr_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND), lhs_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND), rhs_json)));
+ // Different timezones should be fine
+ ValidateBetween(
+ Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND, "Africa/Cairo"),
arr_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND, "America/Chicago"),
lhs_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND, "Asia/Beijing"),
rhs_json)));
+ ValidateBetween(
+ Datum(ArrayFromJSON(timestamp(TimeUnit::NANO, "Europe/Berlin"),
arr_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::NANO, "America/Phoenix"),
lhs_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::NANO, "Africa/Nairobi"),
rhs_json)));
+ // Different units should be fine
+ ValidateBetween(Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND), arr_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::MILLI), lhs_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND), rhs_json)));
+ ValidateBetween(Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND), arr_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::NANO), lhs_json)),
+ Datum(ArrayFromJSON(timestamp(TimeUnit::SECOND), rhs_json)));
+ // But comparing naive to zoned is not OK
+ for (auto inclusive :
+ {BetweenOptions::Inclusive::BOTH, BetweenOptions::Inclusive::LEFT,
+ BetweenOptions::Inclusive::RIGHT, BetweenOptions::Inclusive::NEITHER})
{
+ auto options = BetweenOptions(inclusive);
+ EXPECT_RAISES_WITH_MESSAGE_THAT(
+ TypeError,
+ ::testing::HasSubstr(
+ "Cannot compare timestamp with timezone to timestamp without
timezone"),
+ Between(ArrayFromJSON(timestamp(TimeUnit::SECOND), arr_json),
+ ArrayFromJSON(timestamp(TimeUnit::SECOND), lhs_json),
+ ArrayFromJSON(timestamp(TimeUnit::SECOND, "Asia/Tokyo"),
rhs_json),
+ options, nullptr));
+ EXPECT_RAISES_WITH_MESSAGE_THAT(
+ TypeError,
+ ::testing::HasSubstr(
+ "Cannot compare timestamp with timezone to timestamp without
timezone"),
+ Between(ArrayFromJSON(timestamp(TimeUnit::SECOND), arr_json),
+ ArrayFromJSON(timestamp(TimeUnit::SECOND, "America/New_York"),
lhs_json),
+ ArrayFromJSON(timestamp(TimeUnit::SECOND, "Europe/Berlin"),
rhs_json),
+ options, nullptr));
+ EXPECT_RAISES_WITH_MESSAGE_THAT(
+ TypeError,
+ ::testing::HasSubstr(
+ "Cannot compare timestamp with timezone to timestamp without
timezone"),
+ Between(ArrayFromJSON(timestamp(TimeUnit::SECOND, "Africa/Nairobi"),
arr_json),
+ ArrayFromJSON(timestamp(TimeUnit::SECOND), lhs_json),
+ ArrayFromJSON(timestamp(TimeUnit::SECOND), rhs_json), options,
nullptr));
+ }
+}
+
+template <typename ArrowType>
+class TestBetweenDecimal : public ::testing::Test {};
+TYPED_TEST_SUITE(TestBetweenDecimal, DecimalArrowTypes);
+
+TYPED_TEST(TestBetweenDecimal, 3Arrays) {
+ auto ty = std::make_shared<TypeParam>(3, 2);
Review comment:
So, for the record, the typed tests compiles a separate test for each
type parameter. To minimize compile time expansion, we can instead iterate over
types at runtime, for example like this:
```c++
for (auto decimal_factory : {decimal128, decimal256}) {
ty = decimal_factory(3, 2);
ARROW_SCOPED_TRACE("Type =", ty->ToString());
// etc.
}
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
