philo-he commented on code in PR #12873:
URL: https://github.com/apache/gluten/pull/12873#discussion_r4079918254
##########
cpp/velox/tests/SparkFunctionTest.cc:
##########
@@ -144,3 +151,189 @@ TEST_F(SparkFunctionTest,
expressionLevelLegacyCastIgnoresSessionAnsiOn) {
facebook::velox::test::assertEqualVectors(makeFlatVector<int8_t>({-121}),
evaluate(legacyCast, input));
}
+
+TEST_F(SparkFunctionTest, elt) {
+ // The index picks a different input per row. A NULL in an input that is not
+ // picked does not affect the result, while a NULL in the picked one does.
+ auto index = makeFlatVector<int32_t>({1, 2, 3, 3});
+ auto first = makeNullableFlatVector<StringView>({"a0", std::nullopt, "a2",
"a3"});
+ auto second = makeNullableFlatVector<StringView>({std::nullopt, "b1", "b2",
"b3"});
+ auto third = makeNullableFlatVector<StringView>({"c0", "c1", "c2",
std::nullopt});
+
+ facebook::velox::test::assertEqualVectors(
+ makeNullableFlatVector<StringView>({"a0", "b1", "c2", std::nullopt}),
+ evaluate("elt(c0, c1, c2, c3)", makeRowVector({index, first, second,
third})));
+}
+
+TEST_F(SparkFunctionTest, eltNullIndex) {
+ // A NULL index returns NULL, with ANSI mode on as well.
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"true"}});
+ auto index = makeNullableFlatVector<int32_t>({std::nullopt, 2});
+ auto first = makeFlatVector<StringView>({"a0", "a1"});
+ auto second = makeFlatVector<StringView>({"b0", "b1"});
+
+ facebook::velox::test::assertEqualVectors(
+ makeNullableFlatVector<StringView>({std::nullopt, "b1"}),
+ evaluate("elt(c0, c1, c2)", makeRowVector({index, first, second})));
+}
+
+TEST_F(SparkFunctionTest, eltIndexOutOfRangeAnsiOff) {
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"false"}});
+ auto index = makeFlatVector<int32_t>({0, -1, 3, 2});
+ auto first = makeFlatVector<StringView>({"a0", "a1", "a2", "a3"});
+ auto second = makeFlatVector<StringView>({"b0", "b1", "b2", "b3"});
+
+ facebook::velox::test::assertEqualVectors(
+ makeNullableFlatVector<StringView>({std::nullopt, std::nullopt,
std::nullopt, "b3"}),
+ evaluate("elt(c0, c1, c2)", makeRowVector({index, first, second})));
+}
+
+TEST_F(SparkFunctionTest, eltIndexOutOfRangeAnsiOn) {
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"true"}});
+ auto first = makeFlatVector<StringView>({"a0", "a1"});
+ auto second = makeFlatVector<StringView>({"b0", "b1"});
+
+ auto evaluateWithIndex = [&](const std::vector<int32_t>& indexes) {
+ return evaluate("elt(c0, c1, c2)",
makeRowVector({makeFlatVector<int32_t>(indexes), first, second}));
+ };
+
+ VELOX_ASSERT_THROW(evaluateWithIndex({1, 0}), "The index is out of bounds
for elt. index: 0, number of inputs: 2");
+ VELOX_ASSERT_THROW(evaluateWithIndex({-1, 1}), "The index is out of bounds
for elt. index: -1, number of inputs: 2");
+ VELOX_ASSERT_THROW(evaluateWithIndex({1, 3}), "The index is out of bounds
for elt. index: 3, number of inputs: 2");
+
+ // In-range indexes are unaffected by ANSI mode.
+ facebook::velox::test::assertEqualVectors(makeFlatVector<StringView>({"a0",
"b1"}), evaluateWithIndex({1, 2}));
+}
+
+TEST_F(SparkFunctionTest, eltVarbinary) {
+ auto index = makeFlatVector<int32_t>({2, 1});
+ auto first = makeNullableFlatVector<StringView>({"a0", "a1"}, VARBINARY());
+ auto second = makeNullableFlatVector<StringView>({"b0", std::nullopt},
VARBINARY());
+
+ auto result = evaluate("elt(c0, c1, c2)", makeRowVector({index, first,
second}));
+ ASSERT_EQ(result->type()->kind(), TypeKind::VARBINARY);
+
facebook::velox::test::assertEqualVectors(makeNullableFlatVector<StringView>({"b0",
"a1"}, VARBINARY()), result);
+}
+
+TEST_F(SparkFunctionTest, eltConstantIndexOverDictionaryInput) {
+ // A constant index selects the same input for all rows, and the selected
+ // input may carry an encoding.
+ auto index = makeConstant<int32_t>(2, 3);
+ auto first = makeFlatVector<StringView>({"a0", "a1", "a2"});
+ auto second = makeFlatVector<StringView>({"b0", "b1", "b2"});
+ auto dictionary = BaseVector::wrapInDictionary(nullptr, makeIndices({2, 0,
1}), 3, second);
+
+ facebook::velox::test::assertEqualVectors(
+ makeFlatVector<StringView>({"b2", "b0", "b1"}),
+ evaluate("elt(c0, c1, c2)", makeRowVector({index, first, dictionary})));
+}
+
+TEST_F(SparkFunctionTest, convOverflowAnsiOff) {
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"false"}});
+
+ // Digits that do not fit in an unsigned 64-bit integer saturate, which is
+ // what Spark does with ANSI mode off.
+ EXPECT_EQ(conv("9223372036854775807", 36, 16), "FFFFFFFFFFFFFFFF");
+ EXPECT_EQ(conv("10000000000000000", 16, 10), "18446744073709551615");
+ EXPECT_EQ(conv("-10000000000000000", 16, -10), "-1");
+}
+
+TEST_F(SparkFunctionTest, convOverflowAnsiOn) {
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"true"}});
+
+ VELOX_ASSERT_THROW(conv("9223372036854775807", 36, 16), "Overflow in
function conv()");
+ VELOX_ASSERT_THROW(conv("10000000000000000", 16, 10), "Overflow in function
conv()");
+ VELOX_ASSERT_THROW(conv("-10000000000000000", 16, -10), "Overflow in
function conv()");
+}
+
+TEST_F(SparkFunctionTest, convNoOverflowAnsiOn) {
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"true"}});
+
+ EXPECT_EQ(conv("4", 10, 2), "100");
+ EXPECT_EQ(conv("big", 36, 16), "3A48");
+ // 2^64 - 1 is the largest input that does not overflow.
+ EXPECT_EQ(conv("18446744073709551615", 10, 10), "18446744073709551615");
+ EXPECT_EQ(conv("FFFFFFFFFFFFFFFF", 16, -10), "-1");
+ // The sign is applied after the digits are accumulated, so wrapping around
+ // through a negative input is not an overflow.
+ EXPECT_EQ(conv("-1", 10, 16), "FFFFFFFFFFFFFFFF");
+ EXPECT_EQ(conv("-15", 10, 16), "FFFFFFFFFFFFFFF1");
+ // The digits stop at the first character that is invalid for the base, so a
+ // long tail of invalid characters is not an overflow either.
+ EXPECT_EQ(conv("11abcabcabcabcabcabcabcabc", 10, 16), "B");
+}
+
+TEST_F(SparkFunctionTest, convInvalidInputAnsiOn) {
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"true"}});
+
+ // Invalid input gives NULL, with ANSI mode on as well.
+ EXPECT_EQ(conv("15", 1, 10), std::nullopt);
+ EXPECT_EQ(conv("15", 37, 10), std::nullopt);
+ EXPECT_EQ(conv("15", 10, 1), std::nullopt);
+ EXPECT_EQ(conv("15", 10, -37), std::nullopt);
+ EXPECT_EQ(conv("", 10, 16), std::nullopt);
+ EXPECT_EQ(conv(" ", 10, 16), std::nullopt);
+ EXPECT_EQ(conv(std::nullopt, 10, 16), std::nullopt);
+}
+
+TEST_F(SparkFunctionTest, elementAtArrayOutOfBoundAnsiOff) {
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"false"}});
+ auto array = makeArrayVector<int32_t>({{1, 2, 3}, {1, 2, 3}, {1, 2, 3}, {1,
2, 3}, {1, 2, 3}});
+ auto index = makeFlatVector<int32_t>({1, 3, -1, 4, -4});
+
+ // An index past either end of the array gives NULL with ANSI mode off.
+ facebook::velox::test::assertEqualVectors(
+ makeNullableFlatVector<int32_t>({1, 3, 3, std::nullopt, std::nullopt}),
+ evaluate("element_at(c0, c1)", makeRowVector({array, index})));
+}
+
+TEST_F(SparkFunctionTest, elementAtArrayOutOfBoundAnsiOn) {
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"true"}});
+ auto array = makeArrayVector<int32_t>({{1, 2, 3}, {1, 2, 3}});
+
+ // In-bound indices, including the negative ones counting from the end, are
+ // unaffected by ANSI mode.
+ facebook::velox::test::assertEqualVectors(
+ makeFlatVector<int32_t>({1, 3}),
+ evaluate("element_at(c0, c1)", makeRowVector({array,
makeFlatVector<int32_t>({1, -1})})));
+
+ VELOX_ASSERT_THROW(
+ evaluate("element_at(c0, c1)", makeRowVector({array,
makeFlatVector<int32_t>({1, 4})})),
+ "Array subscript index out of bounds");
+ VELOX_ASSERT_THROW(
+ evaluate("element_at(c0, c1)", makeRowVector({array,
makeFlatVector<int32_t>({-4, 1})})),
+ "Array subscript index out of bounds");
+}
+
+TEST_F(SparkFunctionTest, elementAtArrayZeroIndex) {
+ auto data = makeRowVector({makeArrayVector<int32_t>({{1, 2, 3}}),
makeFlatVector<int32_t>({0})});
+
+ // An index of 0 is an error whatever the ANSI mode is.
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"false"}});
+ VELOX_ASSERT_THROW(evaluate("element_at(c0, c1)", data), "SQL array indices
start at 1");
+
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"true"}});
+ VELOX_ASSERT_THROW(evaluate("element_at(c0, c1)", data), "SQL array indices
start at 1");
+}
+
+TEST_F(SparkFunctionTest, elementAtMapMissingKeyAnsiOn) {
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"true"}});
+ auto map = makeMapVector<int32_t, int32_t>({{{1, 10}, {2, 20}}, {{1, 10},
{2, 20}}});
+ auto key = makeFlatVector<int32_t>({2, 3});
+
+ // A key the map does not contain gives NULL, with ANSI mode on as well.
+ facebook::velox::test::assertEqualVectors(
+ makeNullableFlatVector<int32_t>({20, std::nullopt}),
evaluate("element_at(c0, c1)", makeRowVector({map, key})));
+}
+
+TEST_F(SparkFunctionTest, sizeOfNull) {
+ // Gluten passes Spark's Size.legacySizeOfNull as the second argument, and
+ // Spark derives it from 'spark.sql.legacy.sizeOfNull' AND NOT ANSI mode, so
+ // both behaviors have to work regardless of the session's ANSI mode.
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"true"}});
Review Comment:
Given that the 2nd arg determines the `legacySizeOfNull` for this Velox
function, it seems we don't need to set this ansi config.
##########
cpp/velox/tests/SparkFunctionTest.cc:
##########
@@ -144,3 +151,189 @@ TEST_F(SparkFunctionTest,
expressionLevelLegacyCastIgnoresSessionAnsiOn) {
facebook::velox::test::assertEqualVectors(makeFlatVector<int8_t>({-121}),
evaluate(legacyCast, input));
}
+
+TEST_F(SparkFunctionTest, elt) {
+ // The index picks a different input per row. A NULL in an input that is not
+ // picked does not affect the result, while a NULL in the picked one does.
+ auto index = makeFlatVector<int32_t>({1, 2, 3, 3});
+ auto first = makeNullableFlatVector<StringView>({"a0", std::nullopt, "a2",
"a3"});
+ auto second = makeNullableFlatVector<StringView>({std::nullopt, "b1", "b2",
"b3"});
+ auto third = makeNullableFlatVector<StringView>({"c0", "c1", "c2",
std::nullopt});
+
+ facebook::velox::test::assertEqualVectors(
+ makeNullableFlatVector<StringView>({"a0", "b1", "c2", std::nullopt}),
+ evaluate("elt(c0, c1, c2, c3)", makeRowVector({index, first, second,
third})));
+}
+
+TEST_F(SparkFunctionTest, eltNullIndex) {
+ // A NULL index returns NULL, with ANSI mode on as well.
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"true"}});
+ auto index = makeNullableFlatVector<int32_t>({std::nullopt, 2});
+ auto first = makeFlatVector<StringView>({"a0", "a1"});
+ auto second = makeFlatVector<StringView>({"b0", "b1"});
+
+ facebook::velox::test::assertEqualVectors(
+ makeNullableFlatVector<StringView>({std::nullopt, "b1"}),
+ evaluate("elt(c0, c1, c2)", makeRowVector({index, first, second})));
+}
+
+TEST_F(SparkFunctionTest, eltIndexOutOfRangeAnsiOff) {
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"false"}});
+ auto index = makeFlatVector<int32_t>({0, -1, 3, 2});
+ auto first = makeFlatVector<StringView>({"a0", "a1", "a2", "a3"});
+ auto second = makeFlatVector<StringView>({"b0", "b1", "b2", "b3"});
+
+ facebook::velox::test::assertEqualVectors(
+ makeNullableFlatVector<StringView>({std::nullopt, std::nullopt,
std::nullopt, "b3"}),
+ evaluate("elt(c0, c1, c2)", makeRowVector({index, first, second})));
+}
+
+TEST_F(SparkFunctionTest, eltIndexOutOfRangeAnsiOn) {
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"true"}});
+ auto first = makeFlatVector<StringView>({"a0", "a1"});
+ auto second = makeFlatVector<StringView>({"b0", "b1"});
+
+ auto evaluateWithIndex = [&](const std::vector<int32_t>& indexes) {
+ return evaluate("elt(c0, c1, c2)",
makeRowVector({makeFlatVector<int32_t>(indexes), first, second}));
+ };
+
+ VELOX_ASSERT_THROW(evaluateWithIndex({1, 0}), "The index is out of bounds
for elt. index: 0, number of inputs: 2");
+ VELOX_ASSERT_THROW(evaluateWithIndex({-1, 1}), "The index is out of bounds
for elt. index: -1, number of inputs: 2");
+ VELOX_ASSERT_THROW(evaluateWithIndex({1, 3}), "The index is out of bounds
for elt. index: 3, number of inputs: 2");
+
+ // In-range indexes are unaffected by ANSI mode.
+ facebook::velox::test::assertEqualVectors(makeFlatVector<StringView>({"a0",
"b1"}), evaluateWithIndex({1, 2}));
+}
+
+TEST_F(SparkFunctionTest, eltVarbinary) {
+ auto index = makeFlatVector<int32_t>({2, 1});
+ auto first = makeNullableFlatVector<StringView>({"a0", "a1"}, VARBINARY());
+ auto second = makeNullableFlatVector<StringView>({"b0", std::nullopt},
VARBINARY());
+
+ auto result = evaluate("elt(c0, c1, c2)", makeRowVector({index, first,
second}));
+ ASSERT_EQ(result->type()->kind(), TypeKind::VARBINARY);
+
facebook::velox::test::assertEqualVectors(makeNullableFlatVector<StringView>({"b0",
"a1"}, VARBINARY()), result);
+}
+
+TEST_F(SparkFunctionTest, eltConstantIndexOverDictionaryInput) {
+ // A constant index selects the same input for all rows, and the selected
+ // input may carry an encoding.
+ auto index = makeConstant<int32_t>(2, 3);
+ auto first = makeFlatVector<StringView>({"a0", "a1", "a2"});
+ auto second = makeFlatVector<StringView>({"b0", "b1", "b2"});
+ auto dictionary = BaseVector::wrapInDictionary(nullptr, makeIndices({2, 0,
1}), 3, second);
+
+ facebook::velox::test::assertEqualVectors(
+ makeFlatVector<StringView>({"b2", "b0", "b1"}),
+ evaluate("elt(c0, c1, c2)", makeRowVector({index, first, dictionary})));
+}
+
+TEST_F(SparkFunctionTest, convOverflowAnsiOff) {
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"false"}});
+
+ // Digits that do not fit in an unsigned 64-bit integer saturate, which is
+ // what Spark does with ANSI mode off.
+ EXPECT_EQ(conv("9223372036854775807", 36, 16), "FFFFFFFFFFFFFFFF");
+ EXPECT_EQ(conv("10000000000000000", 16, 10), "18446744073709551615");
+ EXPECT_EQ(conv("-10000000000000000", 16, -10), "-1");
+}
+
+TEST_F(SparkFunctionTest, convOverflowAnsiOn) {
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"true"}});
+
+ VELOX_ASSERT_THROW(conv("9223372036854775807", 36, 16), "Overflow in
function conv()");
+ VELOX_ASSERT_THROW(conv("10000000000000000", 16, 10), "Overflow in function
conv()");
+ VELOX_ASSERT_THROW(conv("-10000000000000000", 16, -10), "Overflow in
function conv()");
+}
+
+TEST_F(SparkFunctionTest, convNoOverflowAnsiOn) {
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"true"}});
+
+ EXPECT_EQ(conv("4", 10, 2), "100");
+ EXPECT_EQ(conv("big", 36, 16), "3A48");
+ // 2^64 - 1 is the largest input that does not overflow.
+ EXPECT_EQ(conv("18446744073709551615", 10, 10), "18446744073709551615");
+ EXPECT_EQ(conv("FFFFFFFFFFFFFFFF", 16, -10), "-1");
+ // The sign is applied after the digits are accumulated, so wrapping around
+ // through a negative input is not an overflow.
+ EXPECT_EQ(conv("-1", 10, 16), "FFFFFFFFFFFFFFFF");
+ EXPECT_EQ(conv("-15", 10, 16), "FFFFFFFFFFFFFFF1");
+ // The digits stop at the first character that is invalid for the base, so a
+ // long tail of invalid characters is not an overflow either.
+ EXPECT_EQ(conv("11abcabcabcabcabcabcabcabc", 10, 16), "B");
+}
+
+TEST_F(SparkFunctionTest, convInvalidInputAnsiOn) {
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"true"}});
+
+ // Invalid input gives NULL, with ANSI mode on as well.
+ EXPECT_EQ(conv("15", 1, 10), std::nullopt);
+ EXPECT_EQ(conv("15", 37, 10), std::nullopt);
+ EXPECT_EQ(conv("15", 10, 1), std::nullopt);
+ EXPECT_EQ(conv("15", 10, -37), std::nullopt);
+ EXPECT_EQ(conv("", 10, 16), std::nullopt);
+ EXPECT_EQ(conv(" ", 10, 16), std::nullopt);
+ EXPECT_EQ(conv(std::nullopt, 10, 16), std::nullopt);
+}
+
+TEST_F(SparkFunctionTest, elementAtArrayOutOfBoundAnsiOff) {
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"false"}});
+ auto array = makeArrayVector<int32_t>({{1, 2, 3}, {1, 2, 3}, {1, 2, 3}, {1,
2, 3}, {1, 2, 3}});
+ auto index = makeFlatVector<int32_t>({1, 3, -1, 4, -4});
+
+ // An index past either end of the array gives NULL with ANSI mode off.
+ facebook::velox::test::assertEqualVectors(
+ makeNullableFlatVector<int32_t>({1, 3, 3, std::nullopt, std::nullopt}),
+ evaluate("element_at(c0, c1)", makeRowVector({array, index})));
+}
+
+TEST_F(SparkFunctionTest, elementAtArrayOutOfBoundAnsiOn) {
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"true"}});
+ auto array = makeArrayVector<int32_t>({{1, 2, 3}, {1, 2, 3}});
+
+ // In-bound indices, including the negative ones counting from the end, are
+ // unaffected by ANSI mode.
+ facebook::velox::test::assertEqualVectors(
+ makeFlatVector<int32_t>({1, 3}),
+ evaluate("element_at(c0, c1)", makeRowVector({array,
makeFlatVector<int32_t>({1, -1})})));
+
+ VELOX_ASSERT_THROW(
+ evaluate("element_at(c0, c1)", makeRowVector({array,
makeFlatVector<int32_t>({1, 4})})),
+ "Array subscript index out of bounds");
+ VELOX_ASSERT_THROW(
+ evaluate("element_at(c0, c1)", makeRowVector({array,
makeFlatVector<int32_t>({-4, 1})})),
+ "Array subscript index out of bounds");
+}
+
+TEST_F(SparkFunctionTest, elementAtArrayZeroIndex) {
+ auto data = makeRowVector({makeArrayVector<int32_t>({{1, 2, 3}}),
makeFlatVector<int32_t>({0})});
+
+ // An index of 0 is an error whatever the ANSI mode is.
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"false"}});
+ VELOX_ASSERT_THROW(evaluate("element_at(c0, c1)", data), "SQL array indices
start at 1");
+
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"true"}});
+ VELOX_ASSERT_THROW(evaluate("element_at(c0, c1)", data), "SQL array indices
start at 1");
+}
+
+TEST_F(SparkFunctionTest, elementAtMapMissingKeyAnsiOn) {
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"true"}});
+ auto map = makeMapVector<int32_t, int32_t>({{{1, 10}, {2, 20}}, {{1, 10},
{2, 20}}});
+ auto key = makeFlatVector<int32_t>({2, 3});
+
+ // A key the map does not contain gives NULL, with ANSI mode on as well.
+ facebook::velox::test::assertEqualVectors(
+ makeNullableFlatVector<int32_t>({20, std::nullopt}),
evaluate("element_at(c0, c1)", makeRowVector({map, key})));
+}
+
+TEST_F(SparkFunctionTest, sizeOfNull) {
+ // Gluten passes Spark's Size.legacySizeOfNull as the second argument, and
+ // Spark derives it from 'spark.sql.legacy.sizeOfNull' AND NOT ANSI mode, so
+ // both behaviors have to work regardless of the session's ANSI mode.
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"true"}});
Review Comment:
This is testing Velox's own function? If so, do we still need this c++ test?
##########
backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala:
##########
@@ -1251,6 +1251,68 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi
with Logging {
GenericExpressionTransformer(substraitExprName, child, expr)
}
+ override def genEltTransformer(
+ substraitExprName: String,
+ children: Seq[ExpressionTransformer],
+ expr: Elt): ExpressionTransformer = {
+ // Velox's elt derives whether an out-of-range index raises an error from
the session's
+ // 'spark.sql.ansi.enabled', while Spark captures it in Elt.failOnError at
analysis time.
+ // The two normally agree; fall back when they don't, so the ANSI behavior
never diverges.
+ if (expr.failOnError != SQLConf.get.ansiEnabled) {
+ GlutenExceptionUtil
+ .throwsNotFullySupported(
+ ExpressionNames.ELT,
+ EltRestrictions.NOT_SUPPORT_FAIL_ON_ERROR_MISMATCH
+ )
+ }
+ GenericExpressionTransformer(substraitExprName, children, expr)
+ }
+
+ override def genConvTransformer(
+ substraitExprName: String,
+ children: Seq[ExpressionTransformer],
+ expr: Conv): ExpressionTransformer = {
+ // Velox derives whether an overflow raises an error from the session's
+ // 'spark.sql.ansi.enabled', while Spark captures it in Conv.ansiEnabled
at analysis time.
+ // The two normally agree; fall back when they don't, so the ANSI behavior
never diverges.
+ if (expr.ansiEnabled != SQLConf.get.ansiEnabled) {
+ GlutenExceptionUtil
+ .throwsNotFullySupported(
+ ExpressionNames.CONV,
+ ConvRestrictions.NOT_SUPPORT_ANSI_ENABLED_MISMATCH
+ )
+ }
+ GenericExpressionTransformer(substraitExprName, children, expr)
+ }
+
+ override def genElementAtTransformer(
+ substraitExprName: String,
+ children: Seq[ExpressionTransformer],
+ expr: ElementAt): ExpressionTransformer = {
+ // Only the array input reads failOnError: Spark returns NULL for a key a
map does not
+ // contain whatever the ANSI mode is, and so does Velox.
+ if (expr.left.dataType.isInstanceOf[ArrayType]) {
+ if (expr.defaultValueOutOfBound.isDefined) {
+ GlutenExceptionUtil
+ .throwsNotFullySupported(
+ ExpressionNames.ELEMENT_AT,
+ ElementAtRestrictions.NOT_SUPPORT_DEFAULT_VALUE_OUT_OF_BOUND
+ )
+ }
+ // Velox derives whether an out-of-bound index raises an error from the
session's
+ // 'spark.sql.ansi.enabled', while Spark captures it in
ElementAt.failOnError at
+ // analysis time. The two normally agree; fall back when they don't.
+ if (expr.failOnError != SQLConf.get.ansiEnabled) {
+ GlutenExceptionUtil
+ .throwsNotFullySupported(
+ ExpressionNames.ELEMENT_AT,
+ ElementAtRestrictions.NOT_SUPPORT_FAIL_ON_ERROR_MISMATCH
Review Comment:
"Spark captures it in ElementAt.failOnError at analysis time", if Spark
already guarantees respecting ANSI when setting expression failOnError, do we
really need this check? Ditto for the other functions.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
