Copilot commented on code in PR #13282:
URL: https://github.com/apache/gluten/pull/13282#discussion_r4583855094
##########
backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala:
##########
@@ -39,17 +40,36 @@ import io.substrait.proto.SimpleExtensionDeclaration
import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer
-class VeloxValidatorApi extends ValidatorApi {
+class VeloxValidatorApi extends ValidatorApi with Logging {
import VeloxValidatorApi._
- /** For velox backend, key validation is on native side. */
- override def doExprValidate(substraitExprName: String, expr: Expression):
Boolean = {
- expr match {
- // Both format_string and printf resolve to Spark's FormatString
expression.
- case formatString: FormatString =>
- supportsFormatString(formatString)
- case _ => true
- }
+ /**
+ * Enforce integral ROUND eligibility on the JVM to preserve Spark's
captured ANSI semantics,
+ * independently of optional native plan validation.
+ */
+ override def doExprValidate(substraitExprName: String, expr: Expression):
Boolean = expr match {
+ // Both format_string and printf resolve to Spark's FormatString
expression.
+ case formatString: FormatString =>
+ supportsFormatString(formatString)
+ case round: Round =>
+ round.child.dataType match {
+ case ByteType | ShortType | IntegerType | LongType =>
+ // Negative scales can overflow, and Spark captures ANSI mode in the
expression.
+ val supported = round.scale match {
+ case Literal(null, IntegerType) => true
+ case Literal(scale: Int, IntegerType) =>
+ scale >= 0 && scale <= MaxQualifiedIntegralRoundScale
+ case _ => false
+ }
Review Comment:
PR description says disabling native validation is a debugging mode that
*bypasses* the compatibility fallback, but this change explicitly enforces
integral ROUND gating on the JVM independently of native validation. Either the
description should be updated to reflect the new behavior (i.e., integral ROUND
fallback still applies even when native validation is disabled), or this JVM
enforcement should be conditioned to match the described semantics.
##########
backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala:
##########
@@ -39,17 +40,36 @@ import io.substrait.proto.SimpleExtensionDeclaration
import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer
-class VeloxValidatorApi extends ValidatorApi {
+class VeloxValidatorApi extends ValidatorApi with Logging {
import VeloxValidatorApi._
- /** For velox backend, key validation is on native side. */
- override def doExprValidate(substraitExprName: String, expr: Expression):
Boolean = {
- expr match {
- // Both format_string and printf resolve to Spark's FormatString
expression.
- case formatString: FormatString =>
- supportsFormatString(formatString)
- case _ => true
- }
+ /**
+ * Enforce integral ROUND eligibility on the JVM to preserve Spark's
captured ANSI semantics,
+ * independently of optional native plan validation.
+ */
+ override def doExprValidate(substraitExprName: String, expr: Expression):
Boolean = expr match {
Review Comment:
PR description says disabling native validation is a debugging mode that
*bypasses* the compatibility fallback, but this change explicitly enforces
integral ROUND gating on the JVM independently of native validation. Either the
description should be updated to reflect the new behavior (i.e., integral ROUND
fallback still applies even when native validation is disabled), or this JVM
enforcement should be conditioned to match the described semantics.
##########
cpp/velox/tests/SparkFunctionTest.cc:
##########
@@ -125,6 +126,114 @@ TEST_F(SparkFunctionTest, roundWithDecimal) {
runRoundWithDecimalTest<int8_t>(testRoundWithDecIntegralData<int8_t>());
}
+TEST_F(SparkFunctionTest, integralRoundUsesSessionAnsi) {
+ const auto input = makeRowVector({makeNullableFlatVector<int8_t>({127, -128,
25, -25, std::nullopt})});
+ for (const auto ansi : {false, true}) {
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(), ansi
? "true" : "false"}});
+ const auto expression = "round(c0, cast(-1 as integer))";
+ if (ansi) {
+ VELOX_ASSERT_USER_THROW(evaluate(expression, input), "Arithmetic
overflow: round(");
Review Comment:
This assertion depends on a specific error-message substring, which tends to
be brittle across Velox/Spark error text changes. If feasible in this test
framework, prefer asserting on a stable error category/code (or the specific
exception type) instead of message text; otherwise, consider narrowing the
match to the minimal stable fragment.
##########
cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc:
##########
@@ -141,17 +141,30 @@ bool
SubstraitToVeloxPlanValidator::flattenDualLevel(const TypePtr& type, std::v
bool SubstraitToVeloxPlanValidator::validateRound(
const ::substrait::Expression::ScalarFunction& scalarFunction,
- const RowTypePtr& inputType) {
+ const std::vector<core::TypedExprPtr>& params) {
const auto& arguments = scalarFunction.arguments();
if (arguments.size() < 2) {
return false;
}
+ if (params.empty() || !params[0]) {
+ LOG_VALIDATION_MSG("Round value expression is unavailable.");
+ return false;
+ }
if (!arguments[1].value().has_literal()) {
- LOG_VALIDATION_MSG("Round scale is expected.");
+ LOG_VALIDATION_MSG("Round scale must be a literal integer for native
execution.");
return false;
}
+ const auto& scale = arguments[1].value().literal();
+ if (scale.has_null() && scale.null().has_i32()) {
+ const auto& valueType = params[0]->type();
+ if (!valueType->isDecimal() &&
+ (valueType->isTinyint() || valueType->isSmallint() ||
valueType->isInteger() || valueType->isBigint())) {
+ return true;
+ }
+ }
+
// Velox has different result with Spark on negative scale.
auto typeCase = arguments[1].value().literal().literal_type_case();
switch (typeCase) {
Review Comment:
validateRound() now allows typed-NULL scales for integral inputs (good), but
it still doesn’t enforce the PR’s documented eligibility bound of scale in [0,
400] for integral inputs. As a result, native validation can accept large
nonnegative scales (e.g., 401/INT_MAX) while the JVM-side validator rejects
them, producing inconsistent behavior depending on which validation layer is in
play. Consider enforcing the same [0, 400] bound here for i32 literal scales
(and ensure the error message/log reflects the bound) so native and JVM
validation stay consistent.
##########
cpp/velox/tests/SparkFunctionTest.cc:
##########
@@ -125,6 +126,114 @@ TEST_F(SparkFunctionTest, roundWithDecimal) {
runRoundWithDecimalTest<int8_t>(testRoundWithDecIntegralData<int8_t>());
}
+TEST_F(SparkFunctionTest, integralRoundUsesSessionAnsi) {
+ const auto input = makeRowVector({makeNullableFlatVector<int8_t>({127, -128,
25, -25, std::nullopt})});
+ for (const auto ansi : {false, true}) {
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(), ansi
? "true" : "false"}});
+ const auto expression = "round(c0, cast(-1 as integer))";
+ if (ansi) {
+ VELOX_ASSERT_USER_THROW(evaluate(expression, input), "Arithmetic
overflow: round(");
+ facebook::velox::test::assertEqualVectors(
+ makeNullableFlatVector<int8_t>({std::nullopt, std::nullopt, 30, -30,
std::nullopt}),
+ evaluate("try(round(c0, cast(-1 as integer)))", input));
+ } else {
+ facebook::velox::test::assertEqualVectors(
+ makeNullableFlatVector<int8_t>({-126, 126, 30, -30, std::nullopt}),
evaluate(expression, input));
+ }
+ }
+}
+
+TEST_F(SparkFunctionTest, integralRoundNullScaleSkipsChild) {
+ queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(),
"true"}});
+ const auto input = makeRowVector({makeNullableFlatVector<int8_t>({127, -128,
25, std::nullopt})});
+ facebook::velox::test::assertEqualVectors(
+ makeNullableFlatVector<int8_t>({std::nullopt, std::nullopt,
std::nullopt, std::nullopt}),
+ evaluate("round(round(c0, cast(-1 as integer)), cast(null as integer))",
input));
+}
+
+TEST_F(SparkFunctionTest, integralRoundScaleValidation) {
+ const auto validate = [&](std::optional<int32_t> scale, bool castInput) {
+ ::substrait::Expression expression;
+ auto* function = expression.mutable_scalar_function();
+ function->set_function_reference(1);
+ auto* value = function->add_arguments()->mutable_value();
+ if (castInput) {
+ auto* cast = value->mutable_cast();
+
cast->mutable_type()->mutable_i8()->set_nullability(::substrait::Type_Nullability_NULLABILITY_NULLABLE);
+ value = cast->mutable_input();
+ }
+
value->mutable_selection()->mutable_direct_reference()->mutable_struct_field()->set_field(0);
+ auto* literal =
function->add_arguments()->mutable_value()->mutable_literal();
+ if (scale.has_value()) {
+ literal->set_i32(scale.value());
+ } else {
+
literal->mutable_null()->mutable_i32()->set_nullability(::substrait::Type_Nullability_NULLABILITY_NULLABLE);
+ }
+
function->mutable_output_type()->mutable_i8()->set_nullability(::substrait::Type_Nullability_NULLABILITY_NULLABLE);
+ gluten::SubstraitToVeloxPlanValidator validator(pool());
+ TypePtr inputType = TINYINT();
+ if (castInput) {
+ inputType = VARCHAR();
+ }
+ return validator.validate(expression, ROW({"value"}, {inputType}), {{1,
"round:opt_i8_i32"}});
+ };
+
+ for (const auto scale : {0, 1, 400, 401,
std::numeric_limits<int32_t>::max()}) {
+ SCOPED_TRACE(scale);
+ EXPECT_TRUE(validate(scale, false));
+ }
+ EXPECT_TRUE(validate(std::nullopt, false));
+ EXPECT_TRUE(validate(std::nullopt, true));
+ for (const auto scale : {std::numeric_limits<int32_t>::min(), -401, -1}) {
Review Comment:
This test asserts that native validation accepts scale 401 and INT_MAX, but
the Scala-side gating and docs in this PR constrain integral ROUND native
eligibility to INTEGER literal scales in [0, 400] (or typed NULL). This
mismatch can mask cross-layer inconsistencies. Align the test expectations with
the intended bound by moving 401/INT_MAX into the rejected set, or (if the
intent is to allow any nonnegative scale natively) update the JVM gating + docs
accordingly.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]