This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 0345fa5d90 [GLUTEN-8108][VL] Correct the logic of null on failure
behavior for try cast (#8107)
0345fa5d90 is described below
commit 0345fa5d90ae07cff75f9668aebaf48bd4391dac
Author: Ankita Victor <[email protected]>
AuthorDate: Thu Dec 19 13:49:49 2024 +0530
[GLUTEN-8108][VL] Correct the logic of null on failure behavior for try
cast (#8107)
---
.../execution/ScalarFunctionsValidateSuite.scala | 38 ++++++++++++++++++++++
.../gluten/substrait/expression/CastNode.java | 8 ++---
.../substrait/expression/ExpressionBuilder.java | 4 +--
.../expression/UnaryExpressionTransformer.scala | 6 +++-
.../gluten/utils/velox/VeloxTestSettings.scala | 10 ++++++
.../catalyst/expressions/GlutenTryCastSuite.scala | 21 ++++++++++++
.../gluten/utils/velox/VeloxTestSettings.scala | 10 ++++++
.../catalyst/expressions/GlutenTryCastSuite.scala | 21 ++++++++++++
8 files changed, 111 insertions(+), 7 deletions(-)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
index 94ea8be520..a0c1f1ec00 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
@@ -1477,4 +1477,42 @@ abstract class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
checkGlutenOperatorMatch[FilterExecTransformer](df)
}
}
+
+ testWithSpecifiedSparkVersion("Test try_cast", Some("3.4")) {
+ withTempView("try_cast_table") {
+ withTempPath {
+ path =>
+ Seq[(String)](("123456"), ("000A1234"))
+ .toDF("str")
+ .write
+ .parquet(path.getCanonicalPath)
+
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("try_cast_table")
+ runQueryAndCompare("select try_cast(str as bigint) from
try_cast_table") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("select try_cast(str as double) from
try_cast_table") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ }
+ }
+ }
+
+ test("Test cast") {
+ withTempView("cast_table") {
+ withTempPath {
+ path =>
+ Seq[(String)](("123456"), ("000A1234"))
+ .toDF("str")
+ .write
+ .parquet(path.getCanonicalPath)
+
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("cast_table")
+ runQueryAndCompare("select cast(str as bigint) from cast_table") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("select cast(str as double) from cast_table") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ }
+ }
+ }
}
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/CastNode.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/CastNode.java
index 1acbcf6aca..3d2def2bd1 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/CastNode.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/CastNode.java
@@ -26,12 +26,12 @@ public class CastNode implements ExpressionNode,
Serializable {
private final TypeNode typeNode;
private final ExpressionNode expressionNode;
- public final boolean ansiEnabled;
+ public final boolean throwOnFailure;
- CastNode(TypeNode typeNode, ExpressionNode expressionNode, boolean
ansiEnabled) {
+ CastNode(TypeNode typeNode, ExpressionNode expressionNode, boolean
throwOnFailure) {
this.typeNode = typeNode;
this.expressionNode = expressionNode;
- this.ansiEnabled = ansiEnabled;
+ this.throwOnFailure = throwOnFailure;
}
@Override
@@ -39,7 +39,7 @@ public class CastNode implements ExpressionNode, Serializable
{
Expression.Cast.Builder castBuilder = Expression.Cast.newBuilder();
castBuilder.setType(typeNode.toProtobuf());
castBuilder.setInput(expressionNode.toProtobuf());
- if (ansiEnabled) {
+ if (throwOnFailure) {
// Throw exception on failure.
castBuilder.setFailureBehaviorValue(2);
} else {
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/ExpressionBuilder.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/ExpressionBuilder.java
index 16ae5412ea..1e6c58f682 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/ExpressionBuilder.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/ExpressionBuilder.java
@@ -263,8 +263,8 @@ public class ExpressionBuilder {
}
public static CastNode makeCast(
- TypeNode typeNode, ExpressionNode expressionNode, boolean ansiEnabled) {
- return new CastNode(typeNode, expressionNode, ansiEnabled);
+ TypeNode typeNode, ExpressionNode expressionNode, boolean
throwOnFailure) {
+ return new CastNode(typeNode, expressionNode, throwOnFailure);
}
public static StringMapNode makeStringMap(Map<String, String> values) {
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala
index bcbac60dec..f9eb1e8eab 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala
@@ -18,6 +18,7 @@ package org.apache.gluten.expression
import org.apache.gluten.backendsapi.BackendsApiManager
import org.apache.gluten.exception.GlutenNotSupportException
+import org.apache.gluten.sql.shims.SparkShimLoader
import org.apache.gluten.substrait.`type`.ListNode
import org.apache.gluten.substrait.`type`.MapNode
import org.apache.gluten.substrait.expression.{ExpressionBuilder,
ExpressionNode, StructLiteralNode}
@@ -43,7 +44,10 @@ case class CastTransformer(substraitExprName: String, child:
ExpressionTransform
extends UnaryExpressionTransformer {
override def doTransform(args: java.lang.Object): ExpressionNode = {
val typeNode = ConverterUtils.getTypeNode(dataType, original.nullable)
- ExpressionBuilder.makeCast(typeNode, child.doTransform(args),
original.ansiEnabled)
+ ExpressionBuilder.makeCast(
+ typeNode,
+ child.doTransform(args),
+ SparkShimLoader.getSparkShims.withAnsiEvalMode(original))
}
}
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index b0446d3ca7..498c2a3b4e 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -91,6 +91,16 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude(
"Process Infinity, -Infinity, NaN in case insensitive manner" // +inf
not supported in folly.
)
+ enableSuite[GlutenTryCastSuite]
+ .exclude(
+ "Process Infinity, -Infinity, NaN in case insensitive manner" // +inf
not supported in folly.
+ )
+ .exclude("ANSI mode: Throw exception on casting out-of-range value to byte
type")
+ .exclude("ANSI mode: Throw exception on casting out-of-range value to
short type")
+ .exclude("ANSI mode: Throw exception on casting out-of-range value to int
type")
+ .exclude("ANSI mode: Throw exception on casting out-of-range value to long
type")
+ .exclude("cast from invalid string to numeric should throw
NumberFormatException")
+ .exclude("SPARK-26218: Fix the corner case of codegen when casting float
to Integer")
enableSuite[GlutenCollectionExpressionsSuite]
// Rewrite in Gluten to replace Seq with Array
.exclude("Shuffle")
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala
new file mode 100644
index 0000000000..b0bddf9354
--- /dev/null
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.catalyst.expressions
+
+import org.apache.spark.sql.GlutenTestsTrait
+
+class GlutenTryCastSuite extends TryCastSuite with GlutenTestsTrait {}
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index a01d0cb4b3..904439e83e 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -92,6 +92,16 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude(
"Process Infinity, -Infinity, NaN in case insensitive manner" // +inf
not supported in folly.
)
+ enableSuite[GlutenTryCastSuite]
+ .exclude(
+ "Process Infinity, -Infinity, NaN in case insensitive manner" // +inf
not supported in folly.
+ )
+ .exclude("ANSI mode: Throw exception on casting out-of-range value to byte
type")
+ .exclude("ANSI mode: Throw exception on casting out-of-range value to
short type")
+ .exclude("ANSI mode: Throw exception on casting out-of-range value to int
type")
+ .exclude("ANSI mode: Throw exception on casting out-of-range value to long
type")
+ .exclude("cast from invalid string to numeric should throw
NumberFormatException")
+ .exclude("SPARK-26218: Fix the corner case of codegen when casting float
to Integer")
enableSuite[GlutenCollectionExpressionsSuite]
// Rewrite in Gluten to replace Seq with Array
.exclude("Shuffle")
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala
new file mode 100644
index 0000000000..b0bddf9354
--- /dev/null
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.catalyst.expressions
+
+import org.apache.spark.sql.GlutenTestsTrait
+
+class GlutenTryCastSuite extends TryCastSuite with GlutenTestsTrait {}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
