This is an automated email from the ASF dual-hosted git repository.
zhangzc pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new b83f51d84 [GLUTEN-5391][CH] Fix equalTo NaN issue (#5391) (#5392)
b83f51d84 is described below
commit b83f51d840b5104005c6beaa3a774fa6bd23bb8b
Author: loudongfeng <[email protected]>
AuthorDate: Tue Apr 16 08:43:22 2024 +0800
[GLUTEN-5391][CH] Fix equalTo NaN issue (#5391) (#5392)
[GLUTEN-5391][CH] Fix equalTo NaN issue
---
.../clickhouse/CHSparkPlanExecApi.scala | 5 ++--
.../apache/spark/sql/catalyst/EqualToRewrite.scala | 31 ++++++++++++++++++++++
.../execution/GlutenFunctionValidateSuite.scala | 12 +++++++++
3 files changed, 46 insertions(+), 2 deletions(-)
diff --git
a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala
b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala
index 791a635b0..a2f7ae984 100644
---
a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala
+++
b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala
@@ -35,7 +35,7 @@ import org.apache.spark.serializer.Serializer
import org.apache.spark.shuffle.{GenShuffleWriterParameters,
GlutenShuffleWriterWrapper, HashPartitioningWrapper}
import org.apache.spark.shuffle.utils.CHShuffleUtil
import org.apache.spark.sql.{SparkSession, Strategy}
-import org.apache.spark.sql.catalyst.CHAggregateFunctionRewriteRule
+import org.apache.spark.sql.catalyst.{CHAggregateFunctionRewriteRule,
EqualToRewrite}
import org.apache.spark.sql.catalyst.catalog.BucketSpec
import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
import org.apache.spark.sql.catalyst.expressions._
@@ -585,7 +585,8 @@ class CHSparkPlanExecApi extends SparkPlanExecApi {
List(
spark => new CommonSubexpressionEliminateRule(spark,
spark.sessionState.conf),
spark => CHAggregateFunctionRewriteRule(spark),
- _ => CountDistinctWithoutExpand
+ _ => CountDistinctWithoutExpand,
+ _ => EqualToRewrite
)
}
diff --git
a/backends-clickhouse/src/main/scala/org/apache/spark/sql/catalyst/EqualToRewrite.scala
b/backends-clickhouse/src/main/scala/org/apache/spark/sql/catalyst/EqualToRewrite.scala
new file mode 100644
index 000000000..241b3f655
--- /dev/null
+++
b/backends-clickhouse/src/main/scala/org/apache/spark/sql/catalyst/EqualToRewrite.scala
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.catalyst
+
+import org.apache.spark.sql.catalyst.expressions.{DoubleLiteral, EqualTo,
FloatLiteral, IsNaN}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.rules.Rule
+
+/** The result of 'equal to NaN' and isNaN is different in CH. */
+object EqualToRewrite extends Rule[LogicalPlan] {
+ def apply(plan: LogicalPlan): LogicalPlan = plan.transformExpressions {
+ case EqualTo(left, FloatLiteral(f)) if f.isNaN => IsNaN(left)
+ case EqualTo(left, DoubleLiteral(d)) if d.isNaN => IsNaN(left)
+ case EqualTo(FloatLiteral(f), right) if f.isNaN => IsNaN(right)
+ case EqualTo(DoubleLiteral(d), right) if d.isNaN => IsNaN(right)
+ }
+}
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala
index 204065e39..8f7dcd645 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala
@@ -655,4 +655,16 @@ class GlutenFunctionValidateSuite extends
GlutenClickHouseWholeStageTransformerS
}
}
+ test("equalTo rewrite to isNaN") {
+ withTable("tb_scrt") {
+ sql("create table tb_scrt(id int) using parquet")
+ sql("""
+ |insert into tb_scrt values (-2147483648),(-2147483648)
+ |""".stripMargin)
+ val q = "select sqrt(id),sqrt(id)='NaN' from tb_scrt"
+ runQueryAndCompare(q)(checkGlutenOperatorMatch[ProjectExecTransformer])
+ }
+
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
