This is an automated email from the ASF dual-hosted git repository.
liuneng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new be0ada5c7 [GLUTEN-6082][CH]Fix lag diff (#6085)
be0ada5c7 is described below
commit be0ada5c73a1002e350c9de2f0855abca9764373
Author: KevinyhZou <[email protected]>
AuthorDate: Tue Jun 18 13:45:58 2024 +0800
[GLUTEN-6082][CH]Fix lag diff (#6085)
What changes were proposed in this pull request?
(Please fill in changes proposed in this fix)
(Fixes: #6082)
How was this patch tested?
TEST BY UT
---
.../gluten/backendsapi/clickhouse/CHBackend.scala | 21 ++++++++++++++++++---
.../GlutenClickHouseTPCHSaltNullParquetSuite.scala | 8 ++++++++
2 files changed, 26 insertions(+), 3 deletions(-)
diff --git
a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala
b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala
index c79d0aaee..1587b9ea3 100644
---
a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala
+++
b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala
@@ -26,7 +26,7 @@ import
org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat._
import org.apache.spark.SparkEnv
import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.catalog.BucketSpec
-import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeReference,
DenseRank, Lag, Lead, NamedExpression, Rank, RowNumber}
+import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeReference,
DenseRank, Expression, Lag, Lead, Literal, NamedExpression, Rank, RowNumber}
import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning,
Partitioning}
import org.apache.spark.sql.execution.SparkPlan
@@ -225,10 +225,25 @@ object CHBackendSettings extends BackendSettingsApi with
Logging {
func => {
val aliasExpr = func.asInstanceOf[Alias]
val wExpression =
WindowFunctionsBuilder.extractWindowExpression(aliasExpr.child)
+
+ def checkLagOrLead(third: Expression): Unit = {
+ third match {
+ case _: Literal =>
+ allSupported = allSupported
+ case _ =>
+ logInfo("Not support lag/lead function with default value not
literal null")
+ allSupported = false
+ break
+ }
+ }
+
wExpression.windowFunction match {
- case _: RowNumber | _: AggregateExpression | _: Rank | _: Lead |
_: Lag |
- _: DenseRank =>
+ case _: RowNumber | _: AggregateExpression | _: Rank | _:
DenseRank =>
allSupported = allSupported
+ case l: Lag =>
+ checkLagOrLead(l.third)
+ case l: Lead =>
+ checkLagOrLead(l.third)
case _ =>
logDebug(s"Not support window function: ${wExpression.getClass}")
allSupported = false
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala
index 84f390139..1d3bbec84 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala
@@ -944,7 +944,15 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends
GlutenClickHouseTPCHAbstr
|from nation
|order by n_regionkey, n_nationkey, n_lag
|""".stripMargin
+ val sql1 =
+ """
+ | select n_regionkey, n_nationkey,
+ | lag(n_nationkey, 1, n_nationkey) OVER (PARTITION BY n_regionkey
ORDER BY n_nationkey) as n_lag
+ |from nation
+ |order by n_regionkey, n_nationkey, n_lag
+ |""".stripMargin
compareResultsAgainstVanillaSpark(sql, true, { _ => })
+ compareResultsAgainstVanillaSpark(sql1, true, { _ => }, false)
}
test("window lag with null value") {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
