This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new bfafa576f6 [GLUTEN-7631][VL] Fall back lead/lag if input is foldable
(#8038)
bfafa576f6 is described below
commit bfafa576f6f7a333301788bffb434b3a5cc5ccf4
Author: Mingliang Zhu <[email protected]>
AuthorDate: Wed Nov 27 11:24:23 2024 +0800
[GLUTEN-7631][VL] Fall back lead/lag if input is foldable (#8038)
---
.../main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala | 4 +++-
.../test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala | 3 ---
.../test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala | 3 ---
.../test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala | 3 ---
.../test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala | 3 ---
5 files changed, 3 insertions(+), 13 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
index e05fd92e32..b1245687b0 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
@@ -400,7 +400,9 @@ object VeloxBackendSettings extends BackendSettingsApi {
}
windowExpression.windowFunction match {
case _: RowNumber | _: Rank | _: CumeDist | _: DenseRank | _:
PercentRank |
- _: NthValue | _: NTile | _: Lag | _: Lead =>
+ _: NthValue | _: NTile =>
+ case l: Lag if !l.input.foldable =>
+ case l: Lead if !l.input.foldable =>
case aggrExpr: AggregateExpression
if
!aggrExpr.aggregateFunction.isInstanceOf[ApproximatePercentile]
&& !aggrExpr.aggregateFunction.isInstanceOf[Percentile] =>
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 363f9c85ed..276a8a0f7a 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -318,9 +318,6 @@ class VeloxTestSettings extends BackendTestSettings {
// spill not supported yet.
enableSuite[GlutenSQLWindowFunctionSuite]
.exclude("test with low buffer spill threshold")
- // https://github.com/apache/incubator-gluten/issues/7631
- .exclude(
- "SPARK-16633: lead/lag should return the default value if the offset row
does not exist")
enableSuite[GlutenSortSuite]
// Sort spill is not supported.
.exclude("sorting does not crash for large inputs")
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 1fc64577d2..08fab69c84 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -886,9 +886,6 @@ class VeloxTestSettings extends BackendTestSettings {
// spill not supported yet.
enableSuite[GlutenSQLWindowFunctionSuite]
.exclude("test with low buffer spill threshold")
- // https://github.com/apache/incubator-gluten/issues/7631
- .exclude(
- "SPARK-16633: lead/lag should return the default value if the offset row
does not exist")
enableSuite[GlutenTakeOrderedAndProjectSuite]
enableSuite[GlutenSessionExtensionSuite]
enableSuite[TestFileSourceScanExecTransformer]
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 8c65afd1a3..df79b663bc 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -886,9 +886,6 @@ class VeloxTestSettings extends BackendTestSettings {
// spill not supported yet.enableSuite[GlutenSQLWindowFunctionSuite]
enableSuite[GlutenSQLWindowFunctionSuite]
.exclude("test with low buffer spill threshold")
- // https://github.com/apache/incubator-gluten/issues/7631
- .exclude(
- "SPARK-16633: lead/lag should return the default value if the offset row
does not exist")
enableSuite[GlutenTakeOrderedAndProjectSuite]
enableSuite[GlutenSessionExtensionSuite]
enableSuite[TestFileSourceScanExecTransformer]
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 18c46a3042..cb2151fe46 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -900,9 +900,6 @@ class VeloxTestSettings extends BackendTestSettings {
// spill not supported yet.
enableSuite[GlutenSQLWindowFunctionSuite]
.exclude("test with low buffer spill threshold")
- // https://github.com/apache/incubator-gluten/issues/7631
- .exclude(
- "SPARK-16633: lead/lag should return the default value if the offset row
does not exist")
enableSuite[GlutenTakeOrderedAndProjectSuite]
enableSuite[GlutenSessionExtensionSuite]
enableSuite[TestFileSourceScanExecTransformer]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
