This is an automated email from the ASF dual-hosted git repository.
richox pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/auron.git
The following commit(s) were added to refs/heads/master by this push:
new 6015ff9b [AURON #1812][Correctness Testing] Spark 3.3 - Add Aggregate
operator related tests (#1819)
6015ff9b is described below
commit 6015ff9b4cadc992aceb9c324b35a5b584278cac
Author: Shreyesh <[email protected]>
AuthorDate: Tue Jan 13 01:11:43 2026 -0800
[AURON #1812][Correctness Testing] Spark 3.3 - Add Aggregate operator
related tests (#1819)
# Which issue does this PR close?
Closes #1812
# Rationale for this change
This change is part of the effort for enhancing Auron's correctness
testing (https://github.com/apache/auron/issues/1745).
# What changes are included in this PR?
This change adds supports for aggregator operator related tests.
# Are there any user-facing changes?
N/A
# How was this patch tested?
Existing unit tests
---------
Co-authored-by: sarangat_LinkedIn <[email protected]>
Co-authored-by: Copilot <[email protected]>
---
.../auron/utils/AuronSparkTestSettings.scala | 14 ++++
.../spark/sql/AuronDataFrameAggregateSuite.scala | 78 ++++++++++++++++++++++
.../sql/AuronDatasetAggregatorSuite.scala} | 24 +------
.../sql/AuronTypedImperativeAggregateSuite.scala} | 26 ++------
4 files changed, 98 insertions(+), 44 deletions(-)
diff --git
a/auron-spark-tests/spark33/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala
b/auron-spark-tests/spark33/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala
index 2bccdc08..4884eda9 100644
---
a/auron-spark-tests/spark33/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala
+++
b/auron-spark-tests/spark33/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala
@@ -28,6 +28,20 @@ class AuronSparkTestSettings extends SparkTestSettings {
// See https://github.com/apache/auron/issues/1724
.exclude("string / binary substring function")
+ enableSuite[AuronDataFrameAggregateSuite]
+ // See https://github.com/apache/auron/issues/1840
+ .excludeByPrefix("collect functions")
+ // A custom version of the SPARK-19471 test has been added to
AuronDataFrameAggregateSuite
+ // with modified plan checks for Auron's native aggregates, so we exclude
the original here.
+ .exclude(
+ "SPARK-19471: AggregationIterator does not initialize the generated
result projection before using it")
+ .exclude(
+ "SPARK-24788: RelationalGroupedDataset.toString with unresolved exprs
should not fail")
+
+ enableSuite[AuronDatasetAggregatorSuite]
+
+ enableSuite[AuronTypedImperativeAggregateSuite]
+
// Will be implemented in the future.
override def getSQLQueryTestSettings = new SQLQueryTestSettings {
override def getResourceFilePath: String = ???
diff --git
a/auron-spark-tests/spark33/src/test/scala/org/apache/spark/sql/AuronDataFrameAggregateSuite.scala
b/auron-spark-tests/spark33/src/test/scala/org/apache/spark/sql/AuronDataFrameAggregateSuite.scala
new file mode 100644
index 00000000..d1361ab7
--- /dev/null
+++
b/auron-spark-tests/spark33/src/test/scala/org/apache/spark/sql/AuronDataFrameAggregateSuite.scala
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql
+
+import scala.util.Random
+
+import org.apache.spark.sql.execution.WholeStageCodegenExec
+import org.apache.spark.sql.execution.aggregate.HashAggregateExec
+import org.apache.spark.sql.execution.auron.plan.NativeAggBase
+import org.apache.spark.sql.functions.{collect_list,
monotonically_increasing_id, rand, randn, spark_partition_id, sum}
+import org.apache.spark.sql.internal.SQLConf
+
+class AuronDataFrameAggregateSuite extends DataFrameAggregateSuite with
SparkQueryTestsBase {
+ import testImplicits._
+
+ // Ported from spark DataFrameAggregateSuite only with plan check changed.
+ private def assertNoExceptions(c: Column): Unit = {
+ for ((wholeStage, useObjectHashAgg) <-
+ Seq((true, true), (true, false), (false, true), (false, false))) {
+ withSQLConf(
+ (SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, wholeStage.toString),
+ (SQLConf.USE_OBJECT_HASH_AGG.key, useObjectHashAgg.toString)) {
+
+ val df = Seq(("1", 1), ("1", 2), ("2", 3), ("2", 4)).toDF("x", "y")
+
+ val hashAggDF = df.groupBy("x").agg(c, sum("y"))
+ hashAggDF.collect()
+ val hashAggPlan = hashAggDF.queryExecution.executedPlan
+ if (wholeStage) {
+ assert(find(hashAggPlan) {
+ case WholeStageCodegenExec(_: HashAggregateExec) => true
+ // If offloaded, Spark whole stage codegen takes no effect and a
native hash agg is
+ // expected to be used.
+ case _: NativeAggBase => true
+ case _ => false
+ }.isDefined)
+ } else {
+ assert(
+ stripAQEPlan(hashAggPlan).isInstanceOf[HashAggregateExec] ||
+ stripAQEPlan(hashAggPlan).find {
+ case _: NativeAggBase => true
+ case _ => false
+ }.isDefined)
+ }
+
+ val objHashAggOrSortAggDF = df.groupBy("x").agg(c, collect_list("y"))
+ objHashAggOrSortAggDF.collect()
+
assert(stripAQEPlan(objHashAggOrSortAggDF.queryExecution.executedPlan).find {
+ case _: NativeAggBase => true
+ case _ => false
+ }.isDefined)
+ }
+ }
+ }
+
+ testAuron(
+ "SPARK-19471: AggregationIterator does not initialize the generated result
projection before using it") {
+ Seq(
+ monotonically_increasing_id(),
+ spark_partition_id(),
+ rand(Random.nextLong()),
+ randn(Random.nextLong())).foreach(assertNoExceptions)
+ }
+}
diff --git
a/auron-spark-tests/spark33/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala
b/auron-spark-tests/spark33/src/test/scala/org/apache/spark/sql/AuronDatasetAggregatorSuite.scala
similarity index 54%
copy from
auron-spark-tests/spark33/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala
copy to
auron-spark-tests/spark33/src/test/scala/org/apache/spark/sql/AuronDatasetAggregatorSuite.scala
index 2bccdc08..b446ab7d 100644
---
a/auron-spark-tests/spark33/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala
+++
b/auron-spark-tests/spark33/src/test/scala/org/apache/spark/sql/AuronDatasetAggregatorSuite.scala
@@ -14,26 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.auron.utils
+package org.apache.spark.sql
-import org.apache.spark.sql._
-
-class AuronSparkTestSettings extends SparkTestSettings {
- {
- // Use Arrow's unsafe implementation.
- System.setProperty("arrow.allocation.manager.type", "Unsafe")
- }
-
- enableSuite[AuronStringFunctionsSuite]
- // See https://github.com/apache/auron/issues/1724
- .exclude("string / binary substring function")
-
- // Will be implemented in the future.
- override def getSQLQueryTestSettings = new SQLQueryTestSettings {
- override def getResourceFilePath: String = ???
-
- override def getSupportedSQLQueryTests: Set[String] = ???
-
- override def getOverwriteSQLQueryTests: Set[String] = ???
- }
-}
+class AuronDatasetAggregatorSuite extends DatasetAggregatorSuite with
SparkQueryTestsBase
diff --git
a/auron-spark-tests/spark33/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala
b/auron-spark-tests/spark33/src/test/scala/org/apache/spark/sql/AuronTypedImperativeAggregateSuite.scala
similarity index 54%
copy from
auron-spark-tests/spark33/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala
copy to
auron-spark-tests/spark33/src/test/scala/org/apache/spark/sql/AuronTypedImperativeAggregateSuite.scala
index 2bccdc08..a6e3af24 100644
---
a/auron-spark-tests/spark33/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala
+++
b/auron-spark-tests/spark33/src/test/scala/org/apache/spark/sql/AuronTypedImperativeAggregateSuite.scala
@@ -14,26 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.auron.utils
+package org.apache.spark.sql
-import org.apache.spark.sql._
-
-class AuronSparkTestSettings extends SparkTestSettings {
- {
- // Use Arrow's unsafe implementation.
- System.setProperty("arrow.allocation.manager.type", "Unsafe")
- }
-
- enableSuite[AuronStringFunctionsSuite]
- // See https://github.com/apache/auron/issues/1724
- .exclude("string / binary substring function")
-
- // Will be implemented in the future.
- override def getSQLQueryTestSettings = new SQLQueryTestSettings {
- override def getResourceFilePath: String = ???
-
- override def getSupportedSQLQueryTests: Set[String] = ???
-
- override def getOverwriteSQLQueryTests: Set[String] = ???
- }
-}
+class AuronTypedImperativeAggregateSuite
+ extends TypedImperativeAggregateSuite
+ with SparkQueryTestsBase
