This is an automated email from the ASF dual-hosted git repository.
csy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/auron.git
The following commit(s) were added to refs/heads/master by this push:
new 4964053c [AURON #1870][BUILD] Add `-Xfatal-warnings` to
scala-maven-plugin and fix related compilation failures (#1872)
4964053c is described below
commit 4964053c12888848956dcf098d2cca09ad827c84
Author: yew1eb <[email protected]>
AuthorDate: Wed Jan 28 19:40:57 2026 +0800
[AURON #1870][BUILD] Add `-Xfatal-warnings` to scala-maven-plugin and fix
related compilation failures (#1872)
# Which issue does this PR close?
Closes #1870
# Rationale for this change
# What changes are included in this PR?
# Are there any user-facing changes?
# How was this patch tested?
---
.../org/apache/spark/sql/SparkQueryTestsBase.scala | 2 +-
.../org/apache/spark/sql/AuronDataFrameSuite.scala | 8 ++++----
.../sql/execution/ui/AuronSQLAppStatusListener.scala | 3 +++
pom.xml | 17 +++++++++++++++++
.../spark/sql/auron/InterceptedValidateSparkPlan.scala | 3 +++
.../scala/org/apache/spark/sql/auron/ShimsImpl.scala | 5 +++++
.../auron/shuffle/AuronBlockStoreShuffleReader.scala | 3 +++
.../auron/shuffle/AuronRssShuffleManagerBase.scala | 3 +++
.../auron/plan/NativeShuffledHashJoinExecProvider.scala | 4 ++++
.../scala/org/apache/auron/AuronFunctionSuite.scala | 4 ----
.../test/scala/org/apache/auron/AuronQuerySuite.scala | 15 ---------------
.../apache/spark/sql/auron/AuronCallNativeWrapper.scala | 3 +++
.../org/apache/spark/sql/auron/NativeConverters.scala | 4 ++--
.../scala/org/apache/spark/sql/auron/NativeHelper.scala | 2 +-
.../main/scala/org/apache/spark/sql/auron/Shims.scala | 3 +++
.../auron/columnar/AuronArrowColumnVector.scala | 3 +++
.../execution/auron/plan/NativeParquetSinkBase.scala | 6 ++++--
.../auron/plan/NativeShuffleExchangeBase.scala | 2 ++
.../auron/plan/NativeShuffledHashJoinBase.scala | 2 ++
.../sql/execution/auron/plan/NativeUnionBase.scala | 4 ++--
.../shuffle/AuronBlockStoreShuffleReaderBase.scala | 2 +-
.../src/main/scala/org/apache/auron/sparkver.scala | 6 ++++--
.../shuffle/uniffle/AuronUniffleShuffleReader.scala | 3 +++
23 files changed, 73 insertions(+), 34 deletions(-)
diff --git
a/auron-spark-tests/common/src/test/scala/org/apache/spark/sql/SparkQueryTestsBase.scala
b/auron-spark-tests/common/src/test/scala/org/apache/spark/sql/SparkQueryTestsBase.scala
index 6c6d839a..8498edcd 100644
---
a/auron-spark-tests/common/src/test/scala/org/apache/spark/sql/SparkQueryTestsBase.scala
+++
b/auron-spark-tests/common/src/test/scala/org/apache/spark/sql/SparkQueryTestsBase.scala
@@ -162,7 +162,7 @@ object AuronQueryTestUtil extends Assertions {
private def genError(
expectedAnswer: Seq[Row],
sparkAnswer: Seq[Row],
- isSorted: Boolean = false): String = {
+ isSorted: Boolean): String = {
val getRowType: Option[Row] => String = row =>
row
.map(row =>
diff --git
a/auron-spark-tests/spark33/src/test/scala/org/apache/spark/sql/AuronDataFrameSuite.scala
b/auron-spark-tests/spark33/src/test/scala/org/apache/spark/sql/AuronDataFrameSuite.scala
index 74e27698..b5374551 100644
---
a/auron-spark-tests/spark33/src/test/scala/org/apache/spark/sql/AuronDataFrameSuite.scala
+++
b/auron-spark-tests/spark33/src/test/scala/org/apache/spark/sql/AuronDataFrameSuite.scala
@@ -186,22 +186,22 @@ class AuronDataFrameSuite extends DataFrameSuite with
SparkQueryTestsBase {
val join = df.join(df, "id")
checkAnswer(join, df)
val shuffleCount = collect(join.queryExecution.executedPlan) {
- case e: NativeShuffleExchangeExec =>
+ case _: NativeShuffleExchangeExec =>
true
}.size
assert(shuffleCount === 1, s"Expected 1 shuffle exchange, got
$shuffleCount")
- assert(collect(join.queryExecution.executedPlan) { case e:
ReusedExchangeExec =>
+ assert(collect(join.queryExecution.executedPlan) { case _:
ReusedExchangeExec =>
true
}.size === 1)
val broadcasted = broadcast(join)
val join2 = join.join(broadcasted, "id").join(broadcasted, "id")
checkAnswer(join2, df)
val shuffleCount2 = collect(join2.queryExecution.executedPlan) {
- case e: NativeShuffleExchangeExec =>
+ case _: NativeShuffleExchangeExec =>
true
}.size
assert(shuffleCount2 == 1, s"Expected 1 shuffle exchange in join2, got
$shuffleCount2")
- assert(collect(join2.queryExecution.executedPlan) { case e:
ReusedExchangeExec =>
+ assert(collect(join2.queryExecution.executedPlan) { case _:
ReusedExchangeExec =>
true
}.size == 4)
}
diff --git
a/auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronSQLAppStatusListener.scala
b/auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronSQLAppStatusListener.scala
index 0da16d4f..651234ac 100644
---
a/auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronSQLAppStatusListener.scala
+++
b/auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronSQLAppStatusListener.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.sql.execution.ui
+import scala.annotation.nowarn
+
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.internal.Logging
import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent}
@@ -23,6 +25,7 @@ import org.apache.spark.status.ElementTrackingStore
import org.apache.auron.spark.ui.AuronBuildInfoEvent
+@nowarn("cat=unused") // conf temporarily unused
class AuronSQLAppStatusListener(conf: SparkConf, kvstore: ElementTrackingStore)
extends SparkListener
with Logging {
diff --git a/pom.xml b/pom.xml
index f59ee507..405b3474 100644
--- a/pom.xml
+++ b/pom.xml
@@ -379,7 +379,10 @@
<target>${javaVersion}</target>
<scalaVersion>${scalaLongVersion}</scalaVersion>
<args>
+ <arg>-deprecation</arg>
+ <arg>-feature</arg>
<arg>-Ywarn-unused</arg>
+ <arg>-Xfatal-warnings</arg>
</args>
</configuration>
<dependencies>
@@ -927,11 +930,25 @@
<artifactId>scala-maven-plugin</artifactId>
<configuration>
<args>
+ <arg>-deprecation</arg>
+ <arg>-feature</arg>
<arg>-Ywarn-unused</arg>
<!-- https://github.com/scalamacros/paradise is no longer
actively developed. -->
<!-- In Scala 2.13, the plugin's functionality has been
included in the compiler -->
<!-- directly under the -Ymacro-annotations flag. -->
<arg>-Ymacro-annotations</arg>
+ <arg>-Xfatal-warnings</arg>
+
+ <arg>-Wconf:cat=deprecation:wv,any:e</arg>
+ <arg>-Wconf:cat=other-nullary-override:s</arg>
+
<arg>-Wconf:msg=^(?=.*?method|value|type|object|trait|inheritance)(?=.*?deprecated)(?=.*?since
2.13).+$:s</arg>
+ <arg>-Wconf:msg=Auto-application to \`\(\)\` is
deprecated:s</arg>
+ <arg>-Wconf:msg=object JavaConverters in package collection is
deprecated:s</arg>
+ <arg>-Wconf:cat=unchecked&msg=outer reference:s</arg>
+ <arg>-Wconf:cat=unchecked&msg=eliminated by erasure:s</arg>
+ <arg>-Wconf:cat=unused-nowarn:s</arg>
+ <arg>-Wconf:msg=early initializers are deprecated:s</arg>
+ <arg>-Wconf:cat=other-match-analysis:s</arg>
</args>
<compilerPlugins>
<compilerPlugin>
diff --git
a/spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/auron/InterceptedValidateSparkPlan.scala
b/spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/auron/InterceptedValidateSparkPlan.scala
index 95aefa65..f90e4865 100644
---
a/spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/auron/InterceptedValidateSparkPlan.scala
+++
b/spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/auron/InterceptedValidateSparkPlan.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.sql.auron
+import scala.annotation.nowarn
+
import org.apache.spark.internal.Logging
import org.apache.spark.sql.execution.SparkPlan
@@ -71,6 +73,7 @@ object InterceptedValidateSparkPlan extends Logging {
}
}
+ @nowarn("cat=unused") // plan unused
@sparkver("3.0 / 3.1")
def validate(plan: SparkPlan): Unit = {
throw new UnsupportedOperationException("validate is not supported in
spark 3.0.3 or 3.1.3")
diff --git
a/spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/auron/ShimsImpl.scala
b/spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/auron/ShimsImpl.scala
index 839f5655..c3a1861d 100644
---
a/spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/auron/ShimsImpl.scala
+++
b/spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/auron/ShimsImpl.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.auron
import java.io.File
import java.util.UUID
+import scala.annotation.nowarn
import scala.collection.mutable
import org.apache.commons.lang3.reflect.FieldUtils
@@ -284,6 +285,7 @@ class ShimsImpl extends Shims with Logging {
child: SparkPlan): NativeGenerateBase =
NativeGenerateExec(generator, requiredChildOutput, outer, generatorOutput,
child)
+ @sparkver("3.4 / 3.5")
private def effectiveLimit(rawLimit: Int): Int =
if (rawLimit == -1) Int.MaxValue else rawLimit
@@ -989,6 +991,7 @@ class ShimsImpl extends Shims with Logging {
}
}
+ @nowarn("cat=unused") // Some params temporarily unused
@sparkver("3.4 / 3.5")
private def convertPromotePrecision(
e: Expression,
@@ -1021,6 +1024,7 @@ class ShimsImpl extends Shims with Logging {
}
}
+ @nowarn("cat=unused") // Some params temporarily unused
@sparkver("3.0 / 3.1 / 3.2")
private def convertBloomFilterAgg(agg: AggregateFunction):
Option[pb.PhysicalAggExprNode] = None
@@ -1047,6 +1051,7 @@ class ShimsImpl extends Shims with Logging {
}
}
+ @nowarn("cat=unused") // Some params temporarily unused
@sparkver("3.0 / 3.1 / 3.2")
private def convertBloomFilterMightContain(
e: Expression,
diff --git
a/spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/AuronBlockStoreShuffleReader.scala
b/spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/AuronBlockStoreShuffleReader.scala
index e08d8490..14ebba48 100644
---
a/spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/AuronBlockStoreShuffleReader.scala
+++
b/spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/AuronBlockStoreShuffleReader.scala
@@ -37,6 +37,9 @@ class AuronBlockStoreShuffleReader[K, C](
extends AuronBlockStoreShuffleReaderBase[K, C](handle, context)
with Logging {
+ // Touch mapOutputTracker to suppress -Xfatal-warnings (used in Spark 3.2+,
unused in 3.0/3.1)
+ private val _ = mapOutputTracker
+
override def readBlocks(): Iterator[InputStream] = {
@sparkver("3.2 / 3.3 / 3.4 / 3.5")
def fetchIterator = new ShuffleBlockFetcherIterator(
diff --git
a/spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/AuronRssShuffleManagerBase.scala
b/spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/AuronRssShuffleManagerBase.scala
index 98863bf6..af92b0e8 100644
---
a/spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/AuronRssShuffleManagerBase.scala
+++
b/spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/AuronRssShuffleManagerBase.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.sql.execution.auron.shuffle
+import scala.annotation.nowarn
+
import org.apache.spark.{ShuffleDependency, SparkConf, TaskContext}
import org.apache.spark.internal.Logging
import org.apache.spark.shuffle._
@@ -23,6 +25,7 @@ import
org.apache.spark.sql.execution.auron.shuffle.AuronShuffleDependency.isArr
import org.apache.auron.sparkver
+@nowarn("cat=unused") // _conf temporarily unused
abstract class AuronRssShuffleManagerBase(_conf: SparkConf) extends
ShuffleManager with Logging {
override def registerShuffle[K, V, C](
shuffleId: Int,
diff --git
a/spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/execution/joins/auron/plan/NativeShuffledHashJoinExecProvider.scala
b/spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/execution/joins/auron/plan/NativeShuffledHashJoinExecProvider.scala
index 0236dd26..5f763eec 100644
---
a/spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/execution/joins/auron/plan/NativeShuffledHashJoinExecProvider.scala
+++
b/spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/execution/joins/auron/plan/NativeShuffledHashJoinExecProvider.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.sql.execution.joins.auron.plan
+import scala.annotation.nowarn
+
import org.apache.spark.sql.auron.join.JoinBuildSides.JoinBuildSide
import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.catalyst.plans.JoinType
@@ -80,6 +82,7 @@ case object NativeShuffledHashJoinExecProvider {
NativeShuffledHashJoinExec(left, right, leftKeys, rightKeys, joinType,
buildSide, isSkewJoin)
}
+ @nowarn("cat=unused") // Some params temporarily unused
@sparkver("3.1")
def provide(
left: SparkPlan,
@@ -127,6 +130,7 @@ case object NativeShuffledHashJoinExecProvider {
NativeShuffledHashJoinExec(left, right, leftKeys, rightKeys, joinType,
buildSide)
}
+ @nowarn("cat=unused") // Some params temporarily unused
@sparkver("3.0")
def provide(
left: SparkPlan,
diff --git
a/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronFunctionSuite.scala
b/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronFunctionSuite.scala
index c7e2f6a9..ae1d2432 100644
---
a/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronFunctionSuite.scala
+++
b/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronFunctionSuite.scala
@@ -278,8 +278,6 @@ class AuronFunctionSuite extends AuronQueryTest with
BaseAuronSQLSuite {
val dateTimeStampMin = format.parse(dateStringMin).getTime
val dateTimeStampMax = format.parse(dateStringMax).getTime
format = new SimpleDateFormat("yyyy-MM-dd")
- val dateString = "2015-01-01"
- val date = format.parse(dateString)
val functions =
s"""
@@ -320,8 +318,6 @@ class AuronFunctionSuite extends AuronQueryTest with
BaseAuronSQLSuite {
val dateTimeStampMin = format.parse(dateStringMin).getTime
val dateTimeStampMax = format.parse(dateStringMax).getTime
format = new SimpleDateFormat("yyyy-MM-dd")
- val dateString = "2015-07-01"
- val date = format.parse(dateString)
val functions =
s"""
diff --git
a/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronQuerySuite.scala
b/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronQuerySuite.scala
index 349b489a..3e990978 100644
---
a/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronQuerySuite.scala
+++
b/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronQuerySuite.scala
@@ -204,18 +204,10 @@ class AuronQuerySuite extends AuronQueryTest with
BaseAuronSQLSuite with AuronSQ
.toDF("c1", "c2")
.write
.orc(path)
- val correctAnswer = Seq(Row(1, 2), Row(3, 4), Row(5, 6), Row(null,
null))
checkSparkAnswerAndOperator(() => spark.read.orc(path))
withTable("t") {
sql(s"CREATE EXTERNAL TABLE t(c3 INT, c2 INT) USING ORC LOCATION
'$path'")
-
- val expected = if (forcePositionalEvolution) {
- correctAnswer
- } else {
- Seq(Row(null, 2), Row(null, 4), Row(null, 6), Row(null, null))
- }
-
checkSparkAnswerAndOperator(() => spark.table("t"))
}
}
@@ -236,7 +228,6 @@ class AuronQuerySuite extends AuronQueryTest with
BaseAuronSQLSuite with AuronSQ
.write
.partitionBy("p")
.orc(path)
- val correctAnswer = Seq(Row(1, 2, 1), Row(3, 4, 2), Row(5, 6, 3),
Row(null, null, 4))
checkSparkAnswerAndOperator(() => spark.read.orc(path))
withTable("t") {
@@ -247,12 +238,6 @@ class AuronQuerySuite extends AuronQueryTest with
BaseAuronSQLSuite with AuronSQ
|LOCATION '$path'
|""".stripMargin)
sql("MSCK REPAIR TABLE t")
- if (forcePositionalEvolution) {
- correctAnswer
- } else {
- Seq(Row(null, 2, 1), Row(null, 4, 2), Row(null, 6, 3),
Row(null, null, 4))
- }
-
checkSparkAnswerAndOperator(() => spark.table("t"))
}
}
diff --git
a/spark-extension/src/main/scala/org/apache/spark/sql/auron/AuronCallNativeWrapper.scala
b/spark-extension/src/main/scala/org/apache/spark/sql/auron/AuronCallNativeWrapper.scala
index bf0918c4..b4028c1e 100644
---
a/spark-extension/src/main/scala/org/apache/spark/sql/auron/AuronCallNativeWrapper.scala
+++
b/spark-extension/src/main/scala/org/apache/spark/sql/auron/AuronCallNativeWrapper.scala
@@ -22,6 +22,7 @@ import java.nio.file.Files
import java.nio.file.StandardCopyOption
import java.util.concurrent.atomic.AtomicReference
+import scala.annotation.nowarn
import scala.collection.mutable.ArrayBuffer
import org.apache.arrow.c.ArrowArray
@@ -53,6 +54,7 @@ import org.apache.auron.protobuf.TaskDefinition
* This class has been deprecated and migrated to {@link
* org.apache.auron.jni.AuronCallNativeWrapper}. Will be removed in the future.
*/
+@nowarn("cat=deprecation") // JniBridge is temporarily used (deprecated)
@Deprecated
case class AuronCallNativeWrapper(
nativePlan: PhysicalPlanNode,
@@ -193,6 +195,7 @@ case class AuronCallNativeWrapper(
}
}
+@nowarn("cat=deprecation") // JniBridge is temporarily used (deprecated)
object AuronCallNativeWrapper extends Logging {
def initNative(): Unit = {
lazyInitNative
diff --git
a/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeConverters.scala
b/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeConverters.scala
index f88c03f2..8b48c39a 100644
---
a/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeConverters.scala
+++
b/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeConverters.scala
@@ -1010,12 +1010,12 @@ object NativeConverters extends Logging {
val children = e.children.map(Cast(_, e.dataType))
buildScalarFunction(pb.ScalarFunction.Coalesce, children, e.dataType)
- case e @ StringLPad(str, len, pad) =>
+ case _ @StringLPad(str, len, pad) =>
buildScalarFunction(
pb.ScalarFunction.Lpad,
Seq(str, castIfNecessary(len, LongType), pad),
StringType)
- case e @ StringRPad(str, len, pad) =>
+ case _ @StringRPad(str, len, pad) =>
buildScalarFunction(
pb.ScalarFunction.Rpad,
Seq(str, castIfNecessary(len, LongType), pad),
diff --git
a/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeHelper.scala
b/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeHelper.scala
index 3ae7669e..e1665647 100644
---
a/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeHelper.scala
+++
b/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeHelper.scala
@@ -100,7 +100,7 @@ object NativeHelper extends Logging {
if (nativePlan == null) {
return Iterator.empty
}
- var auronCallNativeWrapper = new
org.apache.auron.jni.AuronCallNativeWrapper(
+ val auronCallNativeWrapper = new
org.apache.auron.jni.AuronCallNativeWrapper(
ROOT_ALLOCATOR,
nativePlan,
metrics,
diff --git
a/spark-extension/src/main/scala/org/apache/spark/sql/auron/Shims.scala
b/spark-extension/src/main/scala/org/apache/spark/sql/auron/Shims.scala
index 85ab65d5..8d817521 100644
--- a/spark-extension/src/main/scala/org/apache/spark/sql/auron/Shims.scala
+++ b/spark-extension/src/main/scala/org/apache/spark/sql/auron/Shims.scala
@@ -18,6 +18,8 @@ package org.apache.spark.sql.auron
import java.io.File
+import scala.annotation.nowarn
+
import org.apache.spark.ShuffleDependency
import org.apache.spark.SparkContext
import org.apache.spark.TaskContext
@@ -272,6 +274,7 @@ abstract class Shims {
def getMinPartitionNum(sparkSession: SparkSession): Int
+ @nowarn("cat=unused") // Some params temporarily unused
def postTransform(plan: SparkPlan, sc: SparkContext): Unit = {}
def getAdaptiveInputPlan(exec: AdaptiveSparkPlanExec): SparkPlan
diff --git
a/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/columnar/AuronArrowColumnVector.scala
b/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/columnar/AuronArrowColumnVector.scala
index d8be57e6..3dbde795 100644
---
a/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/columnar/AuronArrowColumnVector.scala
+++
b/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/columnar/AuronArrowColumnVector.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.sql.execution.auron.columnar
+import scala.annotation.nowarn
+
import org.apache.arrow.vector.BigIntVector
import org.apache.arrow.vector.BitVector
import org.apache.arrow.vector.DateDayVector
@@ -143,6 +145,7 @@ class AuronArrowColumnVector(vector: ValueVector)
}
object AuronArrowColumnVector {
+ @nowarn("cat=unused") // Data type get methods unimplemented (placeholder)
abstract private class ArrowVectorAccessor(private val vector: ValueVector) {
def isNullAt(rowId: Int): Boolean =
if (vector.getValueCount > 0 && vector.getValidityBuffer.capacity == 0)
false
diff --git
a/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/plan/NativeParquetSinkBase.scala
b/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/plan/NativeParquetSinkBase.scala
index 260ee249..2bd50523 100644
---
a/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/plan/NativeParquetSinkBase.scala
+++
b/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/plan/NativeParquetSinkBase.scala
@@ -21,6 +21,7 @@ import java.security.PrivilegedExceptionAction
import java.util
import java.util.UUID
+import scala.annotation.nowarn
import scala.collection.JavaConverters._
import org.apache.hadoop.conf.Configuration
@@ -80,7 +81,7 @@ abstract class NativeParquetSinkBase(
hiveQlTable.getMetadata)
val tableSchema = table.schema
val hadoopConf = newHadoopConf(tableDesc)
- val job = new Job(hadoopConf)
+ val job = Job.getInstance(hadoopConf)
val parquetFileFormat = new ParquetFileFormat()
parquetFileFormat.prepareWrite(sparkSession, job, Map(), tableSchema)
@@ -114,7 +115,7 @@ abstract class NativeParquetSinkBase(
})
// init parquet schema
- val job = new Job(new JobConf(serializableConf.value))
+ val job = Job.getInstance(new JobConf(serializableConf.value))
val tableProperties = tableDesc.getProperties
val columnNameProperty: String =
tableProperties.getProperty(IOConstants.COLUMNS)
val columnTypeProperty: String =
tableProperties.getProperty(IOConstants.COLUMNS_TYPES)
@@ -157,6 +158,7 @@ abstract class NativeParquetSinkBase(
friendlyName = "NativeRDD.ParquetSink")
}
+ @nowarn("cat=unused") // _tableDesc temporarily unused
protected def newHadoopConf(_tableDesc: TableDesc): Configuration =
sparkSession.sessionState.newHadoopConf()
}
diff --git
a/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/plan/NativeShuffleExchangeBase.scala
b/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/plan/NativeShuffleExchangeBase.scala
index 552dcff3..8b6aa04d 100644
---
a/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/plan/NativeShuffleExchangeBase.scala
+++
b/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/plan/NativeShuffleExchangeBase.scala
@@ -18,6 +18,7 @@ package org.apache.spark.sql.execution.auron.plan
import java.util.UUID
+import scala.annotation.nowarn
import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
@@ -308,6 +309,7 @@ abstract class NativeShuffleExchangeBase(
dependency
}
+ @nowarn("cat=unused") // Some params temporarily unused
private def rangePartitioningBound[K: Ordering: ClassTag, V](
partitions: Int,
rdd: RDD[_ <: Product2[K, V]],
diff --git
a/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/plan/NativeShuffledHashJoinBase.scala
b/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/plan/NativeShuffledHashJoinBase.scala
index 1f8a06c8..17dfbe25 100644
---
a/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/plan/NativeShuffledHashJoinBase.scala
+++
b/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/plan/NativeShuffledHashJoinBase.scala
@@ -82,6 +82,8 @@ abstract class NativeShuffledHashJoinBase(
private def nativeBuildSide = buildSide match {
case JoinBuildLeft => pb.JoinSide.LEFT_SIDE
case JoinBuildRight => pb.JoinSide.RIGHT_SIDE
+ case other =>
+ throw new IllegalArgumentException(s"Unknown Join buildSide: $other")
}
protected def rewriteKeyExprToLong(exprs: Seq[Expression]): Seq[Expression]
diff --git
a/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/plan/NativeUnionBase.scala
b/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/plan/NativeUnionBase.scala
index a62ea6f1..bf3a1549 100644
---
a/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/plan/NativeUnionBase.scala
+++
b/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/plan/NativeUnionBase.scala
@@ -70,7 +70,7 @@ abstract class NativeUnionBase(
val unionInputs = ArrayBuffer[(PhysicalPlanNode, Int)]()
partition match {
case p: UnionPartition[_] =>
- val rdds = unionRDD.asInstanceOf[UnionRDD[_]].rdds
+ val rdds = unionRDD.asInstanceOf[UnionRDD[Any]].rdds
val nativeRDD = rdds(p.parentRddIndex).asInstanceOf[NativeRDD]
val input = nativeRDD.nativePlan(p.parentPartition, taskContext)
for (childIndex <- rdds.indices) {
@@ -81,7 +81,7 @@ abstract class NativeUnionBase(
}
}
case p: PartitionerAwareUnionRDDPartition =>
- val rdds = unionRDD.asInstanceOf[PartitionerAwareUnionRDD[_]].rdds
+ val rdds =
unionRDD.asInstanceOf[PartitionerAwareUnionRDD[Any]].rdds
for ((rdd, partition) <- rdds.zip(p.parents)) {
val nativeRDD = rdd.asInstanceOf[NativeRDD]
unionInputs.append((nativeRDD.nativePlan(partition,
taskContext), partition.index))
diff --git
a/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/AuronBlockStoreShuffleReaderBase.scala
b/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/AuronBlockStoreShuffleReaderBase.scala
index 477c1b5e..eaba3640 100644
---
a/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/AuronBlockStoreShuffleReaderBase.scala
+++
b/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/AuronBlockStoreShuffleReaderBase.scala
@@ -182,5 +182,5 @@ trait BlockObject extends AutoCloseable {
def getFileLength: Long = throw new UnsupportedOperationException
def getByteBuffer: ByteBuffer = throw new UnsupportedOperationException
def getChannel: ReadableByteChannel = throw new UnsupportedOperationException
- def throwFetchFailed(errmsg: String): Unit = throw new
UnsupportedOperationException
+ def throwFetchFailed(errmsg: String): Unit = throw new
UnsupportedOperationException(errmsg)
}
diff --git
a/spark-version-annotation-macros/src/main/scala/org/apache/auron/sparkver.scala
b/spark-version-annotation-macros/src/main/scala/org/apache/auron/sparkver.scala
index 7d1e494d..fd7d4ed9 100644
---
a/spark-version-annotation-macros/src/main/scala/org/apache/auron/sparkver.scala
+++
b/spark-version-annotation-macros/src/main/scala/org/apache/auron/sparkver.scala
@@ -16,8 +16,7 @@
*/
package org.apache.auron
-import scala.annotation.StaticAnnotation
-import scala.annotation.compileTimeOnly
+import scala.annotation.{compileTimeOnly, nowarn, StaticAnnotation}
import scala.language.experimental._
import scala.reflect.macros.whitebox
@@ -96,16 +95,19 @@ object sparkver {
}
}
+@nowarn("cat=unused") // 'vers' is used by macro
@compileTimeOnly("enable macro paradise to expand macro annotations")
final class sparkver(vers: String) extends StaticAnnotation {
def macroTransform(annottees: Any*): Any = macro sparkver.Macros.verEnable
}
+@nowarn("cat=unused") // 'vers' is used by macro
@compileTimeOnly("enable macro paradise to expand macro annotations")
final class sparkverEnableMembers(vers: String) extends StaticAnnotation {
def macroTransform(annottees: Any*): Any = macro
sparkver.Macros.verEnableMembers
}
+@nowarn("cat=unused") // 'vers' is used by macro
@compileTimeOnly("enable macro paradise to expand macro annotations")
final class sparkverEnableOverride(vers: String) extends StaticAnnotation {
def macroTransform(annottees: Any*): Any = macro
sparkver.Macros.verEnableOverride
diff --git
a/thirdparty/auron-uniffle/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/uniffle/AuronUniffleShuffleReader.scala
b/thirdparty/auron-uniffle/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/uniffle/AuronUniffleShuffleReader.scala
index 18745f5c..c4b1ca31 100644
---
a/thirdparty/auron-uniffle/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/uniffle/AuronUniffleShuffleReader.scala
+++
b/thirdparty/auron-uniffle/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/uniffle/AuronUniffleShuffleReader.scala
@@ -20,6 +20,7 @@ import java.io.InputStream
import java.nio.ByteBuffer
import java.util
+import scala.annotation.nowarn
import scala.collection.AbstractIterator
import org.apache.commons.lang3.reflect.FieldUtils
@@ -40,6 +41,7 @@ import org.apache.uniffle.common.config.RssConf
import org.apache.uniffle.common.exception.RssException
import
org.apache.uniffle.shaded.org.roaringbitmap.longlong.Roaring64NavigableMap
+@nowarn("cat=unused") // Some params temporarily unused
class AuronUniffleShuffleReader[K, C](
reader: RssShuffleReader[K, C],
handle: RssShuffleHandleWrapper[K, _, C],
@@ -207,6 +209,7 @@ class AuronUniffleShuffleReader[K, C](
}
}
+ @nowarn("cat=unused") // Some params temporarily unused
private class UniffleInputStream(
iterator: MultiPartitionIterator[_, _],
shuffleId: Int,
