This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new e1bc48b729e [SPARK-45704][BUILD] Fix compile warning - using symbols
inherited from a superclass shadow symbols defined in an outer scope
e1bc48b729e is described below
commit e1bc48b729e40390a4b0f977eec4a9050c7cac77
Author: panbingkun <[email protected]>
AuthorDate: Tue Oct 31 22:02:39 2023 -0700
[SPARK-45704][BUILD] Fix compile warning - using symbols inherited from a
superclass shadow symbols defined in an outer scope
### What changes were proposed in this pull request?
After upgrade to scala 2.13, when using symbols inherited from a superclass
shadow symbols defined in an outer scope, the following warning will appear:
```
[error]
/Users/panbingkun/Developer/spark/spark-community/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala:1315:39:
reference to child is ambiguous;
[error] it is both defined in the enclosing method apply and inherited in
the enclosing anonymous class as value child (defined in class IsNull)
[error] In Scala 2, symbols inherited from a superclass shadow symbols
defined in an outer scope.
[error] Such references are ambiguous in Scala 3. To continue using the
inherited symbol, write `this.child`.
[error] Or use `-Wconf:msg=legacy-binding:s` to silence this warning.
[quickfixable]
[error] Applicable -Wconf / nowarn filters for this fatal warning:
msg=<part of the message>, cat=other,
site=org.apache.spark.sql.catalyst.expressions.IsUnknown.apply
[error] override def sql: String = s"(${child.sql} IS UNKNOWN)"
[error] ^
```
The pr aims to fix it.
### Why are the changes needed?
Prepare for upgrading to scala 3.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
- Pass GA
- Manually test:
```
build/sbt -Phadoop-3 -Pdocker-integration-tests -Pspark-ganglia-lgpl
-Pkinesis-asl -Pkubernetes -Phive-thriftserver -Pconnect -Pyarn -Phive
-Phadoop-cloud -Pvolcano -Pkubernetes-integration-tests Test/package
streaming-kinesis-asl-assembly/assembly connect/assembly
```
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #43593 from panbingkun/SPARK-45704.
Authored-by: panbingkun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../spark/deploy/client/StandaloneAppClient.scala | 6 +++---
.../cluster/CoarseGrainedSchedulerBackend.scala | 2 +-
.../apache/spark/storage/DiskBlockObjectWriter.scala | 2 +-
.../executor/CoarseGrainedExecutorBackendSuite.scala | 20 ++++++++++----------
pom.xml | 7 -------
project/SparkBuild.scala | 5 -----
.../spark/sql/catalyst/expressions/predicates.scala | 4 ++--
.../sql/connector/catalog/InMemoryBaseTable.scala | 2 +-
.../datasources/parquet/ParquetRowConverter.scala | 18 +++++++++---------
.../apache/spark/sql/execution/python/RowQueue.scala | 2 +-
.../spark/sql/internal/BaseSessionStateBuilder.scala | 2 +-
.../command/AlignAssignmentsSuiteBase.scala | 2 +-
.../sql/execution/command/PlanResolutionSuite.scala | 2 +-
.../spark/sql/hive/HiveSessionStateBuilder.scala | 2 +-
14 files changed, 32 insertions(+), 44 deletions(-)
diff --git
a/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClient.scala
b/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClient.scala
index a7e4c1fbab2..b0ee6018970 100644
---
a/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClient.scala
+++
b/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClient.scala
@@ -90,7 +90,7 @@ private[spark] class StandaloneAppClient(
case e: Exception =>
logWarning("Failed to connect to master", e)
markDisconnected()
- stop()
+ this.stop()
}
}
@@ -168,7 +168,7 @@ private[spark] class StandaloneAppClient(
case ApplicationRemoved(message) =>
markDead("Master removed our application: %s".format(message))
- stop()
+ this.stop()
case ExecutorAdded(id: Int, workerId: String, hostPort: String, cores:
Int, memory: Int) =>
val fullId = s"$appId/$id"
@@ -203,7 +203,7 @@ private[spark] class StandaloneAppClient(
markDead("Application has been stopped.")
sendToMaster(UnregisterApplication(appId.get))
context.reply(true)
- stop()
+ this.stop()
case r: RequestExecutors =>
master match {
diff --git
a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
index c49b2411e76..e02dd279370 100644
---
a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
+++
b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
@@ -319,7 +319,7 @@ class CoarseGrainedSchedulerBackend(scheduler:
TaskSchedulerImpl, val rpcEnv: Rp
case StopDriver =>
context.reply(true)
- stop()
+ this.stop()
case UpdateExecutorsLogLevel(logLevel) =>
currentLogLevel = Some(logLevel)
diff --git
a/core/src/main/scala/org/apache/spark/storage/DiskBlockObjectWriter.scala
b/core/src/main/scala/org/apache/spark/storage/DiskBlockObjectWriter.scala
index f8bd73e6561..2096da2fca0 100644
--- a/core/src/main/scala/org/apache/spark/storage/DiskBlockObjectWriter.scala
+++ b/core/src/main/scala/org/apache/spark/storage/DiskBlockObjectWriter.scala
@@ -63,7 +63,7 @@ private[spark] class DiskBlockObjectWriter(
*/
private trait ManualCloseOutputStream extends OutputStream {
abstract override def close(): Unit = {
- flush()
+ this.flush()
}
def manualClose(): Unit = {
diff --git
a/core/src/test/scala/org/apache/spark/executor/CoarseGrainedExecutorBackendSuite.scala
b/core/src/test/scala/org/apache/spark/executor/CoarseGrainedExecutorBackendSuite.scala
index 3ef4da6d3d3..28af0656869 100644
---
a/core/src/test/scala/org/apache/spark/executor/CoarseGrainedExecutorBackendSuite.scala
+++
b/core/src/test/scala/org/apache/spark/executor/CoarseGrainedExecutorBackendSuite.scala
@@ -326,11 +326,11 @@ class CoarseGrainedExecutorBackendSuite extends
SparkFunSuite
def getFakeTaskRunner(taskDescription: TaskDescription):
Executor#TaskRunner = {
new executor.TaskRunner(backend, taskDescription, None) {
override def run(): Unit = {
- logInfo(s"task ${taskDescription.taskId} runs.")
+ logInfo(s"task ${this.taskDescription.taskId} runs.")
}
override def kill(interruptThread: Boolean, reason: String): Unit = {
- logInfo(s"task ${taskDescription.taskId} killed.")
+ logInfo(s"task ${this.taskDescription.taskId} killed.")
}
}
}
@@ -434,13 +434,13 @@ class CoarseGrainedExecutorBackendSuite extends
SparkFunSuite
def getFakeTaskRunner(taskDescription: TaskDescription):
Executor#TaskRunner = {
new executor.TaskRunner(backend, taskDescription, None) {
override def run(): Unit = {
- tasksExecuted.put(taskDescription.taskId, true)
- logInfo(s"task ${taskDescription.taskId} runs.")
+ tasksExecuted.put(this.taskDescription.taskId, true)
+ logInfo(s"task ${this.taskDescription.taskId} runs.")
}
override def kill(interruptThread: Boolean, reason: String): Unit = {
- logInfo(s"task ${taskDescription.taskId} killed.")
- tasksKilled.put(taskDescription.taskId, true)
+ logInfo(s"task ${this.taskDescription.taskId} killed.")
+ tasksKilled.put(this.taskDescription.taskId, true)
}
}
}
@@ -523,13 +523,13 @@ class CoarseGrainedExecutorBackendSuite extends
SparkFunSuite
def getFakeTaskRunner(taskDescription: TaskDescription):
Executor#TaskRunner = {
new executor.TaskRunner(backend, taskDescription, None) {
override def run(): Unit = {
- tasksExecuted.put(taskDescription.taskId, true)
- logInfo(s"task ${taskDescription.taskId} runs.")
+ tasksExecuted.put(this.taskDescription.taskId, true)
+ logInfo(s"task ${this.taskDescription.taskId} runs.")
}
override def kill(interruptThread: Boolean, reason: String): Unit = {
- logInfo(s"task ${taskDescription.taskId} killed.")
- tasksKilled.put(taskDescription.taskId, true)
+ logInfo(s"task ${this.taskDescription.taskId} killed.")
+ tasksKilled.put(this.taskDescription.taskId, true)
}
}
}
diff --git a/pom.xml b/pom.xml
index d545c743928..e29d81f6887 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2985,13 +2985,6 @@
SPARK-40497 Upgrade Scala to 2.13.11 and suppress `Implicit
definition should have explicit type`
-->
<arg>-Wconf:msg=Implicit definition should have explicit
type:s</arg>
- <!--
- SPARK-45331 Upgrade Scala to 2.13.12 and suppress "In Scala 2,
symbols inherited
- from a superclass shadow symbols defined in an outer scope.
Such references are
- ambiguous in Scala 3. To continue using the inherited symbol,
write `this.stop`.
- Or use `-Wconf:msg=legacy-binding:s` to silence this warning.
[quickfixable]"
- -->
- <arg>-Wconf:msg=legacy-binding:s</arg>
<!--
SPARK-45627 Symbol literals are deprecated in Scala 2.13 and
it's a compile error in Scala 3.
-->
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index d9d4a836ab5..d76af6a06cf 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -249,11 +249,6 @@ object SparkBuild extends PomBuild {
"-Wconf:cat=deprecation&msg=procedure syntax is deprecated:e",
// SPARK-40497 Upgrade Scala to 2.13.11 and suppress `Implicit
definition should have explicit type`
"-Wconf:msg=Implicit definition should have explicit type:s",
- // SPARK-45331 Upgrade Scala to 2.13.12 and suppress "In Scala 2,
symbols inherited
- // from a superclass shadow symbols defined in an outer scope. Such
references are
- // ambiguous in Scala 3. To continue using the inherited symbol, write
`this.stop`.
- // Or use `-Wconf:msg=legacy-binding:s` to silence this warning.
[quickfixable]"
- "-Wconf:msg=legacy-binding:s",
// SPARK-45627 Symbol literals are deprecated in Scala 2.13 and it's a
compile error in Scala 3.
"-Wconf:cat=deprecation&msg=symbol literal is deprecated:e",
// SPARK-45627 `enum`, `export` and `given` will become keywords in
Scala 3,
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
index 9eefcef8e17..761bd3f3358 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
@@ -1312,7 +1312,7 @@ object IsUnknown {
def apply(child: Expression): Predicate = {
new IsNull(child) with ExpectsInputTypes {
override def inputTypes: Seq[DataType] = Seq(BooleanType)
- override def sql: String = s"(${child.sql} IS UNKNOWN)"
+ override def sql: String = s"(${this.child.sql} IS UNKNOWN)"
}
}
}
@@ -1321,7 +1321,7 @@ object IsNotUnknown {
def apply(child: Expression): Predicate = {
new IsNotNull(child) with ExpectsInputTypes {
override def inputTypes: Seq[DataType] = Seq(BooleanType)
- override def sql: String = s"(${child.sql} IS NOT UNKNOWN)"
+ override def sql: String = s"(${this.child.sql} IS NOT UNKNOWN)"
}
}
}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryBaseTable.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryBaseTable.scala
index 7765bc26741..cd7f7295d5c 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryBaseTable.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryBaseTable.scala
@@ -452,7 +452,7 @@ abstract class InMemoryBaseTable(
val matchingKeys = values.map { value =>
if (value != null) value.toString else null
}.toSet
- data = data.filter(partition => {
+ this.data = this.data.filter(partition => {
val rows = partition.asInstanceOf[BufferedRows]
rows.key match {
// null partitions are represented as Seq(null)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
index 936339e091d..89c7cae175a 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
@@ -311,7 +311,7 @@ private[parquet] class ParquetRowConverter(
case LongType if isUnsignedIntTypeMatched(32) =>
new ParquetPrimitiveConverter(updater) {
override def addInt(value: Int): Unit =
- updater.setLong(Integer.toUnsignedLong(value))
+ this.updater.setLong(Integer.toUnsignedLong(value))
}
case BooleanType | IntegerType | LongType | FloatType | DoubleType |
BinaryType |
_: AnsiIntervalType =>
@@ -320,13 +320,13 @@ private[parquet] class ParquetRowConverter(
case ByteType =>
new ParquetPrimitiveConverter(updater) {
override def addInt(value: Int): Unit =
- updater.setByte(value.asInstanceOf[PhysicalByteType#InternalType])
+
this.updater.setByte(value.asInstanceOf[PhysicalByteType#InternalType])
}
case ShortType =>
new ParquetPrimitiveConverter(updater) {
override def addInt(value: Int): Unit =
-
updater.setShort(value.asInstanceOf[PhysicalShortType#InternalType])
+
this.updater.setShort(value.asInstanceOf[PhysicalShortType#InternalType])
}
// For INT32 backed decimals
@@ -346,7 +346,7 @@ private[parquet] class ParquetRowConverter(
case _: DecimalType if isUnsignedIntTypeMatched(64) =>
new ParquetPrimitiveConverter(updater) {
override def addLong(value: Long): Unit = {
- updater.set(Decimal(java.lang.Long.toUnsignedString(value)))
+ this.updater.set(Decimal(java.lang.Long.toUnsignedString(value)))
}
}
@@ -391,7 +391,7 @@ private[parquet] class ParquetRowConverter(
.asInstanceOf[TimestampLogicalTypeAnnotation].getUnit ==
TimeUnit.MICROS =>
new ParquetPrimitiveConverter(updater) {
override def addLong(value: Long): Unit = {
- updater.setLong(timestampRebaseFunc(value))
+ this.updater.setLong(timestampRebaseFunc(value))
}
}
@@ -404,7 +404,7 @@ private[parquet] class ParquetRowConverter(
new ParquetPrimitiveConverter(updater) {
override def addLong(value: Long): Unit = {
val micros = DateTimeUtils.millisToMicros(value)
- updater.setLong(timestampRebaseFunc(micros))
+ this.updater.setLong(timestampRebaseFunc(micros))
}
}
@@ -417,7 +417,7 @@ private[parquet] class ParquetRowConverter(
val gregorianMicros = int96RebaseFunc(julianMicros)
val adjTime =
convertTz.map(DateTimeUtils.convertTz(gregorianMicros, _, ZoneOffset.UTC))
.getOrElse(gregorianMicros)
- updater.setLong(adjTime)
+ this.updater.setLong(adjTime)
}
}
@@ -434,14 +434,14 @@ private[parquet] class ParquetRowConverter(
new ParquetPrimitiveConverter(updater) {
override def addLong(value: Long): Unit = {
val micros = DateTimeUtils.millisToMicros(value)
- updater.setLong(micros)
+ this.updater.setLong(micros)
}
}
case DateType =>
new ParquetPrimitiveConverter(updater) {
override def addInt(value: Int): Unit = {
- updater.set(dateRebaseFunc(value))
+ this.updater.set(dateRebaseFunc(value))
}
}
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/RowQueue.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/RowQueue.scala
index 0e3243eac62..5e0c5ff92fd 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/RowQueue.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/RowQueue.scala
@@ -233,7 +233,7 @@ private[python] case class HybridRowQueue(
val buffer = if (page != null) {
new InMemoryRowQueue(page, numFields) {
override def close(): Unit = {
- freePage(page)
+ freePage(this.page)
}
}
} else {
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
index 5543b409d17..1d496b027ef 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
@@ -188,7 +188,7 @@ abstract class BaseSessionStateBuilder(
new ResolveSQLOnFile(session) +:
new FallBackFileSourceV2(session) +:
ResolveEncodersInScalaAgg +:
- new ResolveSessionCatalog(catalogManager) +:
+ new ResolveSessionCatalog(this.catalogManager) +:
ResolveWriteToStream +:
new EvalSubqueriesForTimeTravel +:
customResolutionRules
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlignAssignmentsSuiteBase.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlignAssignmentsSuiteBase.scala
index 6f9cc66f247..2bc747c0abe 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlignAssignmentsSuiteBase.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlignAssignmentsSuiteBase.scala
@@ -191,7 +191,7 @@ abstract class AlignAssignmentsSuiteBase extends
AnalysisTest {
protected def parseAndResolve(query: String): LogicalPlan = {
val analyzer = new CustomAnalyzer(catalogManager) {
override val extendedResolutionRules: Seq[Rule[LogicalPlan]] = Seq(
- new ResolveSessionCatalog(catalogManager))
+ new ResolveSessionCatalog(this.catalogManager))
}
val analyzed = analyzer.execute(CatalystSqlParser.parsePlan(query))
analyzer.checkAnalysis(analyzed)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
index 4eb65305de8..e39cc91d5f0 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
@@ -233,7 +233,7 @@ class PlanResolutionSuite extends AnalysisTest {
}
val analyzer = new Analyzer(catalogManager) {
override val extendedResolutionRules: Seq[Rule[LogicalPlan]] = Seq(
- new ResolveSessionCatalog(catalogManager))
+ new ResolveSessionCatalog(this.catalogManager))
}
// We don't check analysis here by default, as we expect the plan to be
unresolved
// such as `CreateTable`.
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
index 2d0bcdff071..0b5e98d0a3e 100644
---
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
+++
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
@@ -123,7 +123,7 @@ class HiveSessionStateBuilder(
*/
override protected def planner: SparkPlanner = {
new SparkPlanner(session, experimentalMethods) with HiveStrategies {
- override val sparkSession: SparkSession = session
+ override val sparkSession: SparkSession = this.session
override def extraPlanningStrategies: Seq[Strategy] =
super.extraPlanningStrategies ++ customPlanningStrategies ++
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
