HeartSaVioR commented on code in PR #52642:
URL: https://github.com/apache/spark/pull/52642#discussion_r2462487175
##########
sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala:
##########
@@ -1299,37 +1299,6 @@ class StreamSuite extends StreamTest {
}
}
- test("SPARK-34482: correct active SparkSession for logicalPlan") {
- withSQLConf(
- SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
- SQLConf.COALESCE_PARTITIONS_INITIAL_PARTITION_NUM.key -> "10") {
- val df =
spark.readStream.format(classOf[FakeDefaultSource].getName).load()
- var query: StreamExecution = null
- try {
- query =
- df.repartition($"a")
- .writeStream
- .format("memory")
- .queryName("memory")
- .start()
- .asInstanceOf[StreamingQueryWrapper]
- .streamingQuery
- query.awaitInitialization(streamingTimeout.toMillis)
- val plan = query.logicalPlan
- val numPartition = plan
- .find { _.isInstanceOf[RepartitionByExpression] }
- .map(_.asInstanceOf[RepartitionByExpression].numPartitions)
- // Before the fix of SPARK-34482, the numPartition is the value of
- // `COALESCE_PARTITIONS_INITIAL_PARTITION_NUM`.
- assert(numPartition.get ===
spark.sessionState.conf.getConf(SQLConf.SHUFFLE_PARTITIONS))
Review Comment:
Yes, the query is now eligible for AQE. So we are going back to the time
where we didn't fix SPARK-34482.
> // Before the fix of SPARK-34482, the numPartition is the value of
> // `COALESCE_PARTITIONS_INITIAL_PARTITION_NUM`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
