This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.3 by this push: new a52a245d11c [SPARK-38786][SQL][TEST] Bug in StatisticsSuite 'change stats after add/drop partition command' a52a245d11c is described below commit a52a245d11c20b0360d463c973388f3ee05768ac Author: Kazuyuki Tanimura <ktanim...@apple.com> AuthorDate: Tue Apr 5 17:49:44 2022 -0700 [SPARK-38786][SQL][TEST] Bug in StatisticsSuite 'change stats after add/drop partition command' ### What changes were proposed in this pull request? https://github.com/apache/spark/blob/cbffc12f90e45d33e651e38cf886d7ab4bcf96da/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala#L979 It should be `partDir2` instead of `partDir1`. Looks like it is a copy paste bug. ### Why are the changes needed? Due to this test bug, the drop command was dropping a wrong (`partDir1`) underlying file in the test. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Added extra underlying file location check. Closes #36075 from kazuyukitanimura/SPARK-38786. Authored-by: Kazuyuki Tanimura <ktanim...@apple.com> Signed-off-by: Chao Sun <sunc...@apple.com> (cherry picked from commit a6b04f007c07fe00637aa8be33a56f247a494110) Signed-off-by: Dongjoon Hyun <dongj...@apache.org> --- .../src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala index 8afdd50b0c9..46acc9b2f0a 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala @@ -976,7 +976,7 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto s""" |ALTER TABLE $table ADD |PARTITION (ds='2008-04-09', hr='11') LOCATION '${partDir1.toURI.toString}' - |PARTITION (ds='2008-04-09', hr='12') LOCATION '${partDir1.toURI.toString}' + |PARTITION (ds='2008-04-09', hr='12') LOCATION '${partDir2.toURI.toString}' """.stripMargin) if (autoUpdate) { val fetched2 = checkTableStats(table, hasSizeInBytes = true, expectedRowCounts = None) @@ -999,6 +999,7 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto sql(s"ALTER TABLE $table DROP PARTITION (ds='2008-04-08'), PARTITION (hr='12')") assert(spark.sessionState.catalog.listPartitions(TableIdentifier(table)) .map(_.spec).toSet == Set(Map("ds" -> "2008-04-09", "hr" -> "11"))) + assert(partDir1.exists()) // only one partition left if (autoUpdate) { val fetched4 = checkTableStats(table, hasSizeInBytes = true, expectedRowCounts = None) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org