This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new a52a245d11c [SPARK-38786][SQL][TEST] Bug in StatisticsSuite 'change
stats after add/drop partition command'
a52a245d11c is described below
commit a52a245d11c20b0360d463c973388f3ee05768ac
Author: Kazuyuki Tanimura <[email protected]>
AuthorDate: Tue Apr 5 17:49:44 2022 -0700
[SPARK-38786][SQL][TEST] Bug in StatisticsSuite 'change stats after
add/drop partition command'
### What changes were proposed in this pull request?
https://github.com/apache/spark/blob/cbffc12f90e45d33e651e38cf886d7ab4bcf96da/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala#L979
It should be `partDir2` instead of `partDir1`. Looks like it is a copy
paste bug.
### Why are the changes needed?
Due to this test bug, the drop command was dropping a wrong (`partDir1`)
underlying file in the test.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Added extra underlying file location check.
Closes #36075 from kazuyukitanimura/SPARK-38786.
Authored-by: Kazuyuki Tanimura <[email protected]>
Signed-off-by: Chao Sun <[email protected]>
(cherry picked from commit a6b04f007c07fe00637aa8be33a56f247a494110)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
index 8afdd50b0c9..46acc9b2f0a 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
@@ -976,7 +976,7 @@ class StatisticsSuite extends StatisticsCollectionTestBase
with TestHiveSingleto
s"""
|ALTER TABLE $table ADD
|PARTITION (ds='2008-04-09', hr='11') LOCATION
'${partDir1.toURI.toString}'
- |PARTITION (ds='2008-04-09', hr='12') LOCATION
'${partDir1.toURI.toString}'
+ |PARTITION (ds='2008-04-09', hr='12') LOCATION
'${partDir2.toURI.toString}'
""".stripMargin)
if (autoUpdate) {
val fetched2 = checkTableStats(table, hasSizeInBytes = true,
expectedRowCounts = None)
@@ -999,6 +999,7 @@ class StatisticsSuite extends StatisticsCollectionTestBase
with TestHiveSingleto
sql(s"ALTER TABLE $table DROP PARTITION (ds='2008-04-08'),
PARTITION (hr='12')")
assert(spark.sessionState.catalog.listPartitions(TableIdentifier(table))
.map(_.spec).toSet == Set(Map("ds" -> "2008-04-09", "hr" ->
"11")))
+ assert(partDir1.exists())
// only one partition left
if (autoUpdate) {
val fetched4 = checkTableStats(table, hasSizeInBytes = true,
expectedRowCounts = None)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
