RS131419 commented on code in PR #37230:
URL: https://github.com/apache/spark/pull/37230#discussion_r932792260
##########
sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala:
##########
@@ -1611,4 +1611,26 @@ class StatisticsSuite extends
StatisticsCollectionTestBase with TestHiveSingleto
}
}
}
+
+ test("SPARK-33326: partition metadata auto update for dynamic partitions") {
+ val table = "partition_metadata_dynamic_partition"
+ Seq("hive", "parquet").foreach { source =>
+ withSQLConf(SQLConf.AUTO_SIZE_UPDATE_ENABLED.key -> "true") {
+ withTable(table) {
+ sql(s"CREATE TABLE $table (id INT, sp INT, dp INT) USING $source
PARTITIONED BY (sp, dp)")
+ sql(s"INSERT INTO $table PARTITION (sp=0, dp) VALUES (0, 0)")
+ sql(s"INSERT OVERWRITE TABLE $table PARTITION (sp=0, dp) SELECT id,
id FROM range(5)")
+
+ for (i <- 0 until 5) {
+ val partition = spark.sessionState.catalog
+ .getPartition(TableIdentifier(table), Map("sp" -> s"0", "dp" ->
s"$i"))
+ val numFiles = partition.parameters("numFiles")
+ assert(numFiles.nonEmpty && numFiles.toInt > 0)
+ val totalSize = partition.parameters("totalSize")
Review Comment:
Thanks for the reply! This patch will provide the calculated rawDataSize to
hive via the alterPartitions method, but ultimately it fails to take effect,
which I think may be due to hive's own behavior. Again, I did the test via
spark2.3 and the rawDataSize was not updated, so I didn't do the verification
in the unit test.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
