nsivabalan commented on code in PR #10251:
URL: https://github.com/apache/hudi/pull/10251#discussion_r1417816154
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMetadataTableWithSparkDataSource.scala:
##########
@@ -104,9 +127,90 @@ class TestMetadataTableWithSparkDataSource extends
SparkClientFunctionalTestHarn
// Column Stats Index partition of MT
val colStatsDF =
spark.read.format(hudi).load(s"$basePath/.hoodie/metadata/column_stats")
+ // Smoke test
+ colStatsDF.show()
+
+ // lets pick one data file and validate col stats
+ val partitionPathToTest = "2015/03/16"
+ val engineContext = new HoodieSparkEngineContext(jsc())
+ val metadataConfig =
HoodieMetadataConfig.newBuilder().enable(true).withMetadataIndexColumnStats(true).build();
+ val baseTableMetada : HoodieTableMetadata = new
HoodieBackedTableMetadata(engineContext, metadataConfig, s"$basePath", false)
+
+ val fileStatuses = baseTableMetada.getAllFilesInPartition(new
Path(s"$basePath/" + partitionPathToTest))
+ val fileName = fileStatuses.apply(0).getPath.getName
+
+ val partitionFileNamePair :
java.util.List[org.apache.hudi.common.util.collection.Pair[String, String]] =
new util.ArrayList
+
partitionFileNamePair.add(org.apache.hudi.common.util.collection.Pair.of(partitionPathToTest,fileName))
+
+ val colStatsRecords =
baseTableMetada.getColumnStats(partitionFileNamePair, "begin_lat")
+ assertEquals(colStatsRecords.size(), 1)
+ val metadataColStats = colStatsRecords.get(partitionFileNamePair.get(0))
+
+ // read parquet file and verify stats
+ val colRangeMetadataList:
java.util.List[HoodieColumnRangeMetadata[Comparable[_]]] = new ParquetUtils()
+ .readRangeFromParquetMetadata(jsc().hadoopConfiguration(),
fileStatuses.apply(0).getPath, Collections.singletonList("begin_lat"))
+ val columnRangeMetadata = colRangeMetadataList.get(0)
+
+ assertEquals(metadataColStats.getValueCount,
columnRangeMetadata.getValueCount)
+ assertEquals(metadataColStats.getTotalSize,
columnRangeMetadata.getTotalSize)
+
assertEquals(HoodieAvroUtils.unwrapAvroValueWrapper(metadataColStats.getMaxValue),
columnRangeMetadata.getMaxValue)
+
assertEquals(HoodieAvroUtils.unwrapAvroValueWrapper(metadataColStats.getMinValue),
columnRangeMetadata.getMinValue)
+ assertEquals(metadataColStats.getFileName, fileName)
+ }
+
+ private def validateUnPartitionedTable(basePath: String) : Unit = {
+ // Files partition of MT
+ val filesPartitionDF =
spark.read.format(hudi).load(s"$basePath/.hoodie/metadata/files")
+ // Smoke test
+ filesPartitionDF.show()
+ // Query w/ 0 requested columns should be working fine
+ assertEquals(2, filesPartitionDF.count())
+
+ val expectedKeys = Seq(".", "__all_partitions__")
Review Comment:
fixed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
