voonhous commented on code in PR #18961:
URL: https://github.com/apache/hudi/pull/18961#discussion_r3843800420
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/schema/TestVariantDataType.scala:
##########
@@ -1232,6 +1455,32 @@ class TestVariantDataType extends HoodieSparkSqlTestBase
{
}
}
+ /**
+ * Sums the non-null value counts of the leaf columns under
`column`.typed_value across all
+ * blocks of the file, from the block column statistics.
+ */
+ private def typedValueNonNullCount(filePath: String, column: String): Long =
{
+ val conf = spark.sparkContext.hadoopConfiguration
+ val inputFile = HadoopInputFile.fromPath(new HadoopPath(filePath), conf)
+ val reader = ParquetFileReader.open(inputFile)
+ try {
+ val prefix = s"$column.typed_value"
+ reader.getFooter.getBlocks.asScala.flatMap(_.getColumns.asScala)
+ .filter { c =>
+ val dot = c.getPath.toDotString
+ dot == prefix || dot.startsWith(prefix + ".")
Review Comment:
Excluded leaves ending in `.value`; the count is on the typed leaves only.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
