the-other-tim-brown commented on code in PR #818:
URL: https://github.com/apache/incubator-xtable/pull/818#discussion_r3037522228
##########
xtable-core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java:
##########
@@ -151,20 +158,45 @@ private Stream<InternalDataFile>
computeColumnStatsFromMetadataTable(
Map.Entry::getKey,
Collectors.mapping(
Map.Entry::getValue,
CustomCollectors.toList(nameFieldMap.size()))));
- return filePathsToDataFile.entrySet().stream()
- .map(
- pathToDataFile -> {
- Pair<String, String> filePath = pathToDataFile.getKey();
- InternalDataFile file = pathToDataFile.getValue();
- List<Pair<InternalField, HoodieMetadataColumnStats>> fileStats =
- stats.getOrDefault(filePath, Collections.emptyList());
- List<ColumnStat> columnStats =
- fileStats.stream()
- .map(pair -> getColumnStatFromHudiStat(pair.getLeft(),
pair.getRight()))
- .collect(CustomCollectors.toList(fileStats.size()));
- long recordCount = getMaxFromColumnStats(columnStats).orElse(0L);
- return
file.toBuilder().columnStats(columnStats).recordCount(recordCount).build();
- });
+ List<InternalDataFile> withStats = new ArrayList<>();
+ List<InternalDataFile> filesWithoutStats = new ArrayList<>();
+ for (Map.Entry<Pair<String, String>, InternalDataFile> pathToDataFile :
+ filePathsToDataFile.entrySet()) {
+ classifyFileByMetadataStats(pathToDataFile, stats, withStats,
filesWithoutStats);
+ }
+ if (!filesWithoutStats.isEmpty()) {
+ log.warn(
+ "{} file(s) had no column stats in the metadata table for table {};
falling back to parquet footers",
+ filesWithoutStats.size(),
+ metaClient.getBasePathV2());
+ withStats.addAll(
+ computeColumnStatsFromParquetFooters(filesWithoutStats.stream(),
parquetNameFieldMap)
+ .collect(Collectors.toList()));
+ }
+ return withStats.stream();
+ }
+
+ private void classifyFileByMetadataStats(
+ Map.Entry<Pair<String, String>, InternalDataFile> pathToDataFile,
+ Map<Pair<String, String>, List<Pair<InternalField,
HoodieMetadataColumnStats>>> stats,
+ List<InternalDataFile> withStats,
+ List<InternalDataFile> filesWithoutStats) {
+ Pair<String, String> filePath = pathToDataFile.getKey();
+ InternalDataFile file = pathToDataFile.getValue();
+ List<Pair<InternalField, HoodieMetadataColumnStats>> fileStats =
+ stats.getOrDefault(filePath, Collections.emptyList());
+ List<ColumnStat> columnStats =
+ fileStats.stream()
+ .map(pair -> getColumnStatFromHudiStat(pair.getLeft(),
pair.getRight()))
+ .collect(CustomCollectors.toList(fileStats.size()));
+ long recordCount = getMaxFromColumnStats(columnStats).orElse(0L);
+ InternalDataFile result =
+
file.toBuilder().columnStats(columnStats).recordCount(recordCount).build();
Review Comment:
If there are no stats, we should avoid the toBuilder and new object creation.
##########
xtable-core/src/test/java/org/apache/xtable/hudi/TestHudiFileStatsExtractor.java:
##########
@@ -199,6 +209,131 @@ void columnStatsWithoutMetadataTable(@TempDir Path
tempDir) throws IOException {
validateOutput(output);
}
+ @Test
+ void columnStatsWithMetadataTableMissingFallsBackToParquetFooters(@TempDir
Path tempDir) {
+ List<InternalDataFile> inputFiles = generateInputFiles(tempDir, 1);
+
+ HoodieTableConfig mockTableConfig = mock(HoodieTableConfig.class);
+
when(mockTableConfig.isMetadataPartitionAvailable(MetadataPartitionType.COLUMN_STATS))
+ .thenReturn(true);
+
+ HoodieTableMetaClient mockMetaClient = mock(HoodieTableMetaClient.class);
+ when(mockMetaClient.getHadoopConf()).thenReturn(configuration);
+ when(mockMetaClient.getBasePathV2()).thenReturn(new
org.apache.hadoop.fs.Path(tempDir.toUri()));
+ when(mockMetaClient.getTableConfig()).thenReturn(mockTableConfig);
+
+ HoodieTableMetadata mockMetadataTable = mock(HoodieTableMetadata.class);
+ when(mockMetadataTable.getColumnStats(any(),
anyString())).thenReturn(Collections.emptyMap());
+
+ HudiFileStatsExtractor extractor = new
HudiFileStatsExtractor(mockMetaClient);
+ List<InternalDataFile> output =
+ extractor
+ .addStatsToFiles(mockMetadataTable, inputFiles.stream(), schema)
+ .collect(Collectors.toList());
+
+ validateOutput(output);
+ }
+
+ @Test
+ void
columnStatsWithMetadataTablePartialMissingFallsBackToParquetFooters(@TempDir
Path tempDir) {
+ List<InternalDataFile> inputFiles = generateInputFiles(tempDir, 2);
+ InternalDataFile fileWithStats = inputFiles.get(0);
+ InternalDataFile fileWithoutStats = inputFiles.get(1);
+
+ Pair<String, String> fileWithStatsPair =
+ Pair.of("", new
org.apache.hadoop.fs.Path(fileWithStats.getPhysicalPath()).getName());
+
+ HoodieTableConfig mockTableConfig = mock(HoodieTableConfig.class);
+
when(mockTableConfig.isMetadataPartitionAvailable(MetadataPartitionType.COLUMN_STATS))
+ .thenReturn(true);
+
+ HoodieTableMetaClient mockMetaClient = mock(HoodieTableMetaClient.class);
+ when(mockMetaClient.getHadoopConf()).thenReturn(configuration);
+ when(mockMetaClient.getBasePathV2()).thenReturn(new
org.apache.hadoop.fs.Path(tempDir.toUri()));
+ when(mockMetaClient.getTableConfig()).thenReturn(mockTableConfig);
+
+ // Metadata table only returns stats for fileWithStats; fileWithoutStats
is missing entirely
+ HoodieTableMetadata mockMetadataTable = mock(HoodieTableMetadata.class);
+ when(mockMetadataTable.getColumnStats(any(), anyString()))
+ .thenAnswer(
+ invocation -> {
+ String fieldName = invocation.getArgument(1);
+ if (fieldName.equals("long_field")) {
+ Map<Pair<String, String>, HoodieMetadataColumnStats> statsMap
= new HashMap<>();
+ statsMap.put(
+ fileWithStatsPair,
+ HoodieMetadataColumnStats.newBuilder()
+ .setFileName(fileWithStatsPair.getRight())
+ .setColumnName(fieldName)
+ .setValueCount(2L)
+ .setNullCount(1L)
+ .setTotalSize(16L)
+ .setIsDeleted(false)
+ .build());
+ return statsMap;
+ }
+ return Collections.emptyMap();
+ });
+
+ HudiFileStatsExtractor extractor = new
HudiFileStatsExtractor(mockMetaClient);
+ List<InternalDataFile> output =
+ extractor
+ .addStatsToFiles(mockMetadataTable, inputFiles.stream(), schema)
+ .collect(Collectors.toList());
+
+ assertEquals(2, output.size());
+ // fileWithoutStats must have fallen back to parquet footers and have full
stats
+ InternalDataFile fallen =
Review Comment:
Can we call this `fromFooter` to match `fromMeta`?
##########
xtable-core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java:
##########
@@ -151,20 +158,45 @@ private Stream<InternalDataFile>
computeColumnStatsFromMetadataTable(
Map.Entry::getKey,
Collectors.mapping(
Map.Entry::getValue,
CustomCollectors.toList(nameFieldMap.size()))));
- return filePathsToDataFile.entrySet().stream()
- .map(
- pathToDataFile -> {
- Pair<String, String> filePath = pathToDataFile.getKey();
- InternalDataFile file = pathToDataFile.getValue();
- List<Pair<InternalField, HoodieMetadataColumnStats>> fileStats =
- stats.getOrDefault(filePath, Collections.emptyList());
- List<ColumnStat> columnStats =
- fileStats.stream()
- .map(pair -> getColumnStatFromHudiStat(pair.getLeft(),
pair.getRight()))
- .collect(CustomCollectors.toList(fileStats.size()));
- long recordCount = getMaxFromColumnStats(columnStats).orElse(0L);
- return
file.toBuilder().columnStats(columnStats).recordCount(recordCount).build();
- });
+ List<InternalDataFile> withStats = new ArrayList<>();
+ List<InternalDataFile> filesWithoutStats = new ArrayList<>();
+ for (Map.Entry<Pair<String, String>, InternalDataFile> pathToDataFile :
+ filePathsToDataFile.entrySet()) {
+ classifyFileByMetadataStats(pathToDataFile, stats, withStats,
filesWithoutStats);
+ }
+ if (!filesWithoutStats.isEmpty()) {
+ log.warn(
+ "{} file(s) had no column stats in the metadata table for table {};
falling back to parquet footers",
+ filesWithoutStats.size(),
+ metaClient.getBasePathV2());
+ withStats.addAll(
+ computeColumnStatsFromParquetFooters(filesWithoutStats.stream(),
parquetNameFieldMap)
+ .collect(Collectors.toList()));
+ }
+ return withStats.stream();
+ }
+
+ private void classifyFileByMetadataStats(
Review Comment:
I think it would be better to update the flow here to have a method that
returns an `Optional<InternalDataFile>` and then you use that to add to the
lists in the other method. I think this will make the code a bit easier to
follow.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]