the-other-tim-brown commented on code in PR #818:
URL: https://github.com/apache/incubator-xtable/pull/818#discussion_r3037522228


##########
xtable-core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java:
##########
@@ -151,20 +158,45 @@ private Stream<InternalDataFile> 
computeColumnStatsFromMetadataTable(
                     Map.Entry::getKey,
                     Collectors.mapping(
                         Map.Entry::getValue, 
CustomCollectors.toList(nameFieldMap.size()))));
-    return filePathsToDataFile.entrySet().stream()
-        .map(
-            pathToDataFile -> {
-              Pair<String, String> filePath = pathToDataFile.getKey();
-              InternalDataFile file = pathToDataFile.getValue();
-              List<Pair<InternalField, HoodieMetadataColumnStats>> fileStats =
-                  stats.getOrDefault(filePath, Collections.emptyList());
-              List<ColumnStat> columnStats =
-                  fileStats.stream()
-                      .map(pair -> getColumnStatFromHudiStat(pair.getLeft(), 
pair.getRight()))
-                      .collect(CustomCollectors.toList(fileStats.size()));
-              long recordCount = getMaxFromColumnStats(columnStats).orElse(0L);
-              return 
file.toBuilder().columnStats(columnStats).recordCount(recordCount).build();
-            });
+    List<InternalDataFile> withStats = new ArrayList<>();
+    List<InternalDataFile> filesWithoutStats = new ArrayList<>();
+    for (Map.Entry<Pair<String, String>, InternalDataFile> pathToDataFile :
+        filePathsToDataFile.entrySet()) {
+      classifyFileByMetadataStats(pathToDataFile, stats, withStats, 
filesWithoutStats);
+    }
+    if (!filesWithoutStats.isEmpty()) {
+      log.warn(
+          "{} file(s) had no column stats in the metadata table for table {}; 
falling back to parquet footers",
+          filesWithoutStats.size(),
+          metaClient.getBasePathV2());
+      withStats.addAll(
+          computeColumnStatsFromParquetFooters(filesWithoutStats.stream(), 
parquetNameFieldMap)
+              .collect(Collectors.toList()));
+    }
+    return withStats.stream();
+  }
+
+  private void classifyFileByMetadataStats(
+      Map.Entry<Pair<String, String>, InternalDataFile> pathToDataFile,
+      Map<Pair<String, String>, List<Pair<InternalField, 
HoodieMetadataColumnStats>>> stats,
+      List<InternalDataFile> withStats,
+      List<InternalDataFile> filesWithoutStats) {
+    Pair<String, String> filePath = pathToDataFile.getKey();
+    InternalDataFile file = pathToDataFile.getValue();
+    List<Pair<InternalField, HoodieMetadataColumnStats>> fileStats =
+        stats.getOrDefault(filePath, Collections.emptyList());
+    List<ColumnStat> columnStats =
+        fileStats.stream()
+            .map(pair -> getColumnStatFromHudiStat(pair.getLeft(), 
pair.getRight()))
+            .collect(CustomCollectors.toList(fileStats.size()));
+    long recordCount = getMaxFromColumnStats(columnStats).orElse(0L);
+    InternalDataFile result =
+        
file.toBuilder().columnStats(columnStats).recordCount(recordCount).build();

Review Comment:
   If there are no stats, we should avoid the toBuilder and new object creation.



##########
xtable-core/src/test/java/org/apache/xtable/hudi/TestHudiFileStatsExtractor.java:
##########
@@ -199,6 +209,131 @@ void columnStatsWithoutMetadataTable(@TempDir Path 
tempDir) throws IOException {
     validateOutput(output);
   }
 
+  @Test
+  void columnStatsWithMetadataTableMissingFallsBackToParquetFooters(@TempDir 
Path tempDir) {
+    List<InternalDataFile> inputFiles = generateInputFiles(tempDir, 1);
+
+    HoodieTableConfig mockTableConfig = mock(HoodieTableConfig.class);
+    
when(mockTableConfig.isMetadataPartitionAvailable(MetadataPartitionType.COLUMN_STATS))
+        .thenReturn(true);
+
+    HoodieTableMetaClient mockMetaClient = mock(HoodieTableMetaClient.class);
+    when(mockMetaClient.getHadoopConf()).thenReturn(configuration);
+    when(mockMetaClient.getBasePathV2()).thenReturn(new 
org.apache.hadoop.fs.Path(tempDir.toUri()));
+    when(mockMetaClient.getTableConfig()).thenReturn(mockTableConfig);
+
+    HoodieTableMetadata mockMetadataTable = mock(HoodieTableMetadata.class);
+    when(mockMetadataTable.getColumnStats(any(), 
anyString())).thenReturn(Collections.emptyMap());
+
+    HudiFileStatsExtractor extractor = new 
HudiFileStatsExtractor(mockMetaClient);
+    List<InternalDataFile> output =
+        extractor
+            .addStatsToFiles(mockMetadataTable, inputFiles.stream(), schema)
+            .collect(Collectors.toList());
+
+    validateOutput(output);
+  }
+
+  @Test
+  void 
columnStatsWithMetadataTablePartialMissingFallsBackToParquetFooters(@TempDir 
Path tempDir) {
+    List<InternalDataFile> inputFiles = generateInputFiles(tempDir, 2);
+    InternalDataFile fileWithStats = inputFiles.get(0);
+    InternalDataFile fileWithoutStats = inputFiles.get(1);
+
+    Pair<String, String> fileWithStatsPair =
+        Pair.of("", new 
org.apache.hadoop.fs.Path(fileWithStats.getPhysicalPath()).getName());
+
+    HoodieTableConfig mockTableConfig = mock(HoodieTableConfig.class);
+    
when(mockTableConfig.isMetadataPartitionAvailable(MetadataPartitionType.COLUMN_STATS))
+        .thenReturn(true);
+
+    HoodieTableMetaClient mockMetaClient = mock(HoodieTableMetaClient.class);
+    when(mockMetaClient.getHadoopConf()).thenReturn(configuration);
+    when(mockMetaClient.getBasePathV2()).thenReturn(new 
org.apache.hadoop.fs.Path(tempDir.toUri()));
+    when(mockMetaClient.getTableConfig()).thenReturn(mockTableConfig);
+
+    // Metadata table only returns stats for fileWithStats; fileWithoutStats 
is missing entirely
+    HoodieTableMetadata mockMetadataTable = mock(HoodieTableMetadata.class);
+    when(mockMetadataTable.getColumnStats(any(), anyString()))
+        .thenAnswer(
+            invocation -> {
+              String fieldName = invocation.getArgument(1);
+              if (fieldName.equals("long_field")) {
+                Map<Pair<String, String>, HoodieMetadataColumnStats> statsMap 
= new HashMap<>();
+                statsMap.put(
+                    fileWithStatsPair,
+                    HoodieMetadataColumnStats.newBuilder()
+                        .setFileName(fileWithStatsPair.getRight())
+                        .setColumnName(fieldName)
+                        .setValueCount(2L)
+                        .setNullCount(1L)
+                        .setTotalSize(16L)
+                        .setIsDeleted(false)
+                        .build());
+                return statsMap;
+              }
+              return Collections.emptyMap();
+            });
+
+    HudiFileStatsExtractor extractor = new 
HudiFileStatsExtractor(mockMetaClient);
+    List<InternalDataFile> output =
+        extractor
+            .addStatsToFiles(mockMetadataTable, inputFiles.stream(), schema)
+            .collect(Collectors.toList());
+
+    assertEquals(2, output.size());
+    // fileWithoutStats must have fallen back to parquet footers and have full 
stats
+    InternalDataFile fallen =

Review Comment:
   Can we call this `fromFooter` to match `fromMeta`?



##########
xtable-core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java:
##########
@@ -151,20 +158,45 @@ private Stream<InternalDataFile> 
computeColumnStatsFromMetadataTable(
                     Map.Entry::getKey,
                     Collectors.mapping(
                         Map.Entry::getValue, 
CustomCollectors.toList(nameFieldMap.size()))));
-    return filePathsToDataFile.entrySet().stream()
-        .map(
-            pathToDataFile -> {
-              Pair<String, String> filePath = pathToDataFile.getKey();
-              InternalDataFile file = pathToDataFile.getValue();
-              List<Pair<InternalField, HoodieMetadataColumnStats>> fileStats =
-                  stats.getOrDefault(filePath, Collections.emptyList());
-              List<ColumnStat> columnStats =
-                  fileStats.stream()
-                      .map(pair -> getColumnStatFromHudiStat(pair.getLeft(), 
pair.getRight()))
-                      .collect(CustomCollectors.toList(fileStats.size()));
-              long recordCount = getMaxFromColumnStats(columnStats).orElse(0L);
-              return 
file.toBuilder().columnStats(columnStats).recordCount(recordCount).build();
-            });
+    List<InternalDataFile> withStats = new ArrayList<>();
+    List<InternalDataFile> filesWithoutStats = new ArrayList<>();
+    for (Map.Entry<Pair<String, String>, InternalDataFile> pathToDataFile :
+        filePathsToDataFile.entrySet()) {
+      classifyFileByMetadataStats(pathToDataFile, stats, withStats, 
filesWithoutStats);
+    }
+    if (!filesWithoutStats.isEmpty()) {
+      log.warn(
+          "{} file(s) had no column stats in the metadata table for table {}; 
falling back to parquet footers",
+          filesWithoutStats.size(),
+          metaClient.getBasePathV2());
+      withStats.addAll(
+          computeColumnStatsFromParquetFooters(filesWithoutStats.stream(), 
parquetNameFieldMap)
+              .collect(Collectors.toList()));
+    }
+    return withStats.stream();
+  }
+
+  private void classifyFileByMetadataStats(

Review Comment:
   I think it would be better to update the flow here to have a method that 
returns an `Optional<InternalDataFile>` and then you use that to add to the 
lists in the other method. I think this will make the code a bit easier to 
follow. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to