This is an automated email from the ASF dual-hosted git repository. yihua pushed a commit to branch branch-0.x in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 3a0be9cafe9295898b501556cdba5dd8b1debb7f Author: Balaji Varadarajan <[email protected]> AuthorDate: Wed May 15 02:07:16 2024 -0700 [HUDI-7674] Fix Hudi CLI Command "metadata validate-files" to use file listing to validate (#11100) Co-authored-by: Balaji Varadarajan <[email protected]> --- .../java/org/apache/hudi/cli/commands/MetadataCommand.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MetadataCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MetadataCommand.java index b9165c744b3..b9138b14a9f 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MetadataCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MetadataCommand.java @@ -24,6 +24,7 @@ import org.apache.hudi.cli.TableHeader; import org.apache.hudi.cli.utils.SparkUtil; import org.apache.hudi.client.common.HoodieSparkEngineContext; import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.config.SerializableConfiguration; import org.apache.hudi.common.engine.HoodieLocalEngineContext; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.util.HoodieTimer; @@ -31,6 +32,7 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.metadata.FileSystemBackedTableMetadata; import org.apache.hudi.metadata.HoodieBackedTableMetadata; import org.apache.hudi.metadata.HoodieTableMetadata; import org.apache.hudi.metadata.HoodieTableMetadataUtil; @@ -223,7 +225,7 @@ public class MetadataCommand { HoodieTimer timer = HoodieTimer.start(); List<String> partitions = metadata.getAllPartitionPaths(); - LOG.debug("Took " + timer.endTimer() + " ms"); + LOG.debug("Metadata Partition listing took " + timer.endTimer() + " ms"); final List<Comparable[]> rows = new ArrayList<>(); partitions.stream().sorted(Comparator.reverseOrder()).forEach(p -> { @@ -275,7 +277,8 @@ public class MetadataCommand { @ShellMethod(key = "metadata validate-files", value = "Validate all files in all partitions from the metadata") public String validateFiles( - @ShellOption(value = {"--verbose"}, help = "Print all file details", defaultValue = "false") final boolean verbose) throws IOException { + @ShellOption(value = {"--verbose"}, help = "Print all file details", defaultValue = "false") final boolean verbose) + throws IOException { HoodieCLI.getTableMetaClient(); HoodieMetadataConfig config = HoodieMetadataConfig.newBuilder().enable(true).build(); HoodieBackedTableMetadata metadataReader = new HoodieBackedTableMetadata( @@ -285,13 +288,14 @@ public class MetadataCommand { return "[ERROR] Metadata Table not enabled/initialized\n\n"; } + FileSystemBackedTableMetadata fsMetaReader = new FileSystemBackedTableMetadata(new HoodieLocalEngineContext(HoodieCLI.conf), + HoodieCLI.getTableMetaClient().getTableConfig(), new SerializableConfiguration(HoodieCLI.conf), + HoodieCLI.basePath, false); HoodieMetadataConfig fsConfig = HoodieMetadataConfig.newBuilder().enable(false).build(); - HoodieBackedTableMetadata fsMetaReader = new HoodieBackedTableMetadata( - new HoodieLocalEngineContext(HoodieCLI.conf), fsConfig, HoodieCLI.basePath); HoodieTimer timer = HoodieTimer.start(); List<String> metadataPartitions = metadataReader.getAllPartitionPaths(); - LOG.debug("Listing partitions Took " + timer.endTimer() + " ms"); + LOG.debug("Metadata Listing partitions Took " + timer.endTimer() + " ms"); List<String> fsPartitions = fsMetaReader.getAllPartitionPaths(); Collections.sort(fsPartitions); Collections.sort(metadataPartitions);
