(hudi) branch master updated: [HUDI-7761] Make the ManifestWriter Extendable (#11253)

[danny0405]

This is an automated email from the ASF dual-hosted git repository.

danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 0804a79f20c [HUDI-7761] Make the ManifestWriter Extendable (#11253)
0804a79f20c is described below

commit 0804a79f20c680b1d369c5e84d7c9ae4412ad472
Author: csivaguru <csivag...@gmail.com>
AuthorDate: Sat May 18 17:35:18 2024 -0700

    [HUDI-7761] Make the ManifestWriter Extendable (#11253)
---
 .../java/org/apache/hudi/gcp/bigquery/BigQuerySchemaResolver.java  | 4 ++--
 .../java/org/apache/hudi/sync/common/util/ManifestFileWriter.java  | 7 +++----
 .../org/apache/hudi/sync/common/util/TestManifestFileWriter.java   | 3 +--
 3 files changed, 6 insertions(+), 8 deletions(-)

diff --git 
a/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySchemaResolver.java
 
b/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySchemaResolver.java
index 361f869a9fe..27b5143944f 100644
--- 
a/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySchemaResolver.java
+++ 
b/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySchemaResolver.java
@@ -38,7 +38,7 @@ import java.util.stream.Collectors;
 /**
  * Extracts the BigQuery schema from a Hudi table.
  */
-class BigQuerySchemaResolver {
+public class BigQuerySchemaResolver {
   private static final BigQuerySchemaResolver INSTANCE = new 
BigQuerySchemaResolver(TableSchemaResolver::new);
 
   private final Function<HoodieTableMetaClient, TableSchemaResolver> 
tableSchemaResolverSupplier;
@@ -48,7 +48,7 @@ class BigQuerySchemaResolver {
     this.tableSchemaResolverSupplier = tableSchemaResolverSupplier;
   }
 
-  static BigQuerySchemaResolver getInstance() {
+  public static BigQuerySchemaResolver getInstance() {
     return INSTANCE;
   }
 
diff --git 
a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/ManifestFileWriter.java
 
b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/ManifestFileWriter.java
index 3eaf80dddfe..eb52fe4586d 100644
--- 
a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/ManifestFileWriter.java
+++ 
b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/ManifestFileWriter.java
@@ -54,7 +54,7 @@ public class ManifestFileWriter {
   private final HoodieTableMetaClient metaClient;
   private final boolean useFileListingFromMetadata;
 
-  private ManifestFileWriter(HoodieTableMetaClient metaClient, boolean 
useFileListingFromMetadata) {
+  protected ManifestFileWriter(HoodieTableMetaClient metaClient, boolean 
useFileListingFromMetadata) {
     this.metaClient = metaClient;
     this.useFileListingFromMetadata = useFileListingFromMetadata;
   }
@@ -64,7 +64,7 @@ public class ManifestFileWriter {
    */
   public synchronized void writeManifestFile(boolean useAbsolutePath) {
     try {
-      List<String> baseFiles = 
fetchLatestBaseFilesForAllPartitions(metaClient, useFileListingFromMetadata, 
useAbsolutePath)
+      List<String> baseFiles = 
fetchLatestBaseFilesForAllPartitions(useAbsolutePath)
           .collect(Collectors.toList());
       if (baseFiles.isEmpty()) {
         LOG.warn("No base file to generate manifest file.");
@@ -86,8 +86,7 @@ public class ManifestFileWriter {
   }
 
   @VisibleForTesting
-  public static Stream<String> 
fetchLatestBaseFilesForAllPartitions(HoodieTableMetaClient metaClient,
-      boolean useFileListingFromMetadata, boolean useAbsolutePath) {
+  public Stream<String> fetchLatestBaseFilesForAllPartitions(boolean 
useAbsolutePath) {
     try {
       StorageConfiguration storageConf = metaClient.getStorageConf();
       HoodieLocalEngineContext engContext = new 
HoodieLocalEngineContext(storageConf);
diff --git 
a/hudi-sync/hudi-sync-common/src/test/java/org/apache/hudi/sync/common/util/TestManifestFileWriter.java
 
b/hudi-sync/hudi-sync-common/src/test/java/org/apache/hudi/sync/common/util/TestManifestFileWriter.java
index 2db602b65c2..13caefd1d0f 100644
--- 
a/hudi-sync/hudi-sync-common/src/test/java/org/apache/hudi/sync/common/util/TestManifestFileWriter.java
+++ 
b/hudi-sync/hudi-sync-common/src/test/java/org/apache/hudi/sync/common/util/TestManifestFileWriter.java
@@ -34,7 +34,6 @@ import java.util.List;
 import java.util.stream.IntStream;
 
 import static 
org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS;
-import static 
org.apache.hudi.sync.common.util.ManifestFileWriter.fetchLatestBaseFilesForAllPartitions;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -51,7 +50,7 @@ public class TestManifestFileWriter extends 
HoodieCommonTestHarness {
     // Generate 10 files under each partition
     createTestDataForPartitionedTable(metaClient, 10);
     ManifestFileWriter manifestFileWriter = 
ManifestFileWriter.builder().setMetaClient(metaClient).build();
-    assertEquals(30, fetchLatestBaseFilesForAllPartitions(metaClient, false, 
false).count());
+    assertEquals(30, 
manifestFileWriter.fetchLatestBaseFilesForAllPartitions(false).count());
   }
 
   @Test