This is an automated email from the ASF dual-hosted git repository. voonhous pushed a commit to branch release-1.2.1 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 7b457efbae56e9ab8bca7bab4a09958003262049 Author: voonhous <[email protected]> AuthorDate: Wed Jun 3 19:43:00 2026 +0800 refactor: Add Lombok annotations to hudi-common module (part 3) (#17825) * refactor: Add Lombok annotations to hudi-common module (part 3) * Address comments * Address bot comments (cherry picked from commit 438bbdfcaef43abc5d0d21f931a47c5af05a58a8) --- .../table/action/clean/CleanActionExecutor.java | 20 +- .../hudi/utils/TestMetadataConversionUtils.java | 14 +- .../functional/TestExternalPathHandling.java | 10 +- .../apache/hudi/io/TestHoodieTimelineArchiver.java | 15 +- .../java/org/apache/hudi/table/TestCleaner.java | 24 ++- .../hudi/testutils/HoodieCleanerTestBase.java | 31 +-- .../testutils/HoodieSparkClientTestHarness.java | 14 +- hudi-common/pom.xml | 1 + .../org/apache/hudi/common/HoodieCleanStat.java | 210 +++------------------ .../hudi/common/HoodiePendingRollbackInfo.java | 18 +- .../org/apache/hudi/common/HoodieRollbackStat.java | 34 +--- .../common/bloom/InternalDynamicBloomFilter.java | 9 +- .../apache/hudi/common/bloom/InternalFilter.java | 7 +- .../java/org/apache/hudi/common/bloom/Key.java | 8 +- .../bootstrap/index/hfile/HFileBootstrapIndex.java | 15 +- .../index/hfile/HFileBootstrapIndexReader.java | 22 +-- .../index/hfile/HFileBootstrapIndexWriter.java | 17 +- .../apache/hudi/common/config/ConfigGroups.java | 20 +- .../apache/hudi/common/config/ConfigProperty.java | 66 ++----- .../apache/hudi/common/config/HoodieConfig.java | 12 +- .../DirectMarkerBasedDetectionStrategy.java | 21 +-- .../hudi/common/data/HoodieBaseListData.java | 16 +- .../hudi/common/engine/HoodieEngineContext.java | 18 +- .../hudi/common/engine/HoodieReaderContext.java | 128 +++---------- .../apache/hudi/common/engine/RecordContext.java | 12 +- .../java/org/apache/hudi/common/fs/FSUtils.java | 9 +- .../hudi/common/fs/FailSafeConsistencyGuard.java | 16 +- .../hudi/common/fs/OptimisticConsistencyGuard.java | 10 +- .../common/heartbeat/HoodieHeartbeatUtils.java | 7 +- .../apache/hudi/common/schema/HoodieSchema.java | 38 +--- .../common/schema/HoodieSchemaCompatibility.java | 8 +- .../hudi/common/schema/HoodieSchemaField.java | 14 +- .../table/view/TestIncrementalFSViewSync.java | 13 +- .../hudi/common/testutils/HoodieTestTable.java | 24 ++- .../HoodieFileGroupReaderBasedFileFormat.scala | 2 +- 35 files changed, 272 insertions(+), 631 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java index 63a6e7d8f994..2a636221ae60 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java @@ -171,20 +171,18 @@ public class CleanActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I, K, ? partitionCleanStatsMap.get(partitionPath) : new PartitionCleanStat(partitionPath); HoodieActionInstant actionInstant = cleanerPlan.getEarliestInstantToRetain(); - return HoodieCleanStat.newBuilder().withPolicy(config.getCleanerPolicy()).withPartitionPath(partitionPath) - .withEarliestCommitRetained(Option.ofNullable( - actionInstant != null - ? instantGenerator.createNewInstant(HoodieInstant.State.valueOf(actionInstant.getState()), - actionInstant.getAction(), actionInstant.getTimestamp()) - : null)) + return HoodieCleanStat.builder() + .withPolicy(config.getCleanerPolicy()) + .withPartitionPath(partitionPath) + .withEarliestCommitToRetain(actionInstant != null ? actionInstant.getTimestamp() : "") .withLastCompletedCommitTimestamp(cleanerPlan.getLastCompletedCommitTimestamp()) - .withDeletePathPattern(partitionCleanStat.deletePathPatterns()) - .withSuccessfulDeletes(partitionCleanStat.successDeleteFiles()) - .withFailedDeletes(partitionCleanStat.failedDeleteFiles()) + .withDeletePathPatterns(partitionCleanStat.deletePathPatterns()) + .withSuccessDeleteFiles(partitionCleanStat.successDeleteFiles()) + .withFailedDeleteFiles(partitionCleanStat.failedDeleteFiles()) .withDeleteBootstrapBasePathPatterns(partitionCleanStat.getDeleteBootstrapBasePathPatterns()) - .withSuccessfulDeleteBootstrapBaseFiles(partitionCleanStat.getSuccessfulDeleteBootstrapBaseFiles()) + .withSuccessDeleteBootstrapBaseFiles(partitionCleanStat.getSuccessfulDeleteBootstrapBaseFiles()) .withFailedDeleteBootstrapBaseFiles(partitionCleanStat.getFailedDeleteBootstrapBaseFiles()) - .isPartitionDeleted(partitionsToBeDeleted.contains(partitionPath)) + .withPartitionDeleted(partitionsToBeDeleted.contains(partitionPath)) .build(); }).collect(Collectors.toList()); } diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestMetadataConversionUtils.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestMetadataConversionUtils.java index 01af9d3fc3ff..c0374b25e1f1 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestMetadataConversionUtils.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestMetadataConversionUtils.java @@ -405,14 +405,12 @@ public class TestMetadataConversionUtils extends HoodieCommonTestHarness { HoodieCleanFileInfo fileInfo = new HoodieCleanFileInfo("file1", false); HoodieCleanerPlan cleanerPlan = new HoodieCleanerPlan(new HoodieActionInstant("", "", ""), "", "", new HashMap<>(), CleanPlanV2MigrationHandler.VERSION, Collections.singletonMap("key", Collections.singletonList(fileInfo)), new ArrayList<>(), Collections.EMPTY_MAP); - HoodieCleanStat cleanStats = new HoodieCleanStat( - HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS, - HoodieTestUtils.DEFAULT_PARTITION_PATHS[new Random().nextInt(HoodieTestUtils.DEFAULT_PARTITION_PATHS.length)], - Collections.emptyList(), - Collections.emptyList(), - Collections.emptyList(), - instantTime, - ""); + HoodieCleanStat cleanStats = HoodieCleanStat.builder() + .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS) + .withPartitionPath(HoodieTestUtils.DEFAULT_PARTITION_PATHS[new Random().nextInt(HoodieTestUtils.DEFAULT_PARTITION_PATHS.length)]) + .withEarliestCommitToRetain(instantTime) + .withLastCompletedCommitTimestamp("") + .build(); HoodieCleanMetadata cleanMetadata = convertCleanMetadata(instantTime, Option.of(0L), Collections.singletonList(cleanStats), Collections.EMPTY_MAP); HoodieTestTable.of(metaClient).addClean(instantTime, cleanerPlan, cleanMetadata); } diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestExternalPathHandling.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestExternalPathHandling.java index ce94187aacbc..5d7c4aac2416 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestExternalPathHandling.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestExternalPathHandling.java @@ -339,8 +339,14 @@ public class TestExternalPathHandling extends HoodieClientTestBase { } private HoodieCleanStat createCleanStat(String partitionPath, List<String> deletePaths, String earliestCommitToRetain, String lastCompletedCommitTimestamp) { - return new HoodieCleanStat(HoodieCleaningPolicy.KEEP_LATEST_COMMITS, partitionPath, deletePaths, deletePaths, Collections.emptyList(), - earliestCommitToRetain, lastCompletedCommitTimestamp); + return HoodieCleanStat.builder() + .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_COMMITS) + .withPartitionPath(partitionPath) + .withDeletePathPatterns(deletePaths) + .withSuccessDeleteFiles(deletePaths) + .withEarliestCommitToRetain(earliestCommitToRetain) + .withLastCompletedCommitTimestamp(lastCompletedCommitTimestamp) + .build(); } private HoodieCleanerPlan cleanerPlan(HoodieActionInstant earliestInstantToRetain, String latestCommit, Map<String, List<HoodieCleanFileInfo>> filePathsToBeDeletedPerPartition) { diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java index 60db8b8a13b7..5725c571c8dc 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java @@ -2356,15 +2356,12 @@ public class TestHoodieTimelineArchiver extends HoodieSparkClientTestHarness { private void addCleanCommitWithECTR(HoodieTestTable testTable, String cleanInstant, String ectr, String lastCompleted) throws Exception { List<HoodieCleanStat> cleanStatsList = new ArrayList<>(); - cleanStatsList.add(new HoodieCleanStat( - HoodieCleaningPolicy.KEEP_LATEST_COMMITS, - "p1", - Collections.emptyList(), - Collections.emptyList(), - Collections.emptyList(), - ectr, - lastCompleted - )); + cleanStatsList.add(HoodieCleanStat.builder() + .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_COMMITS) + .withPartitionPath("p1") + .withEarliestCommitToRetain(ectr) + .withLastCompletedCommitTimestamp(lastCompleted) + .build()); HoodieCleanMetadata cleanMetadata = CleanerUtils.convertCleanMetadata(cleanInstant, Option.of(0L), cleanStatsList, Collections.emptyMap()); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java index f57ee82385f6..dde8ac5729b3 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java @@ -724,18 +724,30 @@ public class TestCleaner extends HoodieCleanerTestBase { List<String> failedDeleteFiles1 = Collections.singletonList(filePath2); // create partition1 clean stat. - HoodieCleanStat cleanStat1 = new HoodieCleanStat(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS, - partition1, deletePathPatterns1, successDeleteFiles1, - failedDeleteFiles1, instantTime, ""); + HoodieCleanStat cleanStat1 = HoodieCleanStat.builder() + .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS) + .withPartitionPath(partition1) + .withDeletePathPatterns(deletePathPatterns1) + .withSuccessDeleteFiles(successDeleteFiles1) + .withFailedDeleteFiles(failedDeleteFiles1) + .withEarliestCommitToRetain(instantTime) + .withLastCompletedCommitTimestamp("") + .build(); List<String> deletePathPatterns2 = new ArrayList<>(); List<String> successDeleteFiles2 = new ArrayList<>(); List<String> failedDeleteFiles2 = new ArrayList<>(); // create partition2 empty clean stat. - HoodieCleanStat cleanStat2 = new HoodieCleanStat(HoodieCleaningPolicy.KEEP_LATEST_COMMITS, - partition2, deletePathPatterns2, successDeleteFiles2, - failedDeleteFiles2, instantTime, ""); + HoodieCleanStat cleanStat2 = HoodieCleanStat.builder() + .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_COMMITS) + .withPartitionPath(partition2) + .withDeletePathPatterns(deletePathPatterns2) + .withSuccessDeleteFiles(successDeleteFiles2) + .withFailedDeleteFiles(failedDeleteFiles2) + .withEarliestCommitToRetain(instantTime) + .withLastCompletedCommitTimestamp("") + .build(); // map with absolute file path. Map<String, Tuple3> oldExpected = new HashMap<>(); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieCleanerTestBase.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieCleanerTestBase.java index 66f2f595ef4d..5563f39c811b 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieCleanerTestBase.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieCleanerTestBase.java @@ -164,24 +164,27 @@ public class HoodieCleanerTestBase extends HoodieClientTestBase { } Map<String, HoodieCleanStat> cleanStatMap = cleanMetadata1.getPartitionMetadata().values().stream() - .map(x -> new HoodieCleanStat.Builder().withPartitionPath(x.getPartitionPath()) - .withFailedDeletes(x.getFailedDeleteFiles()).withSuccessfulDeletes(x.getSuccessDeleteFiles()) - .withPolicy(HoodieCleaningPolicy.valueOf(x.getPolicy())).withDeletePathPattern(x.getDeletePathPatterns()) - .withEarliestCommitRetained(Option.ofNullable(cleanMetadata1.getEarliestCommitToRetain() != null - ? INSTANT_GENERATOR.createNewInstant(HoodieInstant.State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "000") - : null)) + .map(x -> HoodieCleanStat.builder() + .withPolicy(HoodieCleaningPolicy.valueOf(x.getPolicy())) + .withPartitionPath(x.getPartitionPath()) + .withDeletePathPatterns(x.getDeletePathPatterns()) + .withSuccessDeleteFiles(x.getSuccessDeleteFiles()) + .withFailedDeleteFiles(x.getFailedDeleteFiles()) + .withEarliestCommitToRetain(cleanMetadata1.getEarliestCommitToRetain() != null ? "000" : "") .build()) .collect(Collectors.toMap(HoodieCleanStat::getPartitionPath, x -> x)); cleanMetadata1.getBootstrapPartitionMetadata().values().forEach(x -> { - HoodieCleanStat s = cleanStatMap.get(x.getPartitionPath()); - cleanStatMap.put(x.getPartitionPath(), new HoodieCleanStat.Builder().withPartitionPath(x.getPartitionPath()) - .withFailedDeletes(s.getFailedDeleteFiles()).withSuccessfulDeletes(s.getSuccessDeleteFiles()) - .withPolicy(HoodieCleaningPolicy.valueOf(x.getPolicy())).withDeletePathPattern(s.getDeletePathPatterns()) - .withEarliestCommitRetained(Option.ofNullable(s.getEarliestCommitToRetain()) - .map(y -> INSTANT_GENERATOR.createNewInstant(HoodieInstant.State.COMPLETED, HoodieTimeline.COMMIT_ACTION, y))) - .withSuccessfulDeleteBootstrapBaseFiles(x.getSuccessDeleteFiles()) + cleanStatMap.compute(x.getPartitionPath(), (k, s) -> HoodieCleanStat.builder() + .withPolicy(HoodieCleaningPolicy.valueOf(x.getPolicy())) + .withPartitionPath(x.getPartitionPath()) + .withDeletePathPatterns(s.getDeletePathPatterns()) + .withSuccessDeleteFiles(s.getSuccessDeleteFiles()) + .withFailedDeleteFiles(s.getFailedDeleteFiles()) + .withEarliestCommitToRetain(s.getEarliestCommitToRetain() != null ? s.getEarliestCommitToRetain() : "") + .withDeleteBootstrapBasePathPatterns(x.getDeletePathPatterns()) + .withSuccessDeleteBootstrapBaseFiles(x.getSuccessDeleteFiles()) .withFailedDeleteBootstrapBaseFiles(x.getFailedDeleteFiles()) - .withDeleteBootstrapBasePathPatterns(x.getDeletePathPatterns()).build()); + .build()); }); return new ArrayList<>(cleanStatMap.values()); } diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java index 97bd0e1175d7..296690556327 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java @@ -676,14 +676,12 @@ public abstract class HoodieSparkClientTestHarness extends HoodieWriterClientTes if (inflightOnly) { HoodieTestTable.of(metaClient).addInflightClean(instantTime, cleanerPlan); } else { - HoodieCleanStat cleanStats = new HoodieCleanStat( - HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS, - HoodieTestUtils.DEFAULT_PARTITION_PATHS[new Random().nextInt(HoodieTestUtils.DEFAULT_PARTITION_PATHS.length)], - Collections.emptyList(), - Collections.emptyList(), - Collections.emptyList(), - instantTime, - ""); + HoodieCleanStat cleanStats = HoodieCleanStat.builder() + .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS) + .withPartitionPath(HoodieTestUtils.DEFAULT_PARTITION_PATHS[new Random().nextInt(HoodieTestUtils.DEFAULT_PARTITION_PATHS.length)]) + .withEarliestCommitToRetain(instantTime) + .withLastCompletedCommitTimestamp("") + .build(); HoodieCleanMetadata cleanMetadata = convertCleanMetadata(instantTime, Option.of(0L), Collections.singletonList(cleanStats), Collections.EMPTY_MAP); HoodieTestTable.of(metaClient).addClean(instantTime, cleanerPlan, cleanMetadata, isEmptyForAll, isEmptyCompleted); } diff --git a/hudi-common/pom.xml b/hudi-common/pom.xml index f0416d3cd817..7bb76c74ad5d 100644 --- a/hudi-common/pom.xml +++ b/hudi-common/pom.xml @@ -128,6 +128,7 @@ <dependency> <groupId>org.projectlombok</groupId> <artifactId>lombok</artifactId> + <scope>provided</scope> </dependency> <!-- Logging --> diff --git a/hudi-common/src/main/java/org/apache/hudi/common/HoodieCleanStat.java b/hudi-common/src/main/java/org/apache/hudi/common/HoodieCleanStat.java index 498d430b5c3a..0106dfc96200 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/HoodieCleanStat.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/HoodieCleanStat.java @@ -19,9 +19,10 @@ package org.apache.hudi.common; import org.apache.hudi.common.model.HoodieCleaningPolicy; -import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.util.CollectionUtils; -import org.apache.hudi.common.util.Option; + +import lombok.Builder; +import lombok.Value; import java.io.Serializable; import java.util.List; @@ -29,197 +30,34 @@ import java.util.List; /** * Collects stats about a single partition clean operation. */ +@Builder(setterPrefix = "with") +@Value public class HoodieCleanStat implements Serializable { // Policy used - private final HoodieCleaningPolicy policy; + HoodieCleaningPolicy policy; // Partition path cleaned - private final String partitionPath; + String partitionPath; // The patterns that were generated for the delete operation - private final List<String> deletePathPatterns; - private final List<String> successDeleteFiles; - // Files that could not be deleted - private final List<String> failedDeleteFiles; - // Bootstrap Base Path patterns that were generated for the delete operation - private final List<String> deleteBootstrapBasePathPatterns; - private final List<String> successDeleteBootstrapBaseFiles; + @Builder.Default + List<String> deletePathPatterns = CollectionUtils.createImmutableList(); + @Builder.Default + List<String> successDeleteFiles = CollectionUtils.createImmutableList(); // Files that could not be deleted - private final List<String> failedDeleteBootstrapBaseFiles; + @Builder.Default + List<String> failedDeleteFiles = CollectionUtils.createImmutableList(); // Earliest commit that was retained in this clean - private final String earliestCommitToRetain; + String earliestCommitToRetain; // Last completed commit timestamp before clean - private final String lastCompletedCommitTimestamp; + String lastCompletedCommitTimestamp; + // Bootstrap Base Path patterns that were generated for the delete operation + @Builder.Default + List<String> deleteBootstrapBasePathPatterns = CollectionUtils.createImmutableList(); + @Builder.Default + List<String> successDeleteBootstrapBaseFiles = CollectionUtils.createImmutableList(); + // Files that could not be deleted + @Builder.Default + List<String> failedDeleteBootstrapBaseFiles = CollectionUtils.createImmutableList(); // set to true if partition is deleted - private final boolean isPartitionDeleted; - - public HoodieCleanStat(HoodieCleaningPolicy policy, String partitionPath, List<String> deletePathPatterns, - List<String> successDeleteFiles, List<String> failedDeleteFiles, String earliestCommitToRetain,String lastCompletedCommitTimestamp) { - this(policy, partitionPath, deletePathPatterns, successDeleteFiles, failedDeleteFiles, earliestCommitToRetain, - lastCompletedCommitTimestamp, CollectionUtils.createImmutableList(), CollectionUtils.createImmutableList(), - CollectionUtils.createImmutableList(), false); - } - - public HoodieCleanStat(HoodieCleaningPolicy policy, String partitionPath, List<String> deletePathPatterns, - List<String> successDeleteFiles, List<String> failedDeleteFiles, - String earliestCommitToRetain,String lastCompletedCommitTimestamp, - List<String> deleteBootstrapBasePathPatterns, - List<String> successDeleteBootstrapBaseFiles, - List<String> failedDeleteBootstrapBaseFiles, - boolean isPartitionDeleted) { - this.policy = policy; - this.partitionPath = partitionPath; - this.deletePathPatterns = deletePathPatterns; - this.successDeleteFiles = successDeleteFiles; - this.failedDeleteFiles = failedDeleteFiles; - this.earliestCommitToRetain = earliestCommitToRetain; - this.lastCompletedCommitTimestamp = lastCompletedCommitTimestamp; - this.deleteBootstrapBasePathPatterns = deleteBootstrapBasePathPatterns; - this.successDeleteBootstrapBaseFiles = successDeleteBootstrapBaseFiles; - this.failedDeleteBootstrapBaseFiles = failedDeleteBootstrapBaseFiles; - this.isPartitionDeleted = isPartitionDeleted; - } - - public HoodieCleaningPolicy getPolicy() { - return policy; - } - - public String getPartitionPath() { - return partitionPath; - } - - public List<String> getDeletePathPatterns() { - return deletePathPatterns; - } - - public List<String> getSuccessDeleteFiles() { - return successDeleteFiles; - } - - public List<String> getFailedDeleteFiles() { - return failedDeleteFiles; - } - - public List<String> getDeleteBootstrapBasePathPatterns() { - return deleteBootstrapBasePathPatterns; - } - - public List<String> getSuccessDeleteBootstrapBaseFiles() { - return successDeleteBootstrapBaseFiles; - } - - public List<String> getFailedDeleteBootstrapBaseFiles() { - return failedDeleteBootstrapBaseFiles; - } - - public String getEarliestCommitToRetain() { - return earliestCommitToRetain; - } - - public String getLastCompletedCommitTimestamp() { - return lastCompletedCommitTimestamp; - } - - public boolean isPartitionDeleted() { - return isPartitionDeleted; - } - - public static Builder newBuilder() { - return new Builder(); - } - - /** - * A builder used to build {@link HoodieCleanStat}. - */ - public static class Builder { - - private HoodieCleaningPolicy policy; - private List<String> deletePathPatterns; - private List<String> successDeleteFiles; - private List<String> failedDeleteFiles; - private String partitionPath; - private String earliestCommitToRetain; - private String lastCompletedCommitTimestamp; - private List<String> deleteBootstrapBasePathPatterns; - private List<String> successDeleteBootstrapBaseFiles; - private List<String> failedDeleteBootstrapBaseFiles; - private boolean isPartitionDeleted; - - public Builder withPolicy(HoodieCleaningPolicy policy) { - this.policy = policy; - return this; - } - - public Builder withDeletePathPattern(List<String> deletePathPatterns) { - this.deletePathPatterns = deletePathPatterns; - return this; - } - - public Builder withSuccessfulDeletes(List<String> successDeleteFiles) { - this.successDeleteFiles = successDeleteFiles; - return this; - } - - public Builder withFailedDeletes(List<String> failedDeleteFiles) { - this.failedDeleteFiles = failedDeleteFiles; - return this; - } - - public Builder withDeleteBootstrapBasePathPatterns(List<String> deletePathPatterns) { - this.deleteBootstrapBasePathPatterns = deletePathPatterns; - return this; - } - - public Builder withSuccessfulDeleteBootstrapBaseFiles(List<String> successDeleteFiles) { - this.successDeleteBootstrapBaseFiles = successDeleteFiles; - return this; - } - - public Builder withFailedDeleteBootstrapBaseFiles(List<String> failedDeleteFiles) { - this.failedDeleteBootstrapBaseFiles = failedDeleteFiles; - return this; - } - - public Builder withPartitionPath(String partitionPath) { - this.partitionPath = partitionPath; - return this; - } - - public Builder withEarliestCommitRetained(Option<HoodieInstant> earliestCommitToRetain) { - this.earliestCommitToRetain = - (earliestCommitToRetain.isPresent()) ? earliestCommitToRetain.get().requestedTime() : ""; - return this; - } - - public Builder withLastCompletedCommitTimestamp(String lastCompletedCommitTimestamp) { - this.lastCompletedCommitTimestamp = lastCompletedCommitTimestamp; - return this; - } - - public Builder isPartitionDeleted(boolean isPartitionDeleted) { - this.isPartitionDeleted = isPartitionDeleted; - return this; - } - - public HoodieCleanStat build() { - return new HoodieCleanStat(policy, partitionPath, deletePathPatterns, successDeleteFiles, failedDeleteFiles, - earliestCommitToRetain, lastCompletedCommitTimestamp, deleteBootstrapBasePathPatterns, - successDeleteBootstrapBaseFiles, failedDeleteBootstrapBaseFiles, isPartitionDeleted); - } - } - - @Override - public String toString() { - return "HoodieCleanStat{" - + "policy=" + policy - + ", partitionPath='" + partitionPath + '\'' - + ", deletePathPatterns=" + deletePathPatterns - + ", successDeleteFiles=" + successDeleteFiles - + ", failedDeleteFiles=" + failedDeleteFiles - + ", earliestCommitToRetain='" + earliestCommitToRetain - + ", deleteBootstrapBasePathPatterns=" + deleteBootstrapBasePathPatterns - + ", successDeleteBootstrapBaseFiles=" + successDeleteBootstrapBaseFiles - + ", failedDeleteBootstrapBaseFiles=" + failedDeleteBootstrapBaseFiles - + ", isPartitionDeleted=" + isPartitionDeleted + '\'' - + '}'; - } + boolean partitionDeleted; } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/HoodiePendingRollbackInfo.java b/hudi-common/src/main/java/org/apache/hudi/common/HoodiePendingRollbackInfo.java index c53babf35010..44e61f651b8c 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/HoodiePendingRollbackInfo.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/HoodiePendingRollbackInfo.java @@ -21,24 +21,16 @@ package org.apache.hudi.common; import org.apache.hudi.avro.model.HoodieRollbackPlan; import org.apache.hudi.common.table.timeline.HoodieInstant; +import lombok.AllArgsConstructor; +import lombok.Getter; + /** * Holds rollback instant and rollback plan for a pending rollback. */ +@AllArgsConstructor +@Getter public class HoodiePendingRollbackInfo { private final HoodieInstant rollbackInstant; private final HoodieRollbackPlan rollbackPlan; - - public HoodiePendingRollbackInfo(HoodieInstant rollbackInstant, HoodieRollbackPlan rollbackPlan) { - this.rollbackInstant = rollbackInstant; - this.rollbackPlan = rollbackPlan; - } - - public HoodieInstant getRollbackInstant() { - return rollbackInstant; - } - - public HoodieRollbackPlan getRollbackPlan() { - return rollbackPlan; - } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/HoodieRollbackStat.java b/hudi-common/src/main/java/org/apache/hudi/common/HoodieRollbackStat.java index 59308a43325c..c9be79687854 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/HoodieRollbackStat.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/HoodieRollbackStat.java @@ -20,6 +20,9 @@ package org.apache.hudi.common; import org.apache.hudi.storage.StoragePathInfo; +import lombok.AllArgsConstructor; +import lombok.Getter; + import java.io.Serializable; import java.util.Collections; import java.util.List; @@ -29,6 +32,8 @@ import java.util.stream.Collectors; /** * Collects stats about a single partition clean operation. */ +@AllArgsConstructor +@Getter public class HoodieRollbackStat implements Serializable { // Partition path @@ -41,35 +46,6 @@ public class HoodieRollbackStat implements Serializable { private final Map<String, Long> logFilesFromFailedCommit; - public HoodieRollbackStat(String partitionPath, List<String> successDeleteFiles, List<String> failedDeleteFiles, - Map<StoragePathInfo, Long> commandBlocksCount, Map<String, Long> logFilesFromFailedCommit) { - this.partitionPath = partitionPath; - this.successDeleteFiles = successDeleteFiles; - this.failedDeleteFiles = failedDeleteFiles; - this.commandBlocksCount = commandBlocksCount; - this.logFilesFromFailedCommit = logFilesFromFailedCommit; - } - - public Map<StoragePathInfo, Long> getCommandBlocksCount() { - return commandBlocksCount; - } - - public String getPartitionPath() { - return partitionPath; - } - - public List<String> getSuccessDeleteFiles() { - return successDeleteFiles; - } - - public List<String> getFailedDeleteFiles() { - return failedDeleteFiles; - } - - public Map<String, Long> getLogFilesFromFailedCommit() { - return logFilesFromFailedCommit; - } - public static HoodieRollbackStat.Builder newBuilder() { return new Builder(); } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalDynamicBloomFilter.java b/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalDynamicBloomFilter.java index bf35aeaee61d..fa14e2976f43 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalDynamicBloomFilter.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalDynamicBloomFilter.java @@ -18,6 +18,8 @@ package org.apache.hudi.common.bloom; +import lombok.NoArgsConstructor; + import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; @@ -27,6 +29,7 @@ import java.io.IOException; * with bounds on maximum number of entries. Once the max entries is reached, false positive guarantees are not * honored. */ +@NoArgsConstructor class InternalDynamicBloomFilter extends InternalFilter { /** @@ -47,12 +50,6 @@ class InternalDynamicBloomFilter extends InternalFilter { */ private InternalBloomFilter[] matrix; - /** - * Zero-args constructor for the serialization. - */ - public InternalDynamicBloomFilter() { - } - /** * Constructor. * <p> diff --git a/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalFilter.java b/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalFilter.java index e23255bb4b61..c5b45e7877a8 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalFilter.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalFilter.java @@ -20,6 +20,9 @@ package org.apache.hudi.common.bloom; import org.apache.hudi.common.util.hash.Hash; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; @@ -42,6 +45,7 @@ import java.util.List; * @see Key The general behavior of a key * @see HashFunction A hash function */ +@NoArgsConstructor(access = AccessLevel.PROTECTED) abstract class InternalFilter { private static final int VERSION = -1; // negative to accommodate for old format /** @@ -64,9 +68,6 @@ abstract class InternalFilter { */ protected int hashType; - protected InternalFilter() { - } - /** * Constructor. * diff --git a/hudi-common/src/main/java/org/apache/hudi/common/bloom/Key.java b/hudi-common/src/main/java/org/apache/hudi/common/bloom/Key.java index 013c0f08ea46..ed8096edea34 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/bloom/Key.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/bloom/Key.java @@ -20,6 +20,7 @@ package org.apache.hudi.common.bloom; import lombok.Getter; +import lombok.NoArgsConstructor; import java.io.DataInput; import java.io.DataOutput; @@ -33,6 +34,7 @@ import java.io.IOException; * @see InternalBloomFilter The general behavior of a bloom filter and how the key is used. */ @Getter +@NoArgsConstructor public class Key implements Comparable<Key> { /** * Byte value of key @@ -47,12 +49,6 @@ public class Key implements Comparable<Key> { */ double weight; - /** - * default constructor - use with readFields - */ - public Key() { - } - /** * Constructor. * <p> diff --git a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndex.java b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndex.java index fd342288cf01..79feafbc4784 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndex.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndex.java @@ -30,8 +30,8 @@ import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; @@ -49,12 +49,11 @@ import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes; * on these index files to manage multiple file-groups. */ +@Slf4j public class HFileBootstrapIndex extends BootstrapIndex { private static final long serialVersionUID = 1L; - private static final Logger LOG = LoggerFactory.getLogger(HFileBootstrapIndex.class); - public static final String BOOTSTRAP_INDEX_FILE_ID = "00000000-0000-0000-0000-000000000000-0"; private static final String PARTITION_KEY_PREFIX = "part"; @@ -68,6 +67,7 @@ public class HFileBootstrapIndex extends BootstrapIndex { public static final String INDEX_INFO_KEY_STRING = "INDEX_INFO"; public static final byte[] INDEX_INFO_KEY = getUTF8Bytes(INDEX_INFO_KEY_STRING); + @Getter private final boolean isPresent; public HFileBootstrapIndex(HoodieTableMetaClient metaClient) { @@ -152,7 +152,7 @@ public class HFileBootstrapIndex extends BootstrapIndex { StoragePath[] indexPaths = new StoragePath[] {partitionIndexPath(metaClient), fileIdIndexPath(metaClient)}; for (StoragePath indexPath : indexPaths) { if (metaClient.getStorage().exists(indexPath)) { - LOG.info("Dropping bootstrap index. Deleting file: {}", indexPath); + log.info("Dropping bootstrap index. Deleting file: {}", indexPath); metaClient.getStorage().deleteDirectory(indexPath); } } @@ -160,9 +160,4 @@ public class HFileBootstrapIndex extends BootstrapIndex { throw new HoodieIOException(ioe.getMessage(), ioe); } } - - @Override - public boolean isPresent() { - return isPresent; - } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndexReader.java b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndexReader.java index 53debdb71ba0..320b365e3519 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndexReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndexReader.java @@ -38,8 +38,8 @@ import org.apache.hudi.io.util.IOUtils; import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; import java.util.ArrayList; @@ -59,10 +59,11 @@ import static org.apache.hudi.common.bootstrap.index.hfile.HFileBootstrapIndex.p /** * HFile Based Index Reader. */ +@Slf4j public class HFileBootstrapIndexReader extends BootstrapIndex.IndexReader { - private static final Logger LOG = LoggerFactory.getLogger(HFileBootstrapIndexReader.class); // Base Path of external files. + @Getter private final String bootstrapBasePath; // Well Known Paths for indices private final String indexByPartitionPath; @@ -83,7 +84,7 @@ public class HFileBootstrapIndexReader extends BootstrapIndex.IndexReader { this.indexByFileIdPath = indexByFilePath.toString(); initIndexInfo(); this.bootstrapBasePath = bootstrapIndexInfo.getBootstrapBasePath(); - LOG.info("Loaded HFileBasedBootstrapIndex with source base path :" + bootstrapBasePath); + log.info("Loaded HFileBasedBootstrapIndex with source base path :{}", bootstrapBasePath); } /** @@ -93,7 +94,7 @@ public class HFileBootstrapIndexReader extends BootstrapIndex.IndexReader { * @param storage {@link HoodieStorage} instance. */ private static HFileReader createReader(String hFilePath, HoodieStorage storage) throws IOException { - LOG.info("Opening HFile for reading :" + hFilePath); + log.info("Opening HFile for reading :{}", hFilePath); StoragePath path = new StoragePath(hFilePath); long fileSize = storage.getPathInfo(path).getLength(); SeekableDataInputStream stream = storage.openSeekable(path, false); @@ -118,7 +119,7 @@ public class HFileBootstrapIndexReader extends BootstrapIndex.IndexReader { private synchronized HFileReader partitionIndexReader() throws IOException { if (indexByPartitionReader == null) { - LOG.info("Opening partition index :" + indexByPartitionPath); + log.info("Opening partition index :{}", indexByPartitionPath); this.indexByPartitionReader = createReader(indexByPartitionPath, metaClient.getStorage()); } return indexByPartitionReader; @@ -126,7 +127,7 @@ public class HFileBootstrapIndexReader extends BootstrapIndex.IndexReader { private synchronized HFileReader fileIdIndexReader() throws IOException { if (indexByFileIdReader == null) { - LOG.info("Opening fileId index :" + indexByFileIdPath); + log.info("Opening fileId index :{}", indexByFileIdPath); this.indexByFileIdReader = createReader(indexByFileIdPath, metaClient.getStorage()); } return indexByFileIdReader; @@ -181,7 +182,7 @@ public class HFileBootstrapIndexReader extends BootstrapIndex.IndexReader { .map(e -> new BootstrapFileMapping(bootstrapBasePath, metadata.getBootstrapPartitionPath(), e.getValue(), partition, e.getKey())).collect(Collectors.toList()); } else { - LOG.warn("No value found for partition key ({})", partition); + log.warn("No value found for partition key ({})", partition); return new ArrayList<>(); } } catch (IOException ioe) { @@ -189,11 +190,6 @@ public class HFileBootstrapIndexReader extends BootstrapIndex.IndexReader { } } - @Override - public String getBootstrapBasePath() { - return bootstrapBasePath; - } - @Override public Map<HoodieFileGroupId, BootstrapFileMapping> getSourceFileMappingForFileIds( List<HoodieFileGroupId> ids) { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndexWriter.java b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndexWriter.java index bcd063ff5d0d..8fd232a87402 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndexWriter.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndexWriter.java @@ -35,8 +35,7 @@ import org.apache.hudi.io.hfile.HFileWriter; import org.apache.hudi.io.hfile.HFileWriterImpl; import org.apache.hudi.storage.StoragePath; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; import java.io.OutputStream; @@ -53,8 +52,8 @@ import static org.apache.hudi.common.bootstrap.index.hfile.HFileBootstrapIndex.g import static org.apache.hudi.common.bootstrap.index.hfile.HFileBootstrapIndex.getPartitionKey; import static org.apache.hudi.common.bootstrap.index.hfile.HFileBootstrapIndex.partitionIndexPath; +@Slf4j public class HFileBootstrapIndexWriter extends BootstrapIndex.IndexWriter { - private static final Logger LOG = LoggerFactory.getLogger(HFileBootstrapIndexWriter.class); private final String bootstrapBasePath; private final StoragePath indexByPartitionPath; @@ -80,7 +79,7 @@ public class HFileBootstrapIndexWriter extends BootstrapIndex.IndexWriter { || metaClient.getStorage().exists(indexByFileIdPath)) { String errMsg = "Previous version of bootstrap index exists. Partition Index Path :" + indexByPartitionPath + ", FileId index Path :" + indexByFileIdPath; - LOG.info(errMsg); + log.info(errMsg); throw new HoodieException(errMsg); } } catch (IOException ioe) { @@ -97,9 +96,9 @@ public class HFileBootstrapIndexWriter extends BootstrapIndex.IndexWriter { private void writeNextPartition(String partitionPath, String bootstrapPartitionPath, List<BootstrapFileMapping> bootstrapFileMappings) { try { - LOG.info("Adding bootstrap partition Index entry for partition :" + partitionPath - + ", bootstrap Partition :" + bootstrapPartitionPath + ", Num Entries :" + bootstrapFileMappings.size()); - LOG.info("ADDING entries :" + bootstrapFileMappings); + log.info("Adding bootstrap partition Index entry for partition :{}, bootstrap Partition :{}, Num Entries :{}", + partitionPath, bootstrapPartitionPath, bootstrapFileMappings.size()); + log.info("ADDING entries :{}", bootstrapFileMappings); HoodieBootstrapPartitionMetadata bootstrapPartitionMetadata = new HoodieBootstrapPartitionMetadata(); bootstrapPartitionMetadata.setBootstrapPartitionPath(bootstrapPartitionPath); bootstrapPartitionMetadata.setPartitionPath(partitionPath); @@ -148,14 +147,14 @@ public class HFileBootstrapIndexWriter extends BootstrapIndex.IndexWriter { .setNumKeys(numPartitionKeysAdded) .setBootstrapBasePath(bootstrapBasePath) .build(); - LOG.info("Adding Partition FileInfo :" + partitionIndexInfo); + log.info("Adding Partition FileInfo :{}", partitionIndexInfo); HoodieBootstrapIndexInfo fileIdIndexInfo = HoodieBootstrapIndexInfo.newBuilder() .setCreatedTimestamp(new Date().getTime()) .setNumKeys(numFileIdKeysAdded) .setBootstrapBasePath(bootstrapBasePath) .build(); - LOG.info("Appending FileId FileInfo :" + fileIdIndexInfo); + log.info("Appending FileId FileInfo :{}", fileIdIndexInfo); indexByPartitionWriter.appendFileInfo( INDEX_INFO_KEY_STRING, diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java index f4b6bffd8080..2048b21c7e26 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java @@ -18,6 +18,10 @@ package org.apache.hudi.common.config; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; + /** * In Hudi, we have multiple superclasses, aka Config Classes of {@link HoodieConfig} that maintain * several configs. This class group one or more of these superclasses into higher @@ -29,6 +33,7 @@ public class ConfigGroups { * Config group names. Please add the description of each group in * {@link ConfigGroups#getDescription}. */ + @AllArgsConstructor(access = AccessLevel.PACKAGE) public enum Names { TABLE_CONFIG("Hudi Table Config"), ENVIRONMENT_CONFIG("Environment Config"), @@ -44,12 +49,9 @@ public class ConfigGroups { HUDI_STREAMER("Hudi Streamer Configs"); public final String name; - - Names(String name) { - this.name = name; - } } + @AllArgsConstructor(access = AccessLevel.PACKAGE) public enum SubGroupNames { INDEX( "Index Configs", @@ -80,16 +82,8 @@ public class ConfigGroups { "No subgroup. This description should be hidden."); public final String name; + @Getter private final String description; - - SubGroupNames(String name, String description) { - this.name = name; - this.description = description; - } - - public String getDescription() { - return description; - } } public static String getDescription(Names names) { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigProperty.java b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigProperty.java index 56a087f1e7ea..b93d980eafc1 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigProperty.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigProperty.java @@ -22,6 +22,12 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.exception.HoodieException; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NonNull; +import lombok.experimental.Accessors; + import java.io.Serializable; import java.lang.reflect.Field; import java.util.Arrays; @@ -41,14 +47,21 @@ import java.util.stream.Collectors; * * @param <T> The type of the default value. */ +@AllArgsConstructor(access = AccessLevel.PACKAGE) +@Getter public class ConfigProperty<T> implements Serializable { + @NonNull + @Accessors(fluent = true) // Required so that #key() is generated instead of #getKey() by Lombok private final String key; + @Getter(AccessLevel.NONE) private final T defaultValue; + @Getter(AccessLevel.NONE) private final String docOnDefaultValue; + @Getter(AccessLevel.NONE) private final String doc; private final Option<String> sinceVersion; @@ -57,37 +70,16 @@ public class ConfigProperty<T> implements Serializable { private final List<String> supportedVersions; + // provide the ability to infer config value based on other configs + private final Option<Function<HoodieConfig, Option<T>>> inferFunction; + + @Getter(AccessLevel.NONE) private final Set<String> validValues; private final boolean advanced; private final String[] alternatives; - // provide the ability to infer config value based on other configs - private final Option<Function<HoodieConfig, Option<T>>> inferFunction; - - ConfigProperty(String key, T defaultValue, String docOnDefaultValue, String doc, - Option<String> sinceVersion, Option<String> deprecatedVersion, - List<String> supportedVersions, - Option<Function<HoodieConfig, Option<T>>> inferFunc, Set<String> validValues, - boolean advanced, String... alternatives) { - this.key = Objects.requireNonNull(key); - this.defaultValue = defaultValue; - this.docOnDefaultValue = docOnDefaultValue; - this.doc = doc; - this.sinceVersion = sinceVersion; - this.deprecatedVersion = deprecatedVersion; - this.supportedVersions = supportedVersions; - this.inferFunction = inferFunc; - this.validValues = validValues; - this.advanced = advanced; - this.alternatives = alternatives; - } - - public String key() { - return key; - } - public T defaultValue() { if (defaultValue == null) { throw new HoodieException(String.format("There's no default value for this config: %s", key)); @@ -108,26 +100,10 @@ public class ConfigProperty<T> implements Serializable { return StringUtils.isNullOrEmpty(doc) ? StringUtils.EMPTY_STRING : doc; } - public Option<String> getSinceVersion() { - return sinceVersion; - } - - public Option<String> getDeprecatedVersion() { - return deprecatedVersion; - } - - public List<String> getSupportedVersions() { - return supportedVersions; - } - public boolean hasInferFunction() { return getInferFunction().isPresent(); } - public Option<Function<HoodieConfig, Option<T>>> getInferFunction() { - return inferFunction; - } - public void checkValues(String value) { if (!isValid(value)) { throw new IllegalArgumentException( @@ -144,10 +120,6 @@ public class ConfigProperty<T> implements Serializable { return Arrays.asList(alternatives); } - public boolean isAdvanced() { - return advanced; - } - public ConfigProperty<T> withDocumentation(String doc) { Objects.requireNonNull(doc); return new ConfigProperty<>(key, defaultValue, docOnDefaultValue, doc, sinceVersion, deprecatedVersion, supportedVersions, inferFunction, validValues, advanced, alternatives); @@ -271,7 +243,7 @@ public class ConfigProperty<T> implements Serializable { public <T> ConfigProperty<T> defaultValue(T value, String docOnDefaultValue) { Objects.requireNonNull(docOnDefaultValue); return new ConfigProperty<>(key, value, docOnDefaultValue, "", Option.empty(), - Option.empty(), Collections.emptyList(), Option.empty(), Collections.emptySet(), false); + Option.empty(), Collections.emptyList(), Option.empty(), Collections.emptySet(), false, new String[0]); } public ConfigProperty<String> noDefaultValue() { @@ -280,7 +252,7 @@ public class ConfigProperty<T> implements Serializable { public ConfigProperty<String> noDefaultValue(String docOnDefaultValue) { return new ConfigProperty<>(key, null, docOnDefaultValue, "", Option.empty(), - Option.empty(), Collections.emptyList(), Option.empty(), Collections.emptySet(), false); + Option.empty(), Collections.emptyList(), Option.empty(), Collections.emptySet(), false, new String[0]); } } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java index 498de3821666..6e6668b0284c 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java @@ -24,8 +24,8 @@ import org.apache.hudi.common.util.ReflectionUtils; import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.exception.HoodieException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import java.io.Serializable; import java.lang.reflect.Modifier; @@ -39,16 +39,16 @@ import static org.apache.hudi.common.util.ConfigUtils.loadGlobalProperties; /** * This class deals with {@link ConfigProperty} and provides get/set functionalities. */ +@Slf4j public class HoodieConfig implements Serializable { - private static final Logger LOG = LoggerFactory.getLogger(HoodieConfig.class); - protected static final String CONFIG_VALUES_DELIMITER = ","; // Number of retries while reading the properties file to deal with parallel updates protected static final int MAX_READ_RETRIES = 5; // Delay between retries while reading the properties file protected static final int READ_RETRY_DELAY_MSEC = 1000; + @Getter protected TypedProperties props; public HoodieConfig() { @@ -246,10 +246,6 @@ public class HoodieConfig implements Serializable { return Option.ofNullable(props.getProperty(key)).orElse(defaultVal); } - public TypedProperties getProps() { - return props; - } - public TypedProperties getProps(boolean includeGlobalProps) { if (includeGlobalProps) { TypedProperties mergedProps = loadGlobalProperties(); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/conflict/detection/DirectMarkerBasedDetectionStrategy.java b/hudi-common/src/main/java/org/apache/hudi/common/conflict/detection/DirectMarkerBasedDetectionStrategy.java index 4b11a348bb05..466b8c99769e 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/conflict/detection/DirectMarkerBasedDetectionStrategy.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/conflict/detection/DirectMarkerBasedDetectionStrategy.java @@ -30,8 +30,8 @@ import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.StoragePathInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; import java.util.List; @@ -42,10 +42,10 @@ import java.util.stream.Stream; * This abstract strategy is used for direct marker writers, trying to do early conflict detection. */ @PublicAPIClass(maturity = ApiMaturityLevel.EVOLVING) +@AllArgsConstructor +@Slf4j public abstract class DirectMarkerBasedDetectionStrategy implements EarlyConflictDetectionStrategy { - private static final Logger LOG = LoggerFactory.getLogger(DirectMarkerBasedDetectionStrategy.class); - protected final HoodieStorage storage; protected final String partitionPath; protected final String fileId; @@ -53,17 +53,6 @@ public abstract class DirectMarkerBasedDetectionStrategy implements EarlyConflic protected final HoodieActiveTimeline activeTimeline; protected final HoodieConfig config; - public DirectMarkerBasedDetectionStrategy(HoodieStorage storage, String partitionPath, String fileId, - String instantTime, - HoodieActiveTimeline activeTimeline, HoodieConfig config) { - this.storage = storage; - this.partitionPath = partitionPath; - this.fileId = fileId; - this.instantTime = instantTime; - this.activeTimeline = activeTimeline; - this.config = config; - } - /** * We need to do list operation here. * In order to reduce the list pressure as much as possible, first we build path prefix in advance: @@ -106,7 +95,7 @@ public abstract class DirectMarkerBasedDetectionStrategy implements EarlyConflic }).count(); if (res != 0L) { - LOG.warn("Detected conflict marker files: {}/{} for {}", partitionPath, fileId, instantTime); + log.warn("Detected conflict marker files: {}/{} for {}", partitionPath, fileId, instantTime); return true; } return false; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/data/HoodieBaseListData.java b/hudi-common/src/main/java/org/apache/hudi/common/data/HoodieBaseListData.java index b603b99d9302..f31a0072ab9d 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/data/HoodieBaseListData.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/data/HoodieBaseListData.java @@ -21,8 +21,9 @@ package org.apache.hudi.common.data; import org.apache.hudi.common.util.Either; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; import java.util.Iterator; import java.util.List; @@ -34,6 +35,7 @@ import java.util.stream.Stream; * * @param <T> Object value type. */ +@Slf4j public abstract class HoodieBaseListData<T> { protected final Either<Stream<T>, List<T>> data; @@ -86,13 +88,11 @@ public abstract class HoodieBaseListData<T> { } } + @AllArgsConstructor(access = AccessLevel.PACKAGE) + @Slf4j static class IteratorCloser implements Runnable { - private static final Logger LOG = LoggerFactory.getLogger(IteratorCloser.class); - private final Iterator<?> iterator; - IteratorCloser(Iterator<?> iterator) { - this.iterator = iterator; - } + private final Iterator<?> iterator; @Override public void run() { @@ -100,7 +100,7 @@ public abstract class HoodieBaseListData<T> { try { ((AutoCloseable) iterator).close(); } catch (Exception ex) { - LOG.warn("Failed to properly close iterator", ex); + log.warn("Failed to properly close iterator", ex); } } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java index cf2893e1249c..b4663509002d 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java @@ -41,6 +41,9 @@ import org.apache.hudi.common.metrics.Registry; import org.apache.hudi.keygen.KeyGenerator; import org.apache.hudi.storage.StorageConfiguration; +import lombok.AllArgsConstructor; +import lombok.Getter; + import java.io.IOException; import java.util.Iterator; import java.util.List; @@ -51,6 +54,8 @@ import java.util.stream.Stream; * Base class contains the context information needed by the engine at runtime. It will be extended by different * engine implementation if needed. */ +@AllArgsConstructor +@Getter public abstract class HoodieEngineContext { /** @@ -60,19 +65,6 @@ public abstract class HoodieEngineContext { protected TaskContextSupplier taskContextSupplier; - public HoodieEngineContext(StorageConfiguration<?> storageConf, TaskContextSupplier taskContextSupplier) { - this.storageConf = storageConf; - this.taskContextSupplier = taskContextSupplier; - } - - public StorageConfiguration<?> getStorageConf() { - return storageConf; - } - - public TaskContextSupplier getTaskContextSupplier() { - return taskContextSupplier; - } - public abstract HoodieAccumulator newAccumulator(); public abstract <T> HoodieData<T> emptyHoodieData(); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieReaderContext.java b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieReaderContext.java index 953014802cf2..14e412ffcb18 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieReaderContext.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieReaderContext.java @@ -53,6 +53,10 @@ import org.apache.hudi.storage.StorageConfiguration; import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.StoragePathInfo; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; + import java.io.IOException; import java.util.List; import java.util.Map; @@ -77,28 +81,53 @@ import static org.apache.hudi.common.table.HoodieTableConfig.inferMergingConfigs * and {@code RowData} in Flink. */ public abstract class HoodieReaderContext<T> { + + @Getter private final StorageConfiguration<?> storageConfiguration; protected final HoodieFileFormat baseFileFormat; // For general predicate pushdown. + @Getter protected final Option<Predicate> keyFilterOpt; protected final HoodieTableConfig tableConfig; + @Setter private String tablePath = null; + @Getter + @Setter private String latestCommitTime = null; + @Getter + @Setter private Option<HoodieRecordMerger> recordMerger = null; + @Getter + @Setter private Boolean hasLogFiles = null; + @Getter + @Setter private Boolean hasBootstrapBaseFile = null; + @Getter + @Setter private Boolean needsBootstrapMerge = null; + @Getter + @Setter // should we do position based merging for mor private Boolean shouldMergeUseRecordPosition = null; protected Option<InstantRange> instantRangeOpt; + @Getter private RecordMergeMode mergeMode; + @Getter protected RecordContext<T> recordContext; + @Getter + @Setter private FileGroupReaderSchemaHandler<T> schemaHandler = null; // the default iterator mode is engine-specific record mode + @Setter private IteratorMode iteratorMode = IteratorMode.ENGINE_RECORD; + @Getter protected final HoodieConfig hoodieReaderConfig; - private boolean enableLogicalTimestampFieldRepair = true; + @Getter + @Setter + @Accessors(fluent = true) + private Boolean enableLogicalTimestampFieldRepair = true; protected HoodieReaderContext(StorageConfiguration<?> storageConfiguration, HoodieTableConfig tableConfig, @@ -123,19 +152,6 @@ public abstract class HoodieReaderContext<T> { this.hoodieReaderConfig = hoodieReaderConfig; } - // Getter and Setter for schemaHandler - public FileGroupReaderSchemaHandler<T> getSchemaHandler() { - return schemaHandler; - } - - public void setSchemaHandler(FileGroupReaderSchemaHandler<T> schemaHandler) { - this.schemaHandler = schemaHandler; - } - - public void setIteratorMode(IteratorMode iteratorMode) { - this.iteratorMode = iteratorMode; - } - public IteratorMode getIteratorMode() { ValidationUtils.checkArgument(iteratorMode != null, "iterator mode should not be null!"); return this.iteratorMode; @@ -148,82 +164,10 @@ public abstract class HoodieReaderContext<T> { return tablePath; } - public void setEnableLogicalTimestampFieldRepair(boolean enableLogicalTimestampFieldRepair) { - this.enableLogicalTimestampFieldRepair = enableLogicalTimestampFieldRepair; - } - - public void setTablePath(String tablePath) { - this.tablePath = tablePath; - } - - public String getLatestCommitTime() { - return latestCommitTime; - } - - public void setLatestCommitTime(String latestCommitTime) { - this.latestCommitTime = latestCommitTime; - } - - public Option<HoodieRecordMerger> getRecordMerger() { - return recordMerger; - } - - public void setRecordMerger(Option<HoodieRecordMerger> recordMerger) { - this.recordMerger = recordMerger; - } - - // Getter and Setter for hasLogFiles - public boolean getHasLogFiles() { - return hasLogFiles; - } - - public void setHasLogFiles(boolean hasLogFiles) { - this.hasLogFiles = hasLogFiles; - } - - // Getter and Setter for hasBootstrapBaseFile - public boolean getHasBootstrapBaseFile() { - return hasBootstrapBaseFile; - } - - public void setHasBootstrapBaseFile(boolean hasBootstrapBaseFile) { - this.hasBootstrapBaseFile = hasBootstrapBaseFile; - } - - // Getter and Setter for needsBootstrapMerge - public boolean getNeedsBootstrapMerge() { - return needsBootstrapMerge; - } - - public boolean enableLogicalTimestampFieldRepair() { - return enableLogicalTimestampFieldRepair; - } - - public void setNeedsBootstrapMerge(boolean needsBootstrapMerge) { - this.needsBootstrapMerge = needsBootstrapMerge; - } - - // Getter and Setter for useRecordPosition - public boolean getShouldMergeUseRecordPosition() { - return shouldMergeUseRecordPosition; - } - - public void setShouldMergeUseRecordPosition(boolean shouldMergeUseRecordPosition) { - this.shouldMergeUseRecordPosition = shouldMergeUseRecordPosition; - } - - public StorageConfiguration<?> getStorageConfiguration() { - return storageConfiguration; - } - public TypedProperties getMergeProps(TypedProperties props) { return ConfigUtils.getMergeProps(props, this.tableConfig); } - public Option<Predicate> getKeyFilterOpt() { - return keyFilterOpt; - } - public SizeEstimator<BufferedRecord<T>> getRecordSizeEstimator() { return new HoodieRecordSizeEstimator<>(getSchemaHandler().getSchemaForUpdates()); } @@ -232,14 +176,6 @@ public abstract class HoodieReaderContext<T> { return new DefaultSerializer<>(); } - public RecordContext<T> getRecordContext() { - return recordContext; - } - - public HoodieConfig getHoodieReaderConfig() { - return hoodieReaderConfig; - } - /** * Gets the record iterator based on the type of engine-specific record representation from the * file. @@ -335,10 +271,6 @@ public abstract class HoodieReaderContext<T> { properties.getString(RECORD_MERGE_IMPL_CLASSES_DEPRECATED_WRITE_CONFIG_KEY, ""))); } - public RecordMergeMode getMergeMode() { - return mergeMode; - } - /** * Get the {@link InstantRange} filter. */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/engine/RecordContext.java b/hudi-common/src/main/java/org/apache/hudi/common/engine/RecordContext.java index b1fa5d97892e..d63c9bd63408 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/engine/RecordContext.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/engine/RecordContext.java @@ -35,6 +35,8 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieKeyException; import org.apache.hudi.keygen.KeyGenerator; +import lombok.Getter; +import lombok.Setter; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; @@ -65,7 +67,9 @@ public abstract class RecordContext<T> implements Serializable { // for encoding and decoding schemas to the spillable map private final LocalHoodieSchemaCache localSchemaCache = LocalHoodieSchemaCache.getInstance(); + @Getter protected final JavaTypeConverter typeConverter; + @Setter protected String partitionPath; protected RecordContext(HoodieTableConfig tableConfig, JavaTypeConverter typeConverter) { @@ -84,10 +88,6 @@ public abstract class RecordContext<T> implements Serializable { this.typeConverter = typeConverter; } - public void setPartitionPath(String partitionPath) { - this.partitionPath = partitionPath; - } - public T extractDataFromRecord(HoodieRecord record, HoodieSchema schema, Properties properties) { return (T) record.getData(); } @@ -170,10 +170,6 @@ public abstract class RecordContext<T> implements Serializable { */ public abstract T constructEngineRecord(HoodieSchema recordSchema, Object[] fieldValues); - public JavaTypeConverter getTypeConverter() { - return typeConverter; - } - /** * Gets the record key in String. * diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java index 56c0cc3a5b46..1b5f3ea62e78 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java @@ -45,8 +45,7 @@ import org.apache.hudi.storage.StoragePathFilter; import org.apache.hudi.storage.StoragePathInfo; import org.apache.hudi.storage.inline.InLineFSUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.File; import java.io.FileNotFoundException; @@ -72,9 +71,9 @@ import java.util.stream.Stream; /** * Utility functions related to accessing the file storage. */ +@Slf4j public class FSUtils { - private static final Logger LOG = LoggerFactory.getLogger(FSUtils.class); // Log files are of this pattern - .b5068208-e1a4-11e6-bf01-fe55135034f3_20170101134598.log.1_1-0-1 // Archive log files are of this pattern - .commits_.archive.1_1-0-1 public static final String PATH_SEPARATOR = "/"; @@ -624,7 +623,7 @@ public class FSUtils { pairOfSubPathAndConf.getKey(), pairOfSubPathAndConf.getValue(), true) ); boolean result = storage.deleteDirectory(dirPath); - LOG.info("Removed directory at {}", dirPath); + log.info("Removed directory at {}", dirPath); return result; } } catch (IOException ioe) { @@ -803,7 +802,7 @@ public class FSUtils { if (!ignoreFailed) { throw new HoodieIOException("Failed to delete : " + file, e); } else { - LOG.info("Ignore failed deleting : {}", file); + log.info("Ignore failed deleting : {}", file); return true; } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/FailSafeConsistencyGuard.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/FailSafeConsistencyGuard.java index ed82343ee3be..ada10aaa2fd0 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/fs/FailSafeConsistencyGuard.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/FailSafeConsistencyGuard.java @@ -23,8 +23,7 @@ import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.StoragePathInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.FileNotFoundException; import java.io.IOException; @@ -36,10 +35,9 @@ import java.util.stream.Collectors; /** * A consistency checker that fails if it is unable to meet the required condition within a specified timeout. */ +@Slf4j public class FailSafeConsistencyGuard implements ConsistencyGuard { - private static final Logger LOG = LoggerFactory.getLogger(FailSafeConsistencyGuard.class); - protected final HoodieStorage storage; protected final ConsistencyGuardConfig consistencyGuardConfig; @@ -129,7 +127,7 @@ public class FailSafeConsistencyGuard implements ConsistencyGuard { return; } } catch (IOException ioe) { - LOG.warn("Got IOException waiting for file visibility. Retrying", ioe); + log.warn("Got IOException waiting for file visibility. Retrying", ioe); } sleepSafe(waitMs); @@ -152,7 +150,7 @@ public class FailSafeConsistencyGuard implements ConsistencyGuard { throws TimeoutException { long waitMs = consistencyGuardConfig.getInitialConsistencyCheckIntervalMs(); int attempt = 0; - LOG.info("Max Attempts=" + consistencyGuardConfig.getMaxConsistencyChecks()); + log.info("Max Attempts={}", consistencyGuardConfig.getMaxConsistencyChecks()); while (attempt < consistencyGuardConfig.getMaxConsistencyChecks()) { boolean success = checkFilesVisibility(attempt, dir, files, event); if (success) { @@ -178,7 +176,7 @@ public class FailSafeConsistencyGuard implements ConsistencyGuard { protected boolean checkFilesVisibility(int retryNum, StoragePath dir, List<String> files, FileVisibility event) { try { - LOG.info("Trying " + retryNum); + log.info("Trying {}", retryNum); List<StoragePathInfo> entries = storage.listDirectEntries(dir); List<String> gotFiles = entries.stream() .map(e -> e.getPath().getPathWithoutSchemeAndAuthority()) @@ -188,7 +186,7 @@ public class FailSafeConsistencyGuard implements ConsistencyGuard { switch (event) { case DISAPPEAR: - LOG.info("Following files are visible" + candidateFiles); + log.info("Following files are visible{}", candidateFiles); // If no candidate files gets removed, it means all of them have disappeared return !altered; case APPEAR: @@ -197,7 +195,7 @@ public class FailSafeConsistencyGuard implements ConsistencyGuard { return candidateFiles.isEmpty(); } } catch (IOException ioe) { - LOG.warn("Got IOException waiting for file event. Have tried {} time(s)", retryNum, ioe); + log.warn("Got IOException waiting for file event. Have tried {} time(s)", retryNum, ioe); } return false; } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/OptimisticConsistencyGuard.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/OptimisticConsistencyGuard.java index dfc58c6dd11a..297c9941b9de 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/fs/OptimisticConsistencyGuard.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/OptimisticConsistencyGuard.java @@ -21,8 +21,7 @@ package org.apache.hudi.common.fs; import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; import java.util.List; @@ -49,10 +48,9 @@ import java.util.concurrent.TimeoutException; * With this, if any files that was created, should be available within configured threshold(eventual consistency). * Delete() will return false if FileNotFound. So, both cases are taken care of this {@link ConsistencyGuard}. */ +@Slf4j public class OptimisticConsistencyGuard extends FailSafeConsistencyGuard { - private static final Logger LOG = LoggerFactory.getLogger(OptimisticConsistencyGuard.class); - public OptimisticConsistencyGuard(HoodieStorage storage, ConsistencyGuardConfig consistencyGuardConfig) { super(storage, consistencyGuardConfig); @@ -65,7 +63,7 @@ public class OptimisticConsistencyGuard extends FailSafeConsistencyGuard { Thread.sleep(consistencyGuardConfig.getOptimisticConsistencyGuardSleepTimeMs()); } } catch (IOException | InterruptedException ioe) { - LOG.warn("Got IOException or InterruptedException waiting for file visibility. Ignoring", + log.warn("Got IOException or InterruptedException waiting for file visibility. Ignoring", ioe); } } @@ -83,7 +81,7 @@ public class OptimisticConsistencyGuard extends FailSafeConsistencyGuard { Thread.sleep(consistencyGuardConfig.getOptimisticConsistencyGuardSleepTimeMs()); } } catch (InterruptedException ie) { - LOG.warn("Got InterruptedException waiting for file visibility. Ignoring", ie); + log.warn("Got InterruptedException waiting for file visibility. Ignoring", ie); } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/heartbeat/HoodieHeartbeatUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/heartbeat/HoodieHeartbeatUtils.java index a51914554d71..fab4511c703f 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/heartbeat/HoodieHeartbeatUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/heartbeat/HoodieHeartbeatUtils.java @@ -23,16 +23,15 @@ import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; /** * Common utils for Hudi heartbeat */ +@Slf4j public class HoodieHeartbeatUtils { - private static final Logger LOG = LoggerFactory.getLogger(HoodieHeartbeatUtils.class); /** * Use modification time as last heart beat time. @@ -72,7 +71,7 @@ public class HoodieHeartbeatUtils { Long currentTime = System.currentTimeMillis(); Long lastHeartbeatTime = getLastHeartbeatTime(storage, basePath, instantTime); if (currentTime - lastHeartbeatTime > maxAllowableHeartbeatIntervalInMs) { - LOG.warn("Heartbeat expired, for instant: {}", instantTime); + log.warn("Heartbeat expired, for instant: {}", instantTime); return true; } return false; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchema.java b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchema.java index dff06a4c0e0f..0b0978d552a2 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchema.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchema.java @@ -89,6 +89,7 @@ import static org.apache.hudi.avro.HoodieAvroUtils.createNewSchemaField; * * @since 1.2.0 */ +@Getter public class HoodieSchema implements Serializable { private static final long serialVersionUID = 1L; @@ -1567,18 +1568,6 @@ public class HoodieSchema implements Serializable { return (path.charAt(next) == '.') ? next + 1 : -1; } - /** - * Returns the underlying Avro schema for compatibility purposes. - * - * <p>This method is provided for gradual migration and should be used - * sparingly. New code should prefer the HoodieSchema API.</p> - * - * @return the wrapped Avro Schema - */ - public Schema getAvroSchema() { - return avroSchema; - } - /** * Converts this HoodieSchema to an Avro Schema. * This is an alias for getAvroSchema() provided for API consistency. @@ -1909,7 +1898,9 @@ public class HoodieSchema implements Serializable { } public static class Decimal extends HoodieSchema { + @Getter private final int precision; + @Getter private final int scale; private final Option<Integer> fixedSize; @@ -1934,14 +1925,6 @@ public class HoodieSchema implements Serializable { } } - public int getPrecision() { - return precision; - } - - public int getScale() { - return scale; - } - @Override public String getName() { return String.format("decimal(%d,%d)", precision, scale); @@ -2224,7 +2207,9 @@ public class HoodieSchema implements Serializable { } public static class Timestamp extends HoodieSchema { + @Getter private final boolean isUtcAdjusted; + @Getter private final TimePrecision precision; /** @@ -2255,14 +2240,6 @@ public class HoodieSchema implements Serializable { } } - public TimePrecision getPrecision() { - return precision; - } - - public boolean isUtcAdjusted() { - return isUtcAdjusted; - } - @Override public String getName() { if (isUtcAdjusted) { @@ -2299,6 +2276,7 @@ public class HoodieSchema implements Serializable { } public static class Time extends HoodieSchema { + @Getter private final TimePrecision precision; /** @@ -2321,10 +2299,6 @@ public class HoodieSchema implements Serializable { } } - public TimePrecision getPrecision() { - return precision; - } - @Override public String getName() { if (precision == TimePrecision.MILLIS) { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaCompatibility.java b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaCompatibility.java index 9842cdd098c8..0fd96e55233e 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaCompatibility.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaCompatibility.java @@ -24,6 +24,9 @@ import org.apache.hudi.exception.MissingSchemaFieldException; import org.apache.hudi.exception.SchemaBackwardsCompatibilityException; import org.apache.hudi.internal.schema.HoodieSchemaException; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; @@ -45,12 +48,9 @@ import java.util.stream.Collectors; * <li>Metadata field handling during schema checks</li> * </ul> */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) public final class HoodieSchemaCompatibility { - // Prevent instantiation - private HoodieSchemaCompatibility() { - } - public static boolean areSchemasCompatible(HoodieSchema tableSchema, HoodieSchema writerSchema) { return HoodieSchemaCompatibilityChecker.checkReaderWriterCompatibility(tableSchema, writerSchema, false).getType() == HoodieSchemaCompatibilityChecker.SchemaCompatibilityType.COMPATIBLE; } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaField.java b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaField.java index 190e180b4968..9b1d16952192 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaField.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaField.java @@ -22,6 +22,7 @@ import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ValidationUtils; +import lombok.Getter; import org.apache.avro.Schema; import java.io.Serializable; @@ -55,6 +56,7 @@ public class HoodieSchemaField implements Serializable { private static final long serialVersionUID = 1L; + @Getter private final Schema.Field avroField; private final HoodieSchema fieldSchema; @@ -251,18 +253,6 @@ public class HoodieSchemaField implements Serializable { return avroField.aliases(); } - /** - * Returns the underlying Avro field for compatibility purposes. - * - * <p>This method is provided for gradual migration and should be used - * sparingly. New code should prefer the HoodieSchemaField API.</p> - * - * @return the wrapped Avro Schema.Field - */ - public Schema.Field getAvroField() { - return avroField; - } - /** * Creates a copy of this field with a new name. * diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java index d01669b507f3..6535f6be700e 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java @@ -629,10 +629,15 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness { throws IOException { Map<String, List<String>> partitionToFiles = deleteFiles(files); List<HoodieCleanStat> cleanStats = partitionToFiles.entrySet().stream().map(e -> - new HoodieCleanStat(HoodieCleaningPolicy.KEEP_LATEST_COMMITS, e.getKey(), e.getValue(), e.getValue(), - new ArrayList<>(), - instant.length() < 3 ? String.valueOf(Integer.parseInt(instant) + 1) : HoodieInstantTimeGenerator.instantTimePlusMillis(instant, 1), - "")).collect(Collectors.toList()); + HoodieCleanStat.builder() + .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_COMMITS) + .withPartitionPath(e.getKey()) + .withDeletePathPatterns(e.getValue()) + .withSuccessDeleteFiles(e.getValue()) + .withFailedDeleteFiles(Collections.emptyList()) + .withEarliestCommitToRetain(instant.length() < 3 ? String.valueOf(Integer.parseInt(instant) + 1) : HoodieInstantTimeGenerator.instantTimePlusMillis(instant, 1)) + .withLastCompletedCommitTimestamp("") + .build()).collect(Collectors.toList()); HoodieInstant cleanInflightInstant = INSTANT_GENERATOR.createNewInstant(State.INFLIGHT, HoodieTimeline.CLEAN_ACTION, cleanInstant); metaClient.getActiveTimeline().createNewInstant(cleanInflightInstant); diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java index d0ec62e0abb2..46b5248e6f33 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java @@ -451,14 +451,12 @@ public class HoodieTestTable implements AutoCloseable { public HoodieTestTable addClean(String instantTime) throws IOException { HoodieCleanerPlan cleanerPlan = new HoodieCleanerPlan(new HoodieActionInstant(EMPTY_STRING, EMPTY_STRING, EMPTY_STRING), EMPTY_STRING, EMPTY_STRING, new HashMap<>(), CleanPlanV2MigrationHandler.VERSION, new HashMap<>(), new ArrayList<>(), Collections.EMPTY_MAP); - HoodieCleanStat cleanStats = new HoodieCleanStat( - HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS, - HoodieTestUtils.DEFAULT_PARTITION_PATHS[RANDOM.nextInt(HoodieTestUtils.DEFAULT_PARTITION_PATHS.length)], - Collections.emptyList(), - Collections.emptyList(), - Collections.emptyList(), - instantTime, - ""); + HoodieCleanStat cleanStats = HoodieCleanStat.builder() + .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS) + .withPartitionPath(HoodieTestUtils.DEFAULT_PARTITION_PATHS[RANDOM.nextInt(HoodieTestUtils.DEFAULT_PARTITION_PATHS.length)]) + .withEarliestCommitToRetain(instantTime) + .withLastCompletedCommitTimestamp("") + .build(); HoodieCleanMetadata cleanMetadata = convertCleanMetadata(instantTime, Option.of(0L), Collections.singletonList(cleanStats), Collections.EMPTY_MAP); return HoodieTestTable.of(metaClient).addClean(instantTime, cleanerPlan, cleanMetadata); } @@ -468,8 +466,14 @@ public class HoodieTestTable implements AutoCloseable { EMPTY_STRING, EMPTY_STRING, new HashMap<>(), CleanPlanV2MigrationHandler.VERSION, new HashMap<>(), new ArrayList<>(), Collections.EMPTY_MAP); List<HoodieCleanStat> cleanStats = new ArrayList<>(); for (Map.Entry<String, List<String>> entry : testTableState.getPartitionToFileIdMapForCleaner(commitTime).entrySet()) { - cleanStats.add(new HoodieCleanStat(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS, - entry.getKey(), entry.getValue(), entry.getValue(), Collections.emptyList(), commitTime, "")); + cleanStats.add(HoodieCleanStat.builder() + .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS) + .withPartitionPath(entry.getKey()) + .withDeletePathPatterns(entry.getValue()) + .withSuccessDeleteFiles(entry.getValue()) + .withEarliestCommitToRetain(commitTime) + .withLastCompletedCommitTimestamp("") + .build()); } return Pair.of(cleanerPlan, convertCleanMetadata(commitTime, Option.of(0L), cleanStats, Collections.EMPTY_MAP)); } diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedFileFormat.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedFileFormat.scala index da5bef9f785c..3fe8c6ff62f7 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedFileFormat.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedFileFormat.scala @@ -304,7 +304,7 @@ class HoodieFileGroupReaderBasedFileFormat(tablePath: String, val readerContext = new SparkFileFormatInternalRowReaderContext( fileGroupBaseFileReader.value, filters, requiredFilters, storageConf, metaClient.getTableConfig, sparkRequiredSchema = Some(requiredSchema)) - readerContext.setEnableLogicalTimestampFieldRepair(storageConf.getBoolean(ENABLE_LOGICAL_TIMESTAMP_REPAIR, true)) + readerContext.enableLogicalTimestampFieldRepair(storageConf.getBoolean(ENABLE_LOGICAL_TIMESTAMP_REPAIR, true)) val props = metaClient.getTableConfig.getProps options.foreach(kv => props.setProperty(kv._1, kv._2)) props.put(HoodieMemoryConfig.MAX_MEMORY_FOR_MERGE.key(), String.valueOf(maxMemoryPerCompaction))
