This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 438bbdfcaef4 refactor: Add Lombok annotations to hudi-common module
(part 3) (#17825)
438bbdfcaef4 is described below
commit 438bbdfcaef43abc5d0d21f931a47c5af05a58a8
Author: voonhous <[email protected]>
AuthorDate: Wed Jun 3 19:43:00 2026 +0800
refactor: Add Lombok annotations to hudi-common module (part 3) (#17825)
* refactor: Add Lombok annotations to hudi-common module (part 3)
* Address comments
* Address bot comments
---
.../table/action/clean/CleanActionExecutor.java | 20 +-
.../hudi/utils/TestMetadataConversionUtils.java | 14 +-
.../functional/TestExternalPathHandling.java | 10 +-
.../apache/hudi/io/TestHoodieTimelineArchiver.java | 15 +-
.../java/org/apache/hudi/table/TestCleaner.java | 24 ++-
.../hudi/testutils/HoodieCleanerTestBase.java | 31 +--
.../testutils/HoodieSparkClientTestHarness.java | 14 +-
hudi-common/pom.xml | 1 +
.../org/apache/hudi/common/HoodieCleanStat.java | 210 +++------------------
.../hudi/common/HoodiePendingRollbackInfo.java | 18 +-
.../org/apache/hudi/common/HoodieRollbackStat.java | 34 +---
.../common/bloom/InternalDynamicBloomFilter.java | 9 +-
.../apache/hudi/common/bloom/InternalFilter.java | 7 +-
.../java/org/apache/hudi/common/bloom/Key.java | 8 +-
.../bootstrap/index/hfile/HFileBootstrapIndex.java | 15 +-
.../index/hfile/HFileBootstrapIndexReader.java | 22 +--
.../index/hfile/HFileBootstrapIndexWriter.java | 17 +-
.../apache/hudi/common/config/ConfigGroups.java | 20 +-
.../apache/hudi/common/config/ConfigProperty.java | 66 ++-----
.../apache/hudi/common/config/HoodieConfig.java | 12 +-
.../DirectMarkerBasedDetectionStrategy.java | 21 +--
.../hudi/common/data/HoodieBaseListData.java | 16 +-
.../hudi/common/engine/HoodieEngineContext.java | 18 +-
.../hudi/common/engine/HoodieReaderContext.java | 128 +++----------
.../apache/hudi/common/engine/RecordContext.java | 12 +-
.../java/org/apache/hudi/common/fs/FSUtils.java | 9 +-
.../hudi/common/fs/FailSafeConsistencyGuard.java | 16 +-
.../hudi/common/fs/OptimisticConsistencyGuard.java | 10 +-
.../common/heartbeat/HoodieHeartbeatUtils.java | 7 +-
.../apache/hudi/common/schema/HoodieSchema.java | 38 +---
.../common/schema/HoodieSchemaCompatibility.java | 8 +-
.../hudi/common/schema/HoodieSchemaField.java | 14 +-
.../table/view/TestIncrementalFSViewSync.java | 13 +-
.../hudi/common/testutils/HoodieTestTable.java | 24 ++-
.../HoodieFileGroupReaderBasedFileFormat.scala | 2 +-
35 files changed, 272 insertions(+), 631 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java
index 63a6e7d8f994..2a636221ae60 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java
@@ -171,20 +171,18 @@ public class CleanActionExecutor<T, I, K, O> extends
BaseActionExecutor<T, I, K,
? partitionCleanStatsMap.get(partitionPath)
: new PartitionCleanStat(partitionPath);
HoodieActionInstant actionInstant =
cleanerPlan.getEarliestInstantToRetain();
- return
HoodieCleanStat.newBuilder().withPolicy(config.getCleanerPolicy()).withPartitionPath(partitionPath)
- .withEarliestCommitRetained(Option.ofNullable(
- actionInstant != null
- ?
instantGenerator.createNewInstant(HoodieInstant.State.valueOf(actionInstant.getState()),
- actionInstant.getAction(), actionInstant.getTimestamp())
- : null))
+ return HoodieCleanStat.builder()
+ .withPolicy(config.getCleanerPolicy())
+ .withPartitionPath(partitionPath)
+ .withEarliestCommitToRetain(actionInstant != null ?
actionInstant.getTimestamp() : "")
.withLastCompletedCommitTimestamp(cleanerPlan.getLastCompletedCommitTimestamp())
- .withDeletePathPattern(partitionCleanStat.deletePathPatterns())
- .withSuccessfulDeletes(partitionCleanStat.successDeleteFiles())
- .withFailedDeletes(partitionCleanStat.failedDeleteFiles())
+ .withDeletePathPatterns(partitionCleanStat.deletePathPatterns())
+ .withSuccessDeleteFiles(partitionCleanStat.successDeleteFiles())
+ .withFailedDeleteFiles(partitionCleanStat.failedDeleteFiles())
.withDeleteBootstrapBasePathPatterns(partitionCleanStat.getDeleteBootstrapBasePathPatterns())
-
.withSuccessfulDeleteBootstrapBaseFiles(partitionCleanStat.getSuccessfulDeleteBootstrapBaseFiles())
+
.withSuccessDeleteBootstrapBaseFiles(partitionCleanStat.getSuccessfulDeleteBootstrapBaseFiles())
.withFailedDeleteBootstrapBaseFiles(partitionCleanStat.getFailedDeleteBootstrapBaseFiles())
- .isPartitionDeleted(partitionsToBeDeleted.contains(partitionPath))
+ .withPartitionDeleted(partitionsToBeDeleted.contains(partitionPath))
.build();
}).collect(Collectors.toList());
}
diff --git
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestMetadataConversionUtils.java
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestMetadataConversionUtils.java
index 01af9d3fc3ff..c0374b25e1f1 100644
---
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestMetadataConversionUtils.java
+++
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestMetadataConversionUtils.java
@@ -405,14 +405,12 @@ public class TestMetadataConversionUtils extends
HoodieCommonTestHarness {
HoodieCleanFileInfo fileInfo = new HoodieCleanFileInfo("file1", false);
HoodieCleanerPlan cleanerPlan = new HoodieCleanerPlan(new
HoodieActionInstant("", "", ""),
"", "", new HashMap<>(), CleanPlanV2MigrationHandler.VERSION,
Collections.singletonMap("key", Collections.singletonList(fileInfo)), new
ArrayList<>(), Collections.EMPTY_MAP);
- HoodieCleanStat cleanStats = new HoodieCleanStat(
- HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS,
- HoodieTestUtils.DEFAULT_PARTITION_PATHS[new
Random().nextInt(HoodieTestUtils.DEFAULT_PARTITION_PATHS.length)],
- Collections.emptyList(),
- Collections.emptyList(),
- Collections.emptyList(),
- instantTime,
- "");
+ HoodieCleanStat cleanStats = HoodieCleanStat.builder()
+ .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS)
+ .withPartitionPath(HoodieTestUtils.DEFAULT_PARTITION_PATHS[new
Random().nextInt(HoodieTestUtils.DEFAULT_PARTITION_PATHS.length)])
+ .withEarliestCommitToRetain(instantTime)
+ .withLastCompletedCommitTimestamp("")
+ .build();
HoodieCleanMetadata cleanMetadata = convertCleanMetadata(instantTime,
Option.of(0L), Collections.singletonList(cleanStats), Collections.EMPTY_MAP);
HoodieTestTable.of(metaClient).addClean(instantTime, cleanerPlan,
cleanMetadata);
}
diff --git
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestExternalPathHandling.java
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestExternalPathHandling.java
index ce94187aacbc..5d7c4aac2416 100644
---
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestExternalPathHandling.java
+++
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestExternalPathHandling.java
@@ -339,8 +339,14 @@ public class TestExternalPathHandling extends
HoodieClientTestBase {
}
private HoodieCleanStat createCleanStat(String partitionPath, List<String>
deletePaths, String earliestCommitToRetain, String
lastCompletedCommitTimestamp) {
- return new HoodieCleanStat(HoodieCleaningPolicy.KEEP_LATEST_COMMITS,
partitionPath, deletePaths, deletePaths, Collections.emptyList(),
- earliestCommitToRetain, lastCompletedCommitTimestamp);
+ return HoodieCleanStat.builder()
+ .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_COMMITS)
+ .withPartitionPath(partitionPath)
+ .withDeletePathPatterns(deletePaths)
+ .withSuccessDeleteFiles(deletePaths)
+ .withEarliestCommitToRetain(earliestCommitToRetain)
+ .withLastCompletedCommitTimestamp(lastCompletedCommitTimestamp)
+ .build();
}
private HoodieCleanerPlan cleanerPlan(HoodieActionInstant
earliestInstantToRetain, String latestCommit, Map<String,
List<HoodieCleanFileInfo>> filePathsToBeDeletedPerPartition) {
diff --git
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java
index 1fa1b7883e33..e514e99f941f 100644
---
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java
+++
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java
@@ -2356,15 +2356,12 @@ public class TestHoodieTimelineArchiver extends
HoodieSparkClientTestHarness {
private void addCleanCommitWithECTR(HoodieTestTable testTable, String
cleanInstant, String ectr, String lastCompleted) throws Exception {
List<HoodieCleanStat> cleanStatsList = new ArrayList<>();
- cleanStatsList.add(new HoodieCleanStat(
- HoodieCleaningPolicy.KEEP_LATEST_COMMITS,
- "p1",
- Collections.emptyList(),
- Collections.emptyList(),
- Collections.emptyList(),
- ectr,
- lastCompleted
- ));
+ cleanStatsList.add(HoodieCleanStat.builder()
+ .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_COMMITS)
+ .withPartitionPath("p1")
+ .withEarliestCommitToRetain(ectr)
+ .withLastCompletedCommitTimestamp(lastCompleted)
+ .build());
HoodieCleanMetadata cleanMetadata =
CleanerUtils.convertCleanMetadata(cleanInstant, Option.of(0L),
cleanStatsList, Collections.emptyMap());
diff --git
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java
index b57b91e174ae..1e821b3b6f75 100644
---
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java
+++
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java
@@ -724,18 +724,30 @@ public class TestCleaner extends HoodieCleanerTestBase {
List<String> failedDeleteFiles1 = Collections.singletonList(filePath2);
// create partition1 clean stat.
- HoodieCleanStat cleanStat1 = new
HoodieCleanStat(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS,
- partition1, deletePathPatterns1, successDeleteFiles1,
- failedDeleteFiles1, instantTime, "");
+ HoodieCleanStat cleanStat1 = HoodieCleanStat.builder()
+ .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS)
+ .withPartitionPath(partition1)
+ .withDeletePathPatterns(deletePathPatterns1)
+ .withSuccessDeleteFiles(successDeleteFiles1)
+ .withFailedDeleteFiles(failedDeleteFiles1)
+ .withEarliestCommitToRetain(instantTime)
+ .withLastCompletedCommitTimestamp("")
+ .build();
List<String> deletePathPatterns2 = new ArrayList<>();
List<String> successDeleteFiles2 = new ArrayList<>();
List<String> failedDeleteFiles2 = new ArrayList<>();
// create partition2 empty clean stat.
- HoodieCleanStat cleanStat2 = new
HoodieCleanStat(HoodieCleaningPolicy.KEEP_LATEST_COMMITS,
- partition2, deletePathPatterns2, successDeleteFiles2,
- failedDeleteFiles2, instantTime, "");
+ HoodieCleanStat cleanStat2 = HoodieCleanStat.builder()
+ .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_COMMITS)
+ .withPartitionPath(partition2)
+ .withDeletePathPatterns(deletePathPatterns2)
+ .withSuccessDeleteFiles(successDeleteFiles2)
+ .withFailedDeleteFiles(failedDeleteFiles2)
+ .withEarliestCommitToRetain(instantTime)
+ .withLastCompletedCommitTimestamp("")
+ .build();
// map with absolute file path.
Map<String, Tuple3> oldExpected = new HashMap<>();
diff --git
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieCleanerTestBase.java
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieCleanerTestBase.java
index 66f2f595ef4d..5563f39c811b 100644
---
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieCleanerTestBase.java
+++
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieCleanerTestBase.java
@@ -164,24 +164,27 @@ public class HoodieCleanerTestBase extends
HoodieClientTestBase {
}
Map<String, HoodieCleanStat> cleanStatMap =
cleanMetadata1.getPartitionMetadata().values().stream()
- .map(x -> new
HoodieCleanStat.Builder().withPartitionPath(x.getPartitionPath())
-
.withFailedDeletes(x.getFailedDeleteFiles()).withSuccessfulDeletes(x.getSuccessDeleteFiles())
-
.withPolicy(HoodieCleaningPolicy.valueOf(x.getPolicy())).withDeletePathPattern(x.getDeletePathPatterns())
-
.withEarliestCommitRetained(Option.ofNullable(cleanMetadata1.getEarliestCommitToRetain()
!= null
- ?
INSTANT_GENERATOR.createNewInstant(HoodieInstant.State.COMPLETED,
HoodieTimeline.COMMIT_ACTION, "000")
- : null))
+ .map(x -> HoodieCleanStat.builder()
+ .withPolicy(HoodieCleaningPolicy.valueOf(x.getPolicy()))
+ .withPartitionPath(x.getPartitionPath())
+ .withDeletePathPatterns(x.getDeletePathPatterns())
+ .withSuccessDeleteFiles(x.getSuccessDeleteFiles())
+ .withFailedDeleteFiles(x.getFailedDeleteFiles())
+
.withEarliestCommitToRetain(cleanMetadata1.getEarliestCommitToRetain() != null
? "000" : "")
.build())
.collect(Collectors.toMap(HoodieCleanStat::getPartitionPath, x -> x));
cleanMetadata1.getBootstrapPartitionMetadata().values().forEach(x -> {
- HoodieCleanStat s = cleanStatMap.get(x.getPartitionPath());
- cleanStatMap.put(x.getPartitionPath(), new
HoodieCleanStat.Builder().withPartitionPath(x.getPartitionPath())
-
.withFailedDeletes(s.getFailedDeleteFiles()).withSuccessfulDeletes(s.getSuccessDeleteFiles())
-
.withPolicy(HoodieCleaningPolicy.valueOf(x.getPolicy())).withDeletePathPattern(s.getDeletePathPatterns())
-
.withEarliestCommitRetained(Option.ofNullable(s.getEarliestCommitToRetain())
- .map(y ->
INSTANT_GENERATOR.createNewInstant(HoodieInstant.State.COMPLETED,
HoodieTimeline.COMMIT_ACTION, y)))
- .withSuccessfulDeleteBootstrapBaseFiles(x.getSuccessDeleteFiles())
+ cleanStatMap.compute(x.getPartitionPath(), (k, s) ->
HoodieCleanStat.builder()
+ .withPolicy(HoodieCleaningPolicy.valueOf(x.getPolicy()))
+ .withPartitionPath(x.getPartitionPath())
+ .withDeletePathPatterns(s.getDeletePathPatterns())
+ .withSuccessDeleteFiles(s.getSuccessDeleteFiles())
+ .withFailedDeleteFiles(s.getFailedDeleteFiles())
+ .withEarliestCommitToRetain(s.getEarliestCommitToRetain() != null ?
s.getEarliestCommitToRetain() : "")
+ .withDeleteBootstrapBasePathPatterns(x.getDeletePathPatterns())
+ .withSuccessDeleteBootstrapBaseFiles(x.getSuccessDeleteFiles())
.withFailedDeleteBootstrapBaseFiles(x.getFailedDeleteFiles())
-
.withDeleteBootstrapBasePathPatterns(x.getDeletePathPatterns()).build());
+ .build());
});
return new ArrayList<>(cleanStatMap.values());
}
diff --git
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java
index 97bd0e1175d7..296690556327 100644
---
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java
+++
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java
@@ -676,14 +676,12 @@ public abstract class HoodieSparkClientTestHarness
extends HoodieWriterClientTes
if (inflightOnly) {
HoodieTestTable.of(metaClient).addInflightClean(instantTime,
cleanerPlan);
} else {
- HoodieCleanStat cleanStats = new HoodieCleanStat(
- HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS,
- HoodieTestUtils.DEFAULT_PARTITION_PATHS[new
Random().nextInt(HoodieTestUtils.DEFAULT_PARTITION_PATHS.length)],
- Collections.emptyList(),
- Collections.emptyList(),
- Collections.emptyList(),
- instantTime,
- "");
+ HoodieCleanStat cleanStats = HoodieCleanStat.builder()
+ .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS)
+ .withPartitionPath(HoodieTestUtils.DEFAULT_PARTITION_PATHS[new
Random().nextInt(HoodieTestUtils.DEFAULT_PARTITION_PATHS.length)])
+ .withEarliestCommitToRetain(instantTime)
+ .withLastCompletedCommitTimestamp("")
+ .build();
HoodieCleanMetadata cleanMetadata = convertCleanMetadata(instantTime,
Option.of(0L), Collections.singletonList(cleanStats), Collections.EMPTY_MAP);
HoodieTestTable.of(metaClient).addClean(instantTime, cleanerPlan,
cleanMetadata, isEmptyForAll, isEmptyCompleted);
}
diff --git a/hudi-common/pom.xml b/hudi-common/pom.xml
index 17288103e946..95ef3417b950 100644
--- a/hudi-common/pom.xml
+++ b/hudi-common/pom.xml
@@ -128,6 +128,7 @@
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
+ <scope>provided</scope>
</dependency>
<!-- Logging -->
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/HoodieCleanStat.java
b/hudi-common/src/main/java/org/apache/hudi/common/HoodieCleanStat.java
index 498d430b5c3a..0106dfc96200 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/HoodieCleanStat.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/HoodieCleanStat.java
@@ -19,9 +19,10 @@
package org.apache.hudi.common;
import org.apache.hudi.common.model.HoodieCleaningPolicy;
-import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.util.CollectionUtils;
-import org.apache.hudi.common.util.Option;
+
+import lombok.Builder;
+import lombok.Value;
import java.io.Serializable;
import java.util.List;
@@ -29,197 +30,34 @@ import java.util.List;
/**
* Collects stats about a single partition clean operation.
*/
+@Builder(setterPrefix = "with")
+@Value
public class HoodieCleanStat implements Serializable {
// Policy used
- private final HoodieCleaningPolicy policy;
+ HoodieCleaningPolicy policy;
// Partition path cleaned
- private final String partitionPath;
+ String partitionPath;
// The patterns that were generated for the delete operation
- private final List<String> deletePathPatterns;
- private final List<String> successDeleteFiles;
- // Files that could not be deleted
- private final List<String> failedDeleteFiles;
- // Bootstrap Base Path patterns that were generated for the delete operation
- private final List<String> deleteBootstrapBasePathPatterns;
- private final List<String> successDeleteBootstrapBaseFiles;
+ @Builder.Default
+ List<String> deletePathPatterns = CollectionUtils.createImmutableList();
+ @Builder.Default
+ List<String> successDeleteFiles = CollectionUtils.createImmutableList();
// Files that could not be deleted
- private final List<String> failedDeleteBootstrapBaseFiles;
+ @Builder.Default
+ List<String> failedDeleteFiles = CollectionUtils.createImmutableList();
// Earliest commit that was retained in this clean
- private final String earliestCommitToRetain;
+ String earliestCommitToRetain;
// Last completed commit timestamp before clean
- private final String lastCompletedCommitTimestamp;
+ String lastCompletedCommitTimestamp;
+ // Bootstrap Base Path patterns that were generated for the delete operation
+ @Builder.Default
+ List<String> deleteBootstrapBasePathPatterns =
CollectionUtils.createImmutableList();
+ @Builder.Default
+ List<String> successDeleteBootstrapBaseFiles =
CollectionUtils.createImmutableList();
+ // Files that could not be deleted
+ @Builder.Default
+ List<String> failedDeleteBootstrapBaseFiles =
CollectionUtils.createImmutableList();
// set to true if partition is deleted
- private final boolean isPartitionDeleted;
-
- public HoodieCleanStat(HoodieCleaningPolicy policy, String partitionPath,
List<String> deletePathPatterns,
- List<String> successDeleteFiles, List<String> failedDeleteFiles, String
earliestCommitToRetain,String lastCompletedCommitTimestamp) {
- this(policy, partitionPath, deletePathPatterns, successDeleteFiles,
failedDeleteFiles, earliestCommitToRetain,
- lastCompletedCommitTimestamp, CollectionUtils.createImmutableList(),
CollectionUtils.createImmutableList(),
- CollectionUtils.createImmutableList(), false);
- }
-
- public HoodieCleanStat(HoodieCleaningPolicy policy, String partitionPath,
List<String> deletePathPatterns,
- List<String> successDeleteFiles, List<String>
failedDeleteFiles,
- String earliestCommitToRetain,String
lastCompletedCommitTimestamp,
- List<String> deleteBootstrapBasePathPatterns,
- List<String> successDeleteBootstrapBaseFiles,
- List<String> failedDeleteBootstrapBaseFiles,
- boolean isPartitionDeleted) {
- this.policy = policy;
- this.partitionPath = partitionPath;
- this.deletePathPatterns = deletePathPatterns;
- this.successDeleteFiles = successDeleteFiles;
- this.failedDeleteFiles = failedDeleteFiles;
- this.earliestCommitToRetain = earliestCommitToRetain;
- this.lastCompletedCommitTimestamp = lastCompletedCommitTimestamp;
- this.deleteBootstrapBasePathPatterns = deleteBootstrapBasePathPatterns;
- this.successDeleteBootstrapBaseFiles = successDeleteBootstrapBaseFiles;
- this.failedDeleteBootstrapBaseFiles = failedDeleteBootstrapBaseFiles;
- this.isPartitionDeleted = isPartitionDeleted;
- }
-
- public HoodieCleaningPolicy getPolicy() {
- return policy;
- }
-
- public String getPartitionPath() {
- return partitionPath;
- }
-
- public List<String> getDeletePathPatterns() {
- return deletePathPatterns;
- }
-
- public List<String> getSuccessDeleteFiles() {
- return successDeleteFiles;
- }
-
- public List<String> getFailedDeleteFiles() {
- return failedDeleteFiles;
- }
-
- public List<String> getDeleteBootstrapBasePathPatterns() {
- return deleteBootstrapBasePathPatterns;
- }
-
- public List<String> getSuccessDeleteBootstrapBaseFiles() {
- return successDeleteBootstrapBaseFiles;
- }
-
- public List<String> getFailedDeleteBootstrapBaseFiles() {
- return failedDeleteBootstrapBaseFiles;
- }
-
- public String getEarliestCommitToRetain() {
- return earliestCommitToRetain;
- }
-
- public String getLastCompletedCommitTimestamp() {
- return lastCompletedCommitTimestamp;
- }
-
- public boolean isPartitionDeleted() {
- return isPartitionDeleted;
- }
-
- public static Builder newBuilder() {
- return new Builder();
- }
-
- /**
- * A builder used to build {@link HoodieCleanStat}.
- */
- public static class Builder {
-
- private HoodieCleaningPolicy policy;
- private List<String> deletePathPatterns;
- private List<String> successDeleteFiles;
- private List<String> failedDeleteFiles;
- private String partitionPath;
- private String earliestCommitToRetain;
- private String lastCompletedCommitTimestamp;
- private List<String> deleteBootstrapBasePathPatterns;
- private List<String> successDeleteBootstrapBaseFiles;
- private List<String> failedDeleteBootstrapBaseFiles;
- private boolean isPartitionDeleted;
-
- public Builder withPolicy(HoodieCleaningPolicy policy) {
- this.policy = policy;
- return this;
- }
-
- public Builder withDeletePathPattern(List<String> deletePathPatterns) {
- this.deletePathPatterns = deletePathPatterns;
- return this;
- }
-
- public Builder withSuccessfulDeletes(List<String> successDeleteFiles) {
- this.successDeleteFiles = successDeleteFiles;
- return this;
- }
-
- public Builder withFailedDeletes(List<String> failedDeleteFiles) {
- this.failedDeleteFiles = failedDeleteFiles;
- return this;
- }
-
- public Builder withDeleteBootstrapBasePathPatterns(List<String>
deletePathPatterns) {
- this.deleteBootstrapBasePathPatterns = deletePathPatterns;
- return this;
- }
-
- public Builder withSuccessfulDeleteBootstrapBaseFiles(List<String>
successDeleteFiles) {
- this.successDeleteBootstrapBaseFiles = successDeleteFiles;
- return this;
- }
-
- public Builder withFailedDeleteBootstrapBaseFiles(List<String>
failedDeleteFiles) {
- this.failedDeleteBootstrapBaseFiles = failedDeleteFiles;
- return this;
- }
-
- public Builder withPartitionPath(String partitionPath) {
- this.partitionPath = partitionPath;
- return this;
- }
-
- public Builder withEarliestCommitRetained(Option<HoodieInstant>
earliestCommitToRetain) {
- this.earliestCommitToRetain =
- (earliestCommitToRetain.isPresent()) ?
earliestCommitToRetain.get().requestedTime() : "";
- return this;
- }
-
- public Builder withLastCompletedCommitTimestamp(String
lastCompletedCommitTimestamp) {
- this.lastCompletedCommitTimestamp = lastCompletedCommitTimestamp;
- return this;
- }
-
- public Builder isPartitionDeleted(boolean isPartitionDeleted) {
- this.isPartitionDeleted = isPartitionDeleted;
- return this;
- }
-
- public HoodieCleanStat build() {
- return new HoodieCleanStat(policy, partitionPath, deletePathPatterns,
successDeleteFiles, failedDeleteFiles,
- earliestCommitToRetain, lastCompletedCommitTimestamp,
deleteBootstrapBasePathPatterns,
- successDeleteBootstrapBaseFiles, failedDeleteBootstrapBaseFiles,
isPartitionDeleted);
- }
- }
-
- @Override
- public String toString() {
- return "HoodieCleanStat{"
- + "policy=" + policy
- + ", partitionPath='" + partitionPath + '\''
- + ", deletePathPatterns=" + deletePathPatterns
- + ", successDeleteFiles=" + successDeleteFiles
- + ", failedDeleteFiles=" + failedDeleteFiles
- + ", earliestCommitToRetain='" + earliestCommitToRetain
- + ", deleteBootstrapBasePathPatterns=" +
deleteBootstrapBasePathPatterns
- + ", successDeleteBootstrapBaseFiles=" +
successDeleteBootstrapBaseFiles
- + ", failedDeleteBootstrapBaseFiles=" + failedDeleteBootstrapBaseFiles
- + ", isPartitionDeleted=" + isPartitionDeleted + '\''
- + '}';
- }
+ boolean partitionDeleted;
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/HoodiePendingRollbackInfo.java
b/hudi-common/src/main/java/org/apache/hudi/common/HoodiePendingRollbackInfo.java
index c53babf35010..44e61f651b8c 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/HoodiePendingRollbackInfo.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/HoodiePendingRollbackInfo.java
@@ -21,24 +21,16 @@ package org.apache.hudi.common;
import org.apache.hudi.avro.model.HoodieRollbackPlan;
import org.apache.hudi.common.table.timeline.HoodieInstant;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
/**
* Holds rollback instant and rollback plan for a pending rollback.
*/
+@AllArgsConstructor
+@Getter
public class HoodiePendingRollbackInfo {
private final HoodieInstant rollbackInstant;
private final HoodieRollbackPlan rollbackPlan;
-
- public HoodiePendingRollbackInfo(HoodieInstant rollbackInstant,
HoodieRollbackPlan rollbackPlan) {
- this.rollbackInstant = rollbackInstant;
- this.rollbackPlan = rollbackPlan;
- }
-
- public HoodieInstant getRollbackInstant() {
- return rollbackInstant;
- }
-
- public HoodieRollbackPlan getRollbackPlan() {
- return rollbackPlan;
- }
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/HoodieRollbackStat.java
b/hudi-common/src/main/java/org/apache/hudi/common/HoodieRollbackStat.java
index 59308a43325c..c9be79687854 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/HoodieRollbackStat.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/HoodieRollbackStat.java
@@ -20,6 +20,9 @@ package org.apache.hudi.common;
import org.apache.hudi.storage.StoragePathInfo;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
import java.io.Serializable;
import java.util.Collections;
import java.util.List;
@@ -29,6 +32,8 @@ import java.util.stream.Collectors;
/**
* Collects stats about a single partition clean operation.
*/
+@AllArgsConstructor
+@Getter
public class HoodieRollbackStat implements Serializable {
// Partition path
@@ -41,35 +46,6 @@ public class HoodieRollbackStat implements Serializable {
private final Map<String, Long> logFilesFromFailedCommit;
- public HoodieRollbackStat(String partitionPath, List<String>
successDeleteFiles, List<String> failedDeleteFiles,
- Map<StoragePathInfo, Long> commandBlocksCount,
Map<String, Long> logFilesFromFailedCommit) {
- this.partitionPath = partitionPath;
- this.successDeleteFiles = successDeleteFiles;
- this.failedDeleteFiles = failedDeleteFiles;
- this.commandBlocksCount = commandBlocksCount;
- this.logFilesFromFailedCommit = logFilesFromFailedCommit;
- }
-
- public Map<StoragePathInfo, Long> getCommandBlocksCount() {
- return commandBlocksCount;
- }
-
- public String getPartitionPath() {
- return partitionPath;
- }
-
- public List<String> getSuccessDeleteFiles() {
- return successDeleteFiles;
- }
-
- public List<String> getFailedDeleteFiles() {
- return failedDeleteFiles;
- }
-
- public Map<String, Long> getLogFilesFromFailedCommit() {
- return logFilesFromFailedCommit;
- }
-
public static HoodieRollbackStat.Builder newBuilder() {
return new Builder();
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalDynamicBloomFilter.java
b/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalDynamicBloomFilter.java
index bf35aeaee61d..fa14e2976f43 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalDynamicBloomFilter.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalDynamicBloomFilter.java
@@ -18,6 +18,8 @@
package org.apache.hudi.common.bloom;
+import lombok.NoArgsConstructor;
+
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
@@ -27,6 +29,7 @@ import java.io.IOException;
* with bounds on maximum number of entries. Once the max entries is reached,
false positive guarantees are not
* honored.
*/
+@NoArgsConstructor
class InternalDynamicBloomFilter extends InternalFilter {
/**
@@ -47,12 +50,6 @@ class InternalDynamicBloomFilter extends InternalFilter {
*/
private InternalBloomFilter[] matrix;
- /**
- * Zero-args constructor for the serialization.
- */
- public InternalDynamicBloomFilter() {
- }
-
/**
* Constructor.
* <p>
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalFilter.java
b/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalFilter.java
index e23255bb4b61..c5b45e7877a8 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalFilter.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalFilter.java
@@ -20,6 +20,9 @@ package org.apache.hudi.common.bloom;
import org.apache.hudi.common.util.hash.Hash;
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
@@ -42,6 +45,7 @@ import java.util.List;
* @see Key The general behavior of a key
* @see HashFunction A hash function
*/
+@NoArgsConstructor(access = AccessLevel.PROTECTED)
abstract class InternalFilter {
private static final int VERSION = -1; // negative to accommodate for old
format
/**
@@ -64,9 +68,6 @@ abstract class InternalFilter {
*/
protected int hashType;
- protected InternalFilter() {
- }
-
/**
* Constructor.
*
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/bloom/Key.java
b/hudi-common/src/main/java/org/apache/hudi/common/bloom/Key.java
index 013c0f08ea46..ed8096edea34 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/bloom/Key.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/bloom/Key.java
@@ -20,6 +20,7 @@
package org.apache.hudi.common.bloom;
import lombok.Getter;
+import lombok.NoArgsConstructor;
import java.io.DataInput;
import java.io.DataOutput;
@@ -33,6 +34,7 @@ import java.io.IOException;
* @see InternalBloomFilter The general behavior of a bloom filter and how the
key is used.
*/
@Getter
+@NoArgsConstructor
public class Key implements Comparable<Key> {
/**
* Byte value of key
@@ -47,12 +49,6 @@ public class Key implements Comparable<Key> {
*/
double weight;
- /**
- * default constructor - use with readFields
- */
- public Key() {
- }
-
/**
* Constructor.
* <p>
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndex.java
b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndex.java
index fd342288cf01..79feafbc4784 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndex.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndex.java
@@ -30,8 +30,8 @@ import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
@@ -49,12 +49,11 @@ import static
org.apache.hudi.common.util.StringUtils.getUTF8Bytes;
* on these index files to manage multiple file-groups.
*/
+@Slf4j
public class HFileBootstrapIndex extends BootstrapIndex {
private static final long serialVersionUID = 1L;
- private static final Logger LOG =
LoggerFactory.getLogger(HFileBootstrapIndex.class);
-
public static final String BOOTSTRAP_INDEX_FILE_ID =
"00000000-0000-0000-0000-000000000000-0";
private static final String PARTITION_KEY_PREFIX = "part";
@@ -68,6 +67,7 @@ public class HFileBootstrapIndex extends BootstrapIndex {
public static final String INDEX_INFO_KEY_STRING = "INDEX_INFO";
public static final byte[] INDEX_INFO_KEY =
getUTF8Bytes(INDEX_INFO_KEY_STRING);
+ @Getter
private final boolean isPresent;
public HFileBootstrapIndex(HoodieTableMetaClient metaClient) {
@@ -152,7 +152,7 @@ public class HFileBootstrapIndex extends BootstrapIndex {
StoragePath[] indexPaths = new StoragePath[]
{partitionIndexPath(metaClient), fileIdIndexPath(metaClient)};
for (StoragePath indexPath : indexPaths) {
if (metaClient.getStorage().exists(indexPath)) {
- LOG.info("Dropping bootstrap index. Deleting file: {}", indexPath);
+ log.info("Dropping bootstrap index. Deleting file: {}", indexPath);
metaClient.getStorage().deleteDirectory(indexPath);
}
}
@@ -160,9 +160,4 @@ public class HFileBootstrapIndex extends BootstrapIndex {
throw new HoodieIOException(ioe.getMessage(), ioe);
}
}
-
- @Override
- public boolean isPresent() {
- return isPresent;
- }
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndexReader.java
b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndexReader.java
index 53debdb71ba0..320b365e3519 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndexReader.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndexReader.java
@@ -38,8 +38,8 @@ import org.apache.hudi.io.util.IOUtils;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
import java.util.ArrayList;
@@ -59,10 +59,11 @@ import static
org.apache.hudi.common.bootstrap.index.hfile.HFileBootstrapIndex.p
/**
* HFile Based Index Reader.
*/
+@Slf4j
public class HFileBootstrapIndexReader extends BootstrapIndex.IndexReader {
- private static final Logger LOG =
LoggerFactory.getLogger(HFileBootstrapIndexReader.class);
// Base Path of external files.
+ @Getter
private final String bootstrapBasePath;
// Well Known Paths for indices
private final String indexByPartitionPath;
@@ -83,7 +84,7 @@ public class HFileBootstrapIndexReader extends
BootstrapIndex.IndexReader {
this.indexByFileIdPath = indexByFilePath.toString();
initIndexInfo();
this.bootstrapBasePath = bootstrapIndexInfo.getBootstrapBasePath();
- LOG.info("Loaded HFileBasedBootstrapIndex with source base path :" +
bootstrapBasePath);
+ log.info("Loaded HFileBasedBootstrapIndex with source base path :{}",
bootstrapBasePath);
}
/**
@@ -93,7 +94,7 @@ public class HFileBootstrapIndexReader extends
BootstrapIndex.IndexReader {
* @param storage {@link HoodieStorage} instance.
*/
private static HFileReader createReader(String hFilePath, HoodieStorage
storage) throws IOException {
- LOG.info("Opening HFile for reading :" + hFilePath);
+ log.info("Opening HFile for reading :{}", hFilePath);
StoragePath path = new StoragePath(hFilePath);
long fileSize = storage.getPathInfo(path).getLength();
SeekableDataInputStream stream = storage.openSeekable(path, false);
@@ -118,7 +119,7 @@ public class HFileBootstrapIndexReader extends
BootstrapIndex.IndexReader {
private synchronized HFileReader partitionIndexReader() throws IOException {
if (indexByPartitionReader == null) {
- LOG.info("Opening partition index :" + indexByPartitionPath);
+ log.info("Opening partition index :{}", indexByPartitionPath);
this.indexByPartitionReader = createReader(indexByPartitionPath,
metaClient.getStorage());
}
return indexByPartitionReader;
@@ -126,7 +127,7 @@ public class HFileBootstrapIndexReader extends
BootstrapIndex.IndexReader {
private synchronized HFileReader fileIdIndexReader() throws IOException {
if (indexByFileIdReader == null) {
- LOG.info("Opening fileId index :" + indexByFileIdPath);
+ log.info("Opening fileId index :{}", indexByFileIdPath);
this.indexByFileIdReader = createReader(indexByFileIdPath,
metaClient.getStorage());
}
return indexByFileIdReader;
@@ -181,7 +182,7 @@ public class HFileBootstrapIndexReader extends
BootstrapIndex.IndexReader {
.map(e -> new BootstrapFileMapping(bootstrapBasePath,
metadata.getBootstrapPartitionPath(),
e.getValue(), partition,
e.getKey())).collect(Collectors.toList());
} else {
- LOG.warn("No value found for partition key ({})", partition);
+ log.warn("No value found for partition key ({})", partition);
return new ArrayList<>();
}
} catch (IOException ioe) {
@@ -189,11 +190,6 @@ public class HFileBootstrapIndexReader extends
BootstrapIndex.IndexReader {
}
}
- @Override
- public String getBootstrapBasePath() {
- return bootstrapBasePath;
- }
-
@Override
public Map<HoodieFileGroupId, BootstrapFileMapping>
getSourceFileMappingForFileIds(
List<HoodieFileGroupId> ids) {
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndexWriter.java
b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndexWriter.java
index bcd063ff5d0d..8fd232a87402 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndexWriter.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndexWriter.java
@@ -35,8 +35,7 @@ import org.apache.hudi.io.hfile.HFileWriter;
import org.apache.hudi.io.hfile.HFileWriterImpl;
import org.apache.hudi.storage.StoragePath;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
import java.io.OutputStream;
@@ -53,8 +52,8 @@ import static
org.apache.hudi.common.bootstrap.index.hfile.HFileBootstrapIndex.g
import static
org.apache.hudi.common.bootstrap.index.hfile.HFileBootstrapIndex.getPartitionKey;
import static
org.apache.hudi.common.bootstrap.index.hfile.HFileBootstrapIndex.partitionIndexPath;
+@Slf4j
public class HFileBootstrapIndexWriter extends BootstrapIndex.IndexWriter {
- private static final Logger LOG =
LoggerFactory.getLogger(HFileBootstrapIndexWriter.class);
private final String bootstrapBasePath;
private final StoragePath indexByPartitionPath;
@@ -80,7 +79,7 @@ public class HFileBootstrapIndexWriter extends
BootstrapIndex.IndexWriter {
|| metaClient.getStorage().exists(indexByFileIdPath)) {
String errMsg = "Previous version of bootstrap index exists. Partition
Index Path :" + indexByPartitionPath
+ ", FileId index Path :" + indexByFileIdPath;
- LOG.info(errMsg);
+ log.info(errMsg);
throw new HoodieException(errMsg);
}
} catch (IOException ioe) {
@@ -97,9 +96,9 @@ public class HFileBootstrapIndexWriter extends
BootstrapIndex.IndexWriter {
private void writeNextPartition(String partitionPath, String
bootstrapPartitionPath,
List<BootstrapFileMapping>
bootstrapFileMappings) {
try {
- LOG.info("Adding bootstrap partition Index entry for partition :" +
partitionPath
- + ", bootstrap Partition :" + bootstrapPartitionPath + ", Num
Entries :" + bootstrapFileMappings.size());
- LOG.info("ADDING entries :" + bootstrapFileMappings);
+ log.info("Adding bootstrap partition Index entry for partition :{},
bootstrap Partition :{}, Num Entries :{}",
+ partitionPath, bootstrapPartitionPath, bootstrapFileMappings.size());
+ log.info("ADDING entries :{}", bootstrapFileMappings);
HoodieBootstrapPartitionMetadata bootstrapPartitionMetadata = new
HoodieBootstrapPartitionMetadata();
bootstrapPartitionMetadata.setBootstrapPartitionPath(bootstrapPartitionPath);
bootstrapPartitionMetadata.setPartitionPath(partitionPath);
@@ -148,14 +147,14 @@ public class HFileBootstrapIndexWriter extends
BootstrapIndex.IndexWriter {
.setNumKeys(numPartitionKeysAdded)
.setBootstrapBasePath(bootstrapBasePath)
.build();
- LOG.info("Adding Partition FileInfo :" + partitionIndexInfo);
+ log.info("Adding Partition FileInfo :{}", partitionIndexInfo);
HoodieBootstrapIndexInfo fileIdIndexInfo =
HoodieBootstrapIndexInfo.newBuilder()
.setCreatedTimestamp(new Date().getTime())
.setNumKeys(numFileIdKeysAdded)
.setBootstrapBasePath(bootstrapBasePath)
.build();
- LOG.info("Appending FileId FileInfo :" + fileIdIndexInfo);
+ log.info("Appending FileId FileInfo :{}", fileIdIndexInfo);
indexByPartitionWriter.appendFileInfo(
INDEX_INFO_KEY_STRING,
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java
b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java
index f4b6bffd8080..2048b21c7e26 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java
@@ -18,6 +18,10 @@
package org.apache.hudi.common.config;
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
/**
* In Hudi, we have multiple superclasses, aka Config Classes of {@link
HoodieConfig} that maintain
* several configs. This class group one or more of these superclasses into
higher
@@ -29,6 +33,7 @@ public class ConfigGroups {
* Config group names. Please add the description of each group in
* {@link ConfigGroups#getDescription}.
*/
+ @AllArgsConstructor(access = AccessLevel.PACKAGE)
public enum Names {
TABLE_CONFIG("Hudi Table Config"),
ENVIRONMENT_CONFIG("Environment Config"),
@@ -44,12 +49,9 @@ public class ConfigGroups {
HUDI_STREAMER("Hudi Streamer Configs");
public final String name;
-
- Names(String name) {
- this.name = name;
- }
}
+ @AllArgsConstructor(access = AccessLevel.PACKAGE)
public enum SubGroupNames {
INDEX(
"Index Configs",
@@ -80,16 +82,8 @@ public class ConfigGroups {
"No subgroup. This description should be hidden.");
public final String name;
+ @Getter
private final String description;
-
- SubGroupNames(String name, String description) {
- this.name = name;
- this.description = description;
- }
-
- public String getDescription() {
- return description;
- }
}
public static String getDescription(Names names) {
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigProperty.java
b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigProperty.java
index 56a087f1e7ea..b93d980eafc1 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigProperty.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigProperty.java
@@ -22,6 +22,12 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.exception.HoodieException;
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.experimental.Accessors;
+
import java.io.Serializable;
import java.lang.reflect.Field;
import java.util.Arrays;
@@ -41,14 +47,21 @@ import java.util.stream.Collectors;
*
* @param <T> The type of the default value.
*/
+@AllArgsConstructor(access = AccessLevel.PACKAGE)
+@Getter
public class ConfigProperty<T> implements Serializable {
+ @NonNull
+ @Accessors(fluent = true) // Required so that #key() is generated instead of
#getKey() by Lombok
private final String key;
+ @Getter(AccessLevel.NONE)
private final T defaultValue;
+ @Getter(AccessLevel.NONE)
private final String docOnDefaultValue;
+ @Getter(AccessLevel.NONE)
private final String doc;
private final Option<String> sinceVersion;
@@ -57,37 +70,16 @@ public class ConfigProperty<T> implements Serializable {
private final List<String> supportedVersions;
+ // provide the ability to infer config value based on other configs
+ private final Option<Function<HoodieConfig, Option<T>>> inferFunction;
+
+ @Getter(AccessLevel.NONE)
private final Set<String> validValues;
private final boolean advanced;
private final String[] alternatives;
- // provide the ability to infer config value based on other configs
- private final Option<Function<HoodieConfig, Option<T>>> inferFunction;
-
- ConfigProperty(String key, T defaultValue, String docOnDefaultValue, String
doc,
- Option<String> sinceVersion, Option<String> deprecatedVersion,
- List<String> supportedVersions,
- Option<Function<HoodieConfig, Option<T>>> inferFunc,
Set<String> validValues,
- boolean advanced, String... alternatives) {
- this.key = Objects.requireNonNull(key);
- this.defaultValue = defaultValue;
- this.docOnDefaultValue = docOnDefaultValue;
- this.doc = doc;
- this.sinceVersion = sinceVersion;
- this.deprecatedVersion = deprecatedVersion;
- this.supportedVersions = supportedVersions;
- this.inferFunction = inferFunc;
- this.validValues = validValues;
- this.advanced = advanced;
- this.alternatives = alternatives;
- }
-
- public String key() {
- return key;
- }
-
public T defaultValue() {
if (defaultValue == null) {
throw new HoodieException(String.format("There's no default value for
this config: %s", key));
@@ -108,26 +100,10 @@ public class ConfigProperty<T> implements Serializable {
return StringUtils.isNullOrEmpty(doc) ? StringUtils.EMPTY_STRING : doc;
}
- public Option<String> getSinceVersion() {
- return sinceVersion;
- }
-
- public Option<String> getDeprecatedVersion() {
- return deprecatedVersion;
- }
-
- public List<String> getSupportedVersions() {
- return supportedVersions;
- }
-
public boolean hasInferFunction() {
return getInferFunction().isPresent();
}
- public Option<Function<HoodieConfig, Option<T>>> getInferFunction() {
- return inferFunction;
- }
-
public void checkValues(String value) {
if (!isValid(value)) {
throw new IllegalArgumentException(
@@ -144,10 +120,6 @@ public class ConfigProperty<T> implements Serializable {
return Arrays.asList(alternatives);
}
- public boolean isAdvanced() {
- return advanced;
- }
-
public ConfigProperty<T> withDocumentation(String doc) {
Objects.requireNonNull(doc);
return new ConfigProperty<>(key, defaultValue, docOnDefaultValue, doc,
sinceVersion, deprecatedVersion, supportedVersions, inferFunction, validValues,
advanced, alternatives);
@@ -271,7 +243,7 @@ public class ConfigProperty<T> implements Serializable {
public <T> ConfigProperty<T> defaultValue(T value, String
docOnDefaultValue) {
Objects.requireNonNull(docOnDefaultValue);
return new ConfigProperty<>(key, value, docOnDefaultValue, "",
Option.empty(),
- Option.empty(), Collections.emptyList(), Option.empty(),
Collections.emptySet(), false);
+ Option.empty(), Collections.emptyList(), Option.empty(),
Collections.emptySet(), false, new String[0]);
}
public ConfigProperty<String> noDefaultValue() {
@@ -280,7 +252,7 @@ public class ConfigProperty<T> implements Serializable {
public ConfigProperty<String> noDefaultValue(String docOnDefaultValue) {
return new ConfigProperty<>(key, null, docOnDefaultValue, "",
Option.empty(),
- Option.empty(), Collections.emptyList(), Option.empty(),
Collections.emptySet(), false);
+ Option.empty(), Collections.emptyList(), Option.empty(),
Collections.emptySet(), false, new String[0]);
}
}
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java
b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java
index 498de3821666..6e6668b0284c 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java
@@ -24,8 +24,8 @@ import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.exception.HoodieException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
import java.io.Serializable;
import java.lang.reflect.Modifier;
@@ -39,16 +39,16 @@ import static
org.apache.hudi.common.util.ConfigUtils.loadGlobalProperties;
/**
* This class deals with {@link ConfigProperty} and provides get/set
functionalities.
*/
+@Slf4j
public class HoodieConfig implements Serializable {
- private static final Logger LOG =
LoggerFactory.getLogger(HoodieConfig.class);
-
protected static final String CONFIG_VALUES_DELIMITER = ",";
// Number of retries while reading the properties file to deal with parallel
updates
protected static final int MAX_READ_RETRIES = 5;
// Delay between retries while reading the properties file
protected static final int READ_RETRY_DELAY_MSEC = 1000;
+ @Getter
protected TypedProperties props;
public HoodieConfig() {
@@ -246,10 +246,6 @@ public class HoodieConfig implements Serializable {
return Option.ofNullable(props.getProperty(key)).orElse(defaultVal);
}
- public TypedProperties getProps() {
- return props;
- }
-
public TypedProperties getProps(boolean includeGlobalProps) {
if (includeGlobalProps) {
TypedProperties mergedProps = loadGlobalProperties();
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/conflict/detection/DirectMarkerBasedDetectionStrategy.java
b/hudi-common/src/main/java/org/apache/hudi/common/conflict/detection/DirectMarkerBasedDetectionStrategy.java
index 4b11a348bb05..466b8c99769e 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/conflict/detection/DirectMarkerBasedDetectionStrategy.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/conflict/detection/DirectMarkerBasedDetectionStrategy.java
@@ -30,8 +30,8 @@ import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
import java.util.List;
@@ -42,10 +42,10 @@ import java.util.stream.Stream;
* This abstract strategy is used for direct marker writers, trying to do
early conflict detection.
*/
@PublicAPIClass(maturity = ApiMaturityLevel.EVOLVING)
+@AllArgsConstructor
+@Slf4j
public abstract class DirectMarkerBasedDetectionStrategy implements
EarlyConflictDetectionStrategy {
- private static final Logger LOG =
LoggerFactory.getLogger(DirectMarkerBasedDetectionStrategy.class);
-
protected final HoodieStorage storage;
protected final String partitionPath;
protected final String fileId;
@@ -53,17 +53,6 @@ public abstract class DirectMarkerBasedDetectionStrategy
implements EarlyConflic
protected final HoodieActiveTimeline activeTimeline;
protected final HoodieConfig config;
- public DirectMarkerBasedDetectionStrategy(HoodieStorage storage, String
partitionPath, String fileId,
- String instantTime,
- HoodieActiveTimeline
activeTimeline, HoodieConfig config) {
- this.storage = storage;
- this.partitionPath = partitionPath;
- this.fileId = fileId;
- this.instantTime = instantTime;
- this.activeTimeline = activeTimeline;
- this.config = config;
- }
-
/**
* We need to do list operation here.
* In order to reduce the list pressure as much as possible, first we build
path prefix in advance:
@@ -106,7 +95,7 @@ public abstract class DirectMarkerBasedDetectionStrategy
implements EarlyConflic
}).count();
if (res != 0L) {
- LOG.warn("Detected conflict marker files: {}/{} for {}", partitionPath,
fileId, instantTime);
+ log.warn("Detected conflict marker files: {}/{} for {}", partitionPath,
fileId, instantTime);
return true;
}
return false;
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/data/HoodieBaseListData.java
b/hudi-common/src/main/java/org/apache/hudi/common/data/HoodieBaseListData.java
index b603b99d9302..f31a0072ab9d 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/data/HoodieBaseListData.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/data/HoodieBaseListData.java
@@ -21,8 +21,9 @@ package org.apache.hudi.common.data;
import org.apache.hudi.common.util.Either;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
import java.util.Iterator;
import java.util.List;
@@ -34,6 +35,7 @@ import java.util.stream.Stream;
*
* @param <T> Object value type.
*/
+@Slf4j
public abstract class HoodieBaseListData<T> {
protected final Either<Stream<T>, List<T>> data;
@@ -86,13 +88,11 @@ public abstract class HoodieBaseListData<T> {
}
}
+ @AllArgsConstructor(access = AccessLevel.PACKAGE)
+ @Slf4j
static class IteratorCloser implements Runnable {
- private static final Logger LOG =
LoggerFactory.getLogger(IteratorCloser.class);
- private final Iterator<?> iterator;
- IteratorCloser(Iterator<?> iterator) {
- this.iterator = iterator;
- }
+ private final Iterator<?> iterator;
@Override
public void run() {
@@ -100,7 +100,7 @@ public abstract class HoodieBaseListData<T> {
try {
((AutoCloseable) iterator).close();
} catch (Exception ex) {
- LOG.warn("Failed to properly close iterator", ex);
+ log.warn("Failed to properly close iterator", ex);
}
}
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java
b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java
index 992ce9f1f99e..f56921797283 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java
@@ -41,6 +41,9 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.keygen.KeyGenerator;
import org.apache.hudi.storage.StorageConfiguration;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
import java.io.IOException;
import java.util.Iterator;
import java.util.List;
@@ -51,6 +54,8 @@ import java.util.stream.Stream;
* Base class contains the context information needed by the engine at
runtime. It will be extended by different
* engine implementation if needed.
*/
+@AllArgsConstructor
+@Getter
public abstract class HoodieEngineContext {
/**
@@ -60,19 +65,6 @@ public abstract class HoodieEngineContext {
protected TaskContextSupplier taskContextSupplier;
- public HoodieEngineContext(StorageConfiguration<?> storageConf,
TaskContextSupplier taskContextSupplier) {
- this.storageConf = storageConf;
- this.taskContextSupplier = taskContextSupplier;
- }
-
- public StorageConfiguration<?> getStorageConf() {
- return storageConf;
- }
-
- public TaskContextSupplier getTaskContextSupplier() {
- return taskContextSupplier;
- }
-
public abstract HoodieAccumulator newAccumulator();
public abstract <T> HoodieData<T> emptyHoodieData();
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieReaderContext.java
b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieReaderContext.java
index 953014802cf2..14e412ffcb18 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieReaderContext.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieReaderContext.java
@@ -53,6 +53,10 @@ import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+
import java.io.IOException;
import java.util.List;
import java.util.Map;
@@ -77,28 +81,53 @@ import static
org.apache.hudi.common.table.HoodieTableConfig.inferMergingConfigs
* and {@code RowData} in Flink.
*/
public abstract class HoodieReaderContext<T> {
+
+ @Getter
private final StorageConfiguration<?> storageConfiguration;
protected final HoodieFileFormat baseFileFormat;
// For general predicate pushdown.
+ @Getter
protected final Option<Predicate> keyFilterOpt;
protected final HoodieTableConfig tableConfig;
+ @Setter
private String tablePath = null;
+ @Getter
+ @Setter
private String latestCommitTime = null;
+ @Getter
+ @Setter
private Option<HoodieRecordMerger> recordMerger = null;
+ @Getter
+ @Setter
private Boolean hasLogFiles = null;
+ @Getter
+ @Setter
private Boolean hasBootstrapBaseFile = null;
+ @Getter
+ @Setter
private Boolean needsBootstrapMerge = null;
+ @Getter
+ @Setter
// should we do position based merging for mor
private Boolean shouldMergeUseRecordPosition = null;
protected Option<InstantRange> instantRangeOpt;
+ @Getter
private RecordMergeMode mergeMode;
+ @Getter
protected RecordContext<T> recordContext;
+ @Getter
+ @Setter
private FileGroupReaderSchemaHandler<T> schemaHandler = null;
// the default iterator mode is engine-specific record mode
+ @Setter
private IteratorMode iteratorMode = IteratorMode.ENGINE_RECORD;
+ @Getter
protected final HoodieConfig hoodieReaderConfig;
- private boolean enableLogicalTimestampFieldRepair = true;
+ @Getter
+ @Setter
+ @Accessors(fluent = true)
+ private Boolean enableLogicalTimestampFieldRepair = true;
protected HoodieReaderContext(StorageConfiguration<?> storageConfiguration,
HoodieTableConfig tableConfig,
@@ -123,19 +152,6 @@ public abstract class HoodieReaderContext<T> {
this.hoodieReaderConfig = hoodieReaderConfig;
}
- // Getter and Setter for schemaHandler
- public FileGroupReaderSchemaHandler<T> getSchemaHandler() {
- return schemaHandler;
- }
-
- public void setSchemaHandler(FileGroupReaderSchemaHandler<T> schemaHandler) {
- this.schemaHandler = schemaHandler;
- }
-
- public void setIteratorMode(IteratorMode iteratorMode) {
- this.iteratorMode = iteratorMode;
- }
-
public IteratorMode getIteratorMode() {
ValidationUtils.checkArgument(iteratorMode != null, "iterator mode should
not be null!");
return this.iteratorMode;
@@ -148,82 +164,10 @@ public abstract class HoodieReaderContext<T> {
return tablePath;
}
- public void setEnableLogicalTimestampFieldRepair(boolean
enableLogicalTimestampFieldRepair) {
- this.enableLogicalTimestampFieldRepair = enableLogicalTimestampFieldRepair;
- }
-
- public void setTablePath(String tablePath) {
- this.tablePath = tablePath;
- }
-
- public String getLatestCommitTime() {
- return latestCommitTime;
- }
-
- public void setLatestCommitTime(String latestCommitTime) {
- this.latestCommitTime = latestCommitTime;
- }
-
- public Option<HoodieRecordMerger> getRecordMerger() {
- return recordMerger;
- }
-
- public void setRecordMerger(Option<HoodieRecordMerger> recordMerger) {
- this.recordMerger = recordMerger;
- }
-
- // Getter and Setter for hasLogFiles
- public boolean getHasLogFiles() {
- return hasLogFiles;
- }
-
- public void setHasLogFiles(boolean hasLogFiles) {
- this.hasLogFiles = hasLogFiles;
- }
-
- // Getter and Setter for hasBootstrapBaseFile
- public boolean getHasBootstrapBaseFile() {
- return hasBootstrapBaseFile;
- }
-
- public void setHasBootstrapBaseFile(boolean hasBootstrapBaseFile) {
- this.hasBootstrapBaseFile = hasBootstrapBaseFile;
- }
-
- // Getter and Setter for needsBootstrapMerge
- public boolean getNeedsBootstrapMerge() {
- return needsBootstrapMerge;
- }
-
- public boolean enableLogicalTimestampFieldRepair() {
- return enableLogicalTimestampFieldRepair;
- }
-
- public void setNeedsBootstrapMerge(boolean needsBootstrapMerge) {
- this.needsBootstrapMerge = needsBootstrapMerge;
- }
-
- // Getter and Setter for useRecordPosition
- public boolean getShouldMergeUseRecordPosition() {
- return shouldMergeUseRecordPosition;
- }
-
- public void setShouldMergeUseRecordPosition(boolean
shouldMergeUseRecordPosition) {
- this.shouldMergeUseRecordPosition = shouldMergeUseRecordPosition;
- }
-
- public StorageConfiguration<?> getStorageConfiguration() {
- return storageConfiguration;
- }
-
public TypedProperties getMergeProps(TypedProperties props) {
return ConfigUtils.getMergeProps(props, this.tableConfig);
}
- public Option<Predicate> getKeyFilterOpt() {
- return keyFilterOpt;
- }
-
public SizeEstimator<BufferedRecord<T>> getRecordSizeEstimator() {
return new
HoodieRecordSizeEstimator<>(getSchemaHandler().getSchemaForUpdates());
}
@@ -232,14 +176,6 @@ public abstract class HoodieReaderContext<T> {
return new DefaultSerializer<>();
}
- public RecordContext<T> getRecordContext() {
- return recordContext;
- }
-
- public HoodieConfig getHoodieReaderConfig() {
- return hoodieReaderConfig;
- }
-
/**
* Gets the record iterator based on the type of engine-specific record
representation from the
* file.
@@ -335,10 +271,6 @@ public abstract class HoodieReaderContext<T> {
properties.getString(RECORD_MERGE_IMPL_CLASSES_DEPRECATED_WRITE_CONFIG_KEY,
"")));
}
- public RecordMergeMode getMergeMode() {
- return mergeMode;
- }
-
/**
* Get the {@link InstantRange} filter.
*/
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/engine/RecordContext.java
b/hudi-common/src/main/java/org/apache/hudi/common/engine/RecordContext.java
index b1fa5d97892e..d63c9bd63408 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/engine/RecordContext.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/engine/RecordContext.java
@@ -35,6 +35,8 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieKeyException;
import org.apache.hudi.keygen.KeyGenerator;
+import lombok.Getter;
+import lombok.Setter;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
@@ -65,7 +67,9 @@ public abstract class RecordContext<T> implements
Serializable {
// for encoding and decoding schemas to the spillable map
private final LocalHoodieSchemaCache localSchemaCache =
LocalHoodieSchemaCache.getInstance();
+ @Getter
protected final JavaTypeConverter typeConverter;
+ @Setter
protected String partitionPath;
protected RecordContext(HoodieTableConfig tableConfig, JavaTypeConverter
typeConverter) {
@@ -84,10 +88,6 @@ public abstract class RecordContext<T> implements
Serializable {
this.typeConverter = typeConverter;
}
- public void setPartitionPath(String partitionPath) {
- this.partitionPath = partitionPath;
- }
-
public T extractDataFromRecord(HoodieRecord record, HoodieSchema schema,
Properties properties) {
return (T) record.getData();
}
@@ -170,10 +170,6 @@ public abstract class RecordContext<T> implements
Serializable {
*/
public abstract T constructEngineRecord(HoodieSchema recordSchema, Object[]
fieldValues);
- public JavaTypeConverter getTypeConverter() {
- return typeConverter;
- }
-
/**
* Gets the record key in String.
*
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
index 56c0cc3a5b46..1b5f3ea62e78 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
@@ -45,8 +45,7 @@ import org.apache.hudi.storage.StoragePathFilter;
import org.apache.hudi.storage.StoragePathInfo;
import org.apache.hudi.storage.inline.InLineFSUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import lombok.extern.slf4j.Slf4j;
import java.io.File;
import java.io.FileNotFoundException;
@@ -72,9 +71,9 @@ import java.util.stream.Stream;
/**
* Utility functions related to accessing the file storage.
*/
+@Slf4j
public class FSUtils {
- private static final Logger LOG = LoggerFactory.getLogger(FSUtils.class);
// Log files are of this pattern -
.b5068208-e1a4-11e6-bf01-fe55135034f3_20170101134598.log.1_1-0-1
// Archive log files are of this pattern - .commits_.archive.1_1-0-1
public static final String PATH_SEPARATOR = "/";
@@ -624,7 +623,7 @@ public class FSUtils {
pairOfSubPathAndConf.getKey(),
pairOfSubPathAndConf.getValue(), true)
);
boolean result = storage.deleteDirectory(dirPath);
- LOG.info("Removed directory at {}", dirPath);
+ log.info("Removed directory at {}", dirPath);
return result;
}
} catch (IOException ioe) {
@@ -803,7 +802,7 @@ public class FSUtils {
if (!ignoreFailed) {
throw new HoodieIOException("Failed to delete : " + file, e);
} else {
- LOG.info("Ignore failed deleting : {}", file);
+ log.info("Ignore failed deleting : {}", file);
return true;
}
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/fs/FailSafeConsistencyGuard.java
b/hudi-common/src/main/java/org/apache/hudi/common/fs/FailSafeConsistencyGuard.java
index ed82343ee3be..ada10aaa2fd0 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/fs/FailSafeConsistencyGuard.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/fs/FailSafeConsistencyGuard.java
@@ -23,8 +23,7 @@ import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import lombok.extern.slf4j.Slf4j;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -36,10 +35,9 @@ import java.util.stream.Collectors;
/**
* A consistency checker that fails if it is unable to meet the required
condition within a specified timeout.
*/
+@Slf4j
public class FailSafeConsistencyGuard implements ConsistencyGuard {
- private static final Logger LOG =
LoggerFactory.getLogger(FailSafeConsistencyGuard.class);
-
protected final HoodieStorage storage;
protected final ConsistencyGuardConfig consistencyGuardConfig;
@@ -129,7 +127,7 @@ public class FailSafeConsistencyGuard implements
ConsistencyGuard {
return;
}
} catch (IOException ioe) {
- LOG.warn("Got IOException waiting for file visibility. Retrying", ioe);
+ log.warn("Got IOException waiting for file visibility. Retrying", ioe);
}
sleepSafe(waitMs);
@@ -152,7 +150,7 @@ public class FailSafeConsistencyGuard implements
ConsistencyGuard {
throws TimeoutException {
long waitMs =
consistencyGuardConfig.getInitialConsistencyCheckIntervalMs();
int attempt = 0;
- LOG.info("Max Attempts=" +
consistencyGuardConfig.getMaxConsistencyChecks());
+ log.info("Max Attempts={}",
consistencyGuardConfig.getMaxConsistencyChecks());
while (attempt < consistencyGuardConfig.getMaxConsistencyChecks()) {
boolean success = checkFilesVisibility(attempt, dir, files, event);
if (success) {
@@ -178,7 +176,7 @@ public class FailSafeConsistencyGuard implements
ConsistencyGuard {
protected boolean checkFilesVisibility(int retryNum, StoragePath dir,
List<String> files,
FileVisibility event) {
try {
- LOG.info("Trying " + retryNum);
+ log.info("Trying {}", retryNum);
List<StoragePathInfo> entries = storage.listDirectEntries(dir);
List<String> gotFiles = entries.stream()
.map(e -> e.getPath().getPathWithoutSchemeAndAuthority())
@@ -188,7 +186,7 @@ public class FailSafeConsistencyGuard implements
ConsistencyGuard {
switch (event) {
case DISAPPEAR:
- LOG.info("Following files are visible" + candidateFiles);
+ log.info("Following files are visible{}", candidateFiles);
// If no candidate files gets removed, it means all of them have
disappeared
return !altered;
case APPEAR:
@@ -197,7 +195,7 @@ public class FailSafeConsistencyGuard implements
ConsistencyGuard {
return candidateFiles.isEmpty();
}
} catch (IOException ioe) {
- LOG.warn("Got IOException waiting for file event. Have tried {}
time(s)", retryNum, ioe);
+ log.warn("Got IOException waiting for file event. Have tried {}
time(s)", retryNum, ioe);
}
return false;
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/fs/OptimisticConsistencyGuard.java
b/hudi-common/src/main/java/org/apache/hudi/common/fs/OptimisticConsistencyGuard.java
index dfc58c6dd11a..297c9941b9de 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/fs/OptimisticConsistencyGuard.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/fs/OptimisticConsistencyGuard.java
@@ -21,8 +21,7 @@ package org.apache.hudi.common.fs;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
import java.util.List;
@@ -49,10 +48,9 @@ import java.util.concurrent.TimeoutException;
* With this, if any files that was created, should be available within
configured threshold(eventual consistency).
* Delete() will return false if FileNotFound. So, both cases are taken care
of this {@link ConsistencyGuard}.
*/
+@Slf4j
public class OptimisticConsistencyGuard extends FailSafeConsistencyGuard {
- private static final Logger LOG =
LoggerFactory.getLogger(OptimisticConsistencyGuard.class);
-
public OptimisticConsistencyGuard(HoodieStorage storage,
ConsistencyGuardConfig
consistencyGuardConfig) {
super(storage, consistencyGuardConfig);
@@ -65,7 +63,7 @@ public class OptimisticConsistencyGuard extends
FailSafeConsistencyGuard {
Thread.sleep(consistencyGuardConfig.getOptimisticConsistencyGuardSleepTimeMs());
}
} catch (IOException | InterruptedException ioe) {
- LOG.warn("Got IOException or InterruptedException waiting for file
visibility. Ignoring",
+ log.warn("Got IOException or InterruptedException waiting for file
visibility. Ignoring",
ioe);
}
}
@@ -83,7 +81,7 @@ public class OptimisticConsistencyGuard extends
FailSafeConsistencyGuard {
Thread.sleep(consistencyGuardConfig.getOptimisticConsistencyGuardSleepTimeMs());
}
} catch (InterruptedException ie) {
- LOG.warn("Got InterruptedException waiting for file visibility.
Ignoring", ie);
+ log.warn("Got InterruptedException waiting for file visibility.
Ignoring", ie);
}
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/heartbeat/HoodieHeartbeatUtils.java
b/hudi-common/src/main/java/org/apache/hudi/common/heartbeat/HoodieHeartbeatUtils.java
index a51914554d71..fab4511c703f 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/heartbeat/HoodieHeartbeatUtils.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/heartbeat/HoodieHeartbeatUtils.java
@@ -23,16 +23,15 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
/**
* Common utils for Hudi heartbeat
*/
+@Slf4j
public class HoodieHeartbeatUtils {
- private static final Logger LOG =
LoggerFactory.getLogger(HoodieHeartbeatUtils.class);
/**
* Use modification time as last heart beat time.
@@ -72,7 +71,7 @@ public class HoodieHeartbeatUtils {
Long currentTime = System.currentTimeMillis();
Long lastHeartbeatTime = getLastHeartbeatTime(storage, basePath,
instantTime);
if (currentTime - lastHeartbeatTime > maxAllowableHeartbeatIntervalInMs) {
- LOG.warn("Heartbeat expired, for instant: {}", instantTime);
+ log.warn("Heartbeat expired, for instant: {}", instantTime);
return true;
}
return false;
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchema.java
b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchema.java
index dff06a4c0e0f..0b0978d552a2 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchema.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchema.java
@@ -89,6 +89,7 @@ import static
org.apache.hudi.avro.HoodieAvroUtils.createNewSchemaField;
*
* @since 1.2.0
*/
+@Getter
public class HoodieSchema implements Serializable {
private static final long serialVersionUID = 1L;
@@ -1567,18 +1568,6 @@ public class HoodieSchema implements Serializable {
return (path.charAt(next) == '.') ? next + 1 : -1;
}
- /**
- * Returns the underlying Avro schema for compatibility purposes.
- *
- * <p>This method is provided for gradual migration and should be used
- * sparingly. New code should prefer the HoodieSchema API.</p>
- *
- * @return the wrapped Avro Schema
- */
- public Schema getAvroSchema() {
- return avroSchema;
- }
-
/**
* Converts this HoodieSchema to an Avro Schema.
* This is an alias for getAvroSchema() provided for API consistency.
@@ -1909,7 +1898,9 @@ public class HoodieSchema implements Serializable {
}
public static class Decimal extends HoodieSchema {
+ @Getter
private final int precision;
+ @Getter
private final int scale;
private final Option<Integer> fixedSize;
@@ -1934,14 +1925,6 @@ public class HoodieSchema implements Serializable {
}
}
- public int getPrecision() {
- return precision;
- }
-
- public int getScale() {
- return scale;
- }
-
@Override
public String getName() {
return String.format("decimal(%d,%d)", precision, scale);
@@ -2224,7 +2207,9 @@ public class HoodieSchema implements Serializable {
}
public static class Timestamp extends HoodieSchema {
+ @Getter
private final boolean isUtcAdjusted;
+ @Getter
private final TimePrecision precision;
/**
@@ -2255,14 +2240,6 @@ public class HoodieSchema implements Serializable {
}
}
- public TimePrecision getPrecision() {
- return precision;
- }
-
- public boolean isUtcAdjusted() {
- return isUtcAdjusted;
- }
-
@Override
public String getName() {
if (isUtcAdjusted) {
@@ -2299,6 +2276,7 @@ public class HoodieSchema implements Serializable {
}
public static class Time extends HoodieSchema {
+ @Getter
private final TimePrecision precision;
/**
@@ -2321,10 +2299,6 @@ public class HoodieSchema implements Serializable {
}
}
- public TimePrecision getPrecision() {
- return precision;
- }
-
@Override
public String getName() {
if (precision == TimePrecision.MILLIS) {
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaCompatibility.java
b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaCompatibility.java
index 9842cdd098c8..0fd96e55233e 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaCompatibility.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaCompatibility.java
@@ -24,6 +24,9 @@ import org.apache.hudi.exception.MissingSchemaFieldException;
import org.apache.hudi.exception.SchemaBackwardsCompatibilityException;
import org.apache.hudi.internal.schema.HoodieSchemaException;
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collections;
@@ -45,12 +48,9 @@ import java.util.stream.Collectors;
* <li>Metadata field handling during schema checks</li>
* </ul>
*/
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
public final class HoodieSchemaCompatibility {
- // Prevent instantiation
- private HoodieSchemaCompatibility() {
- }
-
public static boolean areSchemasCompatible(HoodieSchema tableSchema,
HoodieSchema writerSchema) {
return
HoodieSchemaCompatibilityChecker.checkReaderWriterCompatibility(tableSchema,
writerSchema, false).getType() ==
HoodieSchemaCompatibilityChecker.SchemaCompatibilityType.COMPATIBLE;
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaField.java
b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaField.java
index 190e180b4968..9b1d16952192 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaField.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaField.java
@@ -22,6 +22,7 @@ import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ValidationUtils;
+import lombok.Getter;
import org.apache.avro.Schema;
import java.io.Serializable;
@@ -55,6 +56,7 @@ public class HoodieSchemaField implements Serializable {
private static final long serialVersionUID = 1L;
+ @Getter
private final Schema.Field avroField;
private final HoodieSchema fieldSchema;
@@ -251,18 +253,6 @@ public class HoodieSchemaField implements Serializable {
return avroField.aliases();
}
- /**
- * Returns the underlying Avro field for compatibility purposes.
- *
- * <p>This method is provided for gradual migration and should be used
- * sparingly. New code should prefer the HoodieSchemaField API.</p>
- *
- * @return the wrapped Avro Schema.Field
- */
- public Schema.Field getAvroField() {
- return avroField;
- }
-
/**
* Creates a copy of this field with a new name.
*
diff --git
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java
index d01669b507f3..6535f6be700e 100644
---
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java
+++
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java
@@ -629,10 +629,15 @@ public class TestIncrementalFSViewSync extends
HoodieCommonTestHarness {
throws IOException {
Map<String, List<String>> partitionToFiles = deleteFiles(files);
List<HoodieCleanStat> cleanStats =
partitionToFiles.entrySet().stream().map(e ->
- new HoodieCleanStat(HoodieCleaningPolicy.KEEP_LATEST_COMMITS,
e.getKey(), e.getValue(), e.getValue(),
- new ArrayList<>(),
- instant.length() < 3 ? String.valueOf(Integer.parseInt(instant) +
1) : HoodieInstantTimeGenerator.instantTimePlusMillis(instant, 1),
- "")).collect(Collectors.toList());
+ HoodieCleanStat.builder()
+ .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_COMMITS)
+ .withPartitionPath(e.getKey())
+ .withDeletePathPatterns(e.getValue())
+ .withSuccessDeleteFiles(e.getValue())
+ .withFailedDeleteFiles(Collections.emptyList())
+ .withEarliestCommitToRetain(instant.length() < 3 ?
String.valueOf(Integer.parseInt(instant) + 1) :
HoodieInstantTimeGenerator.instantTimePlusMillis(instant, 1))
+ .withLastCompletedCommitTimestamp("")
+ .build()).collect(Collectors.toList());
HoodieInstant cleanInflightInstant =
INSTANT_GENERATOR.createNewInstant(State.INFLIGHT, HoodieTimeline.CLEAN_ACTION,
cleanInstant);
metaClient.getActiveTimeline().createNewInstant(cleanInflightInstant);
diff --git
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
index d0ec62e0abb2..46b5248e6f33 100644
---
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
+++
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
@@ -451,14 +451,12 @@ public class HoodieTestTable implements AutoCloseable {
public HoodieTestTable addClean(String instantTime) throws IOException {
HoodieCleanerPlan cleanerPlan = new HoodieCleanerPlan(new
HoodieActionInstant(EMPTY_STRING, EMPTY_STRING, EMPTY_STRING),
EMPTY_STRING, EMPTY_STRING, new HashMap<>(),
CleanPlanV2MigrationHandler.VERSION, new HashMap<>(), new ArrayList<>(),
Collections.EMPTY_MAP);
- HoodieCleanStat cleanStats = new HoodieCleanStat(
- HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS,
-
HoodieTestUtils.DEFAULT_PARTITION_PATHS[RANDOM.nextInt(HoodieTestUtils.DEFAULT_PARTITION_PATHS.length)],
- Collections.emptyList(),
- Collections.emptyList(),
- Collections.emptyList(),
- instantTime,
- "");
+ HoodieCleanStat cleanStats = HoodieCleanStat.builder()
+ .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS)
+
.withPartitionPath(HoodieTestUtils.DEFAULT_PARTITION_PATHS[RANDOM.nextInt(HoodieTestUtils.DEFAULT_PARTITION_PATHS.length)])
+ .withEarliestCommitToRetain(instantTime)
+ .withLastCompletedCommitTimestamp("")
+ .build();
HoodieCleanMetadata cleanMetadata = convertCleanMetadata(instantTime,
Option.of(0L), Collections.singletonList(cleanStats), Collections.EMPTY_MAP);
return HoodieTestTable.of(metaClient).addClean(instantTime, cleanerPlan,
cleanMetadata);
}
@@ -468,8 +466,14 @@ public class HoodieTestTable implements AutoCloseable {
EMPTY_STRING, EMPTY_STRING, new HashMap<>(),
CleanPlanV2MigrationHandler.VERSION, new HashMap<>(), new ArrayList<>(),
Collections.EMPTY_MAP);
List<HoodieCleanStat> cleanStats = new ArrayList<>();
for (Map.Entry<String, List<String>> entry :
testTableState.getPartitionToFileIdMapForCleaner(commitTime).entrySet()) {
- cleanStats.add(new
HoodieCleanStat(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS,
- entry.getKey(), entry.getValue(), entry.getValue(),
Collections.emptyList(), commitTime, ""));
+ cleanStats.add(HoodieCleanStat.builder()
+ .withPolicy(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS)
+ .withPartitionPath(entry.getKey())
+ .withDeletePathPatterns(entry.getValue())
+ .withSuccessDeleteFiles(entry.getValue())
+ .withEarliestCommitToRetain(commitTime)
+ .withLastCompletedCommitTimestamp("")
+ .build());
}
return Pair.of(cleanerPlan, convertCleanMetadata(commitTime,
Option.of(0L), cleanStats, Collections.EMPTY_MAP));
}
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedFileFormat.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedFileFormat.scala
index da5bef9f785c..3fe8c6ff62f7 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedFileFormat.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedFileFormat.scala
@@ -304,7 +304,7 @@ class HoodieFileGroupReaderBasedFileFormat(tablePath:
String,
val readerContext = new SparkFileFormatInternalRowReaderContext(
fileGroupBaseFileReader.value, filters, requiredFilters,
storageConf, metaClient.getTableConfig,
sparkRequiredSchema = Some(requiredSchema))
-
readerContext.setEnableLogicalTimestampFieldRepair(storageConf.getBoolean(ENABLE_LOGICAL_TIMESTAMP_REPAIR,
true))
+
readerContext.enableLogicalTimestampFieldRepair(storageConf.getBoolean(ENABLE_LOGICAL_TIMESTAMP_REPAIR,
true))
val props = metaClient.getTableConfig.getProps
options.foreach(kv => props.setProperty(kv._1, kv._2))
props.put(HoodieMemoryConfig.MAX_MEMORY_FOR_MERGE.key(),
String.valueOf(maxMemoryPerCompaction))
