This is an automated email from the ASF dual-hosted git repository. pwason pushed a commit to branch release-0.14.0 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit a948fa091584fa8c4fa01bf2cd5cab8f924a3540 Author: Lokesh Jain <[email protected]> AuthorDate: Fri Sep 8 23:19:12 2023 +0530 [HUDI-6833] Add field for tracking log files from failed commit in rollback metadata (#9653) [HUDI-6833] Add field for tracking log files from failed commit in rollback metadata --- .../hudi/table/action/rollback/RollbackUtils.java | 6 ++++-- .../src/main/avro/HoodieRollbackMetadata.avsc | 13 ++++++++++++- .../org/apache/hudi/common/HoodieRollbackStat.java | 20 ++++++++++++++++++-- .../common/table/timeline/TimelineMetadataUtils.java | 2 +- .../apache/hudi/common/table/TestTimelineUtils.java | 3 ++- .../common/table/view/TestIncrementalFSViewSync.java | 2 +- 6 files changed, 38 insertions(+), 8 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RollbackUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RollbackUtils.java index f350b71da82..c3ee30ed3f4 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RollbackUtils.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RollbackUtils.java @@ -82,14 +82,16 @@ public class RollbackUtils { final List<String> successDeleteFiles = new ArrayList<>(); final List<String> failedDeleteFiles = new ArrayList<>(); final Map<FileStatus, Long> commandBlocksCount = new HashMap<>(); - final Map<FileStatus, Long> writtenLogFileSizeMap = new HashMap<>(); + final Map<String, Long> logFilesFromFailedCommit = new HashMap<>(); Option.ofNullable(stat1.getSuccessDeleteFiles()).ifPresent(successDeleteFiles::addAll); Option.ofNullable(stat2.getSuccessDeleteFiles()).ifPresent(successDeleteFiles::addAll); Option.ofNullable(stat1.getFailedDeleteFiles()).ifPresent(failedDeleteFiles::addAll); Option.ofNullable(stat2.getFailedDeleteFiles()).ifPresent(failedDeleteFiles::addAll); Option.ofNullable(stat1.getCommandBlocksCount()).ifPresent(commandBlocksCount::putAll); Option.ofNullable(stat2.getCommandBlocksCount()).ifPresent(commandBlocksCount::putAll); - return new HoodieRollbackStat(stat1.getPartitionPath(), successDeleteFiles, failedDeleteFiles, commandBlocksCount); + Option.ofNullable(stat1.getLogFilesFromFailedCommit()).ifPresent(logFilesFromFailedCommit::putAll); + Option.ofNullable(stat2.getLogFilesFromFailedCommit()).ifPresent(logFilesFromFailedCommit::putAll); + return new HoodieRollbackStat(stat1.getPartitionPath(), successDeleteFiles, failedDeleteFiles, commandBlocksCount, logFilesFromFailedCommit); } } diff --git a/hudi-common/src/main/avro/HoodieRollbackMetadata.avsc b/hudi-common/src/main/avro/HoodieRollbackMetadata.avsc index 5a300cda9e6..727a1461d99 100644 --- a/hudi-common/src/main/avro/HoodieRollbackMetadata.avsc +++ b/hudi-common/src/main/avro/HoodieRollbackMetadata.avsc @@ -38,7 +38,18 @@ "type": "long", "doc": "Size of this file in bytes" } - }], "default":null } + }], "default":null }, + {"name": "logFilesFromFailedCommit", + "type": ["null", { + "type": "map", + "doc": "Log files from the failed commit(commit to be rolled back)", + "values": { + "type": "long", + "doc": "Size of this file in bytes" + } + }], + "default":null + } ] }}}, { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/HoodieRollbackStat.java b/hudi-common/src/main/java/org/apache/hudi/common/HoodieRollbackStat.java index a3191fa026c..ba546866b54 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/HoodieRollbackStat.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/HoodieRollbackStat.java @@ -39,12 +39,15 @@ public class HoodieRollbackStat implements Serializable { // Count of HoodieLogFile to commandBlocks written for a particular rollback private final Map<FileStatus, Long> commandBlocksCount; + private final Map<String, Long> logFilesFromFailedCommit; + public HoodieRollbackStat(String partitionPath, List<String> successDeleteFiles, List<String> failedDeleteFiles, - Map<FileStatus, Long> commandBlocksCount) { + Map<FileStatus, Long> commandBlocksCount, Map<String, Long> logFilesFromFailedCommit) { this.partitionPath = partitionPath; this.successDeleteFiles = successDeleteFiles; this.failedDeleteFiles = failedDeleteFiles; this.commandBlocksCount = commandBlocksCount; + this.logFilesFromFailedCommit = logFilesFromFailedCommit; } public Map<FileStatus, Long> getCommandBlocksCount() { @@ -63,6 +66,10 @@ public class HoodieRollbackStat implements Serializable { return failedDeleteFiles; } + public Map<String, Long> getLogFilesFromFailedCommit() { + return logFilesFromFailedCommit; + } + public static HoodieRollbackStat.Builder newBuilder() { return new Builder(); } @@ -75,6 +82,7 @@ public class HoodieRollbackStat implements Serializable { private List<String> successDeleteFiles; private List<String> failedDeleteFiles; private Map<FileStatus, Long> commandBlocksCount; + private Map<String, Long> logFilesFromFailedCommit; private String partitionPath; public Builder withDeletedFileResults(Map<FileStatus, Boolean> deletedFiles) { @@ -105,6 +113,11 @@ public class HoodieRollbackStat implements Serializable { return this; } + public Builder withLogFilesFromFailedCommit(Map<String, Long> logFilesFromFailedCommit) { + this.logFilesFromFailedCommit = logFilesFromFailedCommit; + return this; + } + public HoodieRollbackStat build() { if (successDeleteFiles == null) { successDeleteFiles = Collections.EMPTY_LIST; @@ -115,7 +128,10 @@ public class HoodieRollbackStat implements Serializable { if (commandBlocksCount == null) { commandBlocksCount = Collections.EMPTY_MAP; } - return new HoodieRollbackStat(partitionPath, successDeleteFiles, failedDeleteFiles, commandBlocksCount); + if (logFilesFromFailedCommit == null) { + logFilesFromFailedCommit = Collections.EMPTY_MAP; + } + return new HoodieRollbackStat(partitionPath, successDeleteFiles, failedDeleteFiles, commandBlocksCount, logFilesFromFailedCommit); } } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineMetadataUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineMetadataUtils.java index c0550fef6fe..93ace4af3f2 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineMetadataUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineMetadataUtils.java @@ -84,7 +84,7 @@ public class TimelineMetadataUtils { Map<String, Long> rollbackLogFiles = stat.getCommandBlocksCount().keySet().stream() .collect(Collectors.toMap(f -> f.getPath().toString(), FileStatus::getLen)); HoodieRollbackPartitionMetadata metadata = new HoodieRollbackPartitionMetadata(stat.getPartitionPath(), - stat.getSuccessDeleteFiles(), stat.getFailedDeleteFiles(), rollbackLogFiles); + stat.getSuccessDeleteFiles(), stat.getFailedDeleteFiles(), rollbackLogFiles, stat.getLogFilesFromFailedCommit()); partitionMetadataBuilder.put(stat.getPartitionPath(), metadata); totalDeleted += stat.getSuccessDeleteFiles().size(); } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/TestTimelineUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/table/TestTimelineUtils.java index 3d950319a88..21251afec3c 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/TestTimelineUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/TestTimelineUtils.java @@ -503,7 +503,8 @@ public class TestTimelineUtils extends HoodieCommonTestHarness { List<HoodieInstant> rollbacks = new ArrayList<>(); rollbacks.add(new HoodieInstant(false, actionType, commitTs)); - HoodieRollbackStat rollbackStat = new HoodieRollbackStat(partition, deletedFiles, Collections.emptyList(), Collections.emptyMap()); + HoodieRollbackStat rollbackStat = new HoodieRollbackStat(partition, deletedFiles, Collections.emptyList(), + Collections.emptyMap(), Collections.emptyMap()); List<HoodieRollbackStat> rollbackStats = new ArrayList<>(); rollbackStats.add(rollbackStat); return TimelineMetadataUtils.convertRollbackMetadata(commitTs, Option.empty(), rollbacks, rollbackStats); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java index 750f7643f8c..9b56851f3e3 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java @@ -643,7 +643,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness { boolean isRestore) throws IOException { Map<String, List<String>> partitionToFiles = deleteFiles(files); List<HoodieRollbackStat> rollbackStats = partitionToFiles.entrySet().stream().map(e -> - new HoodieRollbackStat(e.getKey(), e.getValue(), new ArrayList<>(), new HashMap<>()) + new HoodieRollbackStat(e.getKey(), e.getValue(), new ArrayList<>(), new HashMap<>(), new HashMap<>()) ).collect(Collectors.toList()); List<HoodieInstant> rollbacks = new ArrayList<>();
