This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 20e7983 [HUDI-3252] Avoid creating empty requestedReplaceCommit in
the startCommit method (#4515)
20e7983 is described below
commit 20e798386684911046a7edef2129fb3496359dda
Author: 董可伦 <[email protected]>
AuthorDate: Tue Jan 18 06:28:18 2022 +0800
[HUDI-3252] Avoid creating empty requestedReplaceCommit in the startCommit
method (#4515)
---
.../org/apache/hudi/client/AbstractHoodieWriteClient.java | 9 +++++++--
.../apache/hudi/client/utils/MetadataConversionUtils.java | 2 +-
.../hudi/common/table/timeline/HoodieActiveTimeline.java | 13 +++++++++++++
.../java/org/apache/hudi/common/util/ClusteringUtils.java | 2 +-
4 files changed, 22 insertions(+), 4 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieWriteClient.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieWriteClient.java
index d431520..c9162de 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieWriteClient.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieWriteClient.java
@@ -828,8 +828,13 @@ public abstract class AbstractHoodieWriteClient<T extends
HoodieRecordPayload, I
if (config.getFailedWritesCleanPolicy().isLazy()) {
this.heartbeatClient.start(instantTime);
}
- metaClient.getActiveTimeline().createNewInstant(new
HoodieInstant(HoodieInstant.State.REQUESTED, actionType,
- instantTime));
+
+ if (actionType.equals(HoodieTimeline.REPLACE_COMMIT_ACTION)) {
+ metaClient.getActiveTimeline().createRequestedReplaceCommit(instantTime,
actionType);
+ } else {
+ metaClient.getActiveTimeline().createNewInstant(new
HoodieInstant(HoodieInstant.State.REQUESTED, actionType,
+ instantTime));
+ }
}
/**
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/MetadataConversionUtils.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/MetadataConversionUtils.java
index 02d2b04..8a9d0b3 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/MetadataConversionUtils.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/MetadataConversionUtils.java
@@ -134,7 +134,7 @@ public class MetadataConversionUtils {
return Option.of(HoodieCommitMetadata.fromBytes(inflightContent.get(),
HoodieCommitMetadata.class));
}
- public static Option<HoodieRequestedReplaceMetadata>
getRequestedReplaceMetadata(HoodieTableMetaClient metaClient, HoodieInstant
instant) throws IOException {
+ private static Option<HoodieRequestedReplaceMetadata>
getRequestedReplaceMetadata(HoodieTableMetaClient metaClient, HoodieInstant
instant) throws IOException {
Option<byte[]> requestedContent =
metaClient.getActiveTimeline().getInstantDetails(instant);
if (!requestedContent.isPresent() || requestedContent.get().length == 0) {
// requested commit files can be empty in some certain cases, e.g.
insert_overwrite or insert_overwrite_table.
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
index 0c7d97b..613cdb5 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
@@ -18,6 +18,7 @@
package org.apache.hudi.common.table.timeline;
+import org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieInstant.State;
import org.apache.hudi.common.util.FileIOUtils;
@@ -156,6 +157,18 @@ public class HoodieActiveTimeline extends
HoodieDefaultTimeline {
createFileInMetaPath(instant.getFileName(), Option.empty(), false);
}
+ public void createRequestedReplaceCommit(String instantTime, String
actionType) {
+ try {
+ HoodieInstant instant = new HoodieInstant(State.REQUESTED, actionType,
instantTime);
+ LOG.info("Creating a new instant " + instant);
+ // Create the request replace file
+ createFileInMetaPath(instant.getFileName(),
+ TimelineMetadataUtils.serializeRequestedReplaceMetadata(new
HoodieRequestedReplaceMetadata()), false);
+ } catch (IOException e) {
+ throw new HoodieIOException("Error create requested replace commit ", e);
+ }
+ }
+
public void saveAsComplete(HoodieInstant instant, Option<byte[]> data) {
LOG.info("Marking instant complete " + instant);
ValidationUtils.checkArgument(instant.isInflight(),
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/util/ClusteringUtils.java
b/hudi-common/src/main/java/org/apache/hudi/common/util/ClusteringUtils.java
index 1964ae7..15e5370 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/ClusteringUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/ClusteringUtils.java
@@ -77,7 +77,7 @@ public class ClusteringUtils {
* @return
* @throws IOException
*/
- public static Option<HoodieRequestedReplaceMetadata>
getRequestedReplaceMetadata(HoodieTableMetaClient metaClient, HoodieInstant
pendingReplaceInstant) throws IOException {
+ private static Option<HoodieRequestedReplaceMetadata>
getRequestedReplaceMetadata(HoodieTableMetaClient metaClient, HoodieInstant
pendingReplaceInstant) throws IOException {
final HoodieInstant requestedInstant;
if (!pendingReplaceInstant.isRequested()) {
// inflight replacecommit files don't have clustering plan.
