This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new ea723504fbd [HUDI-6401] Should not throw exception when create marker
file for log file (#9003)
ea723504fbd is described below
commit ea723504fbdb760fd00b4505fcbc737b4176370f
Author: guanziyue <[email protected]>
AuthorDate: Mon Jun 19 10:32:03 2023 +0800
[HUDI-6401] Should not throw exception when create marker file for log file
(#9003)
---
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 3 +--
.../org/apache/hudi/table/marker/WriteMarkers.java | 28 ++++++++++++++++++++++
2 files changed, 29 insertions(+), 2 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
index 4d5c8baefad..6653d348882 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
@@ -294,9 +294,8 @@ public abstract class HoodieWriteHandle<T, I, K, O> extends
HoodieIOHandle<T, I,
@Override
public boolean preLogFileOpen(HoodieLogFile logFileToAppend) {
- // we use create rather than createIfNotExists because create method can
trigger marker-based early conflict detection.
WriteMarkers writeMarkers =
WriteMarkersFactory.get(config.getMarkersType(), hoodieTable, instantTime);
- return writeMarkers.create(partitionPath, logFileToAppend.getFileName(),
IOType.APPEND,
+ return writeMarkers.createIfNotExists(partitionPath,
logFileToAppend.getFileName(), IOType.APPEND,
config, fileId,
hoodieTable.getMetaClient().getActiveTimeline()).isPresent();
}
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/WriteMarkers.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/WriteMarkers.java
index bbbb2cdfd90..8a9c5c4ad1a 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/WriteMarkers.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/WriteMarkers.java
@@ -107,6 +107,34 @@ public abstract class WriteMarkers implements Serializable
{
return create(partitionPath, fileName, type, true);
}
+ /**
+ * Creates a marker if the marker does not exist.
+ * This can invoke marker-based early conflict detection when enabled for
multi-writers.
+ *
+ * @param partitionPath partition path in the table
+ * @param fileName file name
+ * @param type write IO type
+ * @param writeConfig Hudi write configs.
+ * @param fileId File ID.
+ * @param activeTimeline Active timeline for the write operation.
+ * @return the marker path.
+ */
+ public Option<Path> createIfNotExists(String partitionPath, String fileName,
IOType type, HoodieWriteConfig writeConfig,
+ String fileId, HoodieActiveTimeline
activeTimeline) {
+ if (writeConfig.isEarlyConflictDetectionEnable()
+ &&
writeConfig.getWriteConcurrencyMode().supportsOptimisticConcurrencyControl()) {
+ HoodieTimeline pendingCompactionTimeline =
activeTimeline.filterPendingCompactionTimeline();
+ HoodieTimeline pendingReplaceTimeline =
activeTimeline.filterPendingReplaceTimeline();
+ // TODO If current is compact or clustering then create marker directly
without early conflict detection.
+ // Need to support early conflict detection between table service and
common writers.
+ if (pendingCompactionTimeline.containsInstant(instantTime) ||
pendingReplaceTimeline.containsInstant(instantTime)) {
+ return create(partitionPath, fileName, type, true);
+ }
+ return createWithEarlyConflictDetection(partitionPath, fileName, type,
false, writeConfig, fileId, activeTimeline);
+ }
+ return create(partitionPath, fileName, type, true);
+ }
+
/**
* Quietly deletes the marker directory.
*
