This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 9b3f85e1b4d Revert "[HUDI-8400] apply 'write.ignore.failed' when write
data failed (#12136)" (#12148)
9b3f85e1b4d is described below
commit 9b3f85e1b4d037a6e10c81e4b8d5f3e8a4a01ef6
Author: Danny Chan <[email protected]>
AuthorDate: Wed Oct 23 12:27:46 2024 +0800
Revert "[HUDI-8400] apply 'write.ignore.failed' when write data failed
(#12136)" (#12148)
This reverts commit a9194edd526e733863fef8a1fc1e945898498735.
---
.../java/org/apache/hudi/config/HoodieWriteConfig.java | 18 ------------------
.../java/org/apache/hudi/io/HoodieAppendHandle.java | 1 -
.../java/org/apache/hudi/io/HoodieCreateHandle.java | 1 -
.../java/org/apache/hudi/io/HoodieMergeHandle.java | 2 --
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 6 ------
.../hudi/io/storage/row/HoodieRowDataCreateHandle.java | 5 -----
.../org/apache/hudi/configuration/FlinkOptions.java | 1 -
.../java/org/apache/hudi/util/FlinkWriteClients.java | 3 +--
8 files changed, 1 insertion(+), 36 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
index dd5ee628f54..09df47b4452 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
@@ -811,12 +811,6 @@ public class HoodieWriteConfig extends HoodieConfig {
*/
public static final String WRITES_FILEID_ENCODING =
"_hoodie.writes.fileid.encoding";
- public static final ConfigProperty<Boolean> IGNORE_FAILED = ConfigProperty
- .key("hoodie.write.ignore.failed")
- .defaultValue(false)
- .sinceVersion("")
- .withDocumentation("Flag to indicate whether to ignore any non exception
error (e.g. writestatus error).");
-
private ConsistencyGuardConfig consistencyGuardConfig;
private FileSystemRetryConfig fileSystemRetryConfig;
@@ -2794,13 +2788,6 @@ public class HoodieWriteConfig extends HoodieConfig {
return metadataConfig.getSecondaryIndexParallelism();
}
- /**
- * Whether to ignore the write failed.
- */
- public boolean getIgnoreWriteFailed() {
- return getBooleanOrDefault(IGNORE_FAILED);
- }
-
public static class Builder {
protected final HoodieWriteConfig writeConfig = new HoodieWriteConfig();
@@ -3343,11 +3330,6 @@ public class HoodieWriteConfig extends HoodieConfig {
return this;
}
- public Builder withWriteIgnoreFailed(boolean ignoreFailedWriteData) {
- writeConfig.setValue(IGNORE_FAILED,
String.valueOf(ignoreFailedWriteData));
- return this;
- }
-
protected void setDefaults() {
writeConfig.setDefaultValue(MARKERS_TYPE,
getDefaultMarkersType(engineType));
// Check for mandatory properties
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
index b70415e06f7..cb0c7dd283f 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
@@ -499,7 +499,6 @@ public class HoodieAppendHandle<T, I, K, O> extends
HoodieWriteHandle<T, I, K, O
// for a single record
writeStatus.markFailure(record, t, recordMetadata);
LOG.error("Error writing record " + record, t);
- ignoreWriteFailed(t);
}
}
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
index 93f4716fae7..12406927ae6 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
@@ -170,7 +170,6 @@ public class HoodieCreateHandle<T, I, K, O> extends
HoodieWriteHandle<T, I, K, O
// for a single record
writeStatus.markFailure(record, t, recordMetadata);
LOG.error("Error writing record " + record, t);
- ignoreWriteFailed(t);
}
}
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
index f1c1473dc64..1bf6f6b0138 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
@@ -304,7 +304,6 @@ public class HoodieMergeHandle<T, I, K, O> extends
HoodieWriteHandle<T, I, K, O>
HoodieUpsertException failureEx = new HoodieUpsertException("mismatched
partition path, record partition: "
+ newRecord.getPartitionPath() + " but trying to insert into
partition: " + partitionPath);
writeStatus.markFailure(newRecord, failureEx, recordMetadata);
- ignoreWriteFailed(failureEx);
return false;
}
try {
@@ -334,7 +333,6 @@ public class HoodieMergeHandle<T, I, K, O> extends
HoodieWriteHandle<T, I, K, O>
} catch (Exception e) {
LOG.error("Error writing record " + newRecord, e);
writeStatus.markFailure(newRecord, e, recordMetadata);
- ignoreWriteFailed(e);
}
return false;
}
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
index efb266dd0c4..a251a5f7614 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
@@ -174,12 +174,6 @@ public abstract class HoodieWriteHandle<T, I, K, O>
extends HoodieIOHandle<T, I,
// NO_OP
}
- protected void ignoreWriteFailed(Throwable throwable) {
- if (config.getIgnoreWriteFailed()) {
- throw new HoodieException(throwable.getMessage(), throwable);
- }
- }
-
/**
* Perform the actual writing of the given record into the backing file.
*/
diff --git
a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
index dce64fce11b..b08e814d15c 100644
---
a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
+++
b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
@@ -31,7 +31,6 @@ import org.apache.hudi.common.model.IOType;
import org.apache.hudi.common.util.HoodieTimer;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
-import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieInsertException;
import org.apache.hudi.storage.HoodieStorage;
@@ -143,11 +142,7 @@ public class HoodieRowDataCreateHandle implements
Serializable {
? HoodieRecordDelegate.create(recordKey, partitionPath, null,
newRecordLocation) : null;
writeStatus.markSuccess(recordDelegate, Option.empty());
} catch (Throwable t) {
- LOG.error("Failed to write : key is " + recordKey + ", data is " +
rowData, t);
writeStatus.markFailure(recordKey, partitionPath, t);
- if (!writeConfig.getIgnoreWriteFailed()) {
- throw new HoodieException(t.getMessage(), t);
- }
}
} catch (Throwable ge) {
writeStatus.setGlobalError(ge);
diff --git
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java
index 8d4a3f78b7e..cc7f5f6783a 100644
---
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java
+++
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java
@@ -439,7 +439,6 @@ public class FlinkOptions extends HoodieConfig {
.key("write.ignore.failed")
.booleanType()
.defaultValue(false)
- .withFallbackKeys("hoodie.write.ignore.failed")
.withDescription("Flag to indicate whether to ignore any non exception
error (e.g. writestatus error). within a checkpoint batch. \n"
+ "By default false. Turning this on, could hide the write status
errors while the flink checkpoint moves ahead. \n"
+ "So, would recommend users to use this with caution.");
diff --git
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/FlinkWriteClients.java
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/FlinkWriteClients.java
index f25367392fe..e9d0310d475 100644
---
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/FlinkWriteClients.java
+++
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/FlinkWriteClients.java
@@ -225,8 +225,7 @@ public class FlinkWriteClients {
.withAutoCommit(false)
.withAllowOperationMetadataField(conf.getBoolean(FlinkOptions.CHANGELOG_ENABLED))
.withProps(flinkConf2TypedProperties(conf))
- .withSchema(getSourceSchema(conf).toString())
- .withWriteIgnoreFailed(conf.get(FlinkOptions.IGNORE_FAILED));
+ .withSchema(getSourceSchema(conf).toString());
Option<HoodieLockConfig> lockConfig = getLockConfig(conf);
if (lockConfig.isPresent()) {
