This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 42173629067 [HUDI-6229] HoodieInternalWriteStatus marks failure with
totalErrorRecords increment (#8739)
42173629067 is described below
commit 42173629067de5f99be91f5d997da5ad994b61a9
Author: Nicholas Jiang <[email protected]>
AuthorDate: Thu May 18 09:51:22 2023 +0800
[HUDI-6229] HoodieInternalWriteStatus marks failure with totalErrorRecords
increment (#8739)
---
.../main/java/org/apache/hudi/client/HoodieInternalWriteStatus.java | 5 +----
.../org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java | 2 +-
.../java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java | 2 +-
.../java/org/apache/hudi/client/TestHoodieInternalWriteStatus.java | 2 ++
.../org/apache/hudi/io/storage/row/TestHoodieRowCreateHandle.java | 1 -
.../apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java | 1 -
6 files changed, 5 insertions(+), 8 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieInternalWriteStatus.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieInternalWriteStatus.java
index 808eda5071e..103124bf28e 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieInternalWriteStatus.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieInternalWriteStatus.java
@@ -75,6 +75,7 @@ public class HoodieInternalWriteStatus implements
Serializable {
failedRecordKeys.add(Pair.of(recordKey, t));
}
totalRecords++;
+ totalErrorRecords++;
}
public boolean hasErrors() {
@@ -109,10 +110,6 @@ public class HoodieInternalWriteStatus implements
Serializable {
return successRecordKeys;
}
- public long getFailedRowsSize() {
- return failedRecordKeys.size();
- }
-
public List<Pair<String, Throwable>> getFailedRecordKeys() {
return failedRecordKeys;
}
diff --git
a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
index 70709041ac9..739928d6fe4 100644
---
a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
+++
b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
@@ -166,7 +166,7 @@ public class HoodieRowDataCreateHandle implements
Serializable {
long fileSizeInBytes = FSUtils.getFileSize(table.getMetaClient().getFs(),
path);
stat.setTotalWriteBytes(fileSizeInBytes);
stat.setFileSizeInBytes(fileSizeInBytes);
- stat.setTotalWriteErrors(writeStatus.getFailedRowsSize());
+ stat.setTotalWriteErrors(writeStatus.getTotalErrorRecords());
HoodieWriteStat.RuntimeStats runtimeStats = new
HoodieWriteStat.RuntimeStats();
runtimeStats.setTotalCreateTime(currTimer.endTimer());
stat.setRuntimeStats(runtimeStats);
diff --git
a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java
b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java
index f0f869da15d..a2a553470f2 100644
---
a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java
+++
b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java
@@ -232,7 +232,7 @@ public class HoodieRowCreateHandle implements Serializable {
long fileSizeInBytes = FSUtils.getFileSize(table.getMetaClient().getFs(),
path);
stat.setTotalWriteBytes(fileSizeInBytes);
stat.setFileSizeInBytes(fileSizeInBytes);
- stat.setTotalWriteErrors(writeStatus.getFailedRowsSize());
+ stat.setTotalWriteErrors(writeStatus.getTotalErrorRecords());
HoodieWriteStat.RuntimeStats runtimeStats = new
HoodieWriteStat.RuntimeStats();
runtimeStats.setTotalCreateTime(currTimer.endTimer());
stat.setRuntimeStats(runtimeStats);
diff --git
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestHoodieInternalWriteStatus.java
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestHoodieInternalWriteStatus.java
index 3f69c65b660..c3f31d816cd 100644
---
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestHoodieInternalWriteStatus.java
+++
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestHoodieInternalWriteStatus.java
@@ -44,6 +44,7 @@ public class TestHoodieInternalWriteStatus {
// verification
assertEquals(fileId, status.getFileId());
assertEquals(partitionPath, status.getPartitionPath());
+ assertEquals(1000, status.getTotalErrorRecords());
assertTrue(status.getFailedRecordKeys().size() > 0);
assertTrue(status.getFailedRecordKeys().size() < 150); // 150 instead of
100, to prevent flaky test
assertTrue(status.hasErrors());
@@ -66,6 +67,7 @@ public class TestHoodieInternalWriteStatus {
// verification
assertEquals(fileId, status.getFileId());
assertEquals(partitionPath, status.getPartitionPath());
+ assertEquals(1000, status.getTotalErrorRecords());
assertEquals(1000, status.getFailedRecordKeys().size());
assertTrue(status.hasErrors());
if (trackSuccess) {
diff --git
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/row/TestHoodieRowCreateHandle.java
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/row/TestHoodieRowCreateHandle.java
index ad73a256a61..47b07075a06 100644
---
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/row/TestHoodieRowCreateHandle.java
+++
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/row/TestHoodieRowCreateHandle.java
@@ -218,7 +218,6 @@ public class TestHoodieRowCreateHandle extends
HoodieClientTestHarness {
String instantTime, Dataset<Row> inputRows,
List<String> filenames, List<String> fileAbsPaths, boolean populateMetaFields) {
assertEquals(writeStatus.getPartitionPath(), partitionPath);
assertEquals(writeStatus.getTotalRecords(), size);
- assertEquals(writeStatus.getFailedRowsSize(), 0);
assertEquals(writeStatus.getTotalErrorRecords(), 0);
assertFalse(writeStatus.hasErrors());
assertNull(writeStatus.getGlobalError());
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/test/java/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java
b/hudi-spark-datasource/hudi-spark-common/src/test/java/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java
index 3731096bb4d..30cd8ec8df8 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/test/java/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java
+++
b/hudi-spark-datasource/hudi-spark-common/src/test/java/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java
@@ -122,7 +122,6 @@ public class HoodieBulkInsertInternalWriterTestBase extends
HoodieClientTestHarn
assertEquals(writeStatus.getTotalRecords(),
sizeMap.get(HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS[counter % 3]));
}
assertNull(writeStatus.getGlobalError());
- assertEquals(writeStatus.getFailedRowsSize(), 0);
assertEquals(writeStatus.getTotalErrorRecords(), 0);
assertFalse(writeStatus.hasErrors());
assertNotNull(writeStatus.getFileId());
