danny0405 commented on code in PR #13307:
URL: https://github.com/apache/hudi/pull/13307#discussion_r2127825430
##########
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/SparkRDDWriteClient.java:
##########
@@ -120,27 +119,27 @@ public boolean commit(String instantTime,
JavaRDD<WriteStatus> writeStatuses, Op
return Pair.of(writeStatus.isMetadataTable(), writeStatus);
}
).collect();
- // Compute stats for the writes and invoke callback
+ // Compute stats for the data table writes and invoke callback
AtomicLong totalRecords = new AtomicLong(0);
AtomicLong totalErrorRecords = new AtomicLong(0);
- isMetadataWriteStatusPairs.stream().filter(entry -> isMetadataTable &&
entry.getKey()).forEach(pair -> {
+ // collect record stats for data table
+ isMetadataWriteStatusPairs.stream().filter(pair ->
!pair.getKey()).forEach(pair -> {
totalRecords.getAndAdd(pair.getValue().getTotalRecords());
totalErrorRecords.getAndAdd(pair.getValue().getTotalErrorRecords());
});
// reason why we are passing RDD<WriteStatus> to the writeStatusHandler
callback: At the beginning of this method, we drop all index stats and error
records before collecting in the driver.
// Just incase if there are errors, caller might be interested to fetch
error records in the callback. And so, we are passing the RDD<WriteStatus> as
last argument to the write status
// handler callback.
boolean canProceed = writeStatusValidatorOpt.map(callback ->
callback.validate(totalRecords.get(), totalErrorRecords.get(),
- totalErrorRecords.get() > 0 ?
Option.of(HoodieJavaRDD.of(writeStatuses.filter(status ->
table.isMetadataTable() &&
status.isMetadataTable()).map(WriteStatus::removeMetadataStats))) :
Option.empty()))
+ totalErrorRecords.get() > 0 ?
Option.of(HoodieJavaRDD.of(writeStatuses.filter(pair ->
!pair.isMetadataTable()).map(WriteStatus::removeMetadataStats))) :
Option.empty()))
Review Comment:
pair -> status
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
