nsivabalan commented on code in PR #13236:
URL: https://github.com/apache/hudi/pull/13236#discussion_r2071255848
##########
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java:
##########
@@ -266,25 +329,30 @@ public boolean commitStats(String instantTime,
List<HoodieWriteStat> stats,
commitCallback = HoodieCommitCallbackFactory.create(config);
}
commitCallback.call(new HoodieWriteCommitCallbackMessage(
- instantTime, config.getTableName(), config.getBasePath(), stats,
Option.of(commitActionType), extraMetadata));
+ instantTime, config.getTableName(), config.getBasePath(),
dataTableStats, Option.of(commitActionType), extraMetadata));
}
return true;
}
protected void commit(HoodieTable table, String commitActionType, String
instantTime, HoodieCommitMetadata metadata,
- List<HoodieWriteStat> stats) throws IOException {
+ List<HoodieWriteStat> dataTablestats,
List<HoodieWriteStat> metadataWriteStatsSoFar, WriteOperationType
writeOperationType,
+ boolean skipStreamingWritesToMetadataTable) throws
IOException {
LOG.info("Committing {} action {}", instantTime, commitActionType);
HoodieActiveTimeline activeTimeline = table.getActiveTimeline();
// Finalize write
- finalizeWrite(table, instantTime, stats);
+ finalizeWrite(table, instantTime, dataTablestats);
// do save internal schema to support Implicitly add columns in write
process
if (!metadata.getExtraMetadata().containsKey(SerDeHelper.LATEST_SCHEMA)
&& metadata.getExtraMetadata().containsKey(SCHEMA_KEY) &&
table.getConfig().getSchemaEvolutionEnable()) {
saveInternalSchema(table, instantTime, metadata);
}
- // update Metadata table
- writeTableMetadata(table, instantTime, metadata);
- activeTimeline.saveAsComplete(false,
table.getMetaClient().createNewInstant(HoodieInstant.State.INFLIGHT,
commitActionType, instantTime), Option.of(metadata));
+ // generate Completion time
+ String completionTime = activeTimeline.createCompletionTime();
+ // write to metadata table.
+ writeToMetadataTable(skipStreamingWritesToMetadataTable, config, table,
writeOperationType, instantTime, metadataWriteStatsSoFar, metadata);
Review Comment:
writing to metadata table just before wrapping up the commit in data table.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
