wombatu-kun commented on code in PR #11518:
URL: https://github.com/apache/hudi/pull/11518#discussion_r1678694898
##########
hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java:
##########
@@ -248,30 +245,16 @@ public static <T> T fromJsonString(String jsonStr,
Class<T> clazz) throws Except
* parse the bytes of deltacommit, and get the base file and the log files
belonging to this
* provided file group.
*/
- // TODO: refactor this method to avoid doing the json tree walking
(HUDI-4822).
public static Option<Pair<String, List<String>>>
getFileSliceForFileGroupFromDeltaCommit(byte[] bytes, HoodieFileGroupId
fileGroupId) {
try {
- String jsonStr = fromUTF8Bytes(
- convertCommitMetadataToJsonBytes(deserializeCommitMetadata(bytes),
org.apache.hudi.avro.model.HoodieCommitMetadata.class));
- if (jsonStr.isEmpty()) {
- return Option.empty();
- }
- JsonNode ptToWriteStatsMap =
JsonUtils.getObjectMapper().readTree(jsonStr).get("partitionToWriteStats");
- Iterator<Map.Entry<String, JsonNode>> pts = ptToWriteStatsMap.fields();
- while (pts.hasNext()) {
- Map.Entry<String, JsonNode> ptToWriteStats = pts.next();
- if (ptToWriteStats.getValue().isArray()) {
- for (JsonNode writeStat : ptToWriteStats.getValue()) {
- HoodieFileGroupId fgId = new
HoodieFileGroupId(ptToWriteStats.getKey(), writeStat.get("fileId").asText());
- if (fgId.equals(fileGroupId)) {
- String baseFile = writeStat.get("baseFile").asText();
- ArrayNode logFilesNode = (ArrayNode) writeStat.get("logFiles");
- List<String> logFiles = new ArrayList<>();
- for (JsonNode logFile : logFilesNode) {
- logFiles.add(logFile.asText());
- }
- return Option.of(Pair.of(baseFile, logFiles));
- }
+ org.apache.hudi.avro.model.HoodieCommitMetadata commitMetadata =
deserializeCommitMetadata(bytes);
+ Map<String,List<org.apache.hudi.avro.model.HoodieWriteStat>>
partitionToWriteStatsMap =
+ commitMetadata.getPartitionToWriteStats();
+ for (Map.Entry<String, List<org.apache.hudi.avro.model.HoodieWriteStat>>
partitionToWriteStat: partitionToWriteStatsMap.entrySet()) {
+ for (org.apache.hudi.avro.model.HoodieWriteStat writeStat:
partitionToWriteStat.getValue()) {
+ HoodieFileGroupId fgId = new
HoodieFileGroupId(partitionToWriteStat.getKey(), writeStat.getFileId());
+ if (fgId.equals(fileGroupId)) {
+ return Option.of(Pair.of(writeStat.getBaseFile(),
writeStat.getLogFiles()));
Review Comment:
Is that what you wanted? Any other comments?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
