nsivabalan commented on issue #4604:
URL: https://github.com/apache/hudi/issues/4604#issuecomment-1021536416
I could not reproduce w/ latest master. I added a dummy commit in my test
dataset with almost exact content as the one you see. and archival is succeeds
in my case. I also tested w/ 0.10.0 and it succeeds. could not reproduce your
issue unfortunately.
this is the content of completed commit file that I hand curated and added
to the timeline.
```
{
"partitionToWriteStats" : { },
"compacted" : false,
"extraMetadata" : {
"schema" :
"{\"type\":\"record\",\"name\":\"hudi_trips_cow_record\",\"namespace\":\"hoodie.hudi_trips_cow\",\"fields\":[{\"name\":\"begin_lat\",\"type\":[\"null\",\"double\"],\"default\":null},{\"name\":\"begin_lon\",\"type\":[\"null\",\"double\"],\"default\":null},{\"name\":\"driver\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"end_lat\",\"type\":[\"null\",\"double\"],\"default\":null},{\"name\":\"end_lon\",\"type\":[\"null\",\"double\"],\"default\":null},{\"name\":\"fare\",\"type\":[\"null\",\"double\"],\"default\":null},{\"name\":\"partitionpath\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"rider\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"ts\",\"type\":[\"null\",\"long\"],\"default\":null},{\"name\":\"uuid\",\"type\":[\"null\",\"string\"],\"default\":null}]}"
},
"operationType" : "UPSERT",
"writePartitionPaths" : [ ],
"fileIdAndRelativePaths" : { },
"totalRecordsDeleted" : 0,
"totalLogRecordsCompacted" : 0,
"totalLogFilesCompacted" : 0,
"totalCompactedRecordsUpdated" : 0,
"totalLogFilesSize" : 0,
"totalScanTime" : 0,
"totalCreateTime" : 0,
"totalUpsertTime" : 423,
"minAndMaxEventTime" : {
"Optional.empty" : {
"val" : null,
"present" : false
}
}
}
```
I also added the requested and inflight meta file as well.
excerpt from logs:
```
22/01/25 13:47:00 WARN HoodieTimelineArchiveLog: Converting
[==>20220125101344200__commit__REQUESTED] to avro record for archiving
22/01/25 13:47:00 WARN HoodieTimelineArchiveLog: Converted record
{"hoodieCommitMetadata": {"partitionToWriteStats": {}, "extraMetadata":
{"ROLLING_STAT": ""}, "version": null, "operationType": "UNKNOWN"},
"hoodieCleanMetadata": null, "hoodieCompactionMetadata": null,
"hoodieRollbackMetadata": null, "hoodieSavePointMetadata": null, "commitTime":
"20220125101344200", "actionType": "commit", "version": null,
"hoodieCompactionPlan": null, "hoodieCleanerPlan": null, "actionState":
"REQUESTED", "hoodieReplaceCommitMetadata": null,
"hoodieRequestedReplaceMetadata": null, "HoodieInflightReplaceMetadata": null}
22/01/25 13:47:00 WARN HoodieTimelineArchiveLog: Converting
[==>20220125101344200__commit__INFLIGHT] to avro record for archiving
22/01/25 13:47:00 WARN HoodieTimelineArchiveLog: Converted record
{"hoodieCommitMetadata": {"partitionToWriteStats": {}, "extraMetadata":
{"ROLLING_STAT": ""}, "version": null, "operationType": "UPSERT"},
"hoodieCleanMetadata": null, "hoodieCompactionMetadata": null,
"hoodieRollbackMetadata": null, "hoodieSavePointMetadata": null, "commitTime":
"20220125101344200", "actionType": "commit", "version": null,
"hoodieCompactionPlan": null, "hoodieCleanerPlan": null, "actionState":
"INFLIGHT", "hoodieReplaceCommitMetadata": null,
"hoodieRequestedReplaceMetadata": null, "HoodieInflightReplaceMetadata": null}
22/01/25 13:47:00 WARN HoodieTimelineArchiveLog: Converting
[20220125101344200__commit__COMPLETED] to avro record for archiving
22/01/25 13:47:00 WARN HoodieTimelineArchiveLog: Converted record
{"hoodieCommitMetadata": {"partitionToWriteStats": {}, "extraMetadata":
{"schema":
"{\"type\":\"record\",\"name\":\"hudi_trips_cow_record\",\"namespace\":\"hoodie.hudi_trips_cow\",\"fields\":[{\"name\":\"begin_lat\",\"type\":[\"null\",\"double\"],\"default\":null},{\"name\":\"begin_lon\",\"type\":[\"null\",\"double\"],\"default\":null},{\"name\":\"driver\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"end_lat\",\"type\":[\"null\",\"double\"],\"default\":null},{\"name\":\"end_lon\",\"type\":[\"null\",\"double\"],\"default\":null},{\"name\":\"fare\",\"type\":[\"null\",\"double\"],\"default\":null},{\"name\":\"partitionpath\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"rider\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"ts\",\"type\":[\"null\",\"long\"],\"default\":null},{\"name\":\"uuid\",\"type\":[\"null\",\"string\"],\"default\":null}]}",
"ROLLING_STAT": ""}, "versio
n": null, "operationType": "UPSERT"}, "hoodieCleanMetadata": null,
"hoodieCompactionMetadata": null, "hoodieRollbackMetadata": null,
"hoodieSavePointMetadata": null, "commitTime": "20220125101344200",
"actionType": "commit", "version": null, "hoodieCompactionPlan": null,
"hoodieCleanerPlan": null, "actionState": "COMPLETED",
"hoodieReplaceCommitMetadata": null, "hoodieRequestedReplaceMetadata": null,
"HoodieInflightReplaceMetadata": null}
22/01/25 13:47:00 WARN HoodieTimelineArchiveLog: Converting
[==>20220125101344322__commit__REQUESTED] to avro record for archiving
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
