YuweiXiao commented on code in PR #5733:
URL: https://github.com/apache/hudi/pull/5733#discussion_r887715553
##########
hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java:
##########
@@ -267,43 +265,47 @@ public Option<byte[]> getInstantDetails(HoodieInstant
instant) {
}
/**
- * Get the last instant with valid schema, and convert this to
HoodieCommitMetadata
+ * Returns most recent instant having valid schema in its {@link
HoodieCommitMetadata}
*/
public Option<Pair<HoodieInstant, HoodieCommitMetadata>>
getLastCommitMetadataWithValidSchema() {
- List<HoodieInstant> completed =
getCommitsTimeline().filterCompletedInstants().getInstants()
-
.sorted(Comparator.comparing(HoodieInstant::getTimestamp).reversed()).collect(Collectors.toList());
- for (HoodieInstant instant : completed) {
- try {
- HoodieCommitMetadata commitMetadata = HoodieCommitMetadata.fromBytes(
- getInstantDetails(instant).get(), HoodieCommitMetadata.class);
- if
(!StringUtils.isNullOrEmpty(commitMetadata.getMetadata(HoodieCommitMetadata.SCHEMA_KEY)))
{
- return Option.of(Pair.of(instant, commitMetadata));
- }
- } catch (IOException e) {
- LOG.warn("Failed to convert instant to HoodieCommitMetadata: " +
instant.toString());
- }
- }
- return Option.empty();
+ return Option.fromJavaOptional(
+ getCommitMetadataStream()
+ .filter(instantCommitMetadataPair ->
+
!StringUtils.isNullOrEmpty(instantCommitMetadataPair.getValue().getMetadata(HoodieCommitMetadata.SCHEMA_KEY)))
+ .findFirst()
+ );
}
/**
* Get the last instant with valid data, and convert this to
HoodieCommitMetadata
*/
public Option<Pair<HoodieInstant, HoodieCommitMetadata>>
getLastCommitMetadataWithValidData() {
- List<HoodieInstant> completed =
getCommitsTimeline().filterCompletedInstants().getInstants()
-
.sorted(Comparator.comparing(HoodieInstant::getTimestamp).reversed()).collect(Collectors.toList());
- for (HoodieInstant instant : completed) {
- try {
- HoodieCommitMetadata commitMetadata = HoodieCommitMetadata.fromBytes(
- getInstantDetails(instant).get(), HoodieCommitMetadata.class);
- if (!commitMetadata.getFileIdAndRelativePaths().isEmpty()) {
- return Option.of(Pair.of(instant, commitMetadata));
- }
- } catch (IOException e) {
- LOG.warn("Failed to convert instant to HoodieCommitMetadata: " +
instant.toString());
- }
- }
- return Option.empty();
+ return Option.fromJavaOptional(
+ getCommitMetadataStream()
+ .filter(instantCommitMetadataPair ->
+
!instantCommitMetadataPair.getValue().getFileIdAndRelativePaths().isEmpty())
+ .findFirst()
+ );
+ }
+
+ /**
+ * Returns stream of {@link HoodieCommitMetadata} in order reverse to
chronological (ie most
+ * recent metadata being the first element)
+ */
+ private Stream<Pair<HoodieInstant, HoodieCommitMetadata>>
getCommitMetadataStream() {
+ // NOTE: Streams are lazy
Review Comment:
Hey Alex, could this lazy property save some computations compared to
original implementation? (e.g, construct timestamp list first, and loop over it
to get first valid metadata).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
