skywalker0618 commented on code in PR #20735:
URL: https://github.com/apache/hudi/pull/20735#discussion_r4647968786


##########
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/split/DefaultHoodieSplitDiscover.java:
##########
@@ -65,6 +86,141 @@ public HoodieContinuousSplitBatch discoverSplits(String 
lastInstant) {
     return incrementalInputSplits.inputHoodieSourceSplits(metaClient, 
lastInstant, scanContext.isCdcEnabled());
   }
 
+  @Override
+  public List<String> getUndiscoveredCommits(@Nullable String offset, boolean 
reloadTimeline) {
+    if (metaClient == null || offset == null) {
+      return Collections.emptyList();
+    }
+    if (reloadTimeline) {
+      metaClient.reloadActiveTimeline();
+    }
+    HoodieTimeline timeline = 
IncrementalQueryAnalyzer.filterInstantsAsPerUserConfigs(
+        metaClient,
+        metaClient.getCommitsAndCompactionTimeline().filterCompletedInstants(),
+        scanContext.isSkipCompaction(),
+        scanContext.isSkipClustering(),
+        scanContext.isSkipInsertOverwrite());
+    return timeline.getInstantsAsStream()
+        .map(HoodieInstant::getCompletionTime)
+        .filter(completionTime -> completionTime != null && 
completionTime.compareTo(offset) > 0)

Review Comment:
   Sure, changed.



##########
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/enumerator/HoodieContinuousSplitEnumerator.java:
##########
@@ -90,22 +132,80 @@ public HoodieSplitEnumeratorState snapshotState(long 
checkpointId) throws Except
     return new HoodieSplitEnumeratorState(splitProvider.state(), 
position.get().getIssuedInstant(), position.get().getIssuedOffset());
   }
 
-  private HoodieContinuousSplitBatch discoverSplits() {
+  @Override
+  void onSplitsFinished(Collection<String> finishedSplitIds) {
+    if (lagMetrics != null) {
+      lagMetrics.finishSplits(finishedSplitIds);
+    }
+  }
+
+  @Override
+  void onSplitsAddedBack(List<HoodieSourceSplit> splits) {
+    if (lagMetrics != null) {
+      lagMetrics.addSplits(getSplitLagInfo(splits));
+    }
+  }
+
+  /**
+   * Returns the discovered splits, with their lag info and the completion 
times of the commits left to
+   * discover, which are null when the metrics are disabled or failed to be 
resolved.
+   */
+  private DiscoveryResult discoverSplits() {
+    String issuedOffset = position.get().getIssuedOffset().orElse(null);
     int pendingSplitNumber = splitProvider.pendingSplitCount();
     if (pendingSplitNumber > scanContext.getMaxPendingSplits()) {
       log.info(
           "Pause split discovery as the assigner already has too many pending 
splits: {}",
           pendingSplitNumber);
-      return HoodieContinuousSplitBatch.EMPTY;
+      return new DiscoveryResult(HoodieContinuousSplitBatch.EMPTY, 
Collections.emptyMap(), getUndiscoveredCommits(issuedOffset, true));
+    }
+    HoodieContinuousSplitBatch batch = 
splitDiscover.discoverSplits(issuedOffset);
+    String nextOffset = StringUtils.isNullOrEmpty(batch.getOffset()) ? 
issuedOffset : batch.getOffset();
+    return new DiscoveryResult(batch, resolveLagInfo(batch.getSplits()), 
getUndiscoveredCommits(nextOffset, false));
+  }
+
+  @Nullable
+  private Map<String, HoodieSourceSplitLagInfo> 
resolveLagInfo(Collection<HoodieSourceSplit> splits) {
+    if (lagMetrics == null || splits.isEmpty()) {
+      return null;
+    }
+    try {
+      return splitDiscover.getLagInfo(splits);
+    } catch (Exception e) {
+      log.warn("Failed to resolve the lag info of {} discovered splits", 
splits.size(), e);
+      return null;
+    }
+  }
+
+  @Nullable
+  private List<String> getUndiscoveredCommits(String offset, boolean 
reloadTimeline) {
+    if (lagMetrics == null) {
+      return null;
     }
-    return 
splitDiscover.discoverSplits(position.get().getIssuedOffset().isPresent() ? 
position.get().getIssuedOffset().get() : null);
+    try {
+      return splitDiscover.getUndiscoveredCommits(offset, reloadTimeline);
+    } catch (Exception e) {
+      log.warn("Failed to get the undiscovered commits after {}", offset, e);
+      return null;
+    }
+  }
+
+  private static Map<String, HoodieSourceSplitLagInfo> 
getSplitLagInfo(Collection<HoodieSourceSplit> splits) {
+    return 
splits.stream().collect(Collectors.toMap(HoodieSourceSplit::splitId, 
HoodieSourceSplitLagInfo::of, (first, second) -> first));

Review Comment:
   Refactored



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to