9aman commented on code in PR #14741:
URL: https://github.com/apache/pinot/pull/14741#discussion_r1905050421
##########
pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java:
##########
@@ -544,6 +557,55 @@ private void doAddConsumingSegment(String segmentName)
_logger.info("Added new CONSUMING segment: {}", segmentName);
}
+ @Override
+ public File downloadSegment(SegmentZKMetadata zkMetadata)
+ throws Exception {
+ if (!PauselessConsumptionUtils.isPauselessEnabled(_tableConfig)) {
+ return super.downloadSegment(zkMetadata);
+ }
+
+ final long startTime = System.currentTimeMillis();
+
+ while (System.currentTimeMillis() - startTime < TIMEOUT_MS) {
+ // ZK Metadata may change during segment download process; fetch it on
every retry.
+ zkMetadata = fetchZKMetadata(zkMetadata.getSegmentName());
+
+ if (zkMetadata.getDownloadUrl() != null) {
+ // The downloadSegment() will throw an exception in case there are
some genuine issues.
+ // We don't want to retry in those scenarios and will throw an
exception
+ return downloadSegmentFromDeepStore(zkMetadata);
+ }
+
+ if (_peerDownloadScheme != null) {
+ _logger.info("Peer download is enabled for the segment: {}",
zkMetadata.getSegmentName());
+ try {
+ return downloadSegmentFromPeers(zkMetadata);
+ } catch (Exception e) {
+ // TODO :in this case we just retry as some of the other servers
might be trying to build the
+ // segment
+ _logger.warn("Could not download segment: {} from peer",
zkMetadata.getSegmentName(), e);
+ }
+ }
+
+ long timeElapsed = System.currentTimeMillis() - startTime;
+ long timeRemaining = TIMEOUT_MS - timeElapsed;
+
+ if (timeRemaining <= 0) {
+ break;
+ }
+
+ _logger.info("Sleeping for 30 seconds as the segment url is missing.
Time remaining: {} minutes",
+ Math.round(timeRemaining / 60000.0));
+
+ // Sleep for the shorter of our normal interval or remaining time
+ Thread.sleep(Math.min(SLEEP_INTERVAL_MS, timeRemaining));
+ }
+
+// If we exit the loop without returning, throw an exception
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
