mcvsubbu commented on a change in pull request #4374: Enhance the time boundary
service for backward-compatibility
URL: https://github.com/apache/incubator-pinot/pull/4374#discussion_r298309054
##########
File path:
pinot-broker/src/main/java/org/apache/pinot/broker/routing/HelixExternalViewBasedTimeBoundaryService.java
##########
@@ -111,25 +111,47 @@ public void updateTimeBoundaryService(ExternalView
externalView) {
continue;
}
- // Convert segment end time into table time unit
- // NOTE: for now, time unit in segment ZK metadata should always match
table time unit, but in the future we might
- // want to always use MILLISECONDS as the time unit in segment ZK
metadata
- maxTimeValue = Math.max(maxTimeValue,
tableTimeUnit.convert(segmentEndTime, segmentZKMetadata.getTimeUnit()));
+ if (latestSegmentZKMetadata == null) {
+ latestSegmentZKMetadata = segmentZKMetadata;
+ } else {
+ long segmentEndTimeMs =
segmentZKMetadata.getTimeUnit().toMillis(segmentEndTime);
+ long latestSegmentEndTimeMs =
+
latestSegmentZKMetadata.getTimeUnit().toMillis(latestSegmentZKMetadata.getEndTime());
+ if (segmentEndTimeMs > latestSegmentEndTimeMs) {
+ latestSegmentZKMetadata = segmentZKMetadata;
+ }
+ }
}
- if (maxTimeValue == -1L) {
+ if (latestSegmentZKMetadata == null) {
LOGGER.error("Skipping updating time boundary for table: '{}' because no
segment contains valid end time",
tableNameWithType);
return;
}
- // For HOURLY push table with time unit other than DAYS, use (maxTimeValue
- 1 HOUR) as the time boundary
- // Otherwise, use (maxTimeValue - 1 DAY)
- long timeBoundary;
- if ("HOURLY".equalsIgnoreCase(retentionConfig.getSegmentPushFrequency())
&& tableTimeUnit != TimeUnit.DAYS) {
- timeBoundary = maxTimeValue - tableTimeUnit.convert(1L, TimeUnit.HOURS);
- } else {
- timeBoundary = maxTimeValue - tableTimeUnit.convert(1L, TimeUnit.DAYS);
+ // Convert segment end time into table time unit
+ // NOTE: for now, time unit in segment ZK metadata should always match
table time unit, but in the future we might
+ // want to always use MILLISECONDS as the time unit in segment ZK
metadata
+ long latestSegmentEndTime = latestSegmentZKMetadata.getEndTime();
+ long timeBoundary = tableTimeUnit.convert(latestSegmentEndTime,
latestSegmentZKMetadata.getTimeUnit());
+
+ // Case 1 - segment has the same start and end time: directly use the end
time as the time boundary for
+ // backward-compatibility. There are use cases with time unit other than
DAYS, but have time value rounded
Review comment:
You may want to remove the "backward compabitlity" phrase here. Just mention
in the checkin comments that the other commit (reference the PR) broke backward
compatibility and we are restoring it here.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
