This is an automated email from the ASF dual-hosted git repository.
amatya pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 63bfb3e6c94 Handle half-eternity intervals while fetching segments
with created dates (#15608)
63bfb3e6c94 is described below
commit 63bfb3e6c94e218bc20f738e7f4c285e2518b1f9
Author: AmatyaAvadhanula <[email protected]>
AuthorDate: Mon Jan 8 12:07:11 2024 +0530
Handle half-eternity intervals while fetching segments with created dates
(#15608)
* Handle all intervals while fetching segments with created dates
---
.../IndexerSQLMetadataStorageCoordinator.java | 35 +++++++++++++++-------
.../IndexerSQLMetadataStorageCoordinatorTest.java | 16 ++++++++++
2 files changed, 40 insertions(+), 11 deletions(-)
diff --git
a/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java
b/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java
index 9e4fc578eda..e4f5dd318d6 100644
---
a/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java
+++
b/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java
@@ -180,18 +180,14 @@ public class IndexerSQLMetadataStorageCoordinator
implements IndexerMetadataStor
"SELECT created_date, payload FROM %1$s WHERE dataSource = :dataSource
AND used = true"
);
- boolean hasEternityInterval = false;
- for (Interval interval : intervals) {
- if (Intervals.isEternity(interval)) {
- hasEternityInterval = true;
- break;
- }
- }
+ final boolean compareIntervalEndpointsAsString = intervals.stream()
+
.allMatch(Intervals::canCompareEndpointsAsStrings);
+ final SqlSegmentsMetadataQuery.IntervalMode intervalMode =
SqlSegmentsMetadataQuery.IntervalMode.OVERLAPS;
SqlSegmentsMetadataQuery.appendConditionForIntervalsAndMatchMode(
queryBuilder,
- hasEternityInterval ? Collections.emptyList() : intervals,
- SqlSegmentsMetadataQuery.IntervalMode.OVERLAPS,
+ compareIntervalEndpointsAsString ? intervals : Collections.emptyList(),
+ intervalMode,
connector
);
@@ -202,9 +198,11 @@ public class IndexerSQLMetadataStorageCoordinator
implements IndexerMetadataStor
.createQuery(queryString)
.bind("dataSource", dataSource);
- SqlSegmentsMetadataQuery.bindQueryIntervals(query, intervals);
+ if (compareIntervalEndpointsAsString) {
+ SqlSegmentsMetadataQuery.bindQueryIntervals(query, intervals);
+ }
- return query
+ final List<Pair<DataSegment, String>> segmentsWithCreatedDates =
query
.map((int index, ResultSet r, StatementContext ctx) ->
new Pair<>(
JacksonUtils.readValue(jsonMapper,
r.getBytes("payload"), DataSegment.class),
@@ -212,6 +210,21 @@ public class IndexerSQLMetadataStorageCoordinator
implements IndexerMetadataStor
)
)
.list();
+
+ if (intervals.isEmpty() || compareIntervalEndpointsAsString) {
+ return segmentsWithCreatedDates;
+ } else {
+ return segmentsWithCreatedDates
+ .stream()
+ .filter(pair -> {
+ for (Interval interval : intervals) {
+ if (intervalMode.apply(interval, pair.lhs.getInterval())) {
+ return true;
+ }
+ }
+ return false;
+ }).collect(Collectors.toList());
+ }
}
);
}
diff --git
a/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java
b/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java
index 6b55abf1150..9d2b3d495c6 100644
---
a/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java
+++
b/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java
@@ -3002,6 +3002,14 @@ public class IndexerSQLMetadataStorageCoordinatorTest
List<Pair<DataSegment, String>> resultForEternity =
coordinator.retrieveUsedSegmentsAndCreatedDates(defaultSegment.getDataSource(),
Collections.singletonList(Intervals.ETERNITY));
Assert.assertEquals(resultForExactInterval, resultForEternity);
+
+ List<Pair<DataSegment, String>> resultForFirstHalfEternity =
+
coordinator.retrieveUsedSegmentsAndCreatedDates(defaultSegment.getDataSource(),
Collections.singletonList(firstHalfEternityRangeSegment.getInterval()));
+ Assert.assertEquals(resultForExactInterval, resultForFirstHalfEternity);
+
+ List<Pair<DataSegment, String>> resultForSecondHalfEternity =
+
coordinator.retrieveUsedSegmentsAndCreatedDates(defaultSegment.getDataSource(),
Collections.singletonList(secondHalfEternityRangeSegment.getInterval()));
+ Assert.assertEquals(resultForExactInterval, resultForSecondHalfEternity);
}
@Test
@@ -3017,6 +3025,14 @@ public class IndexerSQLMetadataStorageCoordinatorTest
List<Pair<DataSegment, String>> resultForEternity =
coordinator.retrieveUsedSegmentsAndCreatedDates(defaultSegment.getDataSource(),
Collections.singletonList(eternitySegment.getInterval()));
Assert.assertEquals(3, resultForEternity.size());
+
+ List<Pair<DataSegment, String>> resultForFirstHalfEternity =
+
coordinator.retrieveUsedSegmentsAndCreatedDates(defaultSegment.getDataSource(),
Collections.singletonList(firstHalfEternityRangeSegment.getInterval()));
+ Assert.assertEquals(3, resultForFirstHalfEternity.size());
+
+ List<Pair<DataSegment, String>> resultForSecondHalfEternity =
+
coordinator.retrieveUsedSegmentsAndCreatedDates(defaultSegment.getDataSource(),
Collections.singletonList(secondHalfEternityRangeSegment.getInterval()));
+ Assert.assertEquals(3, resultForSecondHalfEternity.size());
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
