This is an automated email from the ASF dual-hosted git repository.
gian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 24f8f9e1ab Add check for eternity time segment to
SqlSegmentsMetadataQuery (#12844)
24f8f9e1ab is described below
commit 24f8f9e1ab4199fd46e906e2506e35c0a467a65d
Author: Adarsh Sanjeev <[email protected]>
AuthorDate: Fri Aug 5 11:03:08 2022 +0530
Add check for eternity time segment to SqlSegmentsMetadataQuery (#12844)
* Add check for eternity time segment to SqlSegmentsMetadataQuery
* Add check for half eternities
* Add multiple segments test
* Add failing test to document known issue
---
.../druid/metadata/SqlSegmentsMetadataQuery.java | 19 +-
.../IndexerSQLMetadataStorageCoordinatorTest.java | 202 +++++++++++++++++++++
2 files changed, 218 insertions(+), 3 deletions(-)
diff --git
a/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataQuery.java
b/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataQuery.java
index 4ecf7bebe6..0c3ddfc890 100644
---
a/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataQuery.java
+++
b/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataQuery.java
@@ -233,12 +233,25 @@ public class SqlSegmentsMetadataQuery
)
);
- if (i == intervals.size() - 1) {
- sb.append(")");
- } else {
+ // Add a special check for a segment which have one end at eternity
and the other at some finite value. Since
+ // we are using string comparison, a segment with this start or end
will not be returned otherwise.
+ if (matchMode.equals(IntervalMode.OVERLAPS)) {
+ sb.append(StringUtils.format(" OR (start = '%s' AND \"end\" != '%s'
AND \"end\" > :start%d)", Intervals.ETERNITY.getStart(),
Intervals.ETERNITY.getEnd(), i));
+ sb.append(StringUtils.format(" OR (start != '%s' AND \"end\" = '%s'
AND start < :end%d)", Intervals.ETERNITY.getStart(),
Intervals.ETERNITY.getEnd(), i));
+ }
+
+ if (i != intervals.size() - 1) {
sb.append(" OR ");
}
}
+
+ // Add a special check for a single segment with eternity. Since we are
using string comparison, a segment with
+ // this start and end will not be returned otherwise.
+ // Known Issue: https://github.com/apache/druid/issues/12860
+ if (matchMode.equals(IntervalMode.OVERLAPS)) {
+ sb.append(StringUtils.format(" OR (start = '%s' AND \"end\" = '%s')",
Intervals.ETERNITY.getStart(), Intervals.ETERNITY.getEnd()));
+ }
+ sb.append(")");
}
final Query<Map<String, Object>> sql = handle
diff --git
a/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java
b/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java
index fca3366b88..6f8d0caaf8 100644
---
a/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java
+++
b/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java
@@ -53,6 +53,7 @@ import org.joda.time.DateTime;
import org.joda.time.Interval;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -97,6 +98,42 @@ public class IndexerSQLMetadataStorageCoordinatorTest
100
);
+ private final DataSegment eternitySegment = new DataSegment(
+ "fooDataSource",
+ Intervals.ETERNITY,
+ "version",
+ ImmutableMap.of(),
+ ImmutableList.of("dim1"),
+ ImmutableList.of("m1"),
+ new LinearShardSpec(0),
+ 9,
+ 100
+ );
+
+
+ private final DataSegment firstHalfEternityRangeSegment = new DataSegment(
+ "fooDataSource",
+ new Interval(DateTimes.MIN, DateTimes.of("3000")),
+ "version",
+ ImmutableMap.of(),
+ ImmutableList.of("dim1"),
+ ImmutableList.of("m1"),
+ new LinearShardSpec(0),
+ 9,
+ 100
+ );
+
+ private final DataSegment secondHalfEternityRangeSegment = new DataSegment(
+ "fooDataSource",
+ new Interval(DateTimes.of("1970"), DateTimes.MAX),
+ "version",
+ ImmutableMap.of(),
+ ImmutableList.of("dim1"),
+ ImmutableList.of("m1"),
+ new LinearShardSpec(0),
+ 9,
+ 100
+ );
private final DataSegment defaultSegment2 = new DataSegment(
"fooDataSource",
Intervals.of("2015-01-01T00Z/2015-01-02T00Z"),
@@ -254,6 +291,18 @@ public class IndexerSQLMetadataStorageCoordinatorTest
100
);
+ private final DataSegment hugeTimeRangeSegment4 = new DataSegment(
+ "hugeTimeRangeDataSource",
+ Intervals.of("1990-01-01T00Z/19940-01-01T00Z"),
+ "zversion",
+ ImmutableMap.of(),
+ ImmutableList.of("dim1"),
+ ImmutableList.of("m1"),
+ new NumberedShardSpec(0, 1),
+ 9,
+ 100
+ );
+
private final Set<DataSegment> SEGMENTS = ImmutableSet.of(defaultSegment,
defaultSegment2);
private final AtomicLong metadataUpdateCounter = new AtomicLong();
private final AtomicLong segmentTableDropUpdateCounter = new AtomicLong();
@@ -1202,6 +1251,159 @@ public class IndexerSQLMetadataStorageCoordinatorTest
);
}
+
+ @Test
+ public void testEternitySegmentWithStringComparison() throws IOException
+ {
+ coordinator.announceHistoricalSegments(
+ ImmutableSet.of(
+ eternitySegment
+ )
+ );
+
+ Assert.assertEquals(
+ ImmutableSet.of(eternitySegment),
+ ImmutableSet.copyOf(
+ coordinator.retrieveUsedSegmentsForInterval(
+ eternitySegment.getDataSource(),
+ Intervals.of("2020/2021"),
+ Segments.ONLY_VISIBLE
+ )
+ )
+ );
+ }
+
+ @Test
+ public void testEternityMultipleSegmentWithStringComparison() throws
IOException
+ {
+ coordinator.announceHistoricalSegments(
+ ImmutableSet.of(
+ numberedSegment0of0,
+ eternitySegment
+ )
+ );
+
+ Assert.assertEquals(
+ ImmutableSet.of(eternitySegment, numberedSegment0of0),
+ ImmutableSet.copyOf(
+ coordinator.retrieveUsedSegmentsForInterval(
+ eternitySegment.getDataSource(),
+ Intervals.of("2015/2016"),
+ Segments.ONLY_VISIBLE
+ )
+ )
+ );
+ }
+
+ @Test
+ public void testFirstHalfEternitySegmentWithStringComparison() throws
IOException
+ {
+ coordinator.announceHistoricalSegments(
+ ImmutableSet.of(
+ firstHalfEternityRangeSegment
+ )
+ );
+
+ Assert.assertEquals(
+ ImmutableSet.of(firstHalfEternityRangeSegment),
+ ImmutableSet.copyOf(
+ coordinator.retrieveUsedSegmentsForInterval(
+ firstHalfEternityRangeSegment.getDataSource(),
+ Intervals.of("2020/2021"),
+ Segments.ONLY_VISIBLE
+ )
+ )
+ );
+ }
+
+ @Test
+ public void testFirstHalfEternityMultipleSegmentWithStringComparison()
throws IOException
+ {
+ coordinator.announceHistoricalSegments(
+ ImmutableSet.of(
+ numberedSegment0of0,
+ firstHalfEternityRangeSegment
+ )
+ );
+
+ Assert.assertEquals(
+ ImmutableSet.of(numberedSegment0of0, firstHalfEternityRangeSegment),
+ ImmutableSet.copyOf(
+ coordinator.retrieveUsedSegmentsForInterval(
+ firstHalfEternityRangeSegment.getDataSource(),
+ Intervals.of("2015/2016"),
+ Segments.ONLY_VISIBLE
+ )
+ )
+ );
+ }
+
+ @Test
+ public void testSecondHalfEternitySegmentWithStringComparison() throws
IOException
+ {
+ coordinator.announceHistoricalSegments(
+ ImmutableSet.of(
+ secondHalfEternityRangeSegment
+ )
+ );
+
+ Assert.assertEquals(
+ ImmutableSet.of(secondHalfEternityRangeSegment),
+ ImmutableSet.copyOf(
+ coordinator.retrieveUsedSegmentsForInterval(
+ secondHalfEternityRangeSegment.getDataSource(),
+ Intervals.of("2020/2021"),
+ Segments.ONLY_VISIBLE
+ )
+ )
+ );
+ }
+
+ // Known Issue: https://github.com/apache/druid/issues/12860
+ @Ignore
+ @Test
+ public void testLargeIntervalWithStringComparison() throws IOException
+ {
+ coordinator.announceHistoricalSegments(
+ ImmutableSet.of(
+ hugeTimeRangeSegment4
+ )
+ );
+
+ Assert.assertEquals(
+ ImmutableSet.of(hugeTimeRangeSegment4),
+ ImmutableSet.copyOf(
+ coordinator.retrieveUsedSegmentsForInterval(
+ hugeTimeRangeSegment4.getDataSource(),
+ Intervals.of("2020/2021"),
+ Segments.ONLY_VISIBLE
+ )
+ )
+ );
+ }
+
+ @Test
+ public void testSecondHalfEternityMultipleSegmentWithStringComparison()
throws IOException
+ {
+ coordinator.announceHistoricalSegments(
+ ImmutableSet.of(
+ numberedSegment0of0,
+ secondHalfEternityRangeSegment
+ )
+ );
+
+ Assert.assertEquals(
+ ImmutableSet.of(numberedSegment0of0, secondHalfEternityRangeSegment),
+ ImmutableSet.copyOf(
+ coordinator.retrieveUsedSegmentsForInterval(
+ secondHalfEternityRangeSegment.getDataSource(),
+ Intervals.of("2015/2016"),
+ Segments.ONLY_VISIBLE
+ )
+ )
+ );
+ }
+
@Test
public void testDeleteDataSourceMetadata() throws IOException
{
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
