This is an automated email from the ASF dual-hosted git repository.
cheddar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new ca94f7146f Planning correctly for order by queries on time which
previously thre… (#13965)
ca94f7146f is described below
commit ca94f7146f50900aeab2e520bec032ad3ded4368
Author: Soumyava <[email protected]>
AuthorDate: Mon Apr 3 18:30:19 2023 -0700
Planning correctly for order by queries on time which previously thre…
(#13965)
* Planning correctly for order by queries on time which previously threw a
planning error
* Updating toDruidQueryForExplaining on a query data source if there is a
window on the partial query
---
.../druid/sql/calcite/rel/DruidOuterQueryRel.java | 7 +-
.../apache/druid/sql/calcite/rel/DruidQuery.java | 12 +--
.../druid/sql/calcite/CalciteExplainQueryTest.java | 2 +-
.../druid/sql/calcite/CalciteJoinQueryTest.java | 58 ++++++++++++
.../apache/druid/sql/calcite/CalciteQueryTest.java | 103 +++++++++++++++++++--
5 files changed, 162 insertions(+), 20 deletions(-)
diff --git
a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidOuterQueryRel.java
b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidOuterQueryRel.java
index 3171eb58c0..c49fc6bd04 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidOuterQueryRel.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidOuterQueryRel.java
@@ -34,6 +34,7 @@ import org.apache.calcite.rel.type.RelDataType;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.Druids;
import org.apache.druid.query.QueryDataSource;
+import org.apache.druid.query.TableDataSource;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.sql.calcite.planner.PlannerContext;
import org.apache.druid.sql.calcite.table.RowSignatures;
@@ -46,7 +47,9 @@ import java.util.Set;
*/
public class DruidOuterQueryRel extends DruidRel<DruidOuterQueryRel>
{
- private static final QueryDataSource DUMMY_DATA_SOURCE = new QueryDataSource(
+ private static final TableDataSource DUMMY_DATA_SOURCE = new
TableDataSource("__subquery__");
+
+ private static final QueryDataSource DUMMY_QUERY_DATA_SOURCE = new
QueryDataSource(
Druids.newScanQueryBuilder().dataSource("__subquery__").eternityInterval().build()
);
@@ -117,7 +120,7 @@ public class DruidOuterQueryRel extends
DruidRel<DruidOuterQueryRel>
public DruidQuery toDruidQueryForExplaining()
{
return partialQuery.build(
- DUMMY_DATA_SOURCE,
+ partialQuery.getWindow() == null ? DUMMY_DATA_SOURCE :
DUMMY_QUERY_DATA_SOURCE,
RowSignatures.fromRelDataType(
sourceRel.getRowType().getFieldNames(),
sourceRel.getRowType()
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java
b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java
index 2a093e6a30..f24f038145 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java
@@ -1439,7 +1439,8 @@ public class DruidQuery
}
if (!plannerContext.featureAvailable(EngineFeature.SCAN_ORDER_BY_NON_TIME)
&& !orderByColumns.isEmpty()) {
- if (orderByColumns.size() > 1 ||
!ColumnHolder.TIME_COLUMN_NAME.equals(orderByColumns.get(0).getColumnName())) {
+ if (orderByColumns.size() > 1 || orderByColumns.stream()
+ .anyMatch(orderBy ->
!orderBy.getColumnName().equals(ColumnHolder.TIME_COLUMN_NAME))) {
// Cannot handle this ordering.
// Scan cannot ORDER BY non-time columns.
plannerContext.setPlanningError(
@@ -1448,15 +1449,6 @@ public class DruidQuery
);
return null;
}
- if (!dataSource.isConcrete()) {
- // Cannot handle this ordering.
- // Scan cannot ORDER BY non-concrete datasources on _any_ column.
- plannerContext.setPlanningError(
- "SQL query requires order by on non-concrete datasource [%s],
which is not supported.",
- dataSource
- );
- return null;
- }
}
// Compute the list of columns to select, sorted and deduped.
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java
index 70e0320e3a..480fca6aeb 100644
---
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java
+++
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java
@@ -109,7 +109,7 @@ public class CalciteExplainQueryTest extends
BaseCalciteQueryTest
+ " )\n"
+ ")";
final String legacyExplanation =
-
"DruidOuterQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"query\",\"query\":{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"__subquery__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"list\",\"granularity\":{\"type\":\"all\"}}},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\
[...]
+
"DruidOuterQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"__subquery__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dum
[...]
+ " DruidJoinQueryRel(condition=[=(SUBSTRING($2, 1, 1), $8)],
joinType=[inner],
query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"__join__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"default\",\"dimension\":\"dim2\",\"outputName\":\"d0\",\"outputType\":\"STRING\"}],\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"
[...]
+ "
DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-0
[...]
+ "
DruidQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":null}},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"extraction\",\"dimension\":\"dim1\",\"outputName\":\"d0\",\"outputType\":\"STRING\",\"extra
[...]
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java
index 32007ed7ab..be65a64e80 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java
@@ -5264,4 +5264,62 @@ public class CalciteJoinQueryTest extends
BaseCalciteQueryTest
)
);
}
+
+ @Test
+ public void testJoinWithAliasAndOrderByNoGroupBy()
+ {
+ minTopNThreshold = 1;
+ Map<String, Object> context = new HashMap<>(QUERY_CONTEXT_DEFAULT);
+ context.put(PlannerConfig.CTX_KEY_USE_APPROXIMATE_TOPN, false);
+ testQuery(
+ "select t1.__time from druid.foo as t1 join\n"
+ + " druid.numfoo as t2 on t1.dim2 = t2.dim2\n"
+ + " order by t1.__time ASC ",
+ context, // turn on exact topN
+ ImmutableList.of(
+ newScanQueryBuilder()
+ .intervals(querySegmentSpec(Filtration.eternity()))
+ .dataSource(
+ JoinDataSource.create(
+ new TableDataSource(CalciteTests.DATASOURCE1),
+ new QueryDataSource(
+ newScanQueryBuilder()
+ .dataSource(CalciteTests.DATASOURCE3)
+ .intervals(querySegmentSpec(Intervals.of(
+
"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z")))
+ .columns("dim2")
+ .context(context)
+ .build()
+ ),
+ "j0.",
+ "(\"dim2\" == \"j0.dim2\")",
+ JoinType.INNER,
+ null,
+ ExprMacroTable.nil(),
+ CalciteTests.createJoinableFactoryWrapper()
+ )
+ )
+ .columns("__time")
+ .order(ScanQuery.Order.ASCENDING)
+ .context(context)
+ .build()
+ ),
+ NullHandling.sqlCompatible()
+ ? ImmutableList.of(
+ new Object[]{946684800000L},
+ new Object[]{946684800000L},
+ new Object[]{946857600000L},
+ new Object[]{978307200000L},
+ new Object[]{978307200000L},
+ new Object[]{978393600000L}
+ )
+ : ImmutableList.of(
+ new Object[]{946684800000L},
+ new Object[]{946684800000L},
+ new Object[]{978307200000L},
+ new Object[]{978307200000L},
+ new Object[]{978393600000L}
+ )
+ );
+ }
}
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
index 78e2e50f3b..f086aa8fe0 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
@@ -7306,14 +7306,9 @@ public class CalciteQueryTest extends
BaseCalciteQueryTest
)
)
.setInterval(querySegmentSpec(Filtration.eternity()))
- .setLimitSpec(
- new DefaultLimitSpec(
- ImmutableList.of(),
- 1
- )
- )
.setGranularity(Granularities.ALL)
.setAggregatorSpecs(
+ useDefault ?
aggregators(
new LongMaxAggregatorFactory("_a0", "a0"),
new LongMinAggregatorFactory("_a1", "a0"),
@@ -7321,6 +7316,16 @@ public class CalciteQueryTest extends
BaseCalciteQueryTest
new CountAggregatorFactory("_a2:count"),
new LongMaxAggregatorFactory("_a3", "d0"),
new CountAggregatorFactory("_a4")
+ ) : aggregators(
+ new LongMaxAggregatorFactory("_a0", "a0"),
+ new LongMinAggregatorFactory("_a1", "a0"),
+ new LongSumAggregatorFactory("_a2:sum", "a0"),
+ new FilteredAggregatorFactory(
+ new CountAggregatorFactory("_a2:count"),
+ not(selector("a0", null, null))
+ ),
+ new LongMaxAggregatorFactory("_a3", "d0"),
+ new CountAggregatorFactory("_a4")
)
)
.setPostAggregatorSpecs(
@@ -7333,7 +7338,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
new FieldAccessPostAggregator(null,
"_a2:count")
)
),
- expressionPostAgg("s0",
"timestamp_extract(\"_a3\",'EPOCH','UTC')")
+ expressionPostAgg("p0",
"timestamp_extract(\"_a3\",'EPOCH','UTC')")
)
)
.setContext(QUERY_CONTEXT_DEFAULT)
@@ -14604,4 +14609,88 @@ public class CalciteQueryTest extends
BaseCalciteQueryTest
)
);
}
+
+ @Test
+ public void testOrderByAlongWithInternalScanQuery()
+ {
+ testQuery(
+ "select __time as t, m1 from druid.foo where (m1 in (select distinct
m1 from druid.foo)) order by 1 limit 1",
+ ImmutableList.of(
+ newScanQueryBuilder()
+ .intervals(querySegmentSpec(Filtration.eternity()))
+ .dataSource(
+ JoinDataSource.create(
+ new TableDataSource(CalciteTests.DATASOURCE1),
+ new QueryDataSource(
+ GroupByQuery.builder()
+ .setGranularity(Granularities.ALL)
+
.setDataSource(CalciteTests.DATASOURCE1)
+
.setInterval(querySegmentSpec(Intervals.of(
+
"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z")))
+ .setDimensions(new
DefaultDimensionSpec("m1", "d0", ColumnType.FLOAT))
+ .build()
+ ),
+ "j0.",
+ "(\"m1\" == \"j0.d0\")",
+ JoinType.INNER,
+ null,
+ ExprMacroTable.nil(),
+ CalciteTests.createJoinableFactoryWrapper()
+ )
+ )
+ .context(QUERY_CONTEXT_DEFAULT)
+ .intervals(querySegmentSpec(Intervals.of(
+
"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z")))
+ .limit(1)
+ .columns(ImmutableList.of("__time", "m1"))
+ .order(ScanQuery.Order.ASCENDING)
+ .build()
+ ),
+ ImmutableList.of(
+ new Object[]{946684800000L, 1.0f}
+ )
+ );
+ }
+
+ @Test
+ public void testOrderByAlongWithInternalScanQueryNoDistinct()
+ {
+ testQuery(
+ "select __time, m1 from druid.foo where (m1 in (select m1 from
druid.foo)) order by __time DESC limit 1",
+ ImmutableList.of(
+ newScanQueryBuilder()
+ .intervals(querySegmentSpec(Filtration.eternity()))
+ .dataSource(
+ JoinDataSource.create(
+ new TableDataSource(CalciteTests.DATASOURCE1),
+ new QueryDataSource(
+ GroupByQuery.builder()
+ .setGranularity(Granularities.ALL)
+
.setDataSource(CalciteTests.DATASOURCE1)
+
.setInterval(querySegmentSpec(Intervals.of(
+
"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z")))
+ .setDimensions(new
DefaultDimensionSpec("m1", "d0", ColumnType.FLOAT))
+ .build()
+ ),
+ "j0.",
+ "(\"m1\" == \"j0.d0\")",
+ JoinType.INNER,
+ null,
+ ExprMacroTable.nil(),
+ CalciteTests.createJoinableFactoryWrapper()
+ )
+ )
+ .context(QUERY_CONTEXT_DEFAULT)
+ .intervals(querySegmentSpec(Intervals.of(
+
"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z")))
+ .limit(1)
+ .columns(ImmutableList.of("__time", "m1"))
+ .order(ScanQuery.Order.DESCENDING)
+ .build()
+ ),
+ ImmutableList.of(
+ new Object[]{978480000000L, 6.0f}
+ )
+ );
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
