This is an automated email from the ASF dual-hosted git repository.
kfaraz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new b6f8d7a1b3 Add query context param `forceExpressionVirtualColumns` to
always use "expression"-type virtual columns in query plan (#12583)
b6f8d7a1b3 is described below
commit b6f8d7a1b39793f7941e450925952c50e53fba9f
Author: Kashif Faraz <[email protected]>
AuthorDate: Wed Jun 22 15:33:50 2022 +0530
Add query context param `forceExpressionVirtualColumns` to always use
"expression"-type virtual columns in query plan (#12583)
SQL expressions such as those containing `MV_FILTER_ONLY` and
`MV_FILTER_NONE`
are planned as specialized virtual columns instead of the default
`expression`-type virtual columns.
This commit adds a new context parameter to force the `expression`-type
virtual columns.
Changes
- Add query context param `forceExpressionVirtualColumns`
- Use context param to determine if specialized virtual columns should be
used or not
- Moved some tests into `CalciteExplainQueryTest`
---
.../sql/calcite/expression/DruidExpression.java | 9 +
.../druid/sql/calcite/planner/PlannerConfig.java | 23 +-
.../druid/sql/calcite/rel/DruidJoinQueryRel.java | 6 +-
.../apache/druid/sql/calcite/rel/DruidQuery.java | 35 +-
.../sql/calcite/rel/VirtualColumnRegistry.java | 29 +-
.../druid/sql/calcite/CalciteExplainQueryTest.java | 368 +++++++++++++++++++++
.../apache/druid/sql/calcite/CalciteQueryTest.java | 227 -------------
.../calcite/expression/ExpressionTestHelper.java | 2 +-
.../druid/sql/calcite/rel/DruidQueryTest.java | 16 +-
9 files changed, 455 insertions(+), 260 deletions(-)
diff --git
a/sql/src/main/java/org/apache/druid/sql/calcite/expression/DruidExpression.java
b/sql/src/main/java/org/apache/druid/sql/calcite/expression/DruidExpression.java
index 950fc93783..6e043eff2b 100644
---
a/sql/src/main/java/org/apache/druid/sql/calcite/expression/DruidExpression.java
+++
b/sql/src/main/java/org/apache/druid/sql/calcite/expression/DruidExpression.java
@@ -387,6 +387,15 @@ public class DruidExpression
return virtualColumnCreator.create(name, outputType, expression.get(),
macroTable);
}
+ public VirtualColumn toExpressionVirtualColumn(
+ final String name,
+ final ColumnType outputType,
+ final ExprMacroTable macroTable
+ )
+ {
+ return DEFAULT_VIRTUAL_COLUMN_BUILDER.create(name, outputType,
expression.get(), macroTable);
+ }
+
public NodeType getType()
{
return nodeType;
diff --git
a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java
b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java
index 836477ce83..f7ceaf51f6 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java
@@ -34,6 +34,7 @@ public class PlannerConfig
public static final String CTX_KEY_USE_APPROXIMATE_TOPN =
"useApproximateTopN";
public static final String CTX_COMPUTE_INNER_JOIN_COST_AS_FILTER =
"computeInnerJoinCostAsFilter";
public static final String CTX_KEY_USE_NATIVE_QUERY_EXPLAIN =
"useNativeQueryExplain";
+ public static final String CTX_KEY_FORCE_EXPRESSION_VIRTUAL_COLUMNS =
"forceExpressionVirtualColumns";
public static final String CTX_MAX_NUMERIC_IN_FILTERS =
"maxNumericInFilters";
public static final int NUM_FILTER_NOT_USED = -1;
@@ -76,6 +77,9 @@ public class PlannerConfig
@JsonProperty
private boolean useNativeQueryExplain = false;
+ @JsonProperty
+ private boolean forceExpressionVirtualColumns = false;
+
@JsonProperty
private int maxNumericInFilters = NUM_FILTER_NOT_USED;
@@ -156,6 +160,15 @@ public class PlannerConfig
return useNativeQueryExplain;
}
+ /**
+ * @return true if special virtual columns should not be optimized and should
+ * always be of type "expressions", false otherwise.
+ */
+ public boolean isForceExpressionVirtualColumns()
+ {
+ return forceExpressionVirtualColumns;
+ }
+
public PlannerConfig withOverrides(final QueryContext queryContext)
{
if (queryContext.isEmpty()) {
@@ -185,6 +198,10 @@ public class PlannerConfig
CTX_KEY_USE_NATIVE_QUERY_EXPLAIN,
isUseNativeQueryExplain()
);
+ newConfig.forceExpressionVirtualColumns = queryContext.getAsBoolean(
+ CTX_KEY_FORCE_EXPRESSION_VIRTUAL_COLUMNS,
+ isForceExpressionVirtualColumns()
+ );
final int systemConfigMaxNumericInFilters = getMaxNumericInFilters();
final int queryContextMaxNumericInFilters = queryContext.getAsInt(
CTX_MAX_NUMERIC_IN_FILTERS,
@@ -244,7 +261,8 @@ public class PlannerConfig
serializeComplexValues == that.serializeComplexValues &&
Objects.equals(metadataRefreshPeriod, that.metadataRefreshPeriod) &&
Objects.equals(sqlTimeZone, that.sqlTimeZone) &&
- useNativeQueryExplain == that.useNativeQueryExplain;
+ useNativeQueryExplain == that.useNativeQueryExplain &&
+ forceExpressionVirtualColumns == that.forceExpressionVirtualColumns;
}
@Override
@@ -262,7 +280,8 @@ public class PlannerConfig
metadataSegmentCacheEnable,
metadataSegmentPollPeriod,
serializeComplexValues,
- useNativeQueryExplain
+ useNativeQueryExplain,
+ forceExpressionVirtualColumns
);
}
diff --git
a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidJoinQueryRel.java
b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidJoinQueryRel.java
index 87e2e21efe..137803bb46 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidJoinQueryRel.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidJoinQueryRel.java
@@ -163,7 +163,11 @@ public class DruidJoinQueryRel extends
DruidRel<DruidJoinQueryRel>
final Pair<String, RowSignature> prefixSignaturePair =
computeJoinRowSignature(leftSignature, rightSignature);
- VirtualColumnRegistry virtualColumnRegistry =
VirtualColumnRegistry.create(prefixSignaturePair.rhs,
getPlannerContext().getExprMacroTable());
+ VirtualColumnRegistry virtualColumnRegistry = VirtualColumnRegistry.create(
+ prefixSignaturePair.rhs,
+ getPlannerContext().getExprMacroTable(),
+
getPlannerContext().getPlannerConfig().isForceExpressionVirtualColumns()
+ );
getPlannerContext().setJoinExpressionVirtualColumnRegistry(virtualColumnRegistry);
// Generate the condition for this join as a Druid expression.
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java
b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java
index 08aa08ed87..075401eff1 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java
@@ -167,7 +167,11 @@ public class DruidQuery
{
final RelDataType outputRowType = partialQuery.leafRel().getRowType();
if (virtualColumnRegistry == null) {
- virtualColumnRegistry = VirtualColumnRegistry.create(sourceRowSignature,
plannerContext.getExprMacroTable());
+ virtualColumnRegistry = VirtualColumnRegistry.create(
+ sourceRowSignature,
+ plannerContext.getExprMacroTable(),
+ plannerContext.getPlannerConfig().isForceExpressionVirtualColumns()
+ );
}
// Now the fun begins.
@@ -634,24 +638,25 @@ public class DruidQuery
// implementation can be used instead of being composed as part of some
expression tree in an expresson virtual
// column
Set<String> specialized = new HashSet<>();
+ final boolean forceExpressionVirtualColumns =
+ plannerContext.getPlannerConfig().isForceExpressionVirtualColumns();
virtualColumnRegistry.visitAllSubExpressions((expression) -> {
- switch (expression.getType()) {
- case SPECIALIZED:
- // add the expression to the top level of the registry as a
standalone virtual column
- final String name =
virtualColumnRegistry.getOrCreateVirtualColumnForExpression(
- expression,
- expression.getDruidType()
- );
- specialized.add(name);
- // replace with an identifier expression of the new virtual column
name
- return DruidExpression.ofColumn(expression.getDruidType(), name);
- default:
- // do nothing
- return expression;
+ if (!forceExpressionVirtualColumns
+ && expression.getType() == DruidExpression.NodeType.SPECIALIZED) {
+ // add the expression to the top level of the registry as a standalone
virtual column
+ final String name =
virtualColumnRegistry.getOrCreateVirtualColumnForExpression(
+ expression,
+ expression.getDruidType()
+ );
+ specialized.add(name);
+ // replace with an identifier expression of the new virtual column name
+ return DruidExpression.ofColumn(expression.getDruidType(), name);
+ } else {
+ // do nothing
+ return expression;
}
});
-
// we always want to add any virtual columns used by the query level
DimFilter
if (filter != null) {
for (String columnName : filter.getRequiredColumns()) {
diff --git
a/sql/src/main/java/org/apache/druid/sql/calcite/rel/VirtualColumnRegistry.java
b/sql/src/main/java/org/apache/druid/sql/calcite/rel/VirtualColumnRegistry.java
index 6ebb9d0457..d07586ec2f 100644
---
a/sql/src/main/java/org/apache/druid/sql/calcite/rel/VirtualColumnRegistry.java
+++
b/sql/src/main/java/org/apache/druid/sql/calcite/rel/VirtualColumnRegistry.java
@@ -49,11 +49,13 @@ public class VirtualColumnRegistry
private final Map<String, ExpressionAndTypeHint> virtualColumnsByName;
private final String virtualColumnPrefix;
private int virtualColumnCounter;
+ private boolean forceExpressionVirtualColumns;
private VirtualColumnRegistry(
RowSignature baseRowSignature,
ExprMacroTable macroTable,
String virtualColumnPrefix,
+ boolean forceExpressionVirtualColumns,
Map<ExpressionAndTypeHint, String> virtualColumnsByExpression,
Map<String, ExpressionAndTypeHint> virtualColumnsByName
)
@@ -63,14 +65,20 @@ public class VirtualColumnRegistry
this.virtualColumnPrefix = virtualColumnPrefix;
this.virtualColumnsByExpression = virtualColumnsByExpression;
this.virtualColumnsByName = virtualColumnsByName;
+ this.forceExpressionVirtualColumns = forceExpressionVirtualColumns;
}
- public static VirtualColumnRegistry create(final RowSignature rowSignature,
final ExprMacroTable macroTable)
+ public static VirtualColumnRegistry create(
+ final RowSignature rowSignature,
+ final ExprMacroTable macroTable,
+ final boolean forceExpressionVirtualColumns
+ )
{
return new VirtualColumnRegistry(
rowSignature,
macroTable,
Calcites.findUnusedPrefixForDigits("v", rowSignature.getColumnNames()),
+ forceExpressionVirtualColumns,
new HashMap<>(),
new HashMap<>()
);
@@ -124,14 +132,23 @@ public class VirtualColumnRegistry
}
/**
- * Get existing virtual column by column name
+ * Get existing virtual column by column name.
+ *
+ * @return null if a virtual column for the given name does not exist.
*/
@Nullable
public VirtualColumn getVirtualColumn(String virtualColumnName)
{
- return Optional.ofNullable(virtualColumnsByName.get(virtualColumnName))
- .map(v ->
v.getExpression().toVirtualColumn(virtualColumnName, v.getTypeHint(),
macroTable))
- .orElse(null);
+ ExpressionAndTypeHint registeredColumn =
virtualColumnsByName.get(virtualColumnName);
+ if (registeredColumn == null) {
+ return null;
+ }
+
+ DruidExpression expression = registeredColumn.getExpression();
+ ColumnType columnType = registeredColumn.getTypeHint();
+ return forceExpressionVirtualColumns
+ ? expression.toExpressionVirtualColumn(virtualColumnName,
columnType, macroTable)
+ : expression.toVirtualColumn(virtualColumnName, columnType,
macroTable);
}
@Nullable
@@ -223,7 +240,7 @@ public class VirtualColumnRegistry
)
{
final String name = getOrCreateVirtualColumnForExpression(expression,
valueType);
- return virtualColumnsByName.get(name).expression.toVirtualColumn(name,
valueType, macroTable);
+ return getVirtualColumn(name);
}
/**
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java
new file mode 100644
index 0000000000..f07f60980d
--- /dev/null
+++
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java
@@ -0,0 +1,368 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.sql.calcite;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.druid.sql.calcite.planner.PlannerConfig;
+import org.apache.druid.sql.calcite.util.CalciteTests;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Unit tests for EXPLAIN PLAN queries.
+ */
+public class CalciteExplainQueryTest extends BaseCalciteQueryTest
+{
+ @Test
+ public void testExplainCountStarOnView() throws Exception
+ {
+ // Skip vectorization since otherwise the "context" will change for each
subtest.
+ skipVectorize();
+
+ final String query = "EXPLAIN PLAN FOR SELECT COUNT(*) FROM view.aview
WHERE dim1_firstchar <> 'z'";
+ final String legacyExplanation =
"DruidQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"filter\":{\"type\":\"and\",\"fields\":[{\"type\":\"selector\",\"dimension\":\"dim2\",\"value\":\"a\"},{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"z\",\"extractionFn\":{\"type\":\"substri
[...]
+ final String explanation = "[{"
+ + "\"query\":{\"queryType\":\"timeseries\","
+ +
"\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},"
+ +
"\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},"
+ +
"\"filter\":{\"type\":\"and\",\"fields\":[{\"type\":\"selector\",\"dimension\":\"dim2\",\"value\":\"a\"},{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"z\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}}]},"
+ + "\"granularity\":{\"type\":\"all\"},"
+ +
"\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],"
+ +
"\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}},"
+ +
"\"signature\":[{\"name\":\"a0\",\"type\":\"LONG\"}]"
+ + "}]";
+ final String resources = "[{\"name\":\"aview\",\"type\":\"VIEW\"}]";
+
+ testQuery(
+ query,
+ ImmutableList.of(),
+ ImmutableList.of(
+ new Object[]{legacyExplanation, resources}
+ )
+ );
+ testQuery(
+ PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN,
+ query,
+ CalciteTests.REGULAR_USER_AUTH_RESULT,
+ ImmutableList.of(),
+ ImmutableList.of(
+ new Object[]{explanation, resources}
+ )
+ );
+ }
+
+ @Test
+ public void testExplainInformationSchemaColumns() throws Exception
+ {
+ final String explanation =
+ "BindableProject(COLUMN_NAME=[$3], DATA_TYPE=[$7])\n"
+ + " BindableFilter(condition=[AND(=($1, 'druid'), =($2, 'foo'))])\n"
+ + " BindableTableScan(table=[[INFORMATION_SCHEMA, COLUMNS]])\n";
+
+ final String resources = "[]";
+
+ testQuery(
+ "EXPLAIN PLAN FOR\n"
+ + "SELECT COLUMN_NAME, DATA_TYPE\n"
+ + "FROM INFORMATION_SCHEMA.COLUMNS\n"
+ + "WHERE TABLE_SCHEMA = 'druid' AND TABLE_NAME = 'foo'",
+ ImmutableList.of(),
+ ImmutableList.of(
+ new Object[]{explanation, resources}
+ )
+ );
+ }
+
+ @Test
+ public void testExplainExactCountDistinctOfSemiJoinResult() throws Exception
+ {
+ // Skip vectorization since otherwise the "context" will change for each
subtest.
+ skipVectorize();
+
+ final String query = "EXPLAIN PLAN FOR SELECT COUNT(*)\n"
+ + "FROM (\n"
+ + " SELECT DISTINCT dim2\n"
+ + " FROM druid.foo\n"
+ + " WHERE SUBSTRING(dim2, 1, 1) IN (\n"
+ + " SELECT SUBSTRING(dim1, 1, 1) FROM druid.foo
WHERE dim1 IS NOT NULL\n"
+ + " )\n"
+ + ")";
+ final String legacyExplanation =
+
"DruidOuterQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"query\",\"query\":{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"__subquery__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"list\",\"granularity\":{\"type\":\"all\"}}},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\
[...]
+ + " DruidJoinQueryRel(condition=[=(SUBSTRING($3, 1, 1), $8)],
joinType=[inner],
query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"__join__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"default\",\"dimension\":\"dim2\",\"outputName\":\"d0\",\"outputType\":\"STRING\"}],\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"
[...]
+ + "
DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-0
[...]
+ + "
DruidQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":null}},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"extraction\",\"dimension\":\"dim1\",\"outputName\":\"d0\",\"outputType\":\"STRING\",\"extra
[...]
+ final String explanation = "["
+ + "{\"query\":{\"queryType\":\"groupBy\","
+ +
"\"dataSource\":{\"type\":\"query\",\"query\":{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"join\",\"left\":{\"type\":\"table\",\"name\":\"foo\"},\"right\":{\"type\":\"query\",\"query\":{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"d
[...]
+ +
"\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},"
+ + "\"granularity\":{\"type\":\"all\"},"
+ + "\"dimensions\":[],"
+ +
"\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],"
+ + "\"limitSpec\":{\"type\":\"NoopLimitSpec\"},"
+ +
"\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}},"
+ +
"\"signature\":[{\"name\":\"a0\",\"type\":\"LONG\"}]"
+ + "}]";
+ final String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]";
+
+ testQuery(
+ query,
+ ImmutableList.of(),
+ ImmutableList.of(new Object[]{legacyExplanation, resources})
+ );
+
+ testQuery(
+ PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN,
+ query,
+ CalciteTests.REGULAR_USER_AUTH_RESULT,
+ ImmutableList.of(),
+ ImmutableList.of(new Object[]{explanation, resources})
+ );
+ }
+
+ // This testcase has been added here and not in CalciteSelectQueryTests
since this checks if the overrides are working
+ // properly when displaying the output of "EXPLAIN PLAN FOR ..." queries
+ @Test
+ public void testExplainSelectStarWithOverrides() throws Exception
+ {
+ Map<String, Object> useRegularExplainContext = new
HashMap<>(QUERY_CONTEXT_DEFAULT);
+
useRegularExplainContext.put(PlannerConfig.CTX_KEY_USE_NATIVE_QUERY_EXPLAIN,
false);
+
+ Map<String, Object> useNativeQueryExplain = new
HashMap<>(QUERY_CONTEXT_DEFAULT);
+ useNativeQueryExplain.put(PlannerConfig.CTX_KEY_USE_NATIVE_QUERY_EXPLAIN,
true);
+
+
+ // Skip vectorization since otherwise the "context" will change for each
subtest.
+ skipVectorize();
+ String legacyExplanation =
"DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTim
[...]
+ String legacyExplanationWithContext =
"DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sq
[...]
+ String explanation = "[{"
+ + "\"query\":{\"queryType\":\"scan\","
+ +
"\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},"
+ +
"\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},"
+ + "\"resultFormat\":\"compactedList\","
+ +
"\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],"
+ + "\"legacy\":false,"
+ +
"\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},"
+ + "\"granularity\":{\"type\":\"all\"}},"
+ +
"\"signature\":[{\"name\":\"__time\",\"type\":\"LONG\"},{\"name\":\"cnt\",\"type\":\"LONG\"},{\"name\":\"dim1\",\"type\":\"STRING\"},{\"name\":\"dim2\",\"type\":\"STRING\"},{\"name\":\"dim3\",\"type\":\"STRING\"},{\"name\":\"m1\",\"type\":\"FLOAT\"},{\"name\":\"m2\",\"type\":\"DOUBLE\"},{\"name\":\"unique_dim1\",\"type\":\"COMPLEX<hyperUnique>\"}]"
+ + "}]";
+
+ String explanationWithContext = "[{"
+ + "\"query\":{\"queryType\":\"scan\","
+ +
"\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},"
+ +
"\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},"
+ + "\"resultFormat\":\"compactedList\","
+ +
"\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],"
+ + "\"legacy\":false,"
+ +
"\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"useNativeQueryExplain\":true,\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},"
+ + "\"granularity\":{\"type\":\"all\"}},"
+ +
"\"signature\":[{\"name\":\"__time\",\"type\":\"LONG\"},{\"name\":\"cnt\",\"type\":\"LONG\"},{\"name\":\"dim1\",\"type\":\"STRING\"},{\"name\":\"dim2\",\"type\":\"STRING\"},{\"name\":\"dim3\",\"type\":\"STRING\"},{\"name\":\"m1\",\"type\":\"FLOAT\"},{\"name\":\"m2\",\"type\":\"DOUBLE\"},{\"name\":\"unique_dim1\",\"type\":\"COMPLEX<hyperUnique>\"}]"
+ + "}]";
+ String sql = "EXPLAIN PLAN FOR SELECT * FROM druid.foo";
+ String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]";
+
+ // Test when default config and no overrides
+ testQuery(sql, ImmutableList.of(), ImmutableList.of(new
Object[]{legacyExplanation, resources}));
+
+ // Test when default config and useNativeQueryExplain is overridden in the
context
+ testQuery(
+ sql,
+ useNativeQueryExplain,
+ ImmutableList.of(),
+ ImmutableList.of(new Object[]{explanationWithContext, resources})
+ );
+
+ // Test when useNativeQueryExplain enabled by default and no overrides
+ testQuery(
+ PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN,
+ sql,
+ CalciteTests.REGULAR_USER_AUTH_RESULT,
+ ImmutableList.of(),
+ ImmutableList.of(new Object[]{explanation, resources})
+ );
+
+ // Test when useNativeQueryExplain enabled by default but is overriden in
the context
+ testQuery(
+ PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN,
+ useRegularExplainContext,
+ sql,
+ CalciteTests.REGULAR_USER_AUTH_RESULT,
+ ImmutableList.of(),
+ ImmutableList.of(new Object[]{legacyExplanationWithContext, resources})
+ );
+ }
+
+ @Test
+ public void testExplainMultipleTopLevelUnionAllQueries() throws Exception
+ {
+ // Skip vectorization since otherwise the "context" will change for each
subtest.
+ skipVectorize();
+
+ final String query = "EXPLAIN PLAN FOR SELECT dim1 FROM druid.foo\n"
+ + "UNION ALL (SELECT dim1 FROM druid.foo WHERE dim1 =
'42'\n"
+ + "UNION ALL SELECT dim1 FROM druid.foo WHERE dim1 =
'44')";
+ final String legacyExplanation = "DruidUnionRel(limit=[-1])\n"
+ + "
DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"columns\":[\"dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\
[...]
+ + " DruidUnionRel(limit=[-1])\n"
+ + "
DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"filter\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"42\"},\"columns\":[\"dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":922337203685477
[...]
+ + "
DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"filter\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"44\"},\"columns\":[\"dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":922337203685477
[...]
+ final String explanation = "["
+ + "{"
+ +
"\"query\":{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"columns\":[\"dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"f
[...]
+ +
"\"signature\":[{\"name\":\"dim1\",\"type\":\"STRING\"}]"
+ + "},"
+ + "{"
+ +
"\"query\":{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"filter\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"42\"},\"columns\":[\"dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTime
[...]
+ +
"\"signature\":[{\"name\":\"dim1\",\"type\":\"STRING\"}]"
+ + "},"
+ + "{"
+ +
"\"query\":{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"filter\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"44\"},\"columns\":[\"dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTime
[...]
+ +
"\"signature\":[{\"name\":\"dim1\",\"type\":\"STRING\"}]"
+ + "}]";
+ final String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]";
+
+ testQuery(
+ query,
+ ImmutableList.of(),
+ ImmutableList.of(
+ new Object[]{legacyExplanation, resources}
+ )
+ );
+ testQuery(
+ PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN,
+ query,
+ CalciteTests.REGULAR_USER_AUTH_RESULT,
+ ImmutableList.of(),
+ ImmutableList.of(
+ new Object[]{explanation, resources}
+ )
+ );
+ }
+
+ @Test
+ public void testExplainSelectMvfilterExpressions() throws Exception
+ {
+ // Skip vectorization since otherwise the "context" will change for each
subtest.
+ skipVectorize();
+
+ final String explainSql = "EXPLAIN PLAN FOR SELECT"
+ + " MV_FILTER_ONLY(\"dim1\", ARRAY['true',
'false']),"
+ + " MV_FILTER_NONE(\"dim1\", ARRAY['true',
'false'])"
+ + " FROM druid.foo";
+
+ // Test plan as default expressions
+ final Map<String, Object> defaultExprContext = new
HashMap<>(QUERY_CONTEXT_DEFAULT);
+ defaultExprContext.put(PlannerConfig.CTX_KEY_USE_NATIVE_QUERY_EXPLAIN,
true);
+
defaultExprContext.put(PlannerConfig.CTX_KEY_FORCE_EXPRESSION_VIRTUAL_COLUMNS,
true);
+
+ final String expectedPlanWithDefaultExpressions = "[{"
+ +
"\"query\":{\"queryType\":\"scan\","
+ +
"\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},"
+ +
"\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},"
+ + "\"virtualColumns\":["
+ +
"{\"type\":\"expression\",\"name\":\"v0\",\"expression\":\"filter((x) ->
array_contains(array('true','false'), x),
\\\"dim1\\\")\",\"outputType\":\"STRING\"},"
+ +
"{\"type\":\"expression\",\"name\":\"v1\",\"expression\":\"filter((x) ->
!array_contains(array('true','false'), x),
\\\"dim1\\\")\",\"outputType\":\"STRING\"}"
+ + "],"
+ +
"\"resultFormat\":\"compactedList\","
+ +
"\"columns\":[\"v0\",\"v1\"],"
+ + "\"legacy\":false,"
+ +
"\"context\":{\"defaultTimeout\":300000,\"forceExpressionVirtualColumns\":true,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"useNativeQueryExplain\":true,\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},"
+ +
"\"granularity\":{\"type\":\"all\"}},"
+ +
"\"signature\":[{\"name\":\"v0\",\"type\":\"STRING\"},{\"name\":\"v1\",\"type\":\"STRING\"}]"
+ + "}]";
+ final String expectedResources =
"[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]";
+
+ testQuery(
+ explainSql,
+ defaultExprContext,
+ ImmutableList.of(),
+ ImmutableList.of(new Object[]{expectedPlanWithDefaultExpressions,
expectedResources})
+ );
+
+ // Test plan as mv-filtered virtual columns
+ final String expectedPlanWithMvfiltered = "[{"
+ +
"\"query\":{\"queryType\":\"scan\","
+ +
"\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},"
+ +
"\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},"
+ + "\"virtualColumns\":["
+ +
"{\"type\":\"mv-filtered\",\"name\":\"v0\",\"delegate\":{\"type\":\"default\",\"dimension\":\"dim1\",\"outputName\":\"dim1\",\"outputType\":\"STRING\"},\"values\":[\"true\",\"false\"],\"isAllowList\":true},"
+ +
"{\"type\":\"mv-filtered\",\"name\":\"v1\",\"delegate\":{\"type\":\"default\",\"dimension\":\"dim1\",\"outputName\":\"dim1\",\"outputType\":\"STRING\"},\"values\":[\"true\",\"false\"],\"isAllowList\":false}"
+ + "],"
+ +
"\"resultFormat\":\"compactedList\","
+ + "\"columns\":[\"v0\",\"v1\"],"
+ + "\"legacy\":false,"
+ +
"\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"useNativeQueryExplain\":true,\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},"
+ +
"\"granularity\":{\"type\":\"all\"}},"
+ +
"\"signature\":[{\"name\":\"v0\",\"type\":\"STRING\"},{\"name\":\"v1\",\"type\":\"STRING\"}]"
+ + "}]";
+
+ final Map<String, Object> mvFilteredContext = new
HashMap<>(QUERY_CONTEXT_DEFAULT);
+ mvFilteredContext.put(PlannerConfig.CTX_KEY_USE_NATIVE_QUERY_EXPLAIN,
true);
+
+ testQuery(
+ explainSql,
+ mvFilteredContext,
+ ImmutableList.of(),
+ ImmutableList.of(new Object[]{expectedPlanWithMvfiltered,
expectedResources})
+ );
+ }
+
+ @Test
+ public void testExplainSelectTimestampExpression() throws Exception
+ {
+ // Skip vectorization since otherwise the "context" will change for each
subtest.
+ skipVectorize();
+
+ final String explainSql = "EXPLAIN PLAN FOR SELECT"
+ + " TIME_PARSE(dim1)"
+ + " FROM druid.foo";
+
+ final Map<String, Object> queryContext = new
HashMap<>(QUERY_CONTEXT_DEFAULT);
+ queryContext.put(PlannerConfig.CTX_KEY_USE_NATIVE_QUERY_EXPLAIN, true);
+
+ final String expectedPlan = "[{"
+ + "\"query\":{\"queryType\":\"scan\","
+ +
"\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},"
+ +
"\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},"
+ + "\"virtualColumns\":["
+ +
"{\"type\":\"expression\",\"name\":\"v0\",\"expression\":\"timestamp_parse(\\\"dim1\\\",null,'UTC')\",\"outputType\":\"LONG\"}"
+ + "],"
+ + "\"resultFormat\":\"compactedList\","
+ + "\"columns\":[\"v0\"],"
+ + "\"legacy\":false,"
+ +
"\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"useNativeQueryExplain\":true,\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},"
+ + "\"granularity\":{\"type\":\"all\"}},"
+ +
"\"signature\":[{\"name\":\"v0\",\"type\":\"LONG\"}]"
+ + "}]";
+ final String expectedResources =
"[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]";
+
+ // Verify the query plan
+ testQuery(
+ explainSql,
+ queryContext,
+ ImmutableList.of(),
+ ImmutableList.of(new Object[]{expectedPlan, expectedResources})
+ );
+ }
+
+}
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
index 7aef54503f..15680504c6 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
@@ -353,28 +353,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
);
}
- @Test
- public void testExplainInformationSchemaColumns() throws Exception
- {
- final String explanation =
- "BindableProject(COLUMN_NAME=[$3], DATA_TYPE=[$7])\n"
- + " BindableFilter(condition=[AND(=($1, 'druid'), =($2, 'foo'))])\n"
- + " BindableTableScan(table=[[INFORMATION_SCHEMA, COLUMNS]])\n";
-
- final String resources = "[]";
-
- testQuery(
- "EXPLAIN PLAN FOR\n"
- + "SELECT COLUMN_NAME, DATA_TYPE\n"
- + "FROM INFORMATION_SCHEMA.COLUMNS\n"
- + "WHERE TABLE_SCHEMA = 'druid' AND TABLE_NAME = 'foo'",
- ImmutableList.of(),
- ImmutableList.of(
- new Object[]{explanation, resources}
- )
- );
- }
-
@Test
public void testAggregatorsOnInformationSchemaColumns() throws Exception
{
@@ -4361,44 +4339,6 @@ public class CalciteQueryTest extends
BaseCalciteQueryTest
);
}
- @Test
- public void testExplainCountStarOnView() throws Exception
- {
- // Skip vectorization since otherwise the "context" will change for each
subtest.
- skipVectorize();
-
- final String query = "EXPLAIN PLAN FOR SELECT COUNT(*) FROM view.aview
WHERE dim1_firstchar <> 'z'";
- final String legacyExplanation =
"DruidQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"filter\":{\"type\":\"and\",\"fields\":[{\"type\":\"selector\",\"dimension\":\"dim2\",\"value\":\"a\"},{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"z\",\"extractionFn\":{\"type\":\"substri
[...]
- final String explanation = "[{"
- + "\"query\":{\"queryType\":\"timeseries\","
- +
"\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},"
- +
"\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},"
- +
"\"filter\":{\"type\":\"and\",\"fields\":[{\"type\":\"selector\",\"dimension\":\"dim2\",\"value\":\"a\"},{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"z\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}}]},"
- + "\"granularity\":{\"type\":\"all\"},"
- +
"\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],"
- +
"\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}},"
- +
"\"signature\":[{\"name\":\"a0\",\"type\":\"LONG\"}]"
- + "}]";
- final String resources = "[{\"name\":\"aview\",\"type\":\"VIEW\"}]";
-
- testQuery(
- query,
- ImmutableList.of(),
- ImmutableList.of(
- new Object[]{legacyExplanation, resources}
- )
- );
- testQuery(
- PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN,
- query,
- CalciteTests.REGULAR_USER_AUTH_RESULT,
- ImmutableList.of(),
- ImmutableList.of(
- new Object[]{explanation, resources}
- )
- );
- }
-
@Test
public void testCountStarWithLikeFilter() throws Exception
{
@@ -7193,173 +7133,6 @@ public class CalciteQueryTest extends
BaseCalciteQueryTest
);
}
- @Test
- public void testExplainExactCountDistinctOfSemiJoinResult() throws Exception
- {
- // Skip vectorization since otherwise the "context" will change for each
subtest.
- skipVectorize();
-
- final String query = "EXPLAIN PLAN FOR SELECT COUNT(*)\n"
- + "FROM (\n"
- + " SELECT DISTINCT dim2\n"
- + " FROM druid.foo\n"
- + " WHERE SUBSTRING(dim2, 1, 1) IN (\n"
- + " SELECT SUBSTRING(dim1, 1, 1) FROM druid.foo
WHERE dim1 IS NOT NULL\n"
- + " )\n"
- + ")";
- final String legacyExplanation =
-
"DruidOuterQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"query\",\"query\":{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"__subquery__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"list\",\"granularity\":{\"type\":\"all\"}}},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\
[...]
- + " DruidJoinQueryRel(condition=[=(SUBSTRING($3, 1, 1), $8)],
joinType=[inner],
query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"__join__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"default\",\"dimension\":\"dim2\",\"outputName\":\"d0\",\"outputType\":\"STRING\"}],\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"
[...]
- + "
DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-0
[...]
- + "
DruidQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":null}},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"extraction\",\"dimension\":\"dim1\",\"outputName\":\"d0\",\"outputType\":\"STRING\",\"extra
[...]
- final String explanation = "["
- + "{\"query\":{\"queryType\":\"groupBy\","
- +
"\"dataSource\":{\"type\":\"query\",\"query\":{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"join\",\"left\":{\"type\":\"table\",\"name\":\"foo\"},\"right\":{\"type\":\"query\",\"query\":{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"d
[...]
- +
"\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},"
- + "\"granularity\":{\"type\":\"all\"},"
- + "\"dimensions\":[],"
- +
"\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],"
- + "\"limitSpec\":{\"type\":\"NoopLimitSpec\"},"
- +
"\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}},"
- +
"\"signature\":[{\"name\":\"a0\",\"type\":\"LONG\"}]"
- + "}]";
- final String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]";
-
- testQuery(
- query,
- ImmutableList.of(),
- ImmutableList.of(new Object[]{legacyExplanation, resources})
- );
-
- testQuery(
- PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN,
- query,
- CalciteTests.REGULAR_USER_AUTH_RESULT,
- ImmutableList.of(),
- ImmutableList.of(new Object[]{explanation, resources})
- );
- }
-
- // This testcase has been added here and not in CalciteSelectQueryTests
since this checks if the overrides are working
- // properly when displaying the output of "EXPLAIN PLAN FOR ..." queries
- @Test
- public void testExplainSelectStarWithOverrides() throws Exception
- {
- Map<String, Object> useRegularExplainContext = new
HashMap<>(QUERY_CONTEXT_DEFAULT);
-
useRegularExplainContext.put(PlannerConfig.CTX_KEY_USE_NATIVE_QUERY_EXPLAIN,
false);
-
- Map<String, Object> useNativeQueryExplain = new
HashMap<>(QUERY_CONTEXT_DEFAULT);
- useNativeQueryExplain.put(PlannerConfig.CTX_KEY_USE_NATIVE_QUERY_EXPLAIN,
true);
-
-
- // Skip vectorization since otherwise the "context" will change for each
subtest.
- skipVectorize();
- String legacyExplanation =
"DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTim
[...]
- String legacyExplanationWithContext =
"DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sq
[...]
- String explanation = "[{"
- + "\"query\":{\"queryType\":\"scan\","
- +
"\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},"
- +
"\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},"
- + "\"resultFormat\":\"compactedList\","
- +
"\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],"
- + "\"legacy\":false,"
- +
"\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},"
- + "\"granularity\":{\"type\":\"all\"}},"
- +
"\"signature\":[{\"name\":\"__time\",\"type\":\"LONG\"},{\"name\":\"cnt\",\"type\":\"LONG\"},{\"name\":\"dim1\",\"type\":\"STRING\"},{\"name\":\"dim2\",\"type\":\"STRING\"},{\"name\":\"dim3\",\"type\":\"STRING\"},{\"name\":\"m1\",\"type\":\"FLOAT\"},{\"name\":\"m2\",\"type\":\"DOUBLE\"},{\"name\":\"unique_dim1\",\"type\":\"COMPLEX<hyperUnique>\"}]"
- + "}]";
-
- String explanationWithContext = "[{"
- + "\"query\":{\"queryType\":\"scan\","
- +
"\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},"
- +
"\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},"
- + "\"resultFormat\":\"compactedList\","
- +
"\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],"
- + "\"legacy\":false,"
- +
"\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"useNativeQueryExplain\":true,\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},"
- + "\"granularity\":{\"type\":\"all\"}},"
- +
"\"signature\":[{\"name\":\"__time\",\"type\":\"LONG\"},{\"name\":\"cnt\",\"type\":\"LONG\"},{\"name\":\"dim1\",\"type\":\"STRING\"},{\"name\":\"dim2\",\"type\":\"STRING\"},{\"name\":\"dim3\",\"type\":\"STRING\"},{\"name\":\"m1\",\"type\":\"FLOAT\"},{\"name\":\"m2\",\"type\":\"DOUBLE\"},{\"name\":\"unique_dim1\",\"type\":\"COMPLEX<hyperUnique>\"}]"
- + "}]";
- String sql = "EXPLAIN PLAN FOR SELECT * FROM druid.foo";
- String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]";
-
- // Test when default config and no overrides
- testQuery(sql, ImmutableList.of(), ImmutableList.of(new
Object[]{legacyExplanation, resources}));
-
- // Test when default config and useNativeQueryExplain is overridden in the
context
- testQuery(
- sql,
- useNativeQueryExplain,
- ImmutableList.of(),
- ImmutableList.of(new Object[]{explanationWithContext, resources})
- );
-
- // Test when useNativeQueryExplain enabled by default and no overrides
- testQuery(
- PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN,
- sql,
- CalciteTests.REGULAR_USER_AUTH_RESULT,
- ImmutableList.of(),
- ImmutableList.of(new Object[]{explanation, resources})
- );
-
- // Test when useNativeQueryExplain enabled by default but is overriden in
the context
- testQuery(
- PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN,
- useRegularExplainContext,
- sql,
- CalciteTests.REGULAR_USER_AUTH_RESULT,
- ImmutableList.of(),
- ImmutableList.of(new Object[]{legacyExplanationWithContext, resources})
- );
- }
-
- @Test
- public void testExplainMultipleTopLevelUnionAllQueries() throws Exception
- {
- // Skip vectorization since otherwise the "context" will change for each
subtest.
- skipVectorize();
-
- final String query = "EXPLAIN PLAN FOR SELECT dim1 FROM druid.foo\n"
- + "UNION ALL (SELECT dim1 FROM druid.foo WHERE dim1 =
'42'\n"
- + "UNION ALL SELECT dim1 FROM druid.foo WHERE dim1 =
'44')";
- final String legacyExplanation = "DruidUnionRel(limit=[-1])\n"
- + "
DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"columns\":[\"dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\
[...]
- + " DruidUnionRel(limit=[-1])\n"
- + "
DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"filter\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"42\"},\"columns\":[\"dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":922337203685477
[...]
- + "
DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"filter\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"44\"},\"columns\":[\"dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":922337203685477
[...]
- final String explanation = "["
- + "{"
- +
"\"query\":{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"columns\":[\"dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"f
[...]
- +
"\"signature\":[{\"name\":\"dim1\",\"type\":\"STRING\"}]"
- + "},"
- + "{"
- +
"\"query\":{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"filter\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"42\"},\"columns\":[\"dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTime
[...]
- +
"\"signature\":[{\"name\":\"dim1\",\"type\":\"STRING\"}]"
- + "},"
- + "{"
- +
"\"query\":{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"filter\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"44\"},\"columns\":[\"dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTime
[...]
- +
"\"signature\":[{\"name\":\"dim1\",\"type\":\"STRING\"}]"
- + "}]";
- final String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]";
-
- testQuery(
- query,
- ImmutableList.of(),
- ImmutableList.of(
- new Object[]{legacyExplanation, resources}
- )
- );
- testQuery(
- PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN,
- query,
- CalciteTests.REGULAR_USER_AUTH_RESULT,
- ImmutableList.of(),
- ImmutableList.of(
- new Object[]{explanation, resources}
- )
- );
- }
-
@Test
public void testExactCountDistinctUsingSubqueryWithWherePushDown() throws
Exception
{
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionTestHelper.java
b/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionTestHelper.java
index df5002983d..3117610ec8 100644
---
a/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionTestHelper.java
+++
b/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionTestHelper.java
@@ -317,7 +317,7 @@ class ExpressionTestHelper
)
{
final RexNode rexNode = rexBuilder.makeCall(op, exprs);
- final VirtualColumnRegistry virtualColumnRegistry =
VirtualColumnRegistry.create(rowSignature, TestExprMacroTable.INSTANCE);
+ final VirtualColumnRegistry virtualColumnRegistry =
VirtualColumnRegistry.create(rowSignature, TestExprMacroTable.INSTANCE, false);
final DimFilter filter = Expressions.toFilter(PLANNER_CONTEXT,
rowSignature, virtualColumnRegistry, rexNode);
Assert.assertEquals("Filter for: " + rexNode, expectedFilter, filter);
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/rel/DruidQueryTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/rel/DruidQueryTest.java
index 61a4118a1d..68edf3f1f2 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/rel/DruidQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/rel/DruidQueryTest.java
@@ -62,7 +62,7 @@ public class DruidQueryTest
Pair<DataSource, Filtration> pair = DruidQuery.getFiltration(
dataSource,
selectorFilter,
- VirtualColumnRegistry.create(RowSignature.empty(),
TestExprMacroTable.INSTANCE)
+ VirtualColumnRegistry.create(RowSignature.empty(),
TestExprMacroTable.INSTANCE, false)
);
verify(pair, dataSource, selectorFilter, Intervals.ETERNITY);
}
@@ -74,7 +74,7 @@ public class DruidQueryTest
Pair<DataSource, Filtration> pair = DruidQuery.getFiltration(
dataSource,
filterWithInterval,
- VirtualColumnRegistry.create(RowSignature.empty(),
TestExprMacroTable.INSTANCE)
+ VirtualColumnRegistry.create(RowSignature.empty(),
TestExprMacroTable.INSTANCE, false)
);
verify(pair, dataSource, selectorFilter, Intervals.utc(100, 200));
}
@@ -86,7 +86,7 @@ public class DruidQueryTest
Pair<DataSource, Filtration> pair = DruidQuery.getFiltration(
dataSource,
filterWithInterval,
- VirtualColumnRegistry.create(RowSignature.empty(),
TestExprMacroTable.INSTANCE)
+ VirtualColumnRegistry.create(RowSignature.empty(),
TestExprMacroTable.INSTANCE, false)
);
verify(pair, dataSource, selectorFilter, Intervals.utc(100, 200));
}
@@ -99,7 +99,7 @@ public class DruidQueryTest
Pair<DataSource, Filtration> pair = DruidQuery.getFiltration(
dataSource,
otherFilter,
- VirtualColumnRegistry.create(RowSignature.empty(),
TestExprMacroTable.INSTANCE)
+ VirtualColumnRegistry.create(RowSignature.empty(),
TestExprMacroTable.INSTANCE, false)
);
verify(pair, expectedDataSource, otherFilter, Intervals.utc(100, 200));
}
@@ -112,7 +112,7 @@ public class DruidQueryTest
Pair<DataSource, Filtration> pair = DruidQuery.getFiltration(
dataSource,
otherFilter,
- VirtualColumnRegistry.create(RowSignature.empty(),
TestExprMacroTable.INSTANCE)
+ VirtualColumnRegistry.create(RowSignature.empty(),
TestExprMacroTable.INSTANCE, false)
);
verify(pair, expectedDataSource, otherFilter, Intervals.utc(100, 200));
}
@@ -125,7 +125,7 @@ public class DruidQueryTest
Pair<DataSource, Filtration> pair = DruidQuery.getFiltration(
dataSource,
otherFilter,
- VirtualColumnRegistry.create(RowSignature.empty(),
TestExprMacroTable.INSTANCE)
+ VirtualColumnRegistry.create(RowSignature.empty(),
TestExprMacroTable.INSTANCE, false)
);
verify(pair, expectedDataSource, otherFilter, Intervals.utc(100, 200));
}
@@ -138,7 +138,7 @@ public class DruidQueryTest
Pair<DataSource, Filtration> pair = DruidQuery.getFiltration(
dataSource,
otherFilter,
- VirtualColumnRegistry.create(RowSignature.empty(),
TestExprMacroTable.INSTANCE)
+ VirtualColumnRegistry.create(RowSignature.empty(),
TestExprMacroTable.INSTANCE, false)
);
verify(pair, expectedDataSource, otherFilter, Intervals.utc(100, 200));
}
@@ -156,7 +156,7 @@ public class DruidQueryTest
Pair<DataSource, Filtration> pair = DruidQuery.getFiltration(
dataSource,
queryFilter,
- VirtualColumnRegistry.create(RowSignature.empty(),
TestExprMacroTable.INSTANCE)
+ VirtualColumnRegistry.create(RowSignature.empty(),
TestExprMacroTable.INSTANCE, false)
);
verify(pair, expectedDataSource, otherFilter, Intervals.utc(150, 200));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
