This is an automated email from the ASF dual-hosted git repository.
abhishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new c381cae Improve the output of SQL explain message (#11908)
c381cae is described below
commit c381cae51b93251e8ae6e5bd84d15a6bd30e8d28
Author: Laksh Singla <[email protected]>
AuthorDate: Thu Nov 25 21:08:33 2021 +0530
Improve the output of SQL explain message (#11908)
Currently, when we try to do EXPLAIN PLAN FOR, it returns the structure of
the SQL parsed (via Calcite's internal planner util), which is verbose (since
it tries to explain about the nodes in the SQL, instead of the Druid Query),
and not representative of the native Druid query which will get executed on the
broker side.
This PR aims to change the format when user tries to EXPLAIN PLAN FOR for
queries which are executed by converting them into Druid's native queries (i.e.
not sys schemas).
---
docs/configuration/index.md | 1 +
.../druid/sql/calcite/planner/DruidPlanner.java | 72 ++++++++-
.../druid/sql/calcite/planner/PlannerConfig.java | 21 ++-
.../druid/sql/calcite/BaseCalciteQueryTest.java | 9 ++
.../apache/druid/sql/calcite/CalciteQueryTest.java | 167 +++++++++++++++++----
.../druid/sql/calcite/CalciteSelectQueryTest.java | 73 ++++++++-
6 files changed, 304 insertions(+), 39 deletions(-)
diff --git a/docs/configuration/index.md b/docs/configuration/index.md
index 6ffd149..0708c07 100644
--- a/docs/configuration/index.md
+++ b/docs/configuration/index.md
@@ -1834,6 +1834,7 @@ The Druid SQL server is configured through the following
properties on the Broke
|`druid.sql.planner.metadataSegmentCacheEnable`|Whether to keep a cache of
published segments in broker. If true, broker polls coordinator in background
to get segments from metadata store and maintains a local cache. If false,
coordinator's REST API will be invoked when broker needs published segments
info.|false|
|`druid.sql.planner.metadataSegmentPollPeriod`|How often to poll coordinator
for published segments list if `druid.sql.planner.metadataSegmentCacheEnable`
is set to true. Poll period is in milliseconds. |60000|
|`druid.sql.planner.authorizeSystemTablesDirectly`|If true, Druid authorizes
queries against any of the system schema tables (`sys` in SQL) as
`SYSTEM_TABLE` resources which require `READ` access, in addition to
permissions based content filtering.|false|
+|`druid.sql.planner.useNativeQueryExplain`|If true, `EXPLAIN PLAN FOR` will
return the explain plan as a JSON representation of equivalent native query(s),
else it will return the original version of explain plan generated by Calcite.
It can be overridden per query with `useNativeQueryExplain` context key.|false|
|`druid.sql.approxCountDistinct.function`|Implementation to use for the
[`APPROX_COUNT_DISTINCT` function](../querying/sql.md#aggregation-functions).
Without extensions loaded, the only valid value is
`APPROX_COUNT_DISTINCT_BUILTIN` (a HyperLogLog, or HLL, based implementation).
If the [DataSketches
extension](../development/extensions-core/datasketches-extension.md) is loaded,
this can also be `APPROX_COUNT_DISTINCT_DS_HLL` (alternative HLL
implementation) or `APPROX_COUNT_DISTINCT_DS_T [...]
> Previous versions of Druid had properties named
> `druid.sql.planner.maxQueryCount` and
> `druid.sql.planner.maxSemiJoinRowsInMemory`.
diff --git
a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java
b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java
index 2d07239..3a5dbeb 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java
@@ -20,6 +20,9 @@
package org.apache.druid.sql.calcite.planner;
import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.base.Preconditions;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
@@ -63,22 +66,27 @@ import org.apache.calcite.tools.Planner;
import org.apache.calcite.tools.RelConversionException;
import org.apache.calcite.tools.ValidationException;
import org.apache.calcite.util.Pair;
+import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.guava.BaseSequence;
import org.apache.druid.java.util.common.guava.Sequence;
import org.apache.druid.java.util.common.guava.Sequences;
import org.apache.druid.java.util.emitter.EmittingLogger;
+import org.apache.druid.query.Query;
import org.apache.druid.segment.DimensionHandlerUtils;
import org.apache.druid.server.security.Action;
import org.apache.druid.server.security.Resource;
import org.apache.druid.server.security.ResourceAction;
import org.apache.druid.server.security.ResourceType;
import org.apache.druid.sql.calcite.rel.DruidConvention;
+import org.apache.druid.sql.calcite.rel.DruidQuery;
import org.apache.druid.sql.calcite.rel.DruidRel;
+import org.apache.druid.sql.calcite.rel.DruidUnionRel;
import org.apache.druid.sql.calcite.run.QueryMaker;
import org.apache.druid.sql.calcite.run.QueryMakerFactory;
import javax.annotation.Nullable;
+
import java.io.Closeable;
import java.util.ArrayList;
import java.util.HashSet;
@@ -270,7 +278,7 @@ public class DruidPlanner implements Closeable
);
if (explain != null) {
- return planExplanation(druidRel, explain);
+ return planExplanation(druidRel, explain, true);
} else {
final Supplier<Sequence<Object[]>> resultsSupplier = () -> {
// sanity check
@@ -331,7 +339,7 @@ public class DruidPlanner implements Closeable
}
if (explain != null) {
- return planExplanation(bindableRel, explain);
+ return planExplanation(bindableRel, explain, false);
} else {
final BindableRel theRel = bindableRel;
final DataContext dataContext = plannerContext.createDataContext(
@@ -380,12 +388,20 @@ public class DruidPlanner implements Closeable
*/
private PlannerResult planExplanation(
final RelNode rel,
- final SqlExplain explain
+ final SqlExplain explain,
+ final boolean isDruidConventionExplanation
)
{
- final String explanation = RelOptUtil.dumpPlan("", rel,
explain.getFormat(), explain.getDetailLevel());
+ String explanation = RelOptUtil.dumpPlan("", rel, explain.getFormat(),
explain.getDetailLevel());
String resourcesString;
try {
+ if (isDruidConventionExplanation && rel instanceof DruidRel) {
+ // Show the native queries instead of Calcite's explain if the legacy
flag is turned off
+ if (plannerContext.getPlannerConfig().isUseNativeQueryExplain()) {
+ DruidRel<?> druidRel = (DruidRel<?>) rel;
+ explanation = explainSqlPlanAsNativeQueries(druidRel);
+ }
+ }
final Set<Resource> resources =
plannerContext.getResourceActions().stream().map(ResourceAction::getResource).collect(Collectors.toSet());
resourcesString =
plannerContext.getJsonMapper().writeValueAsString(resources);
@@ -395,12 +411,59 @@ public class DruidPlanner implements Closeable
log.error(jpe, "Encountered exception while serializing Resources for
explain output");
resourcesString = null;
}
+ catch (ISE ise) {
+ log.error(ise, "Unable to translate to a native Druid query. Resorting
to legacy Druid explain plan");
+ resourcesString = null;
+ }
final Supplier<Sequence<Object[]>> resultsSupplier = Suppliers.ofInstance(
Sequences.simple(ImmutableList.of(new Object[]{explanation,
resourcesString})));
return new PlannerResult(resultsSupplier,
getExplainStructType(rel.getCluster().getTypeFactory()));
}
/**
+ * This method doesn't utilize the Calcite's internal {@link
RelOptUtil#dumpPlan} since that tends to be verbose
+ * and not indicative of the native Druid Queries which will get executed
+ * This method assumes that the Planner has converted the RelNodes to
DruidRels, and thereby we can implictly cast it
+ *
+ * @param rel Instance of the root {@link DruidRel} which is formed by
running the planner transformations on it
+ * @return A string representing an array of native queries that correspond
to the given SQL query, in JSON format
+ * @throws JsonProcessingException
+ */
+ private String explainSqlPlanAsNativeQueries(DruidRel<?> rel) throws
JsonProcessingException
+ {
+ // Only if rel is an instance of DruidUnionRel, do we run multiple native
queries corresponding to single SQL query
+ // Also, DruidUnionRel can only be a top level node, so we don't need to
check for this condition in the subsequent
+ // child nodes
+ ObjectMapper jsonMapper = plannerContext.getJsonMapper();
+ List<DruidQuery> druidQueryList;
+ if (rel instanceof DruidUnionRel) {
+ druidQueryList = rel.getInputs().stream().map(childRel -> (DruidRel<?>)
childRel).map(childRel -> {
+ if (childRel instanceof DruidUnionRel) {
+ log.error("DruidUnionRel can only be the outermost RelNode. This
error shouldn't be encountered");
+ throw new ISE("DruidUnionRel is only supported at the outermost
RelNode.");
+ }
+ return childRel.toDruidQuery(false);
+ }).collect(Collectors.toList());
+ } else {
+ druidQueryList = ImmutableList.of(rel.toDruidQuery(false));
+ }
+
+ // Putting the queries as object node in an ArrayNode, since directly
returning a list causes issues when
+ // serializing the "queryType"
+ ArrayNode nativeQueriesArrayNode = jsonMapper.createArrayNode();
+
+ for (DruidQuery druidQuery : druidQueryList) {
+ Query<?> nativeQuery = druidQuery.getQuery();
+ ObjectNode objectNode = jsonMapper.createObjectNode();
+ objectNode.put("query", jsonMapper.convertValue(nativeQuery,
ObjectNode.class));
+ objectNode.put("signature",
jsonMapper.convertValue(druidQuery.getOutputRowSignature(), ArrayNode.class));
+ nativeQueriesArrayNode.add(objectNode);
+ }
+
+ return jsonMapper.writeValueAsString(nativeQueriesArrayNode);
+ }
+
+ /**
* This method wraps the root with a {@link LogicalSort} that applies a
limit (no ordering change). If the outer rel
* is already a {@link Sort}, we can merge our outerLimit into it, similar
to what is going on in
* {@link org.apache.druid.sql.calcite.rule.SortCollapseRule}.
@@ -409,7 +472,6 @@ public class DruidPlanner implements Closeable
* the web console, allowing it to apply a limit to queries without
rewriting the original SQL.
*
* @param root root node
- *
* @return root node wrapped with a limiting logical sort if a limit is
specified in the query context.
*/
@Nullable
diff --git
a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java
b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java
index 43dfa5a..6203fdc 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java
@@ -33,6 +33,7 @@ public class PlannerConfig
public static final String CTX_KEY_USE_GROUPING_SET_FOR_EXACT_DISTINCT =
"useGroupingSetForExactDistinct";
public static final String CTX_KEY_USE_APPROXIMATE_TOPN =
"useApproximateTopN";
public static final String CTX_COMPUTE_INNER_JOIN_COST_AS_FILTER =
"computeInnerJoinCostAsFilter";
+ public static final String CTX_KEY_USE_NATIVE_QUERY_EXPLAIN =
"useNativeQueryExplain";
@JsonProperty
private Period metadataRefreshPeriod = new Period("PT1M");
@@ -70,6 +71,9 @@ public class PlannerConfig
@JsonProperty
private boolean authorizeSystemTablesDirectly = false;
+ @JsonProperty
+ private boolean useNativeQueryExplain = false;
+
public long getMetadataSegmentPollPeriod()
{
return metadataSegmentPollPeriod;
@@ -137,6 +141,11 @@ public class PlannerConfig
return authorizeSystemTablesDirectly;
}
+ public boolean isUseNativeQueryExplain()
+ {
+ return useNativeQueryExplain;
+ }
+
public PlannerConfig withOverrides(final Map<String, Object> context)
{
if (context == null) {
@@ -166,6 +175,11 @@ public class PlannerConfig
CTX_COMPUTE_INNER_JOIN_COST_AS_FILTER,
computeInnerJoinCostAsFilter
);
+ newConfig.useNativeQueryExplain = getContextBoolean(
+ context,
+ CTX_KEY_USE_NATIVE_QUERY_EXPLAIN,
+ isUseNativeQueryExplain()
+ );
newConfig.requireTimeCondition = isRequireTimeCondition();
newConfig.sqlTimeZone = getSqlTimeZone();
newConfig.awaitInitializationOnStart = isAwaitInitializationOnStart();
@@ -213,7 +227,8 @@ public class PlannerConfig
metadataSegmentPollPeriod == that.metadataSegmentPollPeriod &&
serializeComplexValues == that.serializeComplexValues &&
Objects.equals(metadataRefreshPeriod, that.metadataRefreshPeriod) &&
- Objects.equals(sqlTimeZone, that.sqlTimeZone);
+ Objects.equals(sqlTimeZone, that.sqlTimeZone) &&
+ useNativeQueryExplain == that.useNativeQueryExplain;
}
@Override
@@ -230,7 +245,8 @@ public class PlannerConfig
sqlTimeZone,
metadataSegmentCacheEnable,
metadataSegmentPollPeriod,
- serializeComplexValues
+ serializeComplexValues,
+ useNativeQueryExplain
);
}
@@ -248,6 +264,7 @@ public class PlannerConfig
", metadataSegmentPollPeriod=" + metadataSegmentPollPeriod +
", sqlTimeZone=" + sqlTimeZone +
", serializeComplexValues=" + serializeComplexValues +
+ ", useNativeQueryExplain=" + useNativeQueryExplain +
'}';
}
}
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java
index 3e26507..ff85165 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java
@@ -191,6 +191,15 @@ public class BaseCalciteQueryTest extends CalciteTestBase
}
};
+ public static final PlannerConfig PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN = new
PlannerConfig()
+ {
+ @Override
+ public boolean isUseNativeQueryExplain()
+ {
+ return true;
+ }
+ };
+
public static final String DUMMY_SQL_ID = "dummy";
public static final String LOS_ANGELES = "America/Los_Angeles";
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
index 2feda99..e2b0a24 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
@@ -4185,25 +4185,35 @@ public class CalciteQueryTest extends
BaseCalciteQueryTest
// Skip vectorization since otherwise the "context" will change for each
subtest.
skipVectorize();
- final String explanation =
- "DruidQueryRel(query=[{"
- + "\"queryType\":\"timeseries\","
- + "\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},"
- +
"\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},"
- + "\"descending\":false,"
- + "\"virtualColumns\":[],"
- +
"\"filter\":{\"type\":\"and\",\"fields\":[{\"type\":\"selector\",\"dimension\":\"dim2\",\"value\":\"a\",\"extractionFn\":null},{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"z\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}}]},"
- + "\"granularity\":{\"type\":\"all\"},"
- + "\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],"
- + "\"postAggregations\":[],"
- + "\"limit\":2147483647,"
- +
"\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}}]"
- + ", signature=[{a0:LONG}])\n";
-
+ final String query = "EXPLAIN PLAN FOR SELECT COUNT(*) FROM view.aview
WHERE dim1_firstchar <> 'z'";
+ final String legacyExplanation =
"DruidQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":{\"type\":\"and\",\"fields\":[{\"type\":\"selector\",\"dimension\":\"dim2\",\"value\":\"a\",\"extractionFn\":null},{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimensio
[...]
+ final String explanation = "[{"
+ + "\"query\":{\"queryType\":\"timeseries\","
+ +
"\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},"
+ +
"\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},"
+ + "\"descending\":false,"
+ + "\"virtualColumns\":[],"
+ +
"\"filter\":{\"type\":\"and\",\"fields\":[{\"type\":\"selector\",\"dimension\":\"dim2\",\"value\":\"a\",\"extractionFn\":null},{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"z\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}}]},"
+ + "\"granularity\":{\"type\":\"all\"},"
+ +
"\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],"
+ + "\"postAggregations\":[],"
+ + "\"limit\":2147483647,"
+ +
"\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}},"
+ +
"\"signature\":[{\"name\":\"a0\",\"type\":\"LONG\"}]"
+ + "}]";
final String resources = "[{\"name\":\"aview\",\"type\":\"VIEW\"}]";
testQuery(
- "EXPLAIN PLAN FOR SELECT COUNT(*) FROM view.aview WHERE dim1_firstchar
<> 'z'",
+ query,
+ ImmutableList.of(),
+ ImmutableList.of(
+ new Object[]{legacyExplanation, resources}
+ )
+ );
+ testQuery(
+ PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN,
+ query,
+ CalciteTests.REGULAR_USER_AUTH_RESULT,
ImmutableList.of(),
ImmutableList.of(
new Object[]{explanation, resources}
@@ -6739,28 +6749,133 @@ public class CalciteQueryTest extends
BaseCalciteQueryTest
// Skip vectorization since otherwise the "context" will change for each
subtest.
skipVectorize();
- final String explanation =
+ final String query = "EXPLAIN PLAN FOR SELECT COUNT(*)\n"
+ + "FROM (\n"
+ + " SELECT DISTINCT dim2\n"
+ + " FROM druid.foo\n"
+ + " WHERE SUBSTRING(dim2, 1, 1) IN (\n"
+ + " SELECT SUBSTRING(dim1, 1, 1) FROM druid.foo
WHERE dim1 IS NOT NULL\n"
+ + " )\n"
+ + ")";
+ final String legacyExplanation =
"DruidOuterQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"__subquery__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"defaultTimeout\":300000,\"maxScatte
[...]
+ " DruidJoinQueryRel(condition=[=(SUBSTRING($3, 1, 1), $8)],
joinType=[inner],
query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"__join__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"default\",\"dimension\":\"dim2\",\"outputName\":\"d0\",\"outputType\":\"STRING\"}],\"aggre
[...]
+ "
DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBy
[...]
+ "
DruidQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":null,\"extractionFn\":null}},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"extraction\",\"dimension\":\"dim1\",\"outputNam
[...]
-
+ final String explanation = "["
+ + "{\"query\":{\"queryType\":\"groupBy\","
+ +
"\"dataSource\":{\"type\":\"query\",\"query\":{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"join\",\"left\":{\"type\":\"table\",\"name\":\"foo\"},\"right\":{\"type\":\"query\",\"query\":{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":{\"type\":\"not\",\"field\":{\"t
[...]
+ +
"\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},"
+ + "\"virtualColumns\":[],"
+ + "\"filter\":null,"
+ + "\"granularity\":{\"type\":\"all\"},"
+ + "\"dimensions\":[],"
+ +
"\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],"
+ + "\"postAggregations\":[],"
+ + "\"having\":null,"
+ + "\"limitSpec\":{\"type\":\"NoopLimitSpec\"},"
+ +
"\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},"
+ + "\"descending\":false},"
+ +
"\"signature\":[{\"name\":\"a0\",\"type\":\"LONG\"}]"
+ + "}]";
final String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]";
testQuery(
- "EXPLAIN PLAN FOR SELECT COUNT(*)\n"
- + "FROM (\n"
- + " SELECT DISTINCT dim2\n"
- + " FROM druid.foo\n"
- + " WHERE SUBSTRING(dim2, 1, 1) IN (\n"
- + " SELECT SUBSTRING(dim1, 1, 1) FROM druid.foo WHERE dim1 IS NOT
NULL\n"
- + " )\n"
- + ")",
+ query,
+ ImmutableList.of(),
+ ImmutableList.of(new Object[]{legacyExplanation, resources})
+ );
+
+ testQuery(
+ PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN,
+ query,
+ CalciteTests.REGULAR_USER_AUTH_RESULT,
ImmutableList.of(),
ImmutableList.of(new Object[]{explanation, resources})
);
}
+ // This testcase has been added here and not in CalciteSelectQueryTests
since this checks if the overrides are working
+ // properly when displaying the output of "EXPLAIN PLAN FOR ..." queries
+ @Test
+ public void testExplainSelectStarWithOverrides() throws Exception
+ {
+ Map<String, Object> useRegularExplainContext = new
HashMap<>(QUERY_CONTEXT_DEFAULT);
+
useRegularExplainContext.put(PlannerConfig.CTX_KEY_USE_NATIVE_QUERY_EXPLAIN,
false);
+
+ Map<String, Object> useNativeQueryExplain = new
HashMap<>(QUERY_CONTEXT_DEFAULT);
+ useNativeQueryExplain.put(PlannerConfig.CTX_KEY_USE_NATIVE_QUERY_EXPLAIN,
true);
+
+
+ // Skip vectorization since otherwise the "context" will change for each
subtest.
+ skipVectorize();
+ String legacyExplanation =
"DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"m
[...]
+ String legacyExplanationWithContext =
"DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\"
[...]
+ String explanation = "[{"
+ + "\"query\":{\"queryType\":\"scan\","
+ +
"\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},"
+ +
"\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},"
+ + "\"virtualColumns\":[],"
+ + "\"resultFormat\":\"compactedList\","
+ + "\"batchSize\":20480,"
+ + "\"filter\":null,"
+ +
"\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],"
+ + "\"legacy\":false,"
+ +
"\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},"
+ + "\"descending\":false,"
+ + "\"granularity\":{\"type\":\"all\"}},"
+ +
"\"signature\":[{\"name\":\"__time\",\"type\":\"LONG\"},{\"name\":\"cnt\",\"type\":\"LONG\"},{\"name\":\"dim1\",\"type\":\"STRING\"},{\"name\":\"dim2\",\"type\":\"STRING\"},{\"name\":\"dim3\",\"type\":\"STRING\"},{\"name\":\"m1\",\"type\":\"FLOAT\"},{\"name\":\"m2\",\"type\":\"DOUBLE\"},{\"name\":\"unique_dim1\",\"type\":\"COMPLEX<hyperUnique>\"}]"
+ + "}]";
+
+ String explanationWithContext = "[{"
+ + "\"query\":{\"queryType\":\"scan\","
+ +
"\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},"
+ +
"\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},"
+ + "\"virtualColumns\":[],"
+ + "\"resultFormat\":\"compactedList\","
+ + "\"batchSize\":20480,"
+ + "\"filter\":null,"
+ +
"\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],"
+ + "\"legacy\":false,"
+ +
"\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"useNativeQueryExplain\":true,\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},"
+ + "\"descending\":false,"
+ + "\"granularity\":{\"type\":\"all\"}},"
+ +
"\"signature\":[{\"name\":\"__time\",\"type\":\"LONG\"},{\"name\":\"cnt\",\"type\":\"LONG\"},{\"name\":\"dim1\",\"type\":\"STRING\"},{\"name\":\"dim2\",\"type\":\"STRING\"},{\"name\":\"dim3\",\"type\":\"STRING\"},{\"name\":\"m1\",\"type\":\"FLOAT\"},{\"name\":\"m2\",\"type\":\"DOUBLE\"},{\"name\":\"unique_dim1\",\"type\":\"COMPLEX<hyperUnique>\"}]"
+ + "}]";
+ String sql = "EXPLAIN PLAN FOR SELECT * FROM druid.foo";
+ String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]";
+
+ // Test when default config and no overrides
+ testQuery(sql, ImmutableList.of(), ImmutableList.of(new
Object[]{legacyExplanation, resources}));
+
+ // Test when default config and useNativeQueryExplain is overridden in the
context
+ testQuery(
+ sql,
+ useNativeQueryExplain,
+ ImmutableList.of(),
+ ImmutableList.of(new Object[]{explanationWithContext, resources})
+ );
+
+ // Test when useNativeQueryExplain enabled by default and no overrides
+ testQuery(
+ PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN,
+ sql,
+ CalciteTests.REGULAR_USER_AUTH_RESULT,
+ ImmutableList.of(),
+ ImmutableList.of(new Object[]{explanation, resources})
+ );
+
+ // Test when useNativeQueryExplain enabled by default but is overriden in
the context
+ testQuery(
+ PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN,
+ useRegularExplainContext,
+ sql,
+ CalciteTests.REGULAR_USER_AUTH_RESULT,
+ ImmutableList.of(),
+ ImmutableList.of(new Object[]{legacyExplanationWithContext, resources})
+ );
+ }
+
@Test
public void testExactCountDistinctUsingSubqueryWithWherePushDown() throws
Exception
{
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java
index c6f9808..a4aaab8 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java
@@ -409,14 +409,44 @@ public class CalciteSelectQueryTest extends
BaseCalciteQueryTest
{
// Skip vectorization since otherwise the "context" will change for each
subtest.
skipVectorize();
+ final String query = "EXPLAIN PLAN FOR SELECT 1 + 1";
+ final String explanation = "[{"
+ + "\"query\":{\"queryType\":\"scan\","
+ +
"\"dataSource\":{\"type\":\"inline\",\"columnNames\":[\"EXPR$0\"],\"columnTypes\":[\"LONG\"],\"rows\":[[2]]},"
+ +
"\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},"
+ + "\"virtualColumns\":[],"
+ + "\"resultFormat\":\"compactedList\","
+ + "\"batchSize\":20480,"
+ + "\"filter\":null,"
+ + "\"columns\":[\"EXPR$0\"],"
+ + "\"legacy\":false,"
+ +
"\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},"
+ + "\"descending\":false,"
+ + "\"granularity\":{\"type\":\"all\"}},"
+ +
"\"signature\":[{\"name\":\"EXPR$0\",\"type\":\"LONG\"}]"
+ + "}]";
+ final String legacyExplanation =
"DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"inline\",\"columnNames\":[\"EXPR$0\"],\"columnTypes\":[\"LONG\"],\"rows\":[[2]]},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"filter\":null,\"columns\":[\"EXPR$0\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxSc
[...]
+ final String resources = "[]";
testQuery(
- "EXPLAIN PLAN FOR SELECT 1 + 1",
+ query,
ImmutableList.of(),
ImmutableList.of(
new Object[]{
-
"DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"inline\",\"columnNames\":[\"EXPR$0\"],\"columnTypes\":[\"LONG\"],\"rows\":[[2]]},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"filter\":null,\"columns\":[\"EXPR$0\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":92
[...]
- "[]"
+ legacyExplanation,
+ resources
+ }
+ )
+ );
+ testQuery(
+ PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN,
+ query,
+ CalciteTests.REGULAR_USER_AUTH_RESULT,
+ ImmutableList.of(),
+ ImmutableList.of(
+ new Object[]{
+ explanation,
+ resources
}
)
);
@@ -1078,13 +1108,44 @@ public class CalciteSelectQueryTest extends
BaseCalciteQueryTest
// Skip vectorization since otherwise the "context" will change for each
subtest.
skipVectorize();
+ final String query = "EXPLAIN PLAN FOR SELECT * FROM druid.foo";
+ final String legacyExplanation =
"DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":3000
[...]
+ final String explanation = "[{"
+ + "\"query\":{\"queryType\":\"scan\","
+ +
"\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},"
+ +
"\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},"
+ + "\"virtualColumns\":[],"
+ + "\"resultFormat\":\"compactedList\","
+ + "\"batchSize\":20480,"
+ + "\"filter\":null,"
+ +
"\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],"
+ + "\"legacy\":false,"
+ +
"\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},"
+ + "\"descending\":false,"
+ + "\"granularity\":{\"type\":\"all\"}},"
+ +
"\"signature\":[{\"name\":\"__time\",\"type\":\"LONG\"},{\"name\":\"cnt\",\"type\":\"LONG\"},{\"name\":\"dim1\",\"type\":\"STRING\"},{\"name\":\"dim2\",\"type\":\"STRING\"},{\"name\":\"dim3\",\"type\":\"STRING\"},{\"name\":\"m1\",\"type\":\"FLOAT\"},{\"name\":\"m2\",\"type\":\"DOUBLE\"},{\"name\":\"unique_dim1\",\"type\":\"COMPLEX<hyperUnique>\"}]"
+ + "}]";
+ final String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]";
+
testQuery(
- "EXPLAIN PLAN FOR SELECT * FROM druid.foo",
+ query,
+ ImmutableList.of(),
+ ImmutableList.of(
+ new Object[]{
+ legacyExplanation,
+ resources
+ }
+ )
+ );
+ testQuery(
+ PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN,
+ query,
+ CalciteTests.REGULAR_USER_AUTH_RESULT,
ImmutableList.of(),
ImmutableList.of(
new Object[]{
-
"DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGather
[...]
- "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]"
+ explanation,
+ resources
}
)
);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
