abhishekrb19 commented on code in PR #20183:
URL: https://github.com/apache/druid/pull/20183#discussion_r3961974547
##########
services/src/main/java/org/apache/druid/cli/CliHistorical.java:
##########
@@ -92,30 +86,22 @@ protected List<? extends Module> getModules()
{
return ImmutableList.of(
new DruidProcessingModule(),
- new QueryableModule(),
- new QueryRunnerFactoryModule(),
- new SegmentWranglerModule(),
- new JoinableFactoryModule(),
Review Comment:
Are these not needed? Wondering if this is general cleanup, outside the
scope of this PR
##########
embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/NativeSysTasksQueryTest.java:
##########
@@ -0,0 +1,344 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testing.embedded.query;
+
+import org.apache.druid.indexing.common.task.NoopTask;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.query.QueryContexts;
+import org.apache.druid.sql.calcite.planner.PlannerContext;
+import org.apache.druid.sql.calcite.run.NativeSqlEngine;
+import org.apache.druid.testing.embedded.EmbeddedBroker;
+import org.apache.druid.testing.embedded.EmbeddedCoordinator;
+import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
+import org.apache.druid.testing.embedded.EmbeddedIndexer;
+import org.apache.druid.testing.embedded.EmbeddedOverlord;
+import org.apache.druid.testing.embedded.junit5.EmbeddedClusterTestBase;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class NativeSysTasksQueryTest extends EmbeddedClusterTestBase
+{
+ private static final String TASK_PREFIX = "native_sys_mvp_";
+
+ private final EmbeddedOverlord overlord = new EmbeddedOverlord();
+ private final EmbeddedBroker broker = new EmbeddedBroker();
+
+ @Override
+ protected EmbeddedDruidCluster createCluster()
+ {
+ return EmbeddedDruidCluster.withEmbeddedDerbyAndZookeeper()
+ .useLatchableEmitter()
+ .addServer(new EmbeddedCoordinator())
+ .addServer(new EmbeddedIndexer()
+
.addProperty("druid.worker.capacity", "5"))
+ .addServer(overlord)
+ .addServer(broker);
+ }
+
+ @BeforeAll
+ public void createTasks()
+ {
+ createTasks("a", "native_sys_a", 2);
+ createTasks("b", "native_sys_b", 3);
+ }
+
+ @ParameterizedTest(name = "plannerStrategy = {0}")
+ @ValueSource(strings = {
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_COUPLED,
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_DECOUPLED
+ })
+ public void testGroupByUsesOverlordProvider(final String plannerStrategy)
+ {
+ final String result = cluster.runSql(
+ "SELECT datasource, COUNT(*) "
+ + "FROM sys.tasks "
+ + "WHERE task_id = 'native_sys_mvp_a_0' AND datasource =
'native_sys_a' "
+ + "GROUP BY datasource",
+ nativeQueryContext(plannerStrategy)
+ );
+
+ final Set<String> rows =
Arrays.stream(result.split("\\n")).collect(Collectors.toSet());
+ Assertions.assertEquals(Set.of("native_sys_a,1"), rows);
+ }
+
+ @ParameterizedTest(name = "plannerStrategy = {0}")
+ @ValueSource(strings = {
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_COUPLED,
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_DECOUPLED
+ })
+ public void testNativeAggregationsSupportDistinctCount(final String
plannerStrategy)
+ {
+ final String result = cluster.runSql(
+ "SELECT COUNT(*), COUNT(DISTINCT task_id), COUNT(DISTINCT datasource),
SUM(1) "
+ + "FROM sys.tasks "
+ + "WHERE datasource IN ('native_sys_a', 'native_sys_b')",
+ nativeQueryContext(plannerStrategy)
+ );
+
+ Assertions.assertEquals("5,5,2,5", result);
+ }
+
+ /**
+ * Verifies execution when native planning represents the inner aggregation
as a query datasource.
+ *
+ * <pre>{@code
+ * SELECT COUNT(*)
+ * FROM
+ * (
+ * SELECT
+ * task_id,
+ * COUNT(*) AS task_count
+ * FROM sys.tasks
+ * WHERE datasource IN ('native_sys_a', 'native_sys_b')
+ * GROUP BY task_id
+ * )
+ * WHERE task_count > 0
+ * }</pre>
+ */
Review Comment:
These javadocs here seem like a repeat of the SQL in the test and the test
name. Can be removed.
##########
docs/querying/sql-metadata-tables.md:
##########
@@ -135,10 +135,58 @@ WHERE "IS_AGGREGATOR" = 'YES'
The "sys" schema provides visibility into Druid segments, servers and tasks.
:::info
- Note: "sys" tables do not currently support Druid-specific functions like
`TIME_PARSE` and
- `APPROX_QUANTILE_DS`. Only standard SQL functions can be used.
+ By default, "sys" tables use the SQL-layer execution path and support only
standard SQL functions. You can enable
+ [native query execution](#native-query-execution) for supported system
tables, which enables expressions and
+ aggregations that the native SQL engine can translate.
:::
+### Native query execution
+
+The native SQL engine can plan supported system tables as native datasources.
To enable this behavior for a query, set
+`useNativeQueryForSystemTables` to `true` in the SQL query context:
+
+```json
+{
+ "query": "SELECT COUNT(DISTINCT task_id) FROM sys.tasks",
+ "context": {
+ "useNativeQueryForSystemTables": true
+ }
+}
+```
+
+For clients that support setting query context parameters with SQL statements,
you can enable native system-table
+execution with `SET`:
+
+```sql
+SET useNativeQueryForSystemTables = 'true';
+SELECT COUNT(DISTINCT datasource) FROM sys.tasks;
+```
+
+Native system-table execution is available when the resolved SQL engine is
`native`. You don't need to explicitly set
+the `engine` context parameter when `native` is already the default engine.
The following tables support native query
+execution:
+
+|Table|Source of rows|
+|-----|--------------|
+|[`sys.tasks`](#tasks-table)|The Overlord that owns task state. Supported
filters are pushed into task storage when the configured task storage
implementation supports filter pushdown.|
+|[`sys.server_properties`](#server_properties-table)|The Druid server
processes discovered in the cluster. Filters on `server` and `service_name` can
avoid reading properties from nodes that don't match.|
+
+After Druid retrieves the system-table rows, the native engine applies the
remaining filters, expressions,
+aggregations, sorting, and result processing. A system table that doesn't
advertise native query support continues to
+use its existing SQL-layer execution path, even when
`useNativeQueryForSystemTables` is `true`.
+
+Native system-table queries sent to the Router use distributed Broker
execution by default. To execute a native
+system-table Scan against only the contacted node, set the HTTP header
+`X-Druid-Native-Query-Route: local`. Local execution uses the authenticated
request identity and applies the table's
+authorization rules. The Broker uses the same header for remote node fan-out
requests. If the Broker itself is
+one of the selected nodes, it executes that node Scan in-process without an
HTTP request. The header
+controls routing and doesn't grant additional permissions.
Review Comment:
Hmm, is this header required only for the `sys.server_properties` table? I
wonder if there's a better way to achieve this without passing in additional
headers.
##########
embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/NativeSysTasksQueryTest.java:
##########
@@ -0,0 +1,344 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testing.embedded.query;
+
+import org.apache.druid.indexing.common.task.NoopTask;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.query.QueryContexts;
+import org.apache.druid.sql.calcite.planner.PlannerContext;
+import org.apache.druid.sql.calcite.run.NativeSqlEngine;
+import org.apache.druid.testing.embedded.EmbeddedBroker;
+import org.apache.druid.testing.embedded.EmbeddedCoordinator;
+import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
+import org.apache.druid.testing.embedded.EmbeddedIndexer;
+import org.apache.druid.testing.embedded.EmbeddedOverlord;
+import org.apache.druid.testing.embedded.junit5.EmbeddedClusterTestBase;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class NativeSysTasksQueryTest extends EmbeddedClusterTestBase
+{
+ private static final String TASK_PREFIX = "native_sys_mvp_";
+
+ private final EmbeddedOverlord overlord = new EmbeddedOverlord();
+ private final EmbeddedBroker broker = new EmbeddedBroker();
+
+ @Override
+ protected EmbeddedDruidCluster createCluster()
+ {
+ return EmbeddedDruidCluster.withEmbeddedDerbyAndZookeeper()
+ .useLatchableEmitter()
+ .addServer(new EmbeddedCoordinator())
+ .addServer(new EmbeddedIndexer()
+
.addProperty("druid.worker.capacity", "5"))
+ .addServer(overlord)
+ .addServer(broker);
+ }
+
+ @BeforeAll
+ public void createTasks()
+ {
+ createTasks("a", "native_sys_a", 2);
+ createTasks("b", "native_sys_b", 3);
+ }
+
+ @ParameterizedTest(name = "plannerStrategy = {0}")
+ @ValueSource(strings = {
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_COUPLED,
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_DECOUPLED
+ })
+ public void testGroupByUsesOverlordProvider(final String plannerStrategy)
+ {
+ final String result = cluster.runSql(
+ "SELECT datasource, COUNT(*) "
+ + "FROM sys.tasks "
+ + "WHERE task_id = 'native_sys_mvp_a_0' AND datasource =
'native_sys_a' "
+ + "GROUP BY datasource",
+ nativeQueryContext(plannerStrategy)
+ );
+
+ final Set<String> rows =
Arrays.stream(result.split("\\n")).collect(Collectors.toSet());
+ Assertions.assertEquals(Set.of("native_sys_a,1"), rows);
+ }
+
+ @ParameterizedTest(name = "plannerStrategy = {0}")
+ @ValueSource(strings = {
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_COUPLED,
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_DECOUPLED
+ })
+ public void testNativeAggregationsSupportDistinctCount(final String
plannerStrategy)
+ {
+ final String result = cluster.runSql(
+ "SELECT COUNT(*), COUNT(DISTINCT task_id), COUNT(DISTINCT datasource),
SUM(1) "
+ + "FROM sys.tasks "
+ + "WHERE datasource IN ('native_sys_a', 'native_sys_b')",
+ nativeQueryContext(plannerStrategy)
+ );
+
+ Assertions.assertEquals("5,5,2,5", result);
+ }
+
+ /**
+ * Verifies execution when native planning represents the inner aggregation
as a query datasource.
+ *
+ * <pre>{@code
+ * SELECT COUNT(*)
+ * FROM
+ * (
+ * SELECT
+ * task_id,
+ * COUNT(*) AS task_count
+ * FROM sys.tasks
+ * WHERE datasource IN ('native_sys_a', 'native_sys_b')
+ * GROUP BY task_id
+ * )
+ * WHERE task_count > 0
+ * }</pre>
+ */
+ @ParameterizedTest(name = "plannerStrategy = {0}")
+ @ValueSource(strings = {
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_COUPLED,
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_DECOUPLED
+ })
+ public void testNestedTaskAggregation(final String plannerStrategy)
+ {
+ final String result = cluster.runSql(
+ "SELECT COUNT(*) "
+ + "FROM ("
+ + " SELECT task_id, COUNT(*) AS task_count "
+ + " FROM sys.tasks "
+ + " WHERE datasource IN ('native_sys_a', 'native_sys_b') "
+ + " GROUP BY task_id"
+ + ") "
+ + "WHERE task_count > 0",
+ nativeQueryContext(plannerStrategy)
+ );
+
+ Assertions.assertEquals("5", result);
+ }
+
+ /**
+ * Verifies that an outer aggregation can consume grouped rows from a native
system-table subquery.
+ *
+ * <pre>{@code
+ * SELECT SUM(task_count)
+ * FROM
+ * (
+ * SELECT
+ * datasource,
+ * COUNT(*) AS task_count
+ * FROM sys.tasks
+ * WHERE datasource IN ('native_sys_a', 'native_sys_b')
+ * GROUP BY datasource
+ * )
+ * }</pre>
+ */
+ @ParameterizedTest(name = "plannerStrategy = {0}")
+ @ValueSource(strings = {
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_COUPLED,
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_DECOUPLED
+ })
+ public void testOuterAggregationOverTaskSubquery(final String
plannerStrategy)
+ {
+ final String result = cluster.runSql(
+ "SELECT SUM(task_count) "
+ + "FROM ("
+ + " SELECT datasource, COUNT(*) AS task_count "
+ + " FROM sys.tasks "
+ + " WHERE datasource IN ('native_sys_a', 'native_sys_b') "
+ + " GROUP BY datasource"
+ + ")",
+ nativeQueryContext(plannerStrategy)
+ );
+
+ Assertions.assertEquals("5", result);
+ }
+
+ @ParameterizedTest(name = "plannerStrategy = {0}")
+ @ValueSource(strings = {
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_COUPLED,
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_DECOUPLED
+ })
+ public void testInAndOrFilters(final String plannerStrategy)
+ {
+ assertTaskFilterQuery(
+ "task_id IN ('native_sys_mvp_a_0', 'native_sys_mvp_b_2') "
+ + "OR task_id = 'native_sys_mvp_missing'",
+ Set.of("native_sys_mvp_a_0", "native_sys_mvp_b_2"),
+ plannerStrategy
+ );
+ }
+
+ @ParameterizedTest(name = "plannerStrategy = {0}")
+ @ValueSource(strings = {
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_COUPLED,
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_DECOUPLED
+ })
+ public void testRangeAndLikeFilters(final String plannerStrategy)
+ {
+ assertTaskFilterQuery(
+ "task_id >= 'native_sys_mvp_b_0' "
+ + "AND task_id < 'native_sys_mvp_b_2' "
+ + "AND task_id LIKE 'native_sys_mvp_b_%'",
+ Set.of("native_sys_mvp_b_0", "native_sys_mvp_b_1"),
+ plannerStrategy
+ );
+ }
+
+ /**
+ * Verifies that expression virtual columns required by a pushed node filter
are preserved.
+ *
+ * <pre>{@code
+ * SELECT task_id
+ * FROM sys.tasks
+ * WHERE UPPER(task_id) = 'NATIVE_SYS_MVP_A_0'
+ * }</pre>
+ */
Review Comment:
Same - the javadocs seem redundant here and a few other tests below
##########
docs/querying/sql-metadata-tables.md:
##########
@@ -135,10 +135,58 @@ WHERE "IS_AGGREGATOR" = 'YES'
The "sys" schema provides visibility into Druid segments, servers and tasks.
:::info
- Note: "sys" tables do not currently support Druid-specific functions like
`TIME_PARSE` and
- `APPROX_QUANTILE_DS`. Only standard SQL functions can be used.
+ By default, "sys" tables use the SQL-layer execution path and support only
standard SQL functions. You can enable
+ [native query execution](#native-query-execution) for supported system
tables, which enables expressions and
+ aggregations that the native SQL engine can translate.
:::
+### Native query execution
+
+The native SQL engine can plan supported system tables as native datasources.
To enable this behavior for a query, set
+`useNativeQueryForSystemTables` to `true` in the SQL query context:
+
+```json
+{
+ "query": "SELECT COUNT(DISTINCT task_id) FROM sys.tasks",
+ "context": {
+ "useNativeQueryForSystemTables": true
+ }
+}
+```
+
+For clients that support setting query context parameters with SQL statements,
you can enable native system-table
+execution with `SET`:
+
+```sql
+SET useNativeQueryForSystemTables = 'true';
+SELECT COUNT(DISTINCT datasource) FROM sys.tasks;
+```
+
+Native system-table execution is available when the resolved SQL engine is
`native`. You don't need to explicitly set
+the `engine` context parameter when `native` is already the default engine.
The following tables support native query
+execution:
+
+|Table|Source of rows|
+|-----|--------------|
+|[`sys.tasks`](#tasks-table)|The Overlord that owns task state. Supported
filters are pushed into task storage when the configured task storage
implementation supports filter pushdown.|
+|[`sys.server_properties`](#server_properties-table)|The Druid server
processes discovered in the cluster. Filters on `server` and `service_name` can
avoid reading properties from nodes that don't match.|
+
+After Druid retrieves the system-table rows, the native engine applies the
remaining filters, expressions,
+aggregations, sorting, and result processing. A system table that doesn't
advertise native query support continues to
+use its existing SQL-layer execution path, even when
`useNativeQueryForSystemTables` is `true`.
+
+Native system-table queries sent to the Router use distributed Broker
execution by default. To execute a native
+system-table Scan against only the contacted node, set the HTTP header
+`X-Druid-Native-Query-Route: local`. Local execution uses the authenticated
request identity and applies the table's
+authorization rules. The Broker uses the same header for remote node fan-out
requests. If the Broker itself is
+one of the selected nodes, it executes that node Scan in-process without an
HTTP request. The header
+controls routing and doesn't grant additional permissions.
+
+The parameter defaults to `false`. During a rolling upgrade, leave it disabled
until the Broker and all nodes
+that serve the native system tables have been upgraded. After the upgrade, you
can enable it by query or set
+`druid.query.default.context.useNativeQueryForSystemTables=true` on Brokers as
the cluster-wide default. For more
+information, see [SQL query context](sql-query-context.md).
Review Comment:
I believe this is a continuation of two paragraphs above:
>A system table that doesn't advertise native query support continues to
use its existing SQL-layer execution path, even when
`useNativeQueryForSystemTables` is `true`.
Consider moving this above the paragraph about the additional header.
##########
embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/NativeSysTasksQueryTest.java:
##########
@@ -0,0 +1,344 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testing.embedded.query;
+
+import org.apache.druid.indexing.common.task.NoopTask;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.query.QueryContexts;
+import org.apache.druid.sql.calcite.planner.PlannerContext;
+import org.apache.druid.sql.calcite.run.NativeSqlEngine;
+import org.apache.druid.testing.embedded.EmbeddedBroker;
+import org.apache.druid.testing.embedded.EmbeddedCoordinator;
+import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
+import org.apache.druid.testing.embedded.EmbeddedIndexer;
+import org.apache.druid.testing.embedded.EmbeddedOverlord;
+import org.apache.druid.testing.embedded.junit5.EmbeddedClusterTestBase;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class NativeSysTasksQueryTest extends EmbeddedClusterTestBase
+{
+ private static final String TASK_PREFIX = "native_sys_mvp_";
+
+ private final EmbeddedOverlord overlord = new EmbeddedOverlord();
+ private final EmbeddedBroker broker = new EmbeddedBroker();
+
+ @Override
+ protected EmbeddedDruidCluster createCluster()
+ {
+ return EmbeddedDruidCluster.withEmbeddedDerbyAndZookeeper()
+ .useLatchableEmitter()
+ .addServer(new EmbeddedCoordinator())
+ .addServer(new EmbeddedIndexer()
+
.addProperty("druid.worker.capacity", "5"))
+ .addServer(overlord)
+ .addServer(broker);
+ }
+
+ @BeforeAll
+ public void createTasks()
+ {
+ createTasks("a", "native_sys_a", 2);
+ createTasks("b", "native_sys_b", 3);
+ }
+
+ @ParameterizedTest(name = "plannerStrategy = {0}")
+ @ValueSource(strings = {
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_COUPLED,
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_DECOUPLED
+ })
+ public void testGroupByUsesOverlordProvider(final String plannerStrategy)
+ {
+ final String result = cluster.runSql(
+ "SELECT datasource, COUNT(*) "
+ + "FROM sys.tasks "
+ + "WHERE task_id = 'native_sys_mvp_a_0' AND datasource =
'native_sys_a' "
+ + "GROUP BY datasource",
+ nativeQueryContext(plannerStrategy)
+ );
+
+ final Set<String> rows =
Arrays.stream(result.split("\\n")).collect(Collectors.toSet());
+ Assertions.assertEquals(Set.of("native_sys_a,1"), rows);
+ }
+
+ @ParameterizedTest(name = "plannerStrategy = {0}")
+ @ValueSource(strings = {
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_COUPLED,
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_DECOUPLED
+ })
+ public void testNativeAggregationsSupportDistinctCount(final String
plannerStrategy)
+ {
+ final String result = cluster.runSql(
+ "SELECT COUNT(*), COUNT(DISTINCT task_id), COUNT(DISTINCT datasource),
SUM(1) "
+ + "FROM sys.tasks "
+ + "WHERE datasource IN ('native_sys_a', 'native_sys_b')",
+ nativeQueryContext(plannerStrategy)
+ );
+
+ Assertions.assertEquals("5,5,2,5", result);
+ }
+
+ /**
+ * Verifies execution when native planning represents the inner aggregation
as a query datasource.
+ *
+ * <pre>{@code
+ * SELECT COUNT(*)
+ * FROM
+ * (
+ * SELECT
+ * task_id,
+ * COUNT(*) AS task_count
+ * FROM sys.tasks
+ * WHERE datasource IN ('native_sys_a', 'native_sys_b')
+ * GROUP BY task_id
+ * )
+ * WHERE task_count > 0
+ * }</pre>
+ */
+ @ParameterizedTest(name = "plannerStrategy = {0}")
+ @ValueSource(strings = {
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_COUPLED,
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_DECOUPLED
+ })
+ public void testNestedTaskAggregation(final String plannerStrategy)
+ {
+ final String result = cluster.runSql(
+ "SELECT COUNT(*) "
+ + "FROM ("
+ + " SELECT task_id, COUNT(*) AS task_count "
+ + " FROM sys.tasks "
+ + " WHERE datasource IN ('native_sys_a', 'native_sys_b') "
+ + " GROUP BY task_id"
+ + ") "
+ + "WHERE task_count > 0",
+ nativeQueryContext(plannerStrategy)
+ );
+
+ Assertions.assertEquals("5", result);
+ }
+
+ /**
+ * Verifies that an outer aggregation can consume grouped rows from a native
system-table subquery.
+ *
+ * <pre>{@code
+ * SELECT SUM(task_count)
+ * FROM
+ * (
+ * SELECT
+ * datasource,
+ * COUNT(*) AS task_count
+ * FROM sys.tasks
+ * WHERE datasource IN ('native_sys_a', 'native_sys_b')
+ * GROUP BY datasource
+ * )
+ * }</pre>
+ */
+ @ParameterizedTest(name = "plannerStrategy = {0}")
+ @ValueSource(strings = {
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_COUPLED,
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_DECOUPLED
+ })
+ public void testOuterAggregationOverTaskSubquery(final String
plannerStrategy)
+ {
+ final String result = cluster.runSql(
+ "SELECT SUM(task_count) "
+ + "FROM ("
+ + " SELECT datasource, COUNT(*) AS task_count "
+ + " FROM sys.tasks "
+ + " WHERE datasource IN ('native_sys_a', 'native_sys_b') "
+ + " GROUP BY datasource"
+ + ")",
+ nativeQueryContext(plannerStrategy)
+ );
+
+ Assertions.assertEquals("5", result);
+ }
+
+ @ParameterizedTest(name = "plannerStrategy = {0}")
+ @ValueSource(strings = {
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_COUPLED,
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_DECOUPLED
+ })
+ public void testInAndOrFilters(final String plannerStrategy)
+ {
+ assertTaskFilterQuery(
+ "task_id IN ('native_sys_mvp_a_0', 'native_sys_mvp_b_2') "
+ + "OR task_id = 'native_sys_mvp_missing'",
+ Set.of("native_sys_mvp_a_0", "native_sys_mvp_b_2"),
+ plannerStrategy
+ );
+ }
+
+ @ParameterizedTest(name = "plannerStrategy = {0}")
+ @ValueSource(strings = {
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_COUPLED,
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_DECOUPLED
+ })
+ public void testRangeAndLikeFilters(final String plannerStrategy)
+ {
+ assertTaskFilterQuery(
+ "task_id >= 'native_sys_mvp_b_0' "
+ + "AND task_id < 'native_sys_mvp_b_2' "
+ + "AND task_id LIKE 'native_sys_mvp_b_%'",
+ Set.of("native_sys_mvp_b_0", "native_sys_mvp_b_1"),
+ plannerStrategy
+ );
+ }
+
+ /**
+ * Verifies that expression virtual columns required by a pushed node filter
are preserved.
+ *
+ * <pre>{@code
+ * SELECT task_id
+ * FROM sys.tasks
+ * WHERE UPPER(task_id) = 'NATIVE_SYS_MVP_A_0'
+ * }</pre>
+ */
+ @ParameterizedTest(name = "plannerStrategy = {0}")
+ @ValueSource(strings = {
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_COUPLED,
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_DECOUPLED
+ })
+ public void testExpressionFilterPreservesVirtualColumn(final String
plannerStrategy)
+ {
+ assertTaskFilterQuery(
+ "UPPER(task_id) = 'NATIVE_SYS_MVP_A_0'",
+ Set.of("native_sys_mvp_a_0"),
+ plannerStrategy
+ );
+ }
+
+ @ParameterizedTest(name = "plannerStrategy = {0}")
+ @ValueSource(strings = {
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_COUPLED,
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_DECOUPLED
+ })
+ public void testNegatedLikeFilter(final String plannerStrategy)
+ {
+ assertTaskFilterQuery(
+ "datasource = 'native_sys_b' AND task_id NOT LIKE '%b_2'",
+ Set.of("native_sys_mvp_b_0", "native_sys_mvp_b_1"),
+ plannerStrategy
+ );
+ }
+
+ @ParameterizedTest(name = "plannerStrategy = {0}")
+ @ValueSource(strings = {
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_COUPLED,
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_DECOUPLED
+ })
+ public void testStatusLikeFilterRemainsForResidualEvaluation(final String
plannerStrategy)
+ {
+ assertTaskFilterQuery(
+ "task_id LIKE 'native_sys_mvp_%' AND status LIKE '%'",
+ Set.of(
+ "native_sys_mvp_a_0",
+ "native_sys_mvp_a_1",
+ "native_sys_mvp_b_0",
+ "native_sys_mvp_b_1",
+ "native_sys_mvp_b_2"
+ ),
+ plannerStrategy
+ );
+ }
+
+ @ParameterizedTest(name = "plannerStrategy = {0}")
+ @ValueSource(strings = {
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_COUPLED,
+ QueryContexts.NATIVE_QUERY_SQL_PLANNING_MODE_DECOUPLED
+ })
+ public void testWebConsoleTasksQueryUsesOverlordProvider(final String
plannerStrategy)
+ {
+ final String result = cluster.runSql(
+ "WITH tasks AS (SELECT\n"
+ + " \"task_id\", \"group_id\", \"type\", \"datasource\",
\"created_time\", \"location\", "
+ + "\"duration\", \"error_msg\",\n"
+ + " CASE WHEN \"error_msg\" IN ('Shutdown request from user', "
+ + "'Canceled: Query canceled by user or by task shutdown.') THEN
'CANCELED' "
+ + "WHEN \"status\" = 'RUNNING' THEN \"runner_status\" ELSE \"status\"
END AS \"status\"\n"
Review Comment:
I think this web-console query would be a good candidate to benchmark
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
