This is an automated email from the ASF dual-hosted git repository.
abhishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 45b30dc07d6 Revert "Change default inSubQueryThreshold (#15336)"
(#15722)
45b30dc07d6 is described below
commit 45b30dc07d69eae59a871b705e27bc558b036a55
Author: Pranav <[email protected]>
AuthorDate: Sun Jan 21 22:04:39 2024 -0800
Revert "Change default inSubQueryThreshold (#15336)" (#15722)
A low value of inSubQueryThreshold can cause queries with IN filter to plan
as joins more commonly. However, some of these join queries may not get planned
as IN filter on data nodes and causes significant perf regression.
---
docs/querying/sql-query-context.md | 2 +-
.../java/org/apache/druid/query/QueryContexts.java | 2 +-
.../apache/druid/sql/calcite/CalciteQueryTest.java | 40 ++++++++++++++++++++++
3 files changed, 42 insertions(+), 2 deletions(-)
diff --git a/docs/querying/sql-query-context.md
b/docs/querying/sql-query-context.md
index e72f1df0e47..f90f995e58f 100644
--- a/docs/querying/sql-query-context.md
+++ b/docs/querying/sql-query-context.md
@@ -51,7 +51,7 @@ Configure Druid SQL query planning using the parameters in
the table below.
|`sqlPullUpLookup`|Whether to consider the [pull-up
rewrite](lookups.md#pull-up) of the `LOOKUP` function during SQL planning.|true|
|`enableJoinLeftTableScanDirect`|`false`|This flag applies to queries which
have joins. For joins, where left child is a simple scan with a filter, by
default, druid will run the scan as a query and the join the results to the
right child on broker. Setting this flag to true overrides that behavior and
druid will attempt to push the join to data servers instead. Please note that
the flag could be applicable to queries even if there is no explicit join.
since queries can internally trans [...]
|`maxNumericInFilters`|`-1`|Max limit for the amount of numeric values that
can be compared for a string type dimension when the entire SQL WHERE clause of
a query translates only to an [OR](../querying/filters.md#or) of [Bound
filter](../querying/filters.md#bound-filter). By default, Druid does not
restrict the amount of of numeric Bound Filters on String columns, although
this situation may block other queries from running. Set this parameter to a
smaller value to prevent Druid from ru [...]
-|`inSubQueryThreshold`|`20`| Threshold for minimum number of values in an IN
clause to convert the query to a JOIN operation on an inlined table rather than
a predicate. A threshold of 0 forces usage of an inline table in all cases; a
threshold of [Integer.MAX_VALUE] forces usage of OR in all cases. |
+|`inSubQueryThreshold`|`2147483647`| Threshold for minimum number of values in
an IN clause to convert the query to a JOIN operation on an inlined table
rather than a predicate. A threshold of 0 forces usage of an inline table in
all cases; a threshold of [Integer.MAX_VALUE] forces usage of OR in all cases. |
## Setting the query context
The query context parameters can be specified as a "context" object in the
[JSON API](../api-reference/sql-api.md) or as a [JDBC connection properties
object](../api-reference/sql-jdbc.md).
diff --git a/processing/src/main/java/org/apache/druid/query/QueryContexts.java
b/processing/src/main/java/org/apache/druid/query/QueryContexts.java
index 0948f951338..7d39edfde66 100644
--- a/processing/src/main/java/org/apache/druid/query/QueryContexts.java
+++ b/processing/src/main/java/org/apache/druid/query/QueryContexts.java
@@ -115,7 +115,7 @@ public class QueryContexts
public static final boolean DEFAULT_USE_FILTER_CNF = false;
public static final boolean DEFAULT_SECONDARY_PARTITION_PRUNING = true;
public static final boolean DEFAULT_ENABLE_DEBUG = false;
- public static final int DEFAULT_IN_SUB_QUERY_THRESHOLD = 20;
+ public static final int DEFAULT_IN_SUB_QUERY_THRESHOLD = Integer.MAX_VALUE;
public static final boolean DEFAULT_ENABLE_TIME_BOUNDARY_PLANNING = false;
@SuppressWarnings("unused") // Used by Jackson serialization
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
index 716955d0cf4..c5e26ca56fc 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
@@ -19,6 +19,7 @@
package org.apache.druid.sql.calcite;
+import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
@@ -131,6 +132,7 @@ import org.junit.Ignore;
import org.junit.Test;
import org.junit.internal.matchers.ThrowableMessageMatcher;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
@@ -5664,6 +5666,44 @@ public class CalciteQueryTest extends
BaseCalciteQueryTest
);
}
+ @Test
+ public void testInFilterWith23Elements()
+ {
+ // Regression test for https://github.com/apache/druid/issues/4203.
+ final List<String> elements = new ArrayList<>();
+ elements.add("abc");
+ elements.add("def");
+ elements.add("ghi");
+ for (int i = 0; i < 20; i++) {
+ elements.add("dummy" + i);
+ }
+
+ final String elementsString = Joiner.on(",").join(elements.stream().map(s
-> "'" + s + "'").iterator());
+
+ testQuery(
+ "SELECT dim1, COUNT(*) FROM druid.foo WHERE dim1 IN (" +
elementsString + ") GROUP BY dim1",
+ ImmutableList.of(
+ GroupByQuery.builder()
+ .setDataSource(CalciteTests.DATASOURCE1)
+ .setInterval(querySegmentSpec(Filtration.eternity()))
+ .setGranularity(Granularities.ALL)
+ .setDimensions(dimensions(new
DefaultDimensionSpec("dim1", "d0")))
+ .setDimFilter(new InDimFilter("dim1", elements, null))
+ .setAggregatorSpecs(
+ aggregators(
+ new CountAggregatorFactory("a0")
+ )
+ )
+ .setContext(QUERY_CONTEXT_DEFAULT)
+ .build()
+ ),
+ ImmutableList.of(
+ new Object[]{"abc", 1L},
+ new Object[]{"def", 1L}
+ )
+ );
+ }
+
@Test
public void testCountStarWithDegenerateFilter()
{
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
