This is an automated email from the ASF dual-hosted git repository.
abhishek pushed a commit to branch 31.0.0
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/31.0.0 by this push:
new 52441c005cc add druid.expressions.allowVectorizeFallback and default
to false (#17248) (#17260)
52441c005cc is described below
commit 52441c005cc0370abbc06d28e2426de30fbcff6e
Author: Abhishek Agarwal <[email protected]>
AuthorDate: Sun Oct 6 12:29:16 2024 +0530
add druid.expressions.allowVectorizeFallback and default to false (#17248)
(#17260)
changes:
adds ExpressionProcessing.allowVectorizeFallback() and
ExpressionProcessingConfig.allowVectorizeFallback(), defaulting to false until
few remaining bugs can be fixed (mostly complex types and some odd interactions
with mixed types)
add cannotVectorizeUnlessFallback functions to make it easy to toggle the
default of this config, and easy to know what to delete when we remove it in
the future
Co-authored-by: Clint Wylie <[email protected]>
---
docs/configuration/index.md | 1 -
.../hll/sql/HllSketchSqlAggregatorTest.java | 2 +
.../theta/sql/ThetaSketchSqlAggregatorTest.java | 2 +
.../main/java/org/apache/druid/math/expr/Expr.java | 7 ++++
.../org/apache/druid/math/expr/ExprMacroTable.java | 2 +-
.../druid/math/expr/ExpressionProcessing.java | 18 +++++++--
.../math/expr/ExpressionProcessingConfig.java | 16 +++++++-
.../org/apache/druid/math/expr/FunctionalExpr.java | 6 +--
.../druid/math/expr/VectorExprSanityTest.java | 18 ++++++---
.../timeseries/TimeseriesQueryRunnerTest.java | 10 +++++
.../druid/segment/filter/BaseFilterTest.java | 14 +++++++
.../druid/segment/filter/EqualityFilterTests.java | 14 +++----
.../segment/virtual/ExpressionPlannerTest.java | 40 ++++++++++++++-----
.../druid/sql/calcite/BaseCalciteQueryTest.java | 11 +++++-
.../druid/sql/calcite/CalciteArraysQueryTest.java | 7 ++++
.../druid/sql/calcite/CalciteJoinQueryTest.java | 1 +
.../calcite/CalciteLookupFunctionQueryTest.java | 3 ++
.../calcite/CalciteMultiValueStringQueryTest.java | 7 ++++
.../sql/calcite/CalciteNestedDataQueryTest.java | 2 +
.../apache/druid/sql/calcite/CalciteQueryTest.java | 45 ++++++++++++++++++++++
.../druid/sql/calcite/CalciteSelectQueryTest.java | 6 +++
.../druid/sql/calcite/CalciteSubqueryTest.java | 2 +
.../sql/calcite/CalciteTimeBoundaryQueryTest.java | 1 +
.../druid/sql/calcite/DecoupledExtension.java | 27 ++++++++-----
24 files changed, 221 insertions(+), 41 deletions(-)
diff --git a/docs/configuration/index.md b/docs/configuration/index.md
index 64797c402c0..a370cd7a411 100644
--- a/docs/configuration/index.md
+++ b/docs/configuration/index.md
@@ -2213,7 +2213,6 @@ Supported query contexts:
|Key|Description|Default|
|---|-----------|-------|
|`druid.expressions.useStrictBooleans`|Controls the behavior of Druid boolean
operators and functions, if set to `true` all boolean values are either `1` or
`0`. This configuration has been deprecated and will be removed in a future
release, taking on the `true` behavior. See [expression
documentation](../querying/math-expr.md#logical-operator-modes) for more
information.|true|
-|`druid.expressions.allowNestedArrays`|If enabled, Druid array expressions can
create nested arrays. This configuration has been deprecated and will be
removed in a future release, taking on the `true` behavior.|true|
### Router
diff --git
a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java
b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java
index 65451a3323e..02359a50b62 100644
---
a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java
+++
b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java
@@ -322,6 +322,7 @@ public class HllSketchSqlAggregatorTest extends
BaseCalciteQueryTest
@Test
public void testApproxCountDistinctHllSketch()
{
+ cannotVectorizeUnlessFallback();
final String sql = "SELECT\n"
+ " SUM(cnt),\n"
+ " APPROX_COUNT_DISTINCT_DS_HLL(dim2),\n" // uppercase
@@ -1138,6 +1139,7 @@ public class HllSketchSqlAggregatorTest extends
BaseCalciteQueryTest
@Test
public void testHllEstimateAsVirtualColumnWithGroupByOrderBy()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT"
+ " HLL_SKETCH_ESTIMATE(hllsketch_dim1), count(*)"
diff --git
a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java
b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java
index 7afd2710ccd..d82863a8bf1 100644
---
a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java
+++
b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java
@@ -177,6 +177,7 @@ public class ThetaSketchSqlAggregatorTest extends
BaseCalciteQueryTest
@Test
public void testApproxCountDistinctThetaSketch()
{
+ cannotVectorizeUnlessFallback();
final String sql = "SELECT\n"
+ " SUM(cnt),\n"
+ " APPROX_COUNT_DISTINCT_DS_THETA(dim2),\n"
@@ -1157,6 +1158,7 @@ public class ThetaSketchSqlAggregatorTest extends
BaseCalciteQueryTest
@Test
public void testThetaEstimateAsVirtualColumnWithGroupByOrderBy()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT"
+ " THETA_SKETCH_ESTIMATE(thetasketch_dim1), count(*)"
diff --git a/processing/src/main/java/org/apache/druid/math/expr/Expr.java
b/processing/src/main/java/org/apache/druid/math/expr/Expr.java
index 8fa025cf914..104cc5b1457 100644
--- a/processing/src/main/java/org/apache/druid/math/expr/Expr.java
+++ b/processing/src/main/java/org/apache/druid/math/expr/Expr.java
@@ -182,6 +182,13 @@ public interface Expr extends Cacheable
return false;
}
+
+ default boolean canFallbackVectorize(InputBindingInspector inspector,
List<Expr> args)
+ {
+ return ExpressionProcessing.allowVectorizeFallback() &&
+ getOutputType(inspector) != null &&
+ inspector.canVectorize(args);
+ }
/**
* Possibly convert the {@link Expr} into an optimized, possibly not
thread-safe {@link Expr}. Does not convert
* child {@link Expr}. Most callers should use {@link
Expr#singleThreaded(Expr, InputBindingInspector)} to convert
diff --git
a/processing/src/main/java/org/apache/druid/math/expr/ExprMacroTable.java
b/processing/src/main/java/org/apache/druid/math/expr/ExprMacroTable.java
index 699eb8967eb..40d8ba1112c 100644
--- a/processing/src/main/java/org/apache/druid/math/expr/ExprMacroTable.java
+++ b/processing/src/main/java/org/apache/druid/math/expr/ExprMacroTable.java
@@ -178,7 +178,7 @@ public class ExprMacroTable
@Override
public boolean canVectorize(InputBindingInspector inspector)
{
- return getOutputType(inspector) != null && inspector.canVectorize(args);
+ return canFallbackVectorize(inspector, args);
}
@Override
diff --git
a/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessing.java
b/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessing.java
index ae1d5fb297b..9a4d2ef4694 100644
---
a/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessing.java
+++
b/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessing.java
@@ -45,19 +45,25 @@ public class ExpressionProcessing
@VisibleForTesting
public static void initializeForTests()
{
- INSTANCE = new ExpressionProcessingConfig(null, null, null);
+ INSTANCE = new ExpressionProcessingConfig(null, null, null, null);
}
@VisibleForTesting
public static void initializeForStrictBooleansTests(boolean useStrict)
{
- INSTANCE = new ExpressionProcessingConfig(useStrict, null, null);
+ INSTANCE = new ExpressionProcessingConfig(useStrict, null, null, null);
}
@VisibleForTesting
public static void initializeForHomogenizeNullMultiValueStrings()
{
- INSTANCE = new ExpressionProcessingConfig(null, null, true);
+ INSTANCE = new ExpressionProcessingConfig(null, null, true, null);
+ }
+
+ @VisibleForTesting
+ public static void initializeForFallback()
+ {
+ INSTANCE = new ExpressionProcessingConfig(null, null, null, true);
}
/**
@@ -90,6 +96,12 @@ public class ExpressionProcessing
return INSTANCE.isHomogenizeNullMultiValueStringArrays();
}
+ public static boolean allowVectorizeFallback()
+ {
+ checkInitialized();
+ return INSTANCE.allowVectorizeFallback();
+ }
+
private static void checkInitialized()
{
// this should only be null in a unit test context, in production this
will be injected by the null handling module
diff --git
a/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessingConfig.java
b/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessingConfig.java
index daae90ef534..3d235fe3dda 100644
---
a/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessingConfig.java
+++
b/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessingConfig.java
@@ -37,6 +37,7 @@ public class ExpressionProcessingConfig
// Coerce 'null', '[]', and '[null]' into '[null]' for backwards compat with
0.22 and earlier
public static final String HOMOGENIZE_NULL_MULTIVALUE_STRING_ARRAYS =
"druid.expressions.homogenizeNullMultiValueStringArrays";
+ public static final String ALLOW_VECTORIZE_FALLBACK =
"druid.expressions.allowVectorizeFallback";
@JsonProperty("useStrictBooleans")
private final boolean useStrictBooleans;
@@ -47,11 +48,15 @@ public class ExpressionProcessingConfig
@JsonProperty("homogenizeNullMultiValueStringArrays")
private final boolean homogenizeNullMultiValueStringArrays;
+ @JsonProperty("allowVectorizeFallback")
+ private final boolean allowVectorizeFallback;
+
@JsonCreator
public ExpressionProcessingConfig(
@JsonProperty("useStrictBooleans") @Nullable Boolean useStrictBooleans,
@JsonProperty("processArraysAsMultiValueStrings") @Nullable Boolean
processArraysAsMultiValueStrings,
- @JsonProperty("homogenizeNullMultiValueStringArrays") @Nullable Boolean
homogenizeNullMultiValueStringArrays
+ @JsonProperty("homogenizeNullMultiValueStringArrays") @Nullable Boolean
homogenizeNullMultiValueStringArrays,
+ @JsonProperty("allowVectorizeFallback") @Nullable Boolean
allowVectorizeFallback
)
{
this.useStrictBooleans = getWithPropertyFallback(
@@ -67,6 +72,10 @@ public class ExpressionProcessingConfig
homogenizeNullMultiValueStringArrays,
HOMOGENIZE_NULL_MULTIVALUE_STRING_ARRAYS
);
+ this.allowVectorizeFallback = getWithPropertyFallbackFalse(
+ allowVectorizeFallback,
+ ALLOW_VECTORIZE_FALLBACK
+ );
String version =
ExpressionProcessingConfig.class.getPackage().getImplementationVersion();
if (version == null || version.contains("SNAPSHOT")) {
version = "latest";
@@ -95,6 +104,11 @@ public class ExpressionProcessingConfig
return homogenizeNullMultiValueStringArrays;
}
+ public boolean allowVectorizeFallback()
+ {
+ return allowVectorizeFallback;
+ }
+
private static boolean getWithPropertyFallbackFalse(@Nullable Boolean value,
String property)
{
return getWithPropertyFallback(value, property, "false");
diff --git
a/processing/src/main/java/org/apache/druid/math/expr/FunctionalExpr.java
b/processing/src/main/java/org/apache/druid/math/expr/FunctionalExpr.java
index a7319e0f028..e95140011a7 100644
--- a/processing/src/main/java/org/apache/druid/math/expr/FunctionalExpr.java
+++ b/processing/src/main/java/org/apache/druid/math/expr/FunctionalExpr.java
@@ -100,8 +100,7 @@ class FunctionExpr implements Expr
@Override
public boolean canVectorize(InputBindingInspector inspector)
{
- return function.canVectorize(inspector, args)
- || (getOutputType(inspector) != null &&
inspector.canVectorize(args));
+ return function.canVectorize(inspector, args) ||
canFallbackVectorize(inspector, args);
}
@Override
@@ -226,7 +225,8 @@ class ApplyFunctionExpr implements Expr
@Override
public boolean canVectorize(InputBindingInspector inspector)
{
- return canVectorizeNative(inspector) || (getOutputType(inspector) != null
&& inspector.canVectorize(argsExpr));
+ return canVectorizeNative(inspector) ||
+ (canFallbackVectorize(inspector, argsExpr) &&
lambdaExpr.canVectorize(inspector));
}
@Override
diff --git
a/processing/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java
b/processing/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java
index cf513b4ffba..2da11bb4e29 100644
---
a/processing/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java
+++
b/processing/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java
@@ -30,6 +30,7 @@ import org.apache.druid.math.expr.vector.ExprVectorProcessor;
import org.apache.druid.query.expression.NestedDataExpressions;
import org.apache.druid.testing.InitializedNullHandlingTest;
import org.junit.Assert;
+import org.junit.Assume;
import org.junit.Test;
import javax.annotation.Nullable;
@@ -264,11 +265,17 @@ public class VectorExprSanityTest extends
InitializedNullHandlingTest
@Test
public void testArrayFns()
{
- testExpression("array(s1, s2)", types);
- testExpression("array(l1, l2)", types);
- testExpression("array(d1, d2)", types);
- testExpression("array(l1, d2)", types);
- testExpression("array(s1, l2)", types);
+ try {
+ ExpressionProcessing.initializeForFallback();
+ testExpression("array(s1, s2)", types);
+ testExpression("array(l1, l2)", types);
+ testExpression("array(d1, d2)", types);
+ testExpression("array(l1, d2)", types);
+ testExpression("array(s1, l2)", types);
+ }
+ finally {
+ ExpressionProcessing.initializeForTests();
+ }
}
@Test
@@ -284,6 +291,7 @@ public class VectorExprSanityTest extends
InitializedNullHandlingTest
@Test
public void testJsonFns()
{
+ Assume.assumeTrue(ExpressionProcessing.allowVectorizeFallback());
testExpression("json_object('k1', s1, 'k2', l1)", types);
}
diff --git
a/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerTest.java
b/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerTest.java
index 781d9d31023..9c89d630852 100644
---
a/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerTest.java
+++
b/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerTest.java
@@ -35,6 +35,7 @@ import
org.apache.druid.java.util.common.granularity.Granularity;
import org.apache.druid.java.util.common.granularity.PeriodGranularity;
import org.apache.druid.java.util.common.guava.Sequence;
import org.apache.druid.java.util.metrics.StubServiceEmitter;
+import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.query.Druids;
import org.apache.druid.query.FinalizeResultsQueryRunner;
import org.apache.druid.query.MetricsEmittingQueryRunner;
@@ -2269,6 +2270,7 @@ public class TimeseriesQueryRunnerTest extends
InitializedNullHandlingTest
@Test
public void testTimeSeriesWithFilteredAggAndExpressionFilteredAgg()
{
+ cannotVectorizeUnlessFallback();
TimeseriesQuery query = Druids
.newTimeseriesQueryBuilder()
.dataSource(QueryRunnerTestHelper.DATA_SOURCE)
@@ -3278,4 +3280,12 @@ public class TimeseriesQueryRunnerTest extends
InitializedNullHandlingTest
expectedException.expectMessage("Cannot vectorize!");
}
}
+
+ protected void cannotVectorizeUnlessFallback()
+ {
+ if (vectorize && !ExpressionProcessing.allowVectorizeFallback()) {
+ expectedException.expect(RuntimeException.class);
+ expectedException.expectMessage("Cannot vectorize!");
+ }
+ }
}
diff --git
a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java
b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java
index aa28106073e..a174bc9e667 100644
---
a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java
@@ -48,6 +48,7 @@ import org.apache.druid.java.util.common.Pair;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprType;
+import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.math.expr.ExpressionType;
import org.apache.druid.math.expr.Parser;
import org.apache.druid.query.QueryContext;
@@ -1197,6 +1198,19 @@ public abstract class BaseFilterTest extends
InitializedNullHandlingTest
}
}
+ protected void assertFilterMatchesSkipVectorizeUnlessFallback(
+ final DimFilter filter,
+ final List<String> expectedRows
+ )
+ {
+ final boolean vectorize = ExpressionProcessing.allowVectorizeFallback();
+ assertFilterMatches(filter, expectedRows, vectorize);
+ // test double inverted
+ if (!StringUtils.toLowerCase(testName).contains("concise")) {
+ assertFilterMatches(NotDimFilter.of(NotDimFilter.of(filter)),
expectedRows, vectorize);
+ }
+ }
+
private void assertFilterMatches(
final DimFilter filter,
final List<String> expectedRows,
diff --git
a/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java
b/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java
index 9e29ff266b1..c76ea3e6b85 100644
---
a/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java
+++
b/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java
@@ -1644,7 +1644,7 @@ public class EqualityFilterTests
"5", .. [null], [123L, 345L], null
*/
- assertFilterMatches(
+ assertFilterMatchesSkipVectorizeUnlessFallback(
new EqualityFilter(
"arrayStringAsMvd",
ColumnType.STRING,
@@ -1653,7 +1653,7 @@ public class EqualityFilterTests
),
ImmutableList.of("0", "3")
);
- assertFilterMatches(
+ assertFilterMatchesSkipVectorizeUnlessFallback(
NotDimFilter.of(
new EqualityFilter(
"arrayStringAsMvd",
@@ -1667,7 +1667,7 @@ public class EqualityFilterTests
: ImmutableList.of("1", "2", "4", "5")
);
- assertFilterMatches(
+ assertFilterMatchesSkipVectorizeUnlessFallback(
new EqualityFilter(
"arrayLongAsMvd",
ColumnType.STRING,
@@ -1676,7 +1676,7 @@ public class EqualityFilterTests
),
ImmutableList.of("0", "2")
);
- assertFilterMatches(
+ assertFilterMatchesSkipVectorizeUnlessFallback(
NotDimFilter.of(
new EqualityFilter(
"arrayLongAsMvd",
@@ -1690,7 +1690,7 @@ public class EqualityFilterTests
: ImmutableList.of("1", "3", "4", "5")
);
- assertFilterMatches(
+ assertFilterMatchesSkipVectorizeUnlessFallback(
new EqualityFilter(
"arrayDoubleAsMvd",
ColumnType.STRING,
@@ -1699,7 +1699,7 @@ public class EqualityFilterTests
),
ImmutableList.of("0", "1")
);
- assertFilterMatches(
+ assertFilterMatchesSkipVectorizeUnlessFallback(
NotDimFilter.of(
new EqualityFilter(
"arrayDoubleAsMvd",
@@ -1713,7 +1713,7 @@ public class EqualityFilterTests
: ImmutableList.of("2", "3", "4", "5")
);
- assertFilterMatches(
+ assertFilterMatchesSkipVectorizeUnlessFallback(
new EqualityFilter(
"arrayConstantAsMvd",
ColumnType.STRING,
diff --git
a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionPlannerTest.java
b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionPlannerTest.java
index a0f3d1e8ecc..3dddbd991c3 100644
---
a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionPlannerTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionPlannerTest.java
@@ -26,6 +26,7 @@ import org.apache.druid.error.DruidException;
import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprEval;
import org.apache.druid.math.expr.ExprMacroTable;
+import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.math.expr.ExpressionType;
import org.apache.druid.math.expr.Parser;
import org.apache.druid.query.expression.TestExprMacroTable;
@@ -1151,8 +1152,7 @@ public class ExpressionPlannerTest extends
InitializedNullHandlingTest
Assert.assertTrue(
thePlan.is(
ExpressionPlan.Trait.NON_SCALAR_INPUTS,
- ExpressionPlan.Trait.NEEDS_APPLIED,
- ExpressionPlan.Trait.VECTORIZABLE
+ ExpressionPlan.Trait.NEEDS_APPLIED
)
);
Assert.assertFalse(
@@ -1164,6 +1164,7 @@ public class ExpressionPlannerTest extends
InitializedNullHandlingTest
ExpressionPlan.Trait.UNKNOWN_INPUTS
)
);
+ assertFallbackVectorizable(thePlan);
Assert.assertEquals(
"array_to_string(map((\"multi_dictionary_string\") ->
array_append(\"scalar_string\", \"multi_dictionary_string\"),
\"multi_dictionary_string\"), ',')",
@@ -1251,8 +1252,7 @@ public class ExpressionPlannerTest extends
InitializedNullHandlingTest
ExpressionPlan thePlan = plan("array(long1, long2)");
Assert.assertTrue(
thePlan.is(
- ExpressionPlan.Trait.NON_SCALAR_OUTPUT,
- ExpressionPlan.Trait.VECTORIZABLE
+ ExpressionPlan.Trait.NON_SCALAR_OUTPUT
)
);
Assert.assertFalse(
@@ -1265,6 +1265,7 @@ public class ExpressionPlannerTest extends
InitializedNullHandlingTest
ExpressionPlan.Trait.NON_SCALAR_INPUTS
)
);
+ assertFallbackVectorizable(thePlan);
Assert.assertEquals(ExpressionType.LONG_ARRAY, thePlan.getOutputType());
thePlan = plan("array(long1, double1)");
@@ -1341,10 +1342,11 @@ public class ExpressionPlannerTest extends
InitializedNullHandlingTest
);
Assert.assertTrue(
thePlan.is(
- ExpressionPlan.Trait.SINGLE_INPUT_SCALAR,
- ExpressionPlan.Trait.VECTORIZABLE
+ ExpressionPlan.Trait.SINGLE_INPUT_SCALAR
)
);
+ assertFallbackVectorizable(thePlan);
+
Assert.assertEquals(ExpressionType.STRING, thePlan.getOutputType());
ColumnCapabilities inferred = thePlan.inferColumnCapabilities(
ExpressionType.toColumnType(thePlan.getOutputType())
@@ -1387,8 +1389,7 @@ public class ExpressionPlannerTest extends
InitializedNullHandlingTest
{
Assert.assertTrue(
thePlan.is(
- ExpressionPlan.Trait.NON_SCALAR_INPUTS,
- ExpressionPlan.Trait.VECTORIZABLE
+ ExpressionPlan.Trait.NON_SCALAR_INPUTS
)
);
Assert.assertFalse(
@@ -1401,6 +1402,7 @@ public class ExpressionPlannerTest extends
InitializedNullHandlingTest
ExpressionPlan.Trait.NEEDS_APPLIED
)
);
+ assertFallbackVectorizable(thePlan);
}
private static void assertArrayInAndOut(ExpressionPlan thePlan)
@@ -1408,8 +1410,7 @@ public class ExpressionPlannerTest extends
InitializedNullHandlingTest
Assert.assertTrue(
thePlan.is(
ExpressionPlan.Trait.NON_SCALAR_INPUTS,
- ExpressionPlan.Trait.NON_SCALAR_OUTPUT,
- ExpressionPlan.Trait.VECTORIZABLE
+ ExpressionPlan.Trait.NON_SCALAR_OUTPUT
)
);
Assert.assertFalse(
@@ -1421,6 +1422,25 @@ public class ExpressionPlannerTest extends
InitializedNullHandlingTest
ExpressionPlan.Trait.NEEDS_APPLIED
)
);
+ assertFallbackVectorizable(thePlan);
+ }
+
+
+ private static void assertFallbackVectorizable(ExpressionPlan thePlan)
+ {
+ if (ExpressionProcessing.allowVectorizeFallback()) {
+ Assert.assertTrue(
+ thePlan.is(
+ ExpressionPlan.Trait.VECTORIZABLE
+ )
+ );
+ } else {
+ Assert.assertFalse(
+ thePlan.is(
+ ExpressionPlan.Trait.VECTORIZABLE
+ )
+ );
+ }
}
private static class TestMacroTable extends ExprMacroTable
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java
index f310edae391..afe3e1cb2b2 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java
@@ -43,6 +43,7 @@ import
org.apache.druid.java.util.common.granularity.Granularity;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.math.expr.Evals;
import org.apache.druid.math.expr.ExprEval;
+import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.query.DataSource;
import org.apache.druid.query.Druids;
import org.apache.druid.query.JoinDataSource;
@@ -282,6 +283,7 @@ public class BaseCalciteQueryTest extends CalciteTestBase
final boolean useDefault = NullHandling.replaceWithDefault();
public boolean cannotVectorize = false;
+ public boolean cannotVectorizeUnlessFallback = false;
public boolean skipVectorize = false;
static {
@@ -869,7 +871,9 @@ public class BaseCalciteQueryTest extends CalciteTestBase
protected QueryTestBuilder testBuilder()
{
return new QueryTestBuilder(new CalciteTestConfig())
- .cannotVectorize(cannotVectorize)
+ .cannotVectorize(
+ cannotVectorize || (!ExpressionProcessing.allowVectorizeFallback()
&& cannotVectorizeUnlessFallback)
+ )
.skipVectorize(skipVectorize);
}
@@ -1288,6 +1292,11 @@ public class BaseCalciteQueryTest extends CalciteTestBase
cannotVectorize = true;
}
+ protected void cannotVectorizeUnlessFallback()
+ {
+ cannotVectorizeUnlessFallback = true;
+ }
+
protected void skipVectorize()
{
skipVectorize = true;
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java
index b2f87b3d633..758186459e7 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java
@@ -1525,6 +1525,7 @@ public class CalciteArraysQueryTest extends
BaseCalciteQueryTest
@Test
public void testArrayLength()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT dim1, ARRAY_LENGTH(dim3), SUM(cnt) FROM druid.numfoo GROUP BY
1, 2 ORDER BY 2 DESC",
ImmutableList.of(
@@ -1737,6 +1738,7 @@ public class CalciteArraysQueryTest extends
BaseCalciteQueryTest
@Test
public void testArrayPrependAppend()
{
+ cannotVectorizeUnlessFallback();
ImmutableList<Object[]> results;
if (useDefault) {
results = ImmutableList.of(
@@ -1855,6 +1857,7 @@ public class CalciteArraysQueryTest extends
BaseCalciteQueryTest
@Test
public void testArrayOffset()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT ARRAY_OFFSET(dim3, 1), SUM(cnt) FROM druid.numfoo GROUP BY 1
ORDER BY 2 DESC",
ImmutableList.of(
@@ -2176,6 +2179,7 @@ public class CalciteArraysQueryTest extends
BaseCalciteQueryTest
@Test
public void testArrayOrdinal()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT ARRAY_ORDINAL(dim3, 2), SUM(cnt) FROM druid.numfoo GROUP BY 1
ORDER BY 2 DESC",
ImmutableList.of(
@@ -2216,6 +2220,7 @@ public class CalciteArraysQueryTest extends
BaseCalciteQueryTest
@Test
public void testArrayOffsetOf()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT ARRAY_OFFSET_OF(dim3, 'b'), SUM(cnt) FROM druid.numfoo GROUP
BY 1 ORDER BY 2 DESC",
ImmutableList.of(
@@ -2262,6 +2267,7 @@ public class CalciteArraysQueryTest extends
BaseCalciteQueryTest
@Test
public void testArrayOrdinalOf()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT ARRAY_ORDINAL_OF(dim3, 'b'), SUM(cnt) FROM druid.numfoo GROUP
BY 1 ORDER BY 2 DESC",
ImmutableList.of(
@@ -2309,6 +2315,7 @@ public class CalciteArraysQueryTest extends
BaseCalciteQueryTest
@Test
public void testArrayToString()
{
+ cannotVectorizeUnlessFallback();
ImmutableList<Object[]> results;
if (useDefault) {
results = ImmutableList.of(
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java
index 04cea5e2c13..f9137656d03 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java
@@ -4218,6 +4218,7 @@ public class CalciteJoinQueryTest extends
BaseCalciteQueryTest
@NotYetSupported(Modes.JOIN_CONDITION_NOT_PUSHED_CONDITION)
public void testSemiJoinWithOuterTimeExtractAggregateWithOrderBy()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT COUNT(DISTINCT dim1), EXTRACT(MONTH FROM __time) FROM
druid.foo\n"
+ " WHERE dim2 IN (\n"
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteLookupFunctionQueryTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteLookupFunctionQueryTest.java
index ba4e32b8101..2c8920fb531 100644
---
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteLookupFunctionQueryTest.java
+++
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteLookupFunctionQueryTest.java
@@ -1936,6 +1936,9 @@ public class CalciteLookupFunctionQueryTest extends
BaseCalciteQueryTest
@Test
public void testDontPullUpLookupWhenUsedByAggregation()
{
+ if (NullHandling.sqlCompatible()) {
+ cannotVectorizeUnlessFallback();
+ }
testQuery(
"SELECT LOOKUP(dim1, 'lookyloo121'), COUNT(LOOKUP(dim1,
'lookyloo121')) FROM druid.foo GROUP BY 1",
ImmutableList.of(
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteMultiValueStringQueryTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteMultiValueStringQueryTest.java
index 25f9ac34e0d..4ce75b6270d 100644
---
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteMultiValueStringQueryTest.java
+++
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteMultiValueStringQueryTest.java
@@ -469,6 +469,7 @@ public class CalciteMultiValueStringQueryTest extends
BaseCalciteQueryTest
@Test
public void testMultiValueStringLength()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT dim1, MV_LENGTH(dim3), SUM(cnt) FROM druid.numfoo GROUP BY 1,
2 ORDER BY 2 DESC",
ImmutableList.of(
@@ -629,6 +630,7 @@ public class CalciteMultiValueStringQueryTest extends
BaseCalciteQueryTest
@Test
public void testMultiValueStringPrependAppend()
{
+ cannotVectorizeUnlessFallback();
ImmutableList<Object[]> results;
if (useDefault) {
results = ImmutableList.of(
@@ -812,6 +814,7 @@ public class CalciteMultiValueStringQueryTest extends
BaseCalciteQueryTest
@Test
public void testMultiValueStringOffset()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT MV_OFFSET(dim3, 1), SUM(cnt) FROM druid.numfoo GROUP BY 1
ORDER BY 2 DESC",
ImmutableList.of(
@@ -848,6 +851,7 @@ public class CalciteMultiValueStringQueryTest extends
BaseCalciteQueryTest
@Test
public void testMultiValueStringOrdinal()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT MV_ORDINAL(dim3, 2), SUM(cnt) FROM druid.numfoo GROUP BY 1
ORDER BY 2 DESC",
ImmutableList.of(
@@ -888,6 +892,7 @@ public class CalciteMultiValueStringQueryTest extends
BaseCalciteQueryTest
@Test
public void testMultiValueStringOffsetOf()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT MV_OFFSET_OF(dim3, 'b'), SUM(cnt) FROM druid.numfoo GROUP BY 1
ORDER BY 2 DESC",
ImmutableList.of(
@@ -934,6 +939,7 @@ public class CalciteMultiValueStringQueryTest extends
BaseCalciteQueryTest
@Test
public void testMultiValueStringOrdinalOf()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT MV_ORDINAL_OF(dim3, 'b'), SUM(cnt) FROM druid.numfoo GROUP BY
1 ORDER BY 2 DESC",
ImmutableList.of(
@@ -981,6 +987,7 @@ public class CalciteMultiValueStringQueryTest extends
BaseCalciteQueryTest
@Test
public void testMultiValueStringToString()
{
+ cannotVectorizeUnlessFallback();
ImmutableList<Object[]> results;
if (useDefault) {
results = ImmutableList.of(
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java
index 88ad05a4a27..bdbb30319b0 100644
---
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java
+++
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java
@@ -2681,6 +2681,7 @@ public class CalciteNestedDataQueryTest extends
BaseCalciteQueryTest
@Test
public void testGroupByPathSelectorFilterCoalesce()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT "
+ "JSON_VALUE(nest, '$.x'), "
@@ -7577,6 +7578,7 @@ public class CalciteNestedDataQueryTest extends
BaseCalciteQueryTest
@Test
public void testToJsonString()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT TO_JSON_STRING(nester) FROM druid.nested GROUP BY 1",
ImmutableList.of(
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
index 162fcfc17c7..0df97e7929d 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
@@ -589,6 +589,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testSafeDivide()
{
+ cannotVectorizeUnlessFallback();
final Map<String, Object> context = new HashMap<>(QUERY_CONTEXT_DEFAULT);
testQuery(
@@ -2744,6 +2745,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testExactCountDistinctWithFilter()
{
+ cannotVectorizeUnlessFallback();
msqIncompatible();
final String sqlQuery = "SELECT COUNT(DISTINCT foo.dim1) FILTER(WHERE
foo.cnt = 1), SUM(foo.cnt) FROM druid.foo";
@@ -2770,6 +2772,9 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testExactCountDistinctWithFilter2()
{
+ if (NullHandling.sqlCompatible()) {
+ cannotVectorizeUnlessFallback();
+ }
final String sqlQuery = "SELECT COUNT(DISTINCT foo.dim1) FILTER(WHERE
foo.cnt = 1), SUM(foo.cnt) FROM druid.foo";
testQuery(
@@ -3259,6 +3264,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testGroupByCaseWhen()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT\n"
+ " CASE EXTRACT(DAY FROM __time)\n"
@@ -3382,6 +3388,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testGroupByCaseWhenOfTripleAnd()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT\n"
+ " CASE WHEN m1 > 1 AND m1 < 5 AND cnt = 1 THEN 'x' ELSE NULL END,"
@@ -4493,6 +4500,9 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testCountStarOnCommonTableExpression()
{
+ if (NullHandling.sqlCompatible()) {
+ cannotVectorizeUnlessFallback();
+ }
Druids.TimeseriesQueryBuilder builder =
Druids.newTimeseriesQueryBuilder()
.dataSource(CalciteTests.DATASOURCE1)
@@ -4530,6 +4540,9 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testCountStarOnView()
{
+ if (NullHandling.sqlCompatible()) {
+ cannotVectorizeUnlessFallback();
+ }
Druids.TimeseriesQueryBuilder builder =
Druids.newTimeseriesQueryBuilder()
.dataSource(CalciteTests.DATASOURCE1)
@@ -4574,6 +4587,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
.aggregators(aggregators(new CountAggregatorFactory("a0")))
.context(QUERY_CONTEXT_DEFAULT);
if (NullHandling.sqlCompatible()) {
+ cannotVectorizeUnlessFallback();
builder = builder.virtualColumns(
expressionVirtualColumn("v0", "substring(\"dim1\",
0, 1)", ColumnType.STRING)
)
@@ -5156,6 +5170,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testFilteredAggregations()
{
+ cannotVectorizeUnlessFallback();
Druids.TimeseriesQueryBuilder builder =
Druids.newTimeseriesQueryBuilder()
.dataSource(CalciteTests.DATASOURCE1)
@@ -5340,6 +5355,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testCaseFilteredAggregationWithGroupBy()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT\n"
+ " cnt,\n"
@@ -5419,6 +5435,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testExpressionAggregations()
{
+ cannotVectorizeUnlessFallback();
final ExprMacroTable macroTable = CalciteTests.createExprMacroTable();
testQuery(
@@ -5849,6 +5866,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testInExpression()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT dim1 IN ('abc', 'def', 'ghi'), COUNT(*)\n"
+ "FROM druid.foo\n"
@@ -5912,6 +5930,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testInOrIsNullExpression()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT dim1 IN ('abc', 'def', 'ghi') OR dim1 IS NULL, COUNT(*)\n"
+ "FROM druid.foo\n"
@@ -5943,6 +5962,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testNotInOrIsNullExpression()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT NOT (dim1 IN ('abc', 'def', 'ghi') OR dim1 IS NULL),
COUNT(*)\n"
+ "FROM druid.foo\n"
@@ -5974,6 +5994,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testNotInAndIsNotNullExpression()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT dim1 NOT IN ('abc', 'def', 'ghi') AND dim1 IS NOT NULL,
COUNT(*)\n"
+ "FROM druid.foo\n"
@@ -6005,6 +6026,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testInOrGreaterThanExpression()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT dim1 IN ('abc', 'def', 'ghi') OR dim1 > 'zzz', COUNT(*)\n"
+ "FROM druid.foo\n"
@@ -6036,6 +6058,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testNotInAndLessThanExpression()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT dim1 NOT IN ('abc', 'def', 'ghi') AND dim1 < 'zzz', COUNT(*)\n"
+ "FROM druid.foo\n"
@@ -6067,6 +6090,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testNotInOrEqualToOneOfThemExpression()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT dim1 NOT IN ('abc', 'def', 'ghi') OR dim1 = 'def', COUNT(*)\n"
+ "FROM druid.foo\n"
@@ -7295,6 +7319,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testSumOfExtractionFn()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT SUM(CAST(SUBSTRING(dim1, 1, 10) AS INTEGER)) FROM druid.foo",
ImmutableList.of(
@@ -8639,6 +8664,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testCountDistinctOfTrim()
{
+ cannotVectorizeUnlessFallback();
// Test a couple different syntax variants of TRIM.
testQuery(
"SELECT COUNT(DISTINCT TRIM(BOTH ' ' FROM dim1)) FROM druid.foo WHERE
TRIM(dim1) <> ''",
@@ -8672,6 +8698,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testSillyQuarters()
{
+ cannotVectorizeUnlessFallback();
// Like FLOOR(__time TO QUARTER) but silly.
testQuery(
"SELECT CAST((EXTRACT(MONTH FROM __time) - 1 ) / 3 + 1 AS INTEGER) AS
quarter, COUNT(*)\n"
@@ -8782,6 +8809,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testRegexpExtractFilterViaNotNullCheck()
{
+ cannotVectorizeUnlessFallback();
Druids.TimeseriesQueryBuilder builder =
Druids.newTimeseriesQueryBuilder()
.dataSource(CalciteTests.DATASOURCE1)
@@ -9216,6 +9244,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testFilterOnTimeExtract()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT COUNT(*) FROM druid.foo\n"
+ "WHERE EXTRACT(YEAR FROM __time) = 2000\n"
@@ -9248,6 +9277,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testFilterOnTimeExtractWithMultipleDays()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT COUNT(*) FROM druid.foo\n"
+ "WHERE EXTRACT(YEAR FROM __time) = 2000\n"
@@ -9288,6 +9318,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testFilterOnTimeExtractWithVariousTimeUnits()
{
+ cannotVectorizeUnlessFallback();
msqIncompatible();
testQuery(
"SELECT COUNT(*) FROM druid.foo4\n"
@@ -9397,6 +9428,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testQueryWithSelectProjectAndIdentityProjectDoesNotRename()
{
+ cannotVectorizeUnlessFallback();
msqIncompatible();
testQuery(
PLANNER_CONFIG_NO_HLL.withOverrides(
@@ -9674,6 +9706,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testGroupByStringLength()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT CHARACTER_LENGTH(dim1), COUNT(*) FROM druid.foo GROUP BY
CHARACTER_LENGTH(dim1)",
ImmutableList.of(
@@ -10913,6 +10946,7 @@ public class CalciteQueryTest extends
BaseCalciteQueryTest
@Test
public void testGroupByExtractYear()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT\n"
+ " EXTRACT(YEAR FROM __time) AS \"year\",\n"
@@ -10961,6 +10995,7 @@ public class CalciteQueryTest extends
BaseCalciteQueryTest
@Test
public void testGroupByFormatYearAndMonth()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT\n"
+ " TIME_FORMAt(__time, 'yyyy MM') AS \"year\",\n"
@@ -11009,6 +11044,7 @@ public class CalciteQueryTest extends
BaseCalciteQueryTest
@Test
public void testGroupByExtractFloorTime()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT\n"
+ "EXTRACT(YEAR FROM FLOOR(__time TO YEAR)) AS \"year\", SUM(cnt)\n"
@@ -11041,6 +11077,7 @@ public class CalciteQueryTest extends
BaseCalciteQueryTest
@Test
public void testGroupByExtractFloorTimeLosAngeles()
{
+ cannotVectorizeUnlessFallback();
testQuery(
PLANNER_CONFIG_DEFAULT,
QUERY_CONTEXT_LOS_ANGELES,
@@ -14085,6 +14122,7 @@ public class CalciteQueryTest extends
BaseCalciteQueryTest
@Test
public void testExpressionCounts()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT\n"
+ " COUNT(reverse(dim2)),\n"
@@ -15080,6 +15118,9 @@ public class CalciteQueryTest extends
BaseCalciteQueryTest
@Test
public void testGreatestFunctionForNumberWithIsNull()
{
+ if (NullHandling.sqlCompatible()) {
+ cannotVectorizeUnlessFallback();
+ }
String query = "SELECT dim1, MAX(GREATEST(l1, l2)) IS NULL FROM
druid.numfoo GROUP BY dim1";
List<Object[]> expectedResult;
@@ -15145,6 +15186,7 @@ public class CalciteQueryTest extends
BaseCalciteQueryTest
@Test
public void testGreatestFunctionForStringWithIsNull()
{
+ cannotVectorizeUnlessFallback();
msqIncompatible();
String query = "SELECT l1, LATEST(GREATEST(dim1, dim2)) IS NULL FROM
druid.numfoo GROUP BY l1";
@@ -15328,6 +15370,7 @@ public class CalciteQueryTest extends
BaseCalciteQueryTest
@Test
public void testComplexDecodeAgg()
{
+ cannotVectorizeUnlessFallback();
msqIncompatible();
testQuery(
"SELECT
APPROX_COUNT_DISTINCT_BUILTIN(COMPLEX_DECODE_BASE64('hyperUnique',PARSE_JSON(TO_JSON_STRING(unique_dim1))))
from druid.foo",
@@ -15361,6 +15404,7 @@ public class CalciteQueryTest extends
BaseCalciteQueryTest
@Test
public void testComplexDecodeAggWithCastedTypeName()
{
+ cannotVectorizeUnlessFallback();
msqIncompatible();
testQuery(
"SELECT "
@@ -16397,6 +16441,7 @@ public class CalciteQueryTest extends
BaseCalciteQueryTest
@Test
public void testGroupingSetsWithAggregateCase()
{
+ cannotVectorizeUnlessFallback();
msqIncompatible();
final Map<String, Object> queryContext = ImmutableMap.of(
PlannerConfig.CTX_KEY_USE_APPROXIMATE_COUNT_DISTINCT, false,
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java
index 02b49af5880..e053da4d744 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java
@@ -650,6 +650,9 @@ public class CalciteSelectQueryTest extends
BaseCalciteQueryTest
@Test
public void testSelectDistinctWithCascadeExtractionFilter()
{
+ if (NullHandling.sqlCompatible()) {
+ cannotVectorizeUnlessFallback();
+ }
testQuery(
"SELECT distinct dim1 FROM druid.foo WHERE substring(substring(dim1,
2), 1, 1) = 'e' OR dim2 = 'a'",
ImmutableList.of(
@@ -694,6 +697,7 @@ public class CalciteSelectQueryTest extends
BaseCalciteQueryTest
@Test
public void testSelectDistinctWithStrlenFilter()
{
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT distinct dim1 FROM druid.foo "
+ "WHERE CHARACTER_LENGTH(dim1) = 3 OR CAST(CHARACTER_LENGTH(dim1)
AS varchar) = 3",
@@ -2016,6 +2020,7 @@ public class CalciteSelectQueryTest extends
BaseCalciteQueryTest
@Test
public void testCountDistinctNonApproximateWithFilter()
{
+ cannotVectorizeUnlessFallback();
testQuery(
PLANNER_CONFIG_DEFAULT.withOverrides(
ImmutableMap.of(
@@ -2054,6 +2059,7 @@ public class CalciteSelectQueryTest extends
BaseCalciteQueryTest
@Test
public void testCountDistinctNonApproximateWithFilterHaving()
{
+ cannotVectorizeUnlessFallback();
testQuery(
PLANNER_CONFIG_DEFAULT.withOverrides(
ImmutableMap.of(
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java
index 03d9357c7d4..b8d638e59ca 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java
@@ -456,6 +456,7 @@ public class CalciteSubqueryTest extends
BaseCalciteQueryTest
if (!queryContext.containsKey(QueryContexts.MAX_SUBQUERY_BYTES_KEY)) {
cannotVectorize();
}
+ cannotVectorizeUnlessFallback();
testQuery(
"SELECT TIME_FORMAT(\"date\", 'yyyy-MM'), SUM(x)\n"
+ "FROM (\n"
@@ -1611,6 +1612,7 @@ public class CalciteSubqueryTest extends
BaseCalciteQueryTest
@ParameterizedTest(name = "{0}")
public void testScalarInArrayToUseHavingFilter(String testName, Map<String,
Object> queryContext)
{
+ cannotVectorizeUnlessFallback();
DimFilter filter = NullHandling.replaceWithDefault()
? new InDimFilter("v0", new
HashSet<>(Arrays.asList("1", "17")))
: new TypedInFilter("v0", ColumnType.LONG, null,
ImmutableList.of(1, 17), null);
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteTimeBoundaryQueryTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteTimeBoundaryQueryTest.java
index 8fde5d166b6..75b786d1f18 100644
---
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteTimeBoundaryQueryTest.java
+++
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteTimeBoundaryQueryTest.java
@@ -141,6 +141,7 @@ public class CalciteTimeBoundaryQueryTest extends
BaseCalciteQueryTest
@Test
public void testMinTimeQueryWithTimeAndExpressionFilters()
{
+ cannotVectorizeUnlessFallback();
HashMap<String, Object> queryContext = new
HashMap<>(QUERY_CONTEXT_DEFAULT);
queryContext.put(QueryContexts.TIME_BOUNDARY_PLANNING_KEY, true);
testQuery(
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledExtension.java
b/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledExtension.java
index 094a23cb1d6..3e47f90c008 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledExtension.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledExtension.java
@@ -21,6 +21,7 @@ package org.apache.druid.sql.calcite;
import com.google.common.collect.ImmutableMap;
import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.query.QueryContexts;
import org.apache.druid.quidem.DruidQTestInfo;
import org.apache.druid.quidem.ProjectPathUtils;
@@ -52,10 +53,16 @@ public class DecoupledExtension implements
BeforeEachCallback
}
private static final ImmutableMap<String, Object> CONTEXT_OVERRIDES =
ImmutableMap.<String, Object>builder()
- .putAll(BaseCalciteQueryTest.QUERY_CONTEXT_DEFAULT)
- .put(PlannerConfig.CTX_NATIVE_QUERY_SQL_PLANNING_MODE,
PlannerConfig.NATIVE_QUERY_SQL_PLANNING_MODE_DECOUPLED)
- .put(QueryContexts.ENABLE_DEBUG, true)
- .build();
+
.putAll(BaseCalciteQueryTest.QUERY_CONTEXT_DEFAULT)
+
.put(
+
PlannerConfig.CTX_NATIVE_QUERY_SQL_PLANNING_MODE,
+
PlannerConfig.NATIVE_QUERY_SQL_PLANNING_MODE_DECOUPLED
+
)
+
.put(
+
QueryContexts.ENABLE_DEBUG,
+
true
+
)
+
.build();
public QueryTestBuilder testBuilder()
{
@@ -93,17 +100,19 @@ public class DecoupledExtension implements
BeforeEachCallback
qCaseDir,
testName,
"quidem testcase reason: " + decTestConfig.quidemReason()
- );
+ );
} else {
return null;
}
}
};
- QueryTestBuilder builder = new QueryTestBuilder(testConfig)
- .cannotVectorize(baseTest.cannotVectorize)
- .skipVectorize(baseTest.skipVectorize);
+ QueryTestBuilder builder = new QueryTestBuilder(testConfig);
- return builder;
+ return builder.cannotVectorize(
+ baseTest.cannotVectorize ||
+ (!ExpressionProcessing.allowVectorizeFallback() &&
baseTest.cannotVectorizeUnlessFallback)
+ )
+ .skipVectorize(baseTest.skipVectorize);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
