This is an automated email from the ASF dual-hosted git repository.
siddteotia pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new d49d1173f9 Support filtering on bool/scalar fields without evaluator
(#8518)
d49d1173f9 is described below
commit d49d1173f9b4b7f1e4e834db593dc6def20a9217
Author: Vivek Iyer Vaidyanathan <[email protected]>
AuthorDate: Mon May 2 16:58:46 2022 -0700
Support filtering on bool/scalar fields without evaluator (#8518)
* Support filtering on bool/scalar fields without evaluator
Fixes the bugs in #8444 and #8487.
Added unit tests.
* Address review comments and refactor code
* Address review comments 2
* RequestContextUtil changes
* Review comments in RequestContextUtils
* Refactor RequestContextUtils
* Refactor ComparisonPredicateRewriter
Co-authored-by: Vivek Iyer Vaidyanathan <[email protected]>
---
.../request/context/RequestContextUtils.java | 69 ++++-
.../rewriter/PredicateComparisonRewriter.java | 89 +++++-
.../pinot/sql/parsers/CalciteSqlCompilerTest.java | 326 +++++++++++++++------
.../core/query/request/context/QueryContext.java | 3 -
.../apache/pinot/queries/BooleanQueriesTest.java | 15 +
.../pinot/queries/ExplainPlanQueriesTest.java | 69 ++++-
.../pinot/queries/FilteredAggregationsTest.java | 22 ++
7 files changed, 465 insertions(+), 128 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/request/context/RequestContextUtils.java
b/pinot-common/src/main/java/org/apache/pinot/common/request/context/RequestContextUtils.java
index 2fb53b0a6f..3dbb3fd046 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/request/context/RequestContextUtils.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/request/context/RequestContextUtils.java
@@ -21,6 +21,7 @@ package org.apache.pinot.common.request.context;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import org.apache.commons.lang3.EnumUtils;
import org.apache.pinot.common.request.Expression;
import org.apache.pinot.common.request.ExpressionType;
import org.apache.pinot.common.request.FilterOperator;
@@ -37,6 +38,7 @@ import
org.apache.pinot.common.request.context.predicate.RegexpLikePredicate;
import org.apache.pinot.common.request.context.predicate.TextMatchPredicate;
import org.apache.pinot.common.utils.RegexpPatternConverterUtils;
import org.apache.pinot.common.utils.request.FilterQueryTree;
+import org.apache.pinot.common.utils.request.RequestUtils;
import org.apache.pinot.pql.parsers.Pql2Compiler;
import org.apache.pinot.pql.parsers.pql2.ast.AstNode;
import org.apache.pinot.pql.parsers.pql2.ast.FilterKind;
@@ -163,10 +165,37 @@ public class RequestContextUtils {
/**
* Converts the given Thrift {@link Expression} into a {@link FilterContext}.
* <p>NOTE: Currently the query engine only accepts string literals as the
right-hand side of the predicate, so we
- * always convert the right-hand side expressions into strings.
+ * always convert the right-hand side expressions into strings. We
also update boolean predicates that are
+ * missing an EQUALS filter operator.
*/
public static FilterContext getFilter(Expression thriftExpression) {
- Function thriftFunction = thriftExpression.getFunctionCall();
+ ExpressionType type = thriftExpression.getType();
+ switch (type) {
+ case FUNCTION:
+ Function thriftFunction = thriftExpression.getFunctionCall();
+ return getFilter(thriftFunction);
+ case IDENTIFIER:
+ // Convert "WHERE a" to "WHERE a = true"
+ return new FilterContext(FilterContext.Type.PREDICATE, null,
+ new EqPredicate(getExpression(thriftExpression),
getStringValue(RequestUtils.getLiteralExpression(true))));
+ case LITERAL:
+ // TODO: Handle literals.
+ throw new IllegalStateException();
+ default:
+ throw new IllegalStateException();
+ }
+ }
+
+ public static FilterContext getFilter(Function thriftFunction) {
+ String functionOperator = thriftFunction.getOperator();
+
+ // convert "WHERE startsWith(col, 'str')" to "WHERE startsWith(col, 'str')
= true"
+ if (!EnumUtils.isValidEnum(FilterKind.class, functionOperator)) {
+ return new FilterContext(FilterContext.Type.PREDICATE, null,
+ new
EqPredicate(ExpressionContext.forFunction(getFunction(thriftFunction)),
+ getStringValue(RequestUtils.getLiteralExpression(true))));
+ }
+
FilterKind filterKind =
FilterKind.valueOf(thriftFunction.getOperator().toUpperCase());
List<Expression> operands = thriftFunction.getOperands();
int numOperands = operands.size();
@@ -185,7 +214,8 @@ public class RequestContextUtils {
return new FilterContext(FilterContext.Type.OR, children, null);
case NOT:
assert numOperands == 1;
- return new FilterContext(FilterContext.Type.NOT, new
ArrayList<>(Collections.singletonList(getFilter(operands.get(0)))), null);
+ return new FilterContext(FilterContext.Type.NOT,
+ new
ArrayList<>(Collections.singletonList(getFilter(operands.get(0)))), null);
case EQUALS:
return new FilterContext(FilterContext.Type.PREDICATE, null,
new EqPredicate(getExpression(operands.get(0)),
getStringValue(operands.get(1))));
@@ -264,10 +294,36 @@ public class RequestContextUtils {
/**
* Converts the given filter {@link ExpressionContext} into a {@link
FilterContext}.
* <p>NOTE: Currently the query engine only accepts string literals as the
right-hand side of the predicate, so we
- * always convert the right-hand side expressions into strings.
+ * always convert the right-hand side expressions into strings. We
also update boolean predicates that are
+ * missing an EQUALS filter operator.
*/
public static FilterContext getFilter(ExpressionContext filterExpression) {
- FunctionContext filterFunction = filterExpression.getFunction();
+ ExpressionContext.Type type = filterExpression.getType();
+ switch (type) {
+ case FUNCTION:
+ FunctionContext filterFunction = filterExpression.getFunction();
+ return getFilter(filterFunction);
+ case IDENTIFIER:
+ return new FilterContext(FilterContext.Type.PREDICATE, null,
+ new EqPredicate(filterExpression,
getStringValue(RequestUtils.getLiteralExpression(true))));
+ case LITERAL:
+ // TODO: Handle literals
+ throw new IllegalStateException();
+ default:
+ throw new IllegalStateException();
+ }
+ }
+
+ public static FilterContext getFilter(FunctionContext filterFunction) {
+ String functionOperator = filterFunction.getFunctionName().toUpperCase();
+
+ // convert "WHERE startsWith(col, 'str')" to "WHERE startsWith(col, 'str')
= true"
+ if (!EnumUtils.isValidEnum(FilterKind.class, functionOperator)) {
+ return new FilterContext(FilterContext.Type.PREDICATE, null,
+ new EqPredicate(ExpressionContext.forFunction(filterFunction),
+ getStringValue(RequestUtils.getLiteralExpression(true))));
+ }
+
FilterKind filterKind =
FilterKind.valueOf(filterFunction.getFunctionName().toUpperCase());
List<ExpressionContext> operands = filterFunction.getArguments();
int numOperands = operands.size();
@@ -286,7 +342,8 @@ public class RequestContextUtils {
return new FilterContext(FilterContext.Type.OR, children, null);
case NOT:
assert numOperands == 1;
- return new FilterContext(FilterContext.Type.NOT, new
ArrayList<>(Collections.singletonList(getFilter(operands.get(0)))), null);
+ return new FilterContext(FilterContext.Type.NOT,
+ new
ArrayList<>(Collections.singletonList(getFilter(operands.get(0)))), null);
case EQUALS:
return new FilterContext(FilterContext.Type.PREDICATE, null,
new EqPredicate(operands.get(0), getStringValue(operands.get(1))));
diff --git
a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/rewriter/PredicateComparisonRewriter.java
b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/rewriter/PredicateComparisonRewriter.java
index ae5c8ae7a6..d24945baf0 100644
---
a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/rewriter/PredicateComparisonRewriter.java
+++
b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/rewriter/PredicateComparisonRewriter.java
@@ -18,9 +18,12 @@
*/
package org.apache.pinot.sql.parsers.rewriter;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import org.apache.commons.lang3.EnumUtils;
import org.apache.pinot.common.request.Expression;
+import org.apache.pinot.common.request.ExpressionType;
import org.apache.pinot.common.request.Function;
import org.apache.pinot.common.request.PinotQuery;
import org.apache.pinot.common.utils.request.RequestUtils;
@@ -33,34 +36,69 @@ public class PredicateComparisonRewriter implements
QueryRewriter {
public PinotQuery rewrite(PinotQuery pinotQuery) {
Expression filterExpression = pinotQuery.getFilterExpression();
if (filterExpression != null) {
-
pinotQuery.setFilterExpression(updateComparisonPredicate(filterExpression));
+ pinotQuery.setFilterExpression(updatePredicate(filterExpression));
}
Expression havingExpression = pinotQuery.getHavingExpression();
if (havingExpression != null) {
-
pinotQuery.setHavingExpression(updateComparisonPredicate(havingExpression));
+ pinotQuery.setHavingExpression(updatePredicate(havingExpression));
}
return pinotQuery;
}
- // This method converts a predicate expression to the what Pinot could
evaluate.
- // For comparison expression, left operand could be any expression, but
right operand only
- // supports literal.
- // E.g. 'WHERE a > b' will be updated to 'WHERE a - b > 0'
- private static Expression updateComparisonPredicate(Expression expression) {
+ /**
+ * This method converts an expression to what Pinot could evaluate.
+ * 1. For comparison expression, left operand could be any expression, but
right operand only
+ * supports literal. E.g. 'WHERE a > b' will be converted to 'WHERE a - b
> 0'
+ * 2. Updates boolean predicates (literals and scalar functions) that are
missing an EQUALS filter.
+ * E.g. 1: 'WHERE a' will be updated to 'WHERE a = true'
+ * E.g. 2: "WHERE startsWith(col, 'str')" will be updated to "WHERE
startsWith(col, 'str') = true"
+ *
+ * @param expression current expression in the expression tree
+ * @return re-written expression.
+ */
+ private static Expression updatePredicate(Expression expression) {
+ ExpressionType type = expression.getType();
+
+ switch (type) {
+ case FUNCTION:
+ return updateFunctionExpression(expression);
+ case IDENTIFIER:
+ return convertPredicateToEqualsBooleanExpression(expression);
+ case LITERAL:
+ // TODO: Convert literals to boolean expressions
+ return expression;
+ default:
+ throw new IllegalStateException();
+ }
+ }
+
+ /**
+ * Rewrites a function expression.
+ *
+ * @param expression
+ * @return re-written expression
+ */
+ private static Expression updateFunctionExpression(Expression expression) {
Function function = expression.getFunctionCall();
- if (function != null) {
- FilterKind filterKind;
- try {
- filterKind = FilterKind.valueOf(function.getOperator());
- } catch (Exception e) {
- throw new SqlCompilationException("Unsupported filter kind: " +
function.getOperator());
- }
+ String functionOperator = function.getOperator();
+
+ if (!EnumUtils.isValidEnum(FilterKind.class, functionOperator)) {
+ // If the function is not of FilterKind, we have to rewrite the function.
+ // Example: A query like "select col1 from table where startsWith(col1,
'myStr') AND col2 > 10;" should be
+ // rewritten to "select col1 from table where startsWith(col1,
'myStr') = true AND col2 > 10;".
+ expression = convertPredicateToEqualsBooleanExpression(expression);
+ return expression;
+ } else {
+ FilterKind filterKind = FilterKind.valueOf(function.getOperator());
List<Expression> operands = function.getOperands();
switch (filterKind) {
case AND:
case OR:
case NOT:
-
operands.replaceAll(PredicateComparisonRewriter::updateComparisonPredicate);
+ for (int i = 0; i < operands.size(); i++) {
+ Expression operand = operands.get(i);
+ operands.set(i, updatePredicate(operand));
+ }
break;
case EQUALS:
case NOT_EQUALS:
@@ -102,9 +140,30 @@ public class PredicateComparisonRewriter implements
QueryRewriter {
break;
}
}
+
return expression;
}
+ /**
+ * Rewrite predicates to boolean expressions with EQUALS operator
+ * Example1: "select * from table where col1" converts to
+ * "select * from table where col1 = true"
+ * Example2: "select * from table where startsWith(col1, 'str')"
converts to
+ * "select * from table where startsWith(col1, 'str') = true"
+ * @param expression Expression
+ * @return Rewritten expression
+ */
+ private static Expression
convertPredicateToEqualsBooleanExpression(Expression expression) {
+ Expression newExpression;
+ newExpression =
RequestUtils.getFunctionExpression(FilterKind.EQUALS.name());
+ List<Expression> operands = new ArrayList<>();
+ operands.add(expression);
+ operands.add(RequestUtils.getLiteralExpression(true));
+ newExpression.getFunctionCall().setOperands(operands);
+
+ return newExpression;
+ }
+
/**
* The purpose of this method is to convert expression "0 < columnA" to
"columnA > 0".
* The conversion would be:
diff --git
a/pinot-common/src/test/java/org/apache/pinot/sql/parsers/CalciteSqlCompilerTest.java
b/pinot-common/src/test/java/org/apache/pinot/sql/parsers/CalciteSqlCompilerTest.java
index 0bb6035986..b206951c00 100644
---
a/pinot-common/src/test/java/org/apache/pinot/sql/parsers/CalciteSqlCompilerTest.java
+++
b/pinot-common/src/test/java/org/apache/pinot/sql/parsers/CalciteSqlCompilerTest.java
@@ -194,45 +194,148 @@ public class CalciteSqlCompilerTest {
@Test
public void testFilterClauses() {
- PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery("select *
from vegetables where a > 1.5");
- Function func = pinotQuery.getFilterExpression().getFunctionCall();
- Assert.assertEquals(func.getOperator(), FilterKind.GREATER_THAN.name());
- Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(),
"a");
-
Assert.assertEquals(func.getOperands().get(1).getLiteral().getDoubleValue(),
1.5);
- pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from
vegetables where b < 100");
- func = pinotQuery.getFilterExpression().getFunctionCall();
- Assert.assertEquals(func.getOperator(), FilterKind.LESS_THAN.name());
- Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(),
"b");
- Assert.assertEquals(func.getOperands().get(1).getLiteral().getLongValue(),
100L);
- pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from
vegetables where c >= 10");
- func = pinotQuery.getFilterExpression().getFunctionCall();
- Assert.assertEquals(func.getOperator(),
FilterKind.GREATER_THAN_OR_EQUAL.name());
- Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(),
"c");
- Assert.assertEquals(func.getOperands().get(1).getLiteral().getLongValue(),
10L);
- pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from
vegetables where d <= 50");
- func = pinotQuery.getFilterExpression().getFunctionCall();
- Assert.assertEquals(func.getOperator(),
FilterKind.LESS_THAN_OR_EQUAL.name());
- Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(),
"d");
- Assert.assertEquals(func.getOperands().get(1).getLiteral().getLongValue(),
50L);
- pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from
vegetables where e BETWEEN 70 AND 80");
- func = pinotQuery.getFilterExpression().getFunctionCall();
- Assert.assertEquals(func.getOperator(), FilterKind.BETWEEN.name());
- Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(),
"e");
- Assert.assertEquals(func.getOperands().get(1).getLiteral().getLongValue(),
70L);
- Assert.assertEquals(func.getOperands().get(2).getLiteral().getLongValue(),
80L);
- pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from
vegetables where regexp_like(E, '^U.*')");
- func = pinotQuery.getFilterExpression().getFunctionCall();
- Assert.assertEquals(func.getOperator(), "REGEXP_LIKE");
- Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(),
"E");
-
Assert.assertEquals(func.getOperands().get(1).getLiteral().getStringValue(),
"^U.*");
- pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from
vegetables where g IN (12, 13, 15.2, 17)");
- func = pinotQuery.getFilterExpression().getFunctionCall();
- Assert.assertEquals(func.getOperator(), FilterKind.IN.name());
- Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(),
"g");
- Assert.assertEquals(func.getOperands().get(1).getLiteral().getLongValue(),
12L);
- Assert.assertEquals(func.getOperands().get(2).getLiteral().getLongValue(),
13L);
-
Assert.assertEquals(func.getOperands().get(3).getLiteral().getDoubleValue(),
15.2);
- Assert.assertEquals(func.getOperands().get(4).getLiteral().getLongValue(),
17L);
+ {
+ PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery("select *
from vegetables where a > 1.5");
+ Function func = pinotQuery.getFilterExpression().getFunctionCall();
+ Assert.assertEquals(func.getOperator(), FilterKind.GREATER_THAN.name());
+ Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(),
"a");
+
Assert.assertEquals(func.getOperands().get(1).getLiteral().getDoubleValue(),
1.5);
+ }
+
+ {
+ PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery("select *
from vegetables where b < 100");
+ Function func = pinotQuery.getFilterExpression().getFunctionCall();
+ Assert.assertEquals(func.getOperator(), FilterKind.LESS_THAN.name());
+ Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(),
"b");
+
Assert.assertEquals(func.getOperands().get(1).getLiteral().getLongValue(),
100L);
+ }
+
+ {
+ PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery("select *
from vegetables where c >= 10");
+ Function func = pinotQuery.getFilterExpression().getFunctionCall();
+ Assert.assertEquals(func.getOperator(),
FilterKind.GREATER_THAN_OR_EQUAL.name());
+ Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(),
"c");
+
Assert.assertEquals(func.getOperands().get(1).getLiteral().getLongValue(), 10L);
+ }
+
+ {
+ PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery("select *
from vegetables where d <= 50");
+ Function func = pinotQuery.getFilterExpression().getFunctionCall();
+ Assert.assertEquals(func.getOperator(),
FilterKind.LESS_THAN_OR_EQUAL.name());
+ Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(),
"d");
+
Assert.assertEquals(func.getOperands().get(1).getLiteral().getLongValue(), 50L);
+ }
+
+ {
+ PinotQuery pinotQuery =
+ CalciteSqlParser.compileToPinotQuery("select * from vegetables where
e BETWEEN 70 AND 80");
+ Function func = pinotQuery.getFilterExpression().getFunctionCall();
+ Assert.assertEquals(func.getOperator(), FilterKind.BETWEEN.name());
+ Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(),
"e");
+
Assert.assertEquals(func.getOperands().get(1).getLiteral().getLongValue(), 70L);
+
Assert.assertEquals(func.getOperands().get(2).getLiteral().getLongValue(), 80L);
+ }
+
+ {
+ PinotQuery pinotQuery =
+ CalciteSqlParser.compileToPinotQuery("select * from vegetables where
regexp_like(E, '^U.*')");
+ Function func = pinotQuery.getFilterExpression().getFunctionCall();
+ Assert.assertEquals(func.getOperator(), "REGEXP_LIKE");
+ Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(),
"E");
+
Assert.assertEquals(func.getOperands().get(1).getLiteral().getStringValue(),
"^U.*");
+ }
+
+ {
+ PinotQuery pinotQuery =
+ CalciteSqlParser.compileToPinotQuery("select * from vegetables where
g IN (12, 13, 15.2, 17)");
+ Function func = pinotQuery.getFilterExpression().getFunctionCall();
+ Assert.assertEquals(func.getOperator(), FilterKind.IN.name());
+ Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(),
"g");
+
Assert.assertEquals(func.getOperands().get(1).getLiteral().getLongValue(), 12L);
+
Assert.assertEquals(func.getOperands().get(2).getLiteral().getLongValue(), 13L);
+
Assert.assertEquals(func.getOperands().get(3).getLiteral().getDoubleValue(),
15.2);
+
Assert.assertEquals(func.getOperands().get(4).getLiteral().getLongValue(), 17L);
+ }
+
+ {
+ PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery("select *
from vegetable where g");
+ Function func = pinotQuery.getFilterExpression().getFunctionCall();
+ Assert.assertEquals(func.getOperator(), FilterKind.EQUALS.name());
+ Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(),
"g");
+ Assert.assertEquals(func.getOperands().get(1).getLiteral(),
Literal.boolValue(true));
+ }
+
+ {
+ PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery("select *
from vegetable where g or f = true");
+ Function func = pinotQuery.getFilterExpression().getFunctionCall();
+ Assert.assertEquals(func.getOperator(), FilterKind.OR.name());
+ List<Expression> operands = func.getOperands();
+ Assert.assertEquals(operands.size(), 2);
+ Assert.assertEquals(operands.get(0).getFunctionCall().getOperator(),
FilterKind.EQUALS.name());
+ List<Expression> eqOperands =
operands.get(0).getFunctionCall().getOperands();
+ Assert.assertEquals(eqOperands.get(0).getIdentifier().getName(), "g");
+ Assert.assertEquals(eqOperands.get(1).getLiteral(),
Literal.boolValue(true));
+ eqOperands = operands.get(1).getFunctionCall().getOperands();
+ Assert.assertEquals(eqOperands.get(0).getIdentifier().getName(), "f");
+ Assert.assertEquals(eqOperands.get(1).getLiteral(),
Literal.stringValue("true"));
+ }
+
+ {
+ PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery("select *
from vegetable where startsWith(g, "
+ + "'str')");
+ Function func = pinotQuery.getFilterExpression().getFunctionCall();
+ Assert.assertEquals(func.getOperator(), FilterKind.EQUALS.name());
+
Assert.assertEquals(func.getOperands().get(0).getFunctionCall().getOperator(),
"startswith");
+ Assert.assertEquals(func.getOperands().get(1).getLiteral(),
Literal.boolValue(true));
+ }
+
+ {
+ PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery("select *
from vegetable where startsWith(g, "
+ + "'str')=true and startsWith(f, 'str')");
+ Function func = pinotQuery.getFilterExpression().getFunctionCall();
+ Assert.assertEquals(func.getOperator(), FilterKind.AND.name());
+ List<Expression> operands = func.getOperands();
+ Assert.assertEquals(operands.size(), 2);
+
+ Assert.assertEquals(operands.get(0).getFunctionCall().getOperator(),
FilterKind.EQUALS.name());
+ List<Expression> eqOperands =
operands.get(0).getFunctionCall().getOperands();
+ Assert.assertEquals(eqOperands.get(0).getFunctionCall().getOperator(),
"startswith");
+ Assert.assertEquals(eqOperands.get(1).getLiteral(),
Literal.stringValue("true"));
+
+ Assert.assertEquals(operands.get(1).getFunctionCall().getOperator(),
FilterKind.EQUALS.name());
+ eqOperands = operands.get(1).getFunctionCall().getOperands();
+ Assert.assertEquals(eqOperands.get(0).getFunctionCall().getOperator(),
"startswith");
+ Assert.assertEquals(eqOperands.get(1).getLiteral(),
Literal.boolValue(true));
+ }
+
+ {
+ PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery("select *
from vegetable where (startsWith(g, "
+ + "'str')=true and startsWith(f, 'str')) AND (e and d=true)");
+ Function func = pinotQuery.getFilterExpression().getFunctionCall();
+ Assert.assertEquals(func.getOperator(), FilterKind.AND.name());
+ List<Expression> operands = func.getOperands();
+ Assert.assertEquals(operands.size(), 4);
+
+ Assert.assertEquals(operands.get(0).getFunctionCall().getOperator(),
FilterKind.EQUALS.name());
+ List<Expression> eqOperands =
operands.get(0).getFunctionCall().getOperands();
+ Assert.assertEquals(eqOperands.get(0).getFunctionCall().getOperator(),
"startswith");
+ Assert.assertEquals(eqOperands.get(1).getLiteral(),
Literal.stringValue("true"));
+
+ Assert.assertEquals(operands.get(1).getFunctionCall().getOperator(),
FilterKind.EQUALS.name());
+ eqOperands = operands.get(1).getFunctionCall().getOperands();
+ Assert.assertEquals(eqOperands.get(0).getFunctionCall().getOperator(),
"startswith");
+ Assert.assertEquals(eqOperands.get(1).getLiteral(),
Literal.boolValue(true));
+
+ Assert.assertEquals(operands.get(2).getFunctionCall().getOperator(),
FilterKind.EQUALS.name());
+ eqOperands = operands.get(2).getFunctionCall().getOperands();
+ Assert.assertEquals(eqOperands.get(0).getIdentifier().getName(), "e");
+ Assert.assertEquals(eqOperands.get(1).getLiteral(),
Literal.boolValue(true));
+
+ Assert.assertEquals(operands.get(3).getFunctionCall().getOperator(),
FilterKind.EQUALS.name());
+ eqOperands = operands.get(3).getFunctionCall().getOperands();
+ Assert.assertEquals(eqOperands.get(0).getIdentifier().getName(), "d");
+ Assert.assertEquals(eqOperands.get(1).getLiteral(),
Literal.stringValue("true"));
+ }
}
@Test
@@ -241,29 +344,30 @@ public class CalciteSqlCompilerTest {
Function func = pinotQuery.getFilterExpression().getFunctionCall();
Assert.assertEquals(func.getOperator(), FilterKind.GREATER_THAN.name());
Assert.assertEquals(func.getOperands().get(0).getFunctionCall().getOperator(),
"minus");
-
Assert.assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(0).getIdentifier().getName(),
- "a");
-
Assert.assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(1).getIdentifier().getName(),
- "b");
+ Assert
+
.assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(0).getIdentifier().getName(),
"a");
+ Assert
+
.assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(1).getIdentifier().getName(),
"b");
Assert.assertEquals(func.getOperands().get(1).getLiteral().getLongValue(),
0L);
pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from
vegetables where 0 < a-b");
func = pinotQuery.getFilterExpression().getFunctionCall();
Assert.assertEquals(func.getOperator(), FilterKind.GREATER_THAN.name());
Assert.assertEquals(func.getOperands().get(0).getFunctionCall().getOperator(),
"minus");
-
Assert.assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(0).getIdentifier().getName(),
- "a");
-
Assert.assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(1).getIdentifier().getName(),
- "b");
+ Assert
+
.assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(0).getIdentifier().getName(),
"a");
+ Assert
+
.assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(1).getIdentifier().getName(),
"b");
Assert.assertEquals(func.getOperands().get(1).getLiteral().getLongValue(),
0L);
pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from
vegetables where b < 100 + c");
func = pinotQuery.getFilterExpression().getFunctionCall();
Assert.assertEquals(func.getOperator(), FilterKind.LESS_THAN.name());
Assert.assertEquals(func.getOperands().get(0).getFunctionCall().getOperator(),
"minus");
-
Assert.assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(0).getIdentifier().getName(),
- "b");
- Assert.assertEquals(
-
func.getOperands().get(0).getFunctionCall().getOperands().get(1).getFunctionCall().getOperator(),
"plus");
+ Assert
+
.assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(0).getIdentifier().getName(),
"b");
+ Assert
+
.assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(1).getFunctionCall().getOperator(),
+ "plus");
Assert.assertEquals(
func.getOperands().get(0).getFunctionCall().getOperands().get(1).getFunctionCall().getOperands().get(0)
.getLiteral().getLongValue(), 100L);
@@ -275,10 +379,11 @@ public class CalciteSqlCompilerTest {
func = pinotQuery.getFilterExpression().getFunctionCall();
Assert.assertEquals(func.getOperator(), FilterKind.LESS_THAN.name());
Assert.assertEquals(func.getOperands().get(0).getFunctionCall().getOperator(),
"minus");
-
Assert.assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(0).getIdentifier().getName(),
- "b");
- Assert.assertEquals(
-
func.getOperands().get(0).getFunctionCall().getOperands().get(1).getFunctionCall().getOperator(),
"plus");
+ Assert
+
.assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(0).getIdentifier().getName(),
"b");
+ Assert
+
.assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(1).getFunctionCall().getOperator(),
+ "plus");
Assert.assertEquals(
func.getOperands().get(0).getFunctionCall().getOperands().get(1).getFunctionCall().getOperands().get(0)
.getLiteral().getLongValue(), 100L);
@@ -292,10 +397,12 @@ public class CalciteSqlCompilerTest {
func = pinotQuery.getFilterExpression().getFunctionCall();
Assert.assertEquals(func.getOperator(),
FilterKind.LESS_THAN_OR_EQUAL.name());
Assert.assertEquals(func.getOperands().get(0).getFunctionCall().getOperator(),
"minus");
- Assert.assertEquals(
-
func.getOperands().get(0).getFunctionCall().getOperands().get(0).getFunctionCall().getOperator(),
"foo1");
- Assert.assertEquals(
-
func.getOperands().get(0).getFunctionCall().getOperands().get(1).getFunctionCall().getOperator(),
"foo2");
+ Assert
+
.assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(0).getFunctionCall().getOperator(),
+ "foo1");
+ Assert
+
.assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(1).getFunctionCall().getOperator(),
+ "foo2");
Assert.assertEquals(
func.getOperands().get(0).getFunctionCall().getOperands().get(0).getFunctionCall().getOperands().get(0)
.getFunctionCall().getOperator(), "bar1");
@@ -330,10 +437,12 @@ public class CalciteSqlCompilerTest {
func = pinotQuery.getFilterExpression().getFunctionCall();
Assert.assertEquals(func.getOperator(),
FilterKind.LESS_THAN_OR_EQUAL.name());
Assert.assertEquals(func.getOperands().get(0).getFunctionCall().getOperator(),
"minus");
- Assert.assertEquals(
-
func.getOperands().get(0).getFunctionCall().getOperands().get(0).getFunctionCall().getOperator(),
"foo1");
- Assert.assertEquals(
-
func.getOperands().get(0).getFunctionCall().getOperands().get(1).getFunctionCall().getOperator(),
"foo2");
+ Assert
+
.assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(0).getFunctionCall().getOperator(),
+ "foo1");
+ Assert
+
.assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(1).getFunctionCall().getOperator(),
+ "foo2");
Assert.assertEquals(
func.getOperands().get(0).getFunctionCall().getOperands().get(0).getFunctionCall().getOperands().get(0)
.getFunctionCall().getOperator(), "bar1");
@@ -676,8 +785,8 @@ public class CalciteSqlCompilerTest {
} catch (SqlCompilationException e) {
// Expected
Assert.assertTrue(e.getCause().getMessage().contains("at line 1, column
31."),
- "Compilation exception should contain line and character for error
message. Error message is "
- + e.getMessage());
+ "Compilation exception should contain line and character for error
message. Error message is " + e
+ .getMessage());
return;
}
@@ -701,8 +810,8 @@ public class CalciteSqlCompilerTest {
Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 0);
Assert.assertNull(pinotQuery.getQueryOptions());
- pinotQuery = CalciteSqlParser.compileToPinotQuery(
- "select * from vegetables where name <> 'Brussels sprouts' OPTION
(delicious=yes)");
+ pinotQuery = CalciteSqlParser
+ .compileToPinotQuery("select * from vegetables where name <> 'Brussels
sprouts' OPTION (delicious=yes)");
Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 1);
Assert.assertTrue(pinotQuery.getQueryOptions().containsKey("delicious"));
Assert.assertEquals(pinotQuery.getQueryOptions().get("delicious"), "yes");
@@ -777,8 +886,8 @@ public class CalciteSqlCompilerTest {
@Test
public void testIdentifierQuoteCharacter() {
- PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery(
- "select avg(attributes.age) as avg_age from person group by
attributes.address_city");
+ PinotQuery pinotQuery = CalciteSqlParser
+ .compileToPinotQuery("select avg(attributes.age) as avg_age from
person group by attributes.address_city");
Assert.assertEquals(
pinotQuery.getSelectList().get(0).getFunctionCall().getOperands().get(0).getFunctionCall().getOperands().get(0)
.getIdentifier().getName(), "attributes.age");
@@ -805,14 +914,15 @@ public class CalciteSqlCompilerTest {
Assert.assertEquals(groupbyList.get(1).getIdentifier().getName(), "bar");
// For UDF, string literal won't be treated as column but as LITERAL
- pinotQuery = CalciteSqlParser.compileToPinotQuery(
- "SELECT SUM(ADD(foo, 'bar')) FROM myTable GROUP BY sub(foo, bar),
SUB(BAR, FOO)");
+ pinotQuery = CalciteSqlParser
+ .compileToPinotQuery("SELECT SUM(ADD(foo, 'bar')) FROM myTable GROUP
BY sub(foo, bar), SUB(BAR, FOO)");
selectFunctionList = pinotQuery.getSelectList();
Assert.assertEquals(selectFunctionList.size(), 1);
Assert.assertEquals(selectFunctionList.get(0).getFunctionCall().getOperator(),
"sum");
Assert.assertEquals(selectFunctionList.get(0).getFunctionCall().getOperands().size(),
1);
- Assert.assertEquals(
-
selectFunctionList.get(0).getFunctionCall().getOperands().get(0).getFunctionCall().getOperator(),
"add");
+ Assert
+
.assertEquals(selectFunctionList.get(0).getFunctionCall().getOperands().get(0).getFunctionCall().getOperator(),
+ "add");
Assert.assertEquals(
selectFunctionList.get(0).getFunctionCall().getOperands().get(0).getFunctionCall().getOperands().size(),
2);
Assert.assertEquals(
@@ -878,8 +988,8 @@ public class CalciteSqlCompilerTest {
@Test
public void testSelectionTransformFunction() {
- PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery(
- " select mapKey(mapField,k1) from baseballStats where
mapKey(mapField,k1) = 'v1'");
+ PinotQuery pinotQuery = CalciteSqlParser
+ .compileToPinotQuery(" select mapKey(mapField,k1) from baseballStats
where mapKey(mapField,k1) = 'v1'");
Assert.assertEquals(pinotQuery.getSelectList().get(0).getFunctionCall().getOperator(),
"mapkey");
Assert.assertEquals(
pinotQuery.getSelectList().get(0).getFunctionCall().getOperands().get(0).getIdentifier().getName(),
"mapField");
@@ -1525,8 +1635,9 @@ public class CalciteSqlCompilerTest {
Assert.assertEquals(
pinotQuery.getSelectList().get(4).getFunctionCall().getOperands().get(1).getIdentifier().getName(),
"avg");
Assert.assertEquals(pinotQuery.getFilterExpression().getFunctionCall().getOperator(),
FilterKind.EQUALS.name());
- Assert.assertEquals(
-
pinotQuery.getFilterExpression().getFunctionCall().getOperands().get(0).getIdentifier().getName(),
"groups");
+ Assert
+
.assertEquals(pinotQuery.getFilterExpression().getFunctionCall().getOperands().get(0).getIdentifier().getName(),
+ "groups");
Assert.assertEquals(
pinotQuery.getFilterExpression().getFunctionCall().getOperands().get(1).getLiteral().getStringValue(),
"foo");
@@ -1946,8 +2057,8 @@ public class CalciteSqlCompilerTest {
expression = pinotQuery.getFilterExpression();
Assert.assertNotNull(expression.getFunctionCall());
Assert.assertEquals(expression.getFunctionCall().getOperator(),
"todatetime");
-
Assert.assertEquals(expression.getFunctionCall().getOperands().get(0).getIdentifier().getName(),
- "millisSinceEpoch");
+ Assert
+
.assertEquals(expression.getFunctionCall().getOperands().get(0).getIdentifier().getName(),
"millisSinceEpoch");
expression = CalciteSqlParser.compileToExpression("encodeUrl('key1=value
1&key2=value@!$2&key3=value%3')");
Assert.assertNotNull(expression.getFunctionCall());
@@ -2047,8 +2158,9 @@ public class CalciteSqlCompilerTest {
pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
brokerRequest = converter.convert(pinotQuery);
Assert.assertEquals(pinotQuery.getFilterExpression().getFunctionCall().getOperator(),
"IS_NOT_NULL");
- Assert.assertEquals(
-
pinotQuery.getFilterExpression().getFunctionCall().getOperands().get(0).getIdentifier().getName(),
"col");
+ Assert
+
.assertEquals(pinotQuery.getFilterExpression().getFunctionCall().getOperands().get(0).getIdentifier().getName(),
+ "col");
Assert.assertEquals(brokerRequest.getFilterQuery().getOperator(),
FilterOperator.IS_NOT_NULL);
Assert.assertEquals(brokerRequest.getFilterQuery().getColumn(), "col");
@@ -2056,8 +2168,9 @@ public class CalciteSqlCompilerTest {
pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
brokerRequest = converter.convert(pinotQuery);
Assert.assertEquals(pinotQuery.getFilterExpression().getFunctionCall().getOperator(),
"IS_NOT_NULL");
- Assert.assertEquals(
-
pinotQuery.getFilterExpression().getFunctionCall().getOperands().get(0).getIdentifier().getName(),
"col");
+ Assert
+
.assertEquals(pinotQuery.getFilterExpression().getFunctionCall().getOperands().get(0).getIdentifier().getName(),
+ "col");
Assert.assertEquals(brokerRequest.getFilterQuery().getOperator(),
FilterOperator.IS_NOT_NULL);
Assert.assertEquals(brokerRequest.getFilterQuery().getColumn(), "col");
@@ -2065,8 +2178,9 @@ public class CalciteSqlCompilerTest {
pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
brokerRequest = converter.convert(pinotQuery);
Assert.assertEquals(pinotQuery.getFilterExpression().getFunctionCall().getOperator(),
"IS_NULL");
- Assert.assertEquals(
-
pinotQuery.getFilterExpression().getFunctionCall().getOperands().get(0).getIdentifier().getName(),
"col");
+ Assert
+
.assertEquals(pinotQuery.getFilterExpression().getFunctionCall().getOperands().get(0).getIdentifier().getName(),
+ "col");
Assert.assertEquals(brokerRequest.getFilterQuery().getOperator(),
FilterOperator.IS_NULL);
Assert.assertEquals(brokerRequest.getFilterQuery().getColumn(), "col");
@@ -2074,8 +2188,9 @@ public class CalciteSqlCompilerTest {
pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
brokerRequest = converter.convert(pinotQuery);
Assert.assertEquals(pinotQuery.getFilterExpression().getFunctionCall().getOperator(),
"IS_NULL");
- Assert.assertEquals(
-
pinotQuery.getFilterExpression().getFunctionCall().getOperands().get(0).getIdentifier().getName(),
"col");
+ Assert
+
.assertEquals(pinotQuery.getFilterExpression().getFunctionCall().getOperands().get(0).getIdentifier().getName(),
+ "col");
Assert.assertEquals(brokerRequest.getFilterQuery().getOperator(),
FilterOperator.IS_NULL);
Assert.assertEquals(brokerRequest.getFilterQuery().getColumn(), "col");
}
@@ -2206,17 +2321,41 @@ public class CalciteSqlCompilerTest {
}
{
- String query = "SELECT * FROM foo WHERE col1 > 0 AND (col2 AND col3 > 0)
AND col4";
+ String query = "SELECT * FROM foo WHERE col1 > 0 AND (col2 AND col3 > 0)
AND startsWith(col4, 'myStr')";
PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
Function functionCall =
pinotQuery.getFilterExpression().getFunctionCall();
Assert.assertEquals(functionCall.getOperator(), FilterKind.AND.name());
List<Expression> operands = functionCall.getOperands();
Assert.assertEquals(operands.size(), 4);
Assert.assertEquals(operands.get(0).getFunctionCall().getOperator(),
FilterKind.GREATER_THAN.name());
- Assert.assertEquals(operands.get(1).getIdentifier().getName(), "col2");
+ Assert.assertEquals(operands.get(1).getFunctionCall().getOperator(),
FilterKind.EQUALS.name());
+ List<Expression> eqOperands =
operands.get(1).getFunctionCall().getOperands();
+ Assert.assertEquals(eqOperands.get(0).getIdentifier(), new
Identifier("col2"));
+ Assert.assertEquals(eqOperands.get(1).getLiteral(),
Literal.boolValue(true));
Assert.assertEquals(operands.get(2).getFunctionCall().getOperator(),
FilterKind.GREATER_THAN.name());
- Assert.assertEquals(operands.get(3).getIdentifier().getName(), "col4");
+ Assert.assertEquals(operands.get(3).getFunctionCall().getOperator(),
FilterKind.EQUALS.name());
+ eqOperands = operands.get(3).getFunctionCall().getOperands();
+ Assert.assertEquals(eqOperands.get(0).getFunctionCall().getOperator(),
"startswith");
+ Assert.assertEquals(eqOperands.get(1).getLiteral(),
Literal.boolValue(true));
+ }
+ {
+ String query = "SELECT * FROM foo WHERE col1 > 0 AND (col2 AND col3 > 0)
AND col4 = true";
+ PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
+ Function functionCall =
pinotQuery.getFilterExpression().getFunctionCall();
+ Assert.assertEquals(functionCall.getOperator(), FilterKind.AND.name());
+ List<Expression> operands = functionCall.getOperands();
+ Assert.assertEquals(operands.size(), 4);
+ Assert.assertEquals(operands.get(0).getFunctionCall().getOperator(),
FilterKind.GREATER_THAN.name());
+ Assert.assertEquals(operands.get(1).getFunctionCall().getOperator(),
FilterKind.EQUALS.name());
+ List<Expression> eqOperands =
operands.get(1).getFunctionCall().getOperands();
+ Assert.assertEquals(eqOperands.get(0).getIdentifier(), new
Identifier("col2"));
+ Assert.assertEquals(eqOperands.get(1).getLiteral(),
Literal.boolValue(true));
+ Assert.assertEquals(operands.get(2).getFunctionCall().getOperator(),
FilterKind.GREATER_THAN.name());
+ Assert.assertEquals(operands.get(3).getFunctionCall().getOperator(),
FilterKind.EQUALS.name());
+ eqOperands = operands.get(3).getFunctionCall().getOperands();
+ Assert.assertEquals(eqOperands.get(0).getIdentifier(), new
Identifier("col4"));
+ Assert.assertEquals(eqOperands.get(1).getLiteral(),
Literal.stringValue("true"));
// NOTE: PQL does not support logical identifier
}
@@ -2249,9 +2388,12 @@ public class CalciteSqlCompilerTest {
List<Expression> operands = functionCall.getOperands();
Assert.assertEquals(operands.size(), 4);
Assert.assertEquals(operands.get(0).getFunctionCall().getOperator(),
FilterKind.LESS_THAN_OR_EQUAL.name());
- Assert.assertEquals(operands.get(1).getIdentifier().getName(), "col2");
+ Assert.assertEquals(operands.get(1).getFunctionCall().getOperator(),
FilterKind.EQUALS.name());
Assert.assertEquals(operands.get(2).getFunctionCall().getOperator(),
FilterKind.LESS_THAN_OR_EQUAL.name());
- Assert.assertEquals(operands.get(3).getIdentifier().getName(), "col4");
+ Assert.assertEquals(operands.get(3).getFunctionCall().getOperator(),
FilterKind.EQUALS.name());
+ List<Expression> eqOperands =
operands.get(3).getFunctionCall().getOperands();
+ Assert.assertEquals(eqOperands.get(0).getIdentifier(), new
Identifier("col4"));
+ Assert.assertEquals(eqOperands.get(1).getLiteral(),
Literal.boolValue(true));
// NOTE: PQL does not support logical identifier
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/request/context/QueryContext.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/request/context/QueryContext.java
index 3121fe2186..1076a8504f 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/request/context/QueryContext.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/request/context/QueryContext.java
@@ -582,9 +582,6 @@ public class QueryContext {
Preconditions.checkState(aggregation != null &&
aggregation.getType() == FunctionContext.Type.AGGREGATION,
"First argument of FILTER must be an aggregation function");
ExpressionContext filterExpression = arguments.get(1);
- Preconditions.checkState(filterExpression.getFunction() != null
- && filterExpression.getFunction().getType() ==
FunctionContext.Type.TRANSFORM,
- "Second argument of FILTER must be a filter expression");
FilterContext filter =
RequestContextUtils.getFilter(filterExpression);
filteredAggregations.add(Pair.of(aggregation, filter));
} else {
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/BooleanQueriesTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/BooleanQueriesTest.java
index dad1ac1455..384a8989ed 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/BooleanQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/BooleanQueriesTest.java
@@ -162,6 +162,21 @@ public class BooleanQueriesTest extends BaseQueriesTest {
assertEquals(row[0], false);
}
}
+ {
+ String query = "SELECT booleanColumn FROM testTable WHERE booleanColumn";
+ BrokerResponseNative brokerResponse = getBrokerResponse(query);
+ ResultTable resultTable = brokerResponse.getResultTable();
+ DataSchema dataSchema = resultTable.getDataSchema();
+ assertEquals(dataSchema,
+ new DataSchema(new String[]{"booleanColumn"}, new
ColumnDataType[]{ColumnDataType.BOOLEAN}));
+ List<Object[]> rows = resultTable.getRows();
+ assertEquals(rows.size(), 10);
+ for (int i = 0; i < 10; i++) {
+ Object[] row = rows.get(i);
+ assertEquals(row.length, 1);
+ assertEquals(row[0], true);
+ }
+ }
{
String query = "SELECT * FROM testTable ORDER BY booleanColumn DESC
LIMIT 20";
BrokerResponseNative brokerResponse = getBrokerResponse(query);
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/ExplainPlanQueriesTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/ExplainPlanQueriesTest.java
index 9c2fe3d940..7e5dc8e9aa 100644
---
a/pinot-core/src/test/java/org/apache/pinot/queries/ExplainPlanQueriesTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/queries/ExplainPlanQueriesTest.java
@@ -55,6 +55,7 @@ public class ExplainPlanQueriesTest extends BaseQueriesTest {
private final static String COL1_NO_INDEX = "noIndexCol1";
private final static String COL2_NO_INDEX = "noIndexCol2";
private final static String COL3_NO_INDEX = "noIndexCol3";
+ private final static String COL4_NO_INDEX = "noIndexCol4";
private final static String COL1_INVERTED_INDEX = "invertedIndexCol1";
private final static String COL2_INVERTED_INDEX = "invertedIndexCol2";
private final static String COL3_INVERTED_INDEX = "invertedIndexCol3";
@@ -69,6 +70,7 @@ public class ExplainPlanQueriesTest extends BaseQueriesTest {
.addSingleValueDimension(COL1_NO_INDEX, FieldSpec.DataType.INT)
.addSingleValueDimension(COL2_NO_INDEX, FieldSpec.DataType.INT)
.addSingleValueDimension(COL3_NO_INDEX, FieldSpec.DataType.INT)
+ .addSingleValueDimension(COL4_NO_INDEX, FieldSpec.DataType.BOOLEAN)
.addSingleValueDimension(COL1_INVERTED_INDEX, FieldSpec.DataType.DOUBLE)
.addSingleValueDimension(COL2_INVERTED_INDEX, FieldSpec.DataType.INT)
.addSingleValueDimension(COL3_INVERTED_INDEX, FieldSpec.DataType.STRING)
@@ -104,14 +106,16 @@ public class ExplainPlanQueriesTest extends
BaseQueriesTest {
return _indexSegments;
}
- GenericRow createMockRecord(int noIndexCol1, int noIndexCol2, int
noIndexCol3, double invertedIndexCol1,
- int invertedIndexCol2, String intervedIndexCol3, double rangeIndexCol1,
int rangeIndexCol2, int rangeIndexCol3,
- double sortedIndexCol1, String jsonIndexCol1, String textIndexCol1) {
+ GenericRow createMockRecord(int noIndexCol1, int noIndexCol2, int
noIndexCol3,
+ boolean noIndexCol4, double invertedIndexCol1, int invertedIndexCol2,
String intervedIndexCol3,
+ double rangeIndexCol1, int rangeIndexCol2, int rangeIndexCol3, double
sortedIndexCol1, String jsonIndexCol1,
+ String textIndexCol1) {
GenericRow record = new GenericRow();
record.putValue(COL1_NO_INDEX, noIndexCol1);
record.putValue(COL2_NO_INDEX, noIndexCol2);
record.putValue(COL3_NO_INDEX, noIndexCol3);
+ record.putValue(COL4_NO_INDEX, noIndexCol4);
record.putValue(COL1_INVERTED_INDEX, invertedIndexCol1);
record.putValue(COL2_INVERTED_INDEX, invertedIndexCol2);
@@ -135,11 +139,11 @@ public class ExplainPlanQueriesTest extends
BaseQueriesTest {
FileUtils.deleteDirectory(INDEX_DIR);
List<GenericRow> records = new ArrayList<>(NUM_RECORDS);
- records.add(createMockRecord(1, 2, 3, 1.1, 2, "daffy", 10.1, 20, 30, 100.1,
+ records.add(createMockRecord(1, 2, 3, true, 1.1, 2, "daffy", 10.1, 20, 30,
100.1,
"{\"first\": \"daffy\", \"last\": " + "\"duck\"}", "daffy"));
- records.add(createMockRecord(0, 1, 2, 0.1, 1, "mickey", 0.1, 10, 20, 100.2,
+ records.add(createMockRecord(0, 1, 2, false, 0.1, 1, "mickey", 0.1, 10,
20, 100.2,
"{\"first\": \"mickey\", \"last\": " + "\"mouse\"}", "mickey"));
- records.add(createMockRecord(3, 4, 5, 2.1, 3, "mickey", 20.1, 30, 40,
100.3,
+ records.add(createMockRecord(3, 4, 5, true, 2.1, 3, "mickey", 20.1, 30,
40, 100.3,
"{\"first\": \"mickey\", \"last\": " + "\"mouse\"}", "mickey"));
IndexingConfig indexingConfig = TABLE_CONFIG.getIndexingConfig();
@@ -191,14 +195,14 @@ public class ExplainPlanQueriesTest extends
BaseQueriesTest {
result1.add(new Object[]{"COMBINE_SELECT", 1, 0});
result1.add(new Object[]{
"SELECT(selectList:invertedIndexCol1, invertedIndexCol2,
invertedIndexCol3, jsonIndexCol1, "
- + "noIndexCol1, noIndexCol2, noIndexCol3, rangeIndexCol1,
rangeIndexCol2, rangeIndexCol3, "
+ + "noIndexCol1, noIndexCol2, noIndexCol3, noIndexCol4,
rangeIndexCol1, rangeIndexCol2, rangeIndexCol3, "
+ "sortedIndexCol1, textIndexCol1)", 2, 1});
result1.add(new Object[]{"TRANSFORM_PASSTHROUGH(invertedIndexCol1,
invertedIndexCol2, invertedIndexCol3, "
- + "jsonIndexCol1, noIndexCol1, noIndexCol2, noIndexCol3,
rangeIndexCol1, rangeIndexCol2, rangeIndexCol3, "
- + "sortedIndexCol1, textIndexCol1)", 3, 2});
- result1.add(new Object[]{"PROJECT(sortedIndexCol1, noIndexCol3,
rangeIndexCol1, rangeIndexCol2, jsonIndexCol1, "
- + "invertedIndexCol1, noIndexCol2, invertedIndexCol2, noIndexCol1,
invertedIndexCol3, rangeIndexCol3, "
- + "textIndexCol1)", 4, 3});
+ + "jsonIndexCol1, noIndexCol1, noIndexCol2, noIndexCol3, noIndexCol4,
rangeIndexCol1, rangeIndexCol2, "
+ + "rangeIndexCol3, sortedIndexCol1, textIndexCol1)", 3, 2});
+ result1.add(new Object[]{"PROJECT(noIndexCol4, sortedIndexCol1,
noIndexCol3, rangeIndexCol1, rangeIndexCol2, "
+ + "invertedIndexCol1, noIndexCol2, invertedIndexCol2, noIndexCol1,
rangeIndexCol3, textIndexCol1, "
+ + "jsonIndexCol1, invertedIndexCol3)", 4, 3});
result1.add(new Object[]{"DOC_ID_SET", 5, 4});
result1.add(new Object[]{"FILTER_MATCH_ENTIRE_SEGMENT(docs:3)", 6, 5});
check(query1, new ResultTable(DATA_SCHEMA, result1));
@@ -337,6 +341,47 @@ public class ExplainPlanQueriesTest extends
BaseQueriesTest {
result3.add(new
Object[]{"FILTER_FULL_SCAN(operator:RANGE,predicate:noIndexCol1 > '1')", 7, 6});
result3.add(new
Object[]{"FILTER_FULL_SCAN(operator:RANGE,predicate:noIndexCol2 BETWEEN '2' AND
'101')", 8, 6});
check(query3, new ResultTable(DATA_SCHEMA, result3));
+
+ String query4 = "EXPLAIN PLAN FOR SELECT invertedIndexCol1, noIndexCol1
FROM testTable WHERE noIndexCol1 > 1 OR "
+ + "contains(textIndexCol1, 'daff') OR noIndexCol2 BETWEEN 2 AND 101
LIMIT 100";
+ List<Object[]> result4 = new ArrayList<>();
+ result4.add(new Object[]{"BROKER_REDUCE(limit:100)", 0, -1});
+ result4.add(new Object[]{"COMBINE_SELECT", 1, 0});
+ result4.add(new Object[]{"SELECT(selectList:invertedIndexCol1,
noIndexCol1)", 2, 1});
+ result4.add(new Object[]{"TRANSFORM_PASSTHROUGH(invertedIndexCol1,
noIndexCol1)", 3, 2});
+ result4.add(new Object[]{"PROJECT(invertedIndexCol1, noIndexCol1)", 4, 3});
+ result4.add(new Object[]{"DOC_ID_SET", 5, 4});
+ result4.add(new Object[]{"FILTER_OR", 6, 5});
+ result4.add(new
Object[]{"FILTER_FULL_SCAN(operator:RANGE,predicate:noIndexCol1 > '1')", 7, 6});
+ result4.add(new
Object[]{"FILTER_EXPRESSION(operator:EQ,predicate:contains(textIndexCol1,'daff')
= 'true')", 8, 6});
+ result4.add(new
Object[]{"FILTER_FULL_SCAN(operator:RANGE,predicate:noIndexCol2 BETWEEN '2' AND
'101')", 9, 6});
+ check(query4, new ResultTable(DATA_SCHEMA, result4));
+
+ String query5 = "EXPLAIN PLAN FOR SELECT invertedIndexCol1, noIndexCol1
FROM testTable WHERE noIndexCol4 LIMIT 100";
+ List<Object[]> result5 = new ArrayList<>();
+ result5.add(new Object[]{"BROKER_REDUCE(limit:100)", 0, -1});
+ result5.add(new Object[]{"COMBINE_SELECT", 1, 0});
+ result5.add(new Object[]{"SELECT(selectList:invertedIndexCol1,
noIndexCol1)", 2, 1});
+ result5.add(new Object[]{"TRANSFORM_PASSTHROUGH(invertedIndexCol1,
noIndexCol1)", 3, 2});
+ result5.add(new Object[]{"PROJECT(invertedIndexCol1, noIndexCol1)", 4, 3});
+ result5.add(new Object[]{"DOC_ID_SET", 5, 4});
+ result5.add(new
Object[]{"FILTER_FULL_SCAN(operator:EQ,predicate:noIndexCol4 = 'true')", 6, 5});
+ check(query5, new ResultTable(DATA_SCHEMA, result5));
+
+ String query6 = "EXPLAIN PLAN FOR SELECT invertedIndexCol1, noIndexCol1
FROM testTable WHERE startsWith "
+ + "(textIndexCol1, 'daff') AND noIndexCol4";
+ List<Object[]> result6 = new ArrayList<>();
+ result6.add(new Object[]{"BROKER_REDUCE(limit:10)", 0, -1});
+ result6.add(new Object[]{"COMBINE_SELECT", 1, 0});
+ result6.add(new Object[]{"SELECT(selectList:invertedIndexCol1,
noIndexCol1)", 2, 1});
+ result6.add(new Object[]{"TRANSFORM_PASSTHROUGH(invertedIndexCol1,
noIndexCol1)", 3, 2});
+ result6.add(new Object[]{"PROJECT(invertedIndexCol1, noIndexCol1)", 4, 3});
+ result6.add(new Object[]{"DOC_ID_SET", 5, 4});
+ result6.add(new Object[]{"FILTER_AND", 6, 5});
+ result6.add(new
Object[]{"FILTER_FULL_SCAN(operator:EQ,predicate:noIndexCol4 = 'true')", 7, 6});
+ result6.add(new
Object[]{"FILTER_EXPRESSION(operator:EQ,predicate:startswith(textIndexCol1,'daff')
= 'true')", 8,
+ 6});
+ check(query6, new ResultTable(DATA_SCHEMA, result6));
}
/** Test case for SQL statements with filter that involves inverted or
sorted index access. */
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/FilteredAggregationsTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/FilteredAggregationsTest.java
index 5b3a1bc153..2fc9ad1fa6 100644
---
a/pinot-core/src/test/java/org/apache/pinot/queries/FilteredAggregationsTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/queries/FilteredAggregationsTest.java
@@ -26,6 +26,8 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang.math.RandomUtils;
import org.apache.pinot.common.response.broker.ResultTable;
import
org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
import
org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
@@ -57,6 +59,8 @@ public class FilteredAggregationsTest extends BaseQueriesTest
{
private static final String INT_COL_NAME = "INT_COL";
private static final String NO_INDEX_INT_COL_NAME = "NO_INDEX_COL";
private static final String STATIC_INT_COL_NAME = "STATIC_INT_COL";
+ private static final String BOOLEAN_COL_NAME = "BOOLEAN_COL";
+ private static final String STRING_COL_NAME = "STRING_COL";
private static final Integer NUM_ROWS = 30000;
private IndexSegment _indexSegment;
@@ -111,6 +115,8 @@ public class FilteredAggregationsTest extends
BaseQueriesTest {
row.putValue(INT_COL_NAME, i);
row.putValue(NO_INDEX_INT_COL_NAME, i);
row.putValue(STATIC_INT_COL_NAME, 10);
+ row.putValue(BOOLEAN_COL_NAME, RandomUtils.nextBoolean());
+ row.putValue(STRING_COL_NAME, RandomStringUtils.randomAlphabetic(4));
rows.add(row);
}
return rows;
@@ -126,6 +132,8 @@ public class FilteredAggregationsTest extends
BaseQueriesTest {
Schema schema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
.addSingleValueDimension(NO_INDEX_INT_COL_NAME, FieldSpec.DataType.INT)
.addSingleValueDimension(STATIC_INT_COL_NAME, FieldSpec.DataType.INT)
+ .addSingleValueDimension(BOOLEAN_COL_NAME, FieldSpec.DataType.BOOLEAN)
+ .addSingleValueDimension(STRING_COL_NAME, FieldSpec.DataType.STRING)
.addMetric(INT_COL_NAME, FieldSpec.DataType.INT).build();
SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig,
schema);
config.setOutDir(INDEX_DIR.getPath());
@@ -185,6 +193,20 @@ public class FilteredAggregationsTest extends
BaseQueriesTest {
+ "FROM MyTable";
nonFilterQuery = "SELECT MIN(INT_COL), MAX(INT_COL) FROM MyTable WHERE
INT_COL > 29990";
testQuery(filterQuery, nonFilterQuery);
+
+ filterQuery = "SELECT SUM(INT_COL) FILTER(WHERE BOOLEAN_COL) FROM MyTable";
+ nonFilterQuery = "SELECT SUM(INT_COL) FROM MyTable WHERE BOOLEAN_COL=true";
+ testQuery(filterQuery, nonFilterQuery);
+
+ filterQuery = "SELECT SUM(INT_COL) FILTER(WHERE BOOLEAN_COL AND
STARTSWITH(STRING_COL, 'abc')) FROM MyTable";
+ nonFilterQuery = "SELECT SUM(INT_COL) FROM MyTable WHERE BOOLEAN_COL=true
AND STARTSWITH(STRING_COL, 'abc')";
+ testQuery(filterQuery, nonFilterQuery);
+
+ filterQuery = "SELECT SUM(INT_COL) FILTER(WHERE BOOLEAN_COL AND
STARTSWITH(REVERSE(STRING_COL), 'abc')) FROM "
+ + "MyTable";
+ nonFilterQuery = "SELECT SUM(INT_COL) FROM MyTable WHERE BOOLEAN_COL=true
AND STARTSWITH(REVERSE(STRING_COL), "
+ + "'abc')";
+ testQuery(filterQuery, nonFilterQuery);
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
