This is an automated email from the ASF dual-hosted git repository.
richardstartin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 088da3f8c2 deduplicate predicates optimistically (#8687)
088da3f8c2 is described below
commit 088da3f8c2f7077c51b7d8531b7b967ad1cf58c6
Author: Richard Startin <[email protected]>
AuthorDate: Thu May 12 21:07:03 2022 +0200
deduplicate predicates optimistically (#8687)
---
.../plan/AggregationGroupByOrderByPlanNode.java | 2 +-
.../pinot/core/plan/AggregationPlanNode.java | 2 +-
.../org/apache/pinot/core/plan/FilterPlanNode.java | 20 ++++++++----------
.../apache/pinot/core/startree/StarTreeUtils.java | 24 ++++++++++++++--------
.../pinot/core/startree/v2/BaseStarTreeV2Test.java | 2 +-
.../org/apache/pinot/perf/BenchmarkQueries.java | 6 +++++-
6 files changed, 31 insertions(+), 25 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/plan/AggregationGroupByOrderByPlanNode.java
b/pinot-core/src/main/java/org/apache/pinot/core/plan/AggregationGroupByOrderByPlanNode.java
index 58118f4960..6451b43c83 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/plan/AggregationGroupByOrderByPlanNode.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/plan/AggregationGroupByOrderByPlanNode.java
@@ -70,7 +70,7 @@ public class AggregationGroupByOrderByPlanNode implements
PlanNode {
if (aggregationFunctionColumnPairs != null) {
Map<String, List<CompositePredicateEvaluator>> predicateEvaluatorsMap =
StarTreeUtils.extractPredicateEvaluatorsMap(_indexSegment,
_queryContext.getFilter(),
- filterPlanNode.getPredicateEvaluatorMap());
+ filterPlanNode.getPredicateEvaluators());
if (predicateEvaluatorsMap != null) {
for (StarTreeV2 starTreeV2 : starTrees) {
if (StarTreeUtils.isFitForStarTree(starTreeV2.getMetadata(),
aggregationFunctionColumnPairs,
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/plan/AggregationPlanNode.java
b/pinot-core/src/main/java/org/apache/pinot/core/plan/AggregationPlanNode.java
index 7ee09c300c..e0ea896609 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/plan/AggregationPlanNode.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/plan/AggregationPlanNode.java
@@ -204,7 +204,7 @@ public class AggregationPlanNode implements PlanNode {
if (aggregationFunctionColumnPairs != null) {
Map<String, List<CompositePredicateEvaluator>> predicateEvaluatorsMap =
StarTreeUtils.extractPredicateEvaluatorsMap(_indexSegment,
_queryContext.getFilter(),
- filterPlanNode.getPredicateEvaluatorMap());
+ filterPlanNode.getPredicateEvaluators());
if (predicateEvaluatorsMap != null) {
for (StarTreeV2 starTreeV2 : starTrees) {
if (StarTreeUtils.isFitForStarTree(starTreeV2.getMetadata(),
aggregationFunctionColumnPairs, null,
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/plan/FilterPlanNode.java
b/pinot-core/src/main/java/org/apache/pinot/core/plan/FilterPlanNode.java
index 7bc1a4f642..14b0146ff1 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/plan/FilterPlanNode.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/plan/FilterPlanNode.java
@@ -21,10 +21,9 @@ package org.apache.pinot.core.plan;
import com.google.common.base.Preconditions;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import javax.annotation.Nullable;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.common.request.context.FilterContext;
import org.apache.pinot.common.request.context.FunctionContext;
@@ -68,7 +67,7 @@ public class FilterPlanNode implements PlanNode {
private final FilterContext _filter;
// Cache the predicate evaluators
- private final Map<Predicate, PredicateEvaluator> _predicateEvaluatorMap =
new HashMap<>();
+ private final List<Pair<Predicate, PredicateEvaluator>> _predicateEvaluators
= new ArrayList<>(4);
public FilterPlanNode(IndexSegment indexSegment, QueryContext queryContext) {
this(indexSegment, queryContext, null);
@@ -106,10 +105,10 @@ public class FilterPlanNode implements PlanNode {
}
/**
- * Returns a map from predicates to their evaluators.
+ * Returns a mapping from predicates to their evaluators.
*/
- public Map<Predicate, PredicateEvaluator> getPredicateEvaluatorMap() {
- return _predicateEvaluatorMap;
+ public List<Pair<Predicate, PredicateEvaluator>> getPredicateEvaluators() {
+ return _predicateEvaluators;
}
/**
@@ -240,10 +239,7 @@ public class FilterPlanNode implements PlanNode {
} else {
String column = lhs.getIdentifier();
DataSource dataSource = _indexSegment.getDataSource(column);
- PredicateEvaluator predicateEvaluator =
_predicateEvaluatorMap.get(predicate);
- if (predicateEvaluator != null) {
- return
FilterOperatorUtils.getLeafFilterOperator(predicateEvaluator, dataSource,
numDocs);
- }
+ PredicateEvaluator predicateEvaluator;
switch (predicate.getType()) {
case TEXT_CONTAINS:
TextIndexReader textIndexReader = dataSource.getTextIndex();
@@ -278,7 +274,7 @@ public class FilterPlanNode implements PlanNode {
PredicateEvaluatorProvider.getPredicateEvaluator(predicate,
dataSource.getDictionary(),
dataSource.getDataSourceMetadata().getDataType());
}
- _predicateEvaluatorMap.put(predicate, predicateEvaluator);
+ _predicateEvaluators.add(Pair.of(predicate, predicateEvaluator));
return
FilterOperatorUtils.getLeafFilterOperator(predicateEvaluator, dataSource,
numDocs);
case JSON_MATCH:
JsonIndexReader jsonIndex = dataSource.getJsonIndex();
@@ -303,7 +299,7 @@ public class FilterPlanNode implements PlanNode {
predicateEvaluator =
PredicateEvaluatorProvider.getPredicateEvaluator(predicate,
dataSource.getDictionary(),
dataSource.getDataSourceMetadata().getDataType());
- _predicateEvaluatorMap.put(predicate, predicateEvaluator);
+ _predicateEvaluators.add(Pair.of(predicate, predicateEvaluator));
return
FilterOperatorUtils.getLeafFilterOperator(predicateEvaluator, dataSource,
numDocs);
}
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/startree/StarTreeUtils.java
b/pinot-core/src/main/java/org/apache/pinot/core/startree/StarTreeUtils.java
index 12c2ee5bd3..ada24ad19b 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/startree/StarTreeUtils.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/startree/StarTreeUtils.java
@@ -18,11 +18,11 @@
*/
package org.apache.pinot.core.startree;
+import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
@@ -94,13 +94,13 @@ public class StarTreeUtils {
*/
@Nullable
public static Map<String, List<CompositePredicateEvaluator>>
extractPredicateEvaluatorsMap(IndexSegment indexSegment,
- @Nullable FilterContext filter, Map<Predicate, PredicateEvaluator>
predicateEvaluatorMap) {
+ @Nullable FilterContext filter, List<Pair<Predicate,
PredicateEvaluator>> predicateEvaluatorMapping) {
if (filter == null) {
return Collections.emptyMap();
}
Map<String, List<CompositePredicateEvaluator>> predicateEvaluatorsMap =
new HashMap<>();
- Queue<FilterContext> queue = new LinkedList<>();
+ Queue<FilterContext> queue = new ArrayDeque<>();
queue.add(filter);
FilterContext filterNode;
while ((filterNode = queue.poll()) != null) {
@@ -110,7 +110,7 @@ public class StarTreeUtils {
break;
case OR:
Pair<String, List<PredicateEvaluator>> pair =
- isOrClauseValidForStarTree(indexSegment, filterNode,
predicateEvaluatorMap);
+ isOrClauseValidForStarTree(indexSegment, filterNode,
predicateEvaluatorMapping);
if (pair == null) {
return null;
}
@@ -126,7 +126,8 @@ public class StarTreeUtils {
return null;
case PREDICATE:
Predicate predicate = filterNode.getPredicate();
- PredicateEvaluator predicateEvaluator =
getPredicateEvaluator(indexSegment, predicate, predicateEvaluatorMap);
+ PredicateEvaluator predicateEvaluator =
getPredicateEvaluator(indexSegment, predicate,
+ predicateEvaluatorMapping);
if (predicateEvaluator == null) {
// The predicate cannot be solved with star-tree
return null;
@@ -187,7 +188,7 @@ public class StarTreeUtils {
*/
@Nullable
private static Pair<String, List<PredicateEvaluator>>
isOrClauseValidForStarTree(IndexSegment indexSegment,
- FilterContext filter, Map<Predicate, PredicateEvaluator>
predicateEvaluatorMap) {
+ FilterContext filter, List<Pair<Predicate, PredicateEvaluator>>
predicateEvaluatorMapping) {
assert filter.getType() == FilterContext.Type.OR;
List<Predicate> predicates = new ArrayList<>();
@@ -198,7 +199,7 @@ public class StarTreeUtils {
String identifier = null;
List<PredicateEvaluator> predicateEvaluators = new ArrayList<>();
for (Predicate predicate : predicates) {
- PredicateEvaluator predicateEvaluator =
getPredicateEvaluator(indexSegment, predicate, predicateEvaluatorMap);
+ PredicateEvaluator predicateEvaluator =
getPredicateEvaluator(indexSegment, predicate, predicateEvaluatorMapping);
if (predicateEvaluator == null) {
// The predicate cannot be solved with star-tree
return null;
@@ -258,7 +259,7 @@ public class StarTreeUtils {
*/
@Nullable
private static PredicateEvaluator getPredicateEvaluator(IndexSegment
indexSegment, Predicate predicate,
- Map<Predicate, PredicateEvaluator> predicateEvaluatorMap) {
+ List<Pair<Predicate, PredicateEvaluator>> predicatesEvaluatorMapping) {
ExpressionContext lhs = predicate.getLhs();
if (lhs.getType() != ExpressionContext.Type.IDENTIFIER) {
// Star-tree does not support non-identifier expression
@@ -283,6 +284,11 @@ public class StarTreeUtils {
default:
break;
}
- return predicateEvaluatorMap.get(predicate);
+ for (Pair<Predicate, PredicateEvaluator> pair :
predicatesEvaluatorMapping) {
+ if (pair.getKey().equals(predicate)) {
+ return pair.getValue();
+ }
+ }
+ return null;
}
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/BaseStarTreeV2Test.java
b/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/BaseStarTreeV2Test.java
index eab94b4594..ae15c597ef 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/BaseStarTreeV2Test.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/startree/v2/BaseStarTreeV2Test.java
@@ -244,7 +244,7 @@ abstract class BaseStarTreeV2Test<R, A> {
filterPlanNode.run();
Map<String, List<CompositePredicateEvaluator>> predicateEvaluatorsMap =
StarTreeUtils.extractPredicateEvaluatorsMap(_indexSegment,
queryContext.getFilter(),
- filterPlanNode.getPredicateEvaluatorMap());
+ filterPlanNode.getPredicateEvaluators());
assertNotNull(predicateEvaluatorsMap);
// Extract values with star-tree
diff --git
a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkQueries.java
b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkQueries.java
index 714a6ede56..f1059b94d9 100644
--- a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkQueries.java
+++ b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkQueries.java
@@ -156,6 +156,10 @@ public class BenchmarkQueries extends BaseQueriesTest {
public static final String STARTREE_SUM_QUERY = "SELECT INT_COL, SORTED_COL,
SUM(RAW_INT_COL) from MyTable "
+ "GROUP BY INT_COL, SORTED_COL ORDER BY SORTED_COL, INT_COL ASC";
+ public static final String STARTREE_FILTER_QUERY = "SELECT INT_COL,
SORTED_COL, SUM(RAW_INT_COL) FROM MyTable "
+ + "WHERE INT_COL = 0 and SORTED_COL = 1"
+ + "GROUP BY INT_COL, SORTED_COL ORDER BY SORTED_COL, INT_COL ASC";
+
@Param("1500000")
private int _numRows;
@Param({"EXP(0.001)", "EXP(0.5)", "EXP(0.999)"})
@@ -164,7 +168,7 @@ public class BenchmarkQueries extends BaseQueriesTest {
MULTI_GROUP_BY_WITH_RAW_QUERY, MULTI_GROUP_BY_WITH_RAW_QUERY_2,
FILTERED_QUERY, NON_FILTERED_QUERY,
SUM_QUERY, NO_INDEX_LIKE_QUERY, MULTI_GROUP_BY_ORDER_BY,
MULTI_GROUP_BY_ORDER_BY_LOW_HIGH, TIME_GROUP_BY,
RAW_COLUMN_SUMMARY_STATS, COUNT_OVER_BITMAP_INDEX_IN,
COUNT_OVER_BITMAP_INDEXES,
- COUNT_OVER_BITMAP_AND_SORTED_INDEXES, COUNT_OVER_BITMAP_INDEX_EQUALS,
STARTREE_SUM_QUERY
+ COUNT_OVER_BITMAP_AND_SORTED_INDEXES, COUNT_OVER_BITMAP_INDEX_EQUALS,
STARTREE_SUM_QUERY, STARTREE_FILTER_QUERY
})
String _query;
private IndexSegment _indexSegment;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
