This is an automated email from the ASF dual-hosted git repository.
mayanks pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 31d4194 Avoid redundant merge of empty sketches. (#5783)
31d4194 is described below
commit 31d4194a2dc8912656a40394a8556e8619b363db
Author: Mayank Shrivastava <[email protected]>
AuthorDate: Fri Jul 31 13:18:40 2020 -0700
Avoid redundant merge of empty sketches. (#5783)
For segments where no rows are selected, the intermediate and the final
merge steps
still create empty Union objects and merge them. This can cause huge
latency degradation
in cases there is a large number of segments, and large value of nominal
entriesf
This PR avoids creation/merging of empty sketches. Doing so, we are seeing
a latency
improvement from 8s to < 500ms for 167 segments, and nominal entries of
`1048576`.
---
.../DistinctCountThetaSketchAggregationFunction.java | 17 +++++++++++++----
1 file changed, 13 insertions(+), 4 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctCountThetaSketchAggregationFunction.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctCountThetaSketchAggregationFunction.java
index c89ed8f..7f02368 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctCountThetaSketchAggregationFunction.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctCountThetaSketchAggregationFunction.java
@@ -20,6 +20,7 @@ package org.apache.pinot.core.query.aggregation.function;
import com.google.common.base.Preconditions;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -384,7 +385,11 @@ public class DistinctCountThetaSketchAggregationFunction
implements AggregationF
@Override
public Map<String, Sketch> extractAggregationResult(AggregationResultHolder
aggregationResultHolder) {
- Map<Predicate, Union> unionMap = getUnionMap(aggregationResultHolder);
+ Map<Predicate, Union> unionMap = aggregationResultHolder.getResult();
+ if (unionMap == null || unionMap.isEmpty()) {
+ return Collections.emptyMap();
+ }
+
Map<String, Sketch> result = new HashMap<>();
for (PredicateInfo predicateInfo : _predicateInfoMap.values()) {
result.put(predicateInfo.getStringPredicate(),
unionMap.get(predicateInfo.getPredicate()).getResult());
@@ -394,7 +399,11 @@ public class DistinctCountThetaSketchAggregationFunction
implements AggregationF
@Override
public Map<String, Sketch> extractGroupByResult(GroupByResultHolder
groupByResultHolder, int groupKey) {
- Map<Predicate, Union> unionMap = getUnionMap(groupByResultHolder,
groupKey);
+ Map<Predicate, Union> unionMap = groupByResultHolder.getResult(groupKey);
+ if (unionMap == null || unionMap.isEmpty()) {
+ return Collections.emptyMap();
+ }
+
Map<String, Sketch> result = new HashMap<>();
for (PredicateInfo predicateInfo : _predicateInfoMap.values()) {
result.put(predicateInfo.getStringPredicate(),
unionMap.get(predicateInfo.getPredicate()).getResult());
@@ -404,9 +413,9 @@ public class DistinctCountThetaSketchAggregationFunction
implements AggregationF
@Override
public Map<String, Sketch> merge(Map<String, Sketch> intermediateResult1,
Map<String, Sketch> intermediateResult2) {
- if (intermediateResult1 == null) {
+ if (intermediateResult1 == null || intermediateResult1.isEmpty()) {
return intermediateResult2;
- } else if (intermediateResult2 == null) {
+ } else if (intermediateResult2 == null || intermediateResult2.isEmpty()) {
return intermediateResult1;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
