This is an automated email from the ASF dual-hosted git repository.

jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new 4a263ea7d9 DISTINCT COUNT/SUM/AVG NULL support. (#11348)
4a263ea7d9 is described below

commit 4a263ea7d91d5dfbddbf2731433680c5fd7bdb4d
Author: Shen Yu <[email protected]>
AuthorDate: Mon Aug 21 16:40:44 2023 +0000

    DISTINCT COUNT/SUM/AVG NULL support. (#11348)
---
 .../function/AggregationFunctionFactory.java       |   6 +-
 .../BaseDistinctAggregateAggregationFunction.java  | 319 ++++++++++++++++++---
 .../function/DistinctAvgAggregationFunction.java   |   4 +-
 .../function/DistinctAvgMVAggregationFunction.java |   2 +-
 .../function/DistinctCountAggregationFunction.java |   4 +-
 .../DistinctCountMVAggregationFunction.java        |   2 +-
 .../function/DistinctSumAggregationFunction.java   |   4 +-
 .../function/DistinctSumMVAggregationFunction.java |   2 +-
 .../apache/pinot/queries/AllNullQueriesTest.java   |   2 +-
 .../queries/NullHandlingEnabledQueriesTest.java    | 207 +++++++++++++
 10 files changed, 494 insertions(+), 58 deletions(-)

diff --git 
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/AggregationFunctionFactory.java
 
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/AggregationFunctionFactory.java
index aadc9fbd63..a79f01a0f1 100644
--- 
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/AggregationFunctionFactory.java
+++ 
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/AggregationFunctionFactory.java
@@ -257,7 +257,7 @@ public class AggregationFunctionFactory {
           case MINMAXRANGE:
             return new MinMaxRangeAggregationFunction(firstArgument);
           case DISTINCTCOUNT:
-            return new DistinctCountAggregationFunction(firstArgument);
+            return new DistinctCountAggregationFunction(firstArgument, 
nullHandlingEnabled);
           case DISTINCTCOUNTBITMAP:
             return new DistinctCountBitmapAggregationFunction(firstArgument);
           case SEGMENTPARTITIONEDDISTINCTCOUNT:
@@ -275,9 +275,9 @@ public class AggregationFunctionFactory {
           case DISTINCTCOUNTRAWTHETASKETCH:
             return new 
DistinctCountRawThetaSketchAggregationFunction(arguments);
           case DISTINCTSUM:
-            return new DistinctSumAggregationFunction(firstArgument);
+            return new DistinctSumAggregationFunction(firstArgument, 
nullHandlingEnabled);
           case DISTINCTAVG:
-            return new DistinctAvgAggregationFunction(firstArgument);
+            return new DistinctAvgAggregationFunction(firstArgument, 
nullHandlingEnabled);
           case IDSET:
             return new IdSetAggregationFunction(arguments);
           case COUNTMV:
diff --git 
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/BaseDistinctAggregateAggregationFunction.java
 
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/BaseDistinctAggregateAggregationFunction.java
index e9bc38e92b..49754b11f0 100644
--- 
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/BaseDistinctAggregateAggregationFunction.java
+++ 
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/BaseDistinctAggregateAggregationFunction.java
@@ -49,11 +49,13 @@ import org.roaringbitmap.RoaringBitmap;
 public abstract class BaseDistinctAggregateAggregationFunction<T extends 
Comparable>
     extends BaseSingleInputAggregationFunction<Set, T> {
   private final AggregationFunctionType _functionType;
+  private final boolean _nullHandlingEnabled;
 
   protected BaseDistinctAggregateAggregationFunction(ExpressionContext 
expression,
-      AggregationFunctionType aggregationFunctionType) {
+      AggregationFunctionType aggregationFunctionType, boolean 
nullHandlingEnabled) {
     super(expression);
     _functionType = aggregationFunctionType;
+    _nullHandlingEnabled = nullHandlingEnabled;
   }
 
   @Override
@@ -144,12 +146,25 @@ public abstract class 
BaseDistinctAggregateAggregationFunction<T extends Compara
   protected void svAggregate(int length, AggregationResultHolder 
aggregationResultHolder,
       Map<ExpressionContext, BlockValSet> blockValSetMap) {
     BlockValSet blockValSet = blockValSetMap.get(_expression);
+    RoaringBitmap nullBitmap = null;
 
     // For dictionary-encoded expression, store dictionary ids into the bitmap
     Dictionary dictionary = blockValSet.getDictionary();
     if (dictionary != null) {
+      if (_nullHandlingEnabled) {
+        nullBitmap = blockValSet.getNullBitmap();
+      }
       int[] dictIds = blockValSet.getDictionaryIdsSV();
-      getDictIdBitmap(aggregationResultHolder, dictionary).addN(dictIds, 0, 
length);
+      if (nullBitmap != null && !nullBitmap.isEmpty()) {
+        RoaringBitmap dictIdBitmap = getDictIdBitmap(aggregationResultHolder, 
dictionary);
+        for (int i = 0; i < length; i++) {
+          if (!nullBitmap.contains(i)) {
+            dictIdBitmap.add(dictIds[i]);
+          }
+        }
+      } else {
+        getDictIdBitmap(aggregationResultHolder, dictionary).addN(dictIds, 0, 
length);
+      }
       return;
     }
 
@@ -160,44 +175,110 @@ public abstract class 
BaseDistinctAggregateAggregationFunction<T extends Compara
       case INT:
         IntOpenHashSet intSet = (IntOpenHashSet) valueSet;
         int[] intValues = blockValSet.getIntValuesSV();
-        for (int i = 0; i < length; i++) {
-          intSet.add(intValues[i]);
+        if (_nullHandlingEnabled) {
+          nullBitmap = blockValSet.getNullBitmap();
+        }
+        if (nullBitmap != null && !nullBitmap.isEmpty()) {
+          for (int i = 0; i < length; i++) {
+            if (!nullBitmap.contains(i)) {
+              intSet.add(intValues[i]);
+            }
+          }
+        } else {
+          for (int i = 0; i < length; i++) {
+            intSet.add(intValues[i]);
+          }
         }
         break;
       case LONG:
         LongOpenHashSet longSet = (LongOpenHashSet) valueSet;
         long[] longValues = blockValSet.getLongValuesSV();
-        for (int i = 0; i < length; i++) {
-          longSet.add(longValues[i]);
+        if (_nullHandlingEnabled) {
+          nullBitmap = blockValSet.getNullBitmap();
+        }
+        if (nullBitmap != null && !nullBitmap.isEmpty()) {
+          for (int i = 0; i < length; i++) {
+            if (!nullBitmap.contains(i)) {
+              longSet.add(longValues[i]);
+            }
+          }
+        } else {
+          for (int i = 0; i < length; i++) {
+            longSet.add(longValues[i]);
+          }
         }
         break;
       case FLOAT:
         FloatOpenHashSet floatSet = (FloatOpenHashSet) valueSet;
         float[] floatValues = blockValSet.getFloatValuesSV();
-        for (int i = 0; i < length; i++) {
-          floatSet.add(floatValues[i]);
+        if (_nullHandlingEnabled) {
+          nullBitmap = blockValSet.getNullBitmap();
+        }
+        if (nullBitmap != null && !nullBitmap.isEmpty()) {
+          for (int i = 0; i < length; i++) {
+            if (!nullBitmap.contains(i)) {
+              floatSet.add(floatValues[i]);
+            }
+          }
+        } else {
+          for (int i = 0; i < length; i++) {
+            floatSet.add(floatValues[i]);
+          }
         }
         break;
       case DOUBLE:
         DoubleOpenHashSet doubleSet = (DoubleOpenHashSet) valueSet;
         double[] doubleValues = blockValSet.getDoubleValuesSV();
-        for (int i = 0; i < length; i++) {
-          doubleSet.add(doubleValues[i]);
+        if (_nullHandlingEnabled) {
+          nullBitmap = blockValSet.getNullBitmap();
+        }
+        if (nullBitmap != null && !nullBitmap.isEmpty()) {
+          for (int i = 0; i < length; i++) {
+            if (!nullBitmap.contains(i)) {
+              doubleSet.add(doubleValues[i]);
+            }
+          }
+        } else {
+          for (int i = 0; i < length; i++) {
+            doubleSet.add(doubleValues[i]);
+          }
         }
         break;
       case STRING:
         ObjectOpenHashSet<String> stringSet = (ObjectOpenHashSet<String>) 
valueSet;
         String[] stringValues = blockValSet.getStringValuesSV();
         //noinspection ManualArrayToCollectionCopy
-        for (int i = 0; i < length; i++) {
-          stringSet.add(stringValues[i]);
+        if (_nullHandlingEnabled) {
+          nullBitmap = blockValSet.getNullBitmap();
+        }
+        if (nullBitmap != null && !nullBitmap.isEmpty()) {
+          for (int i = 0; i < length; i++) {
+            if (!nullBitmap.contains(i)) {
+              stringSet.add(stringValues[i]);
+            }
+          }
+        } else {
+          for (int i = 0; i < length; i++) {
+            stringSet.add(stringValues[i]);
+          }
         }
         break;
       case BYTES:
         ObjectOpenHashSet<ByteArray> bytesSet = (ObjectOpenHashSet<ByteArray>) 
valueSet;
         byte[][] bytesValues = blockValSet.getBytesValuesSV();
-        for (int i = 0; i < length; i++) {
-          bytesSet.add(new ByteArray(bytesValues[i]));
+        if (_nullHandlingEnabled) {
+          nullBitmap = blockValSet.getNullBitmap();
+        }
+        if (nullBitmap != null && !nullBitmap.isEmpty()) {
+          for (int i = 0; i < length; i++) {
+            if (!nullBitmap.contains(i)) {
+              bytesSet.add(new ByteArray(bytesValues[i]));
+            }
+          }
+        } else {
+          for (int i = 0; i < length; i++) {
+            bytesSet.add(new ByteArray(bytesValues[i]));
+          }
         }
         break;
       default:
@@ -287,13 +368,25 @@ public abstract class 
BaseDistinctAggregateAggregationFunction<T extends Compara
   protected void svAggregateGroupBySV(int length, int[] groupKeyArray, 
GroupByResultHolder groupByResultHolder,
       Map<ExpressionContext, BlockValSet> blockValSetMap) {
     BlockValSet blockValSet = blockValSetMap.get(_expression);
+    RoaringBitmap nullBitmap = null;
 
     // For dictionary-encoded expression, store dictionary ids into the bitmap
     Dictionary dictionary = blockValSet.getDictionary();
     if (dictionary != null) {
+      if (_nullHandlingEnabled) {
+        nullBitmap = blockValSet.getNullBitmap();
+      }
       int[] dictIds = blockValSet.getDictionaryIdsSV();
-      for (int i = 0; i < length; i++) {
-        getDictIdBitmap(groupByResultHolder, groupKeyArray[i], 
dictionary).add(dictIds[i]);
+      if (nullBitmap != null && !nullBitmap.isEmpty()) {
+        for (int i = 0; i < length; i++) {
+          if (!nullBitmap.contains(i)) {
+            getDictIdBitmap(groupByResultHolder, groupKeyArray[i], 
dictionary).add(dictIds[i]);
+          }
+        }
+      } else {
+        for (int i = 0; i < length; i++) {
+          getDictIdBitmap(groupByResultHolder, groupKeyArray[i], 
dictionary).add(dictIds[i]);
+        }
       }
       return;
     }
@@ -303,41 +396,111 @@ public abstract class 
BaseDistinctAggregateAggregationFunction<T extends Compara
     switch (storedType) {
       case INT:
         int[] intValues = blockValSet.getIntValuesSV();
-        for (int i = 0; i < length; i++) {
-          ((IntOpenHashSet) getValueSet(groupByResultHolder, groupKeyArray[i], 
DataType.INT)).add(intValues[i]);
+        if (_nullHandlingEnabled) {
+          nullBitmap = blockValSet.getNullBitmap();
+        }
+        if (nullBitmap != null && !nullBitmap.isEmpty()) {
+          for (int i = 0; i < length; i++) {
+            if (!nullBitmap.contains(i)) {
+              ((IntOpenHashSet) getValueSet(groupByResultHolder, 
groupKeyArray[i], DataType.INT)).add(intValues[i]);
+            }
+          }
+        } else {
+          for (int i = 0; i < length; i++) {
+            ((IntOpenHashSet) getValueSet(groupByResultHolder, 
groupKeyArray[i], DataType.INT)).add(intValues[i]);
+          }
         }
         break;
       case LONG:
         long[] longValues = blockValSet.getLongValuesSV();
-        for (int i = 0; i < length; i++) {
-          ((LongOpenHashSet) getValueSet(groupByResultHolder, 
groupKeyArray[i], DataType.LONG)).add(longValues[i]);
+        if (_nullHandlingEnabled) {
+          nullBitmap = blockValSet.getNullBitmap();
+        }
+        if (nullBitmap != null && !nullBitmap.isEmpty()) {
+          for (int i = 0; i < length; i++) {
+            if (!nullBitmap.contains(i)) {
+              ((LongOpenHashSet) getValueSet(groupByResultHolder, 
groupKeyArray[i], DataType.LONG)).add(longValues[i]);
+            }
+          }
+        } else {
+          for (int i = 0; i < length; i++) {
+            ((LongOpenHashSet) getValueSet(groupByResultHolder, 
groupKeyArray[i], DataType.LONG)).add(longValues[i]);
+          }
         }
         break;
       case FLOAT:
         float[] floatValues = blockValSet.getFloatValuesSV();
-        for (int i = 0; i < length; i++) {
-          ((FloatOpenHashSet) getValueSet(groupByResultHolder, 
groupKeyArray[i], DataType.FLOAT)).add(floatValues[i]);
+        if (_nullHandlingEnabled) {
+          nullBitmap = blockValSet.getNullBitmap();
+        }
+        if (nullBitmap != null && !nullBitmap.isEmpty()) {
+          for (int i = 0; i < length; i++) {
+            if (!nullBitmap.contains(i)) {
+              ((FloatOpenHashSet) getValueSet(groupByResultHolder, 
groupKeyArray[i], DataType.FLOAT)).add(
+                  floatValues[i]);
+            }
+          }
+        } else {
+          for (int i = 0; i < length; i++) {
+            ((FloatOpenHashSet) getValueSet(groupByResultHolder, 
groupKeyArray[i], DataType.FLOAT)).add(floatValues[i]);
+          }
         }
         break;
       case DOUBLE:
         double[] doubleValues = blockValSet.getDoubleValuesSV();
-        for (int i = 0; i < length; i++) {
-          ((DoubleOpenHashSet) getValueSet(groupByResultHolder, 
groupKeyArray[i], DataType.DOUBLE)).add(
-              doubleValues[i]);
+        if (_nullHandlingEnabled) {
+          nullBitmap = blockValSet.getNullBitmap();
+        }
+        if (nullBitmap != null && !nullBitmap.isEmpty()) {
+          for (int i = 0; i < length; i++) {
+            if (!nullBitmap.contains(i)) {
+              ((DoubleOpenHashSet) getValueSet(groupByResultHolder, 
groupKeyArray[i], DataType.DOUBLE)).add(
+                  doubleValues[i]);
+            }
+          }
+        } else {
+          for (int i = 0; i < length; i++) {
+            ((DoubleOpenHashSet) getValueSet(groupByResultHolder, 
groupKeyArray[i], DataType.DOUBLE)).add(
+                doubleValues[i]);
+          }
         }
         break;
       case STRING:
         String[] stringValues = blockValSet.getStringValuesSV();
-        for (int i = 0; i < length; i++) {
-          ((ObjectOpenHashSet<String>) getValueSet(groupByResultHolder, 
groupKeyArray[i], DataType.STRING)).add(
-              stringValues[i]);
+        if (_nullHandlingEnabled) {
+          nullBitmap = blockValSet.getNullBitmap();
+        }
+        if (nullBitmap != null && !nullBitmap.isEmpty()) {
+          for (int i = 0; i < length; i++) {
+            if (!nullBitmap.contains(i)) {
+              ((ObjectOpenHashSet<String>) getValueSet(groupByResultHolder, 
groupKeyArray[i], DataType.STRING)).add(
+                  stringValues[i]);
+            }
+          }
+        } else {
+          for (int i = 0; i < length; i++) {
+            ((ObjectOpenHashSet<String>) getValueSet(groupByResultHolder, 
groupKeyArray[i], DataType.STRING)).add(
+                stringValues[i]);
+          }
         }
         break;
       case BYTES:
         byte[][] bytesValues = blockValSet.getBytesValuesSV();
-        for (int i = 0; i < length; i++) {
-          ((ObjectOpenHashSet<ByteArray>) getValueSet(groupByResultHolder, 
groupKeyArray[i], DataType.BYTES)).add(
-              new ByteArray(bytesValues[i]));
+        if (_nullHandlingEnabled) {
+          nullBitmap = blockValSet.getNullBitmap();
+        }
+        if (nullBitmap != null && !nullBitmap.isEmpty()) {
+          for (int i = 0; i < length; i++) {
+            if (!nullBitmap.contains(i)) {
+              ((ObjectOpenHashSet<ByteArray>) getValueSet(groupByResultHolder, 
groupKeyArray[i], DataType.BYTES)).add(
+                  new ByteArray(bytesValues[i]));
+            }
+          }
+        } else {
+          for (int i = 0; i < length; i++) {
+            ((ObjectOpenHashSet<ByteArray>) getValueSet(groupByResultHolder, 
groupKeyArray[i], DataType.BYTES)).add(
+                new ByteArray(bytesValues[i]));
+          }
         }
         break;
       default:
@@ -428,13 +591,25 @@ public abstract class 
BaseDistinctAggregateAggregationFunction<T extends Compara
   protected void svAggregateGroupByMV(int length, int[][] groupKeysArray, 
GroupByResultHolder groupByResultHolder,
       Map<ExpressionContext, BlockValSet> blockValSetMap) {
     BlockValSet blockValSet = blockValSetMap.get(_expression);
+    RoaringBitmap nullBitmap = null;
 
     // For dictionary-encoded expression, store dictionary ids into the bitmap
     Dictionary dictionary = blockValSet.getDictionary();
     if (dictionary != null) {
+      if (_nullHandlingEnabled) {
+        nullBitmap = blockValSet.getNullBitmap();
+      }
       int[] dictIds = blockValSet.getDictionaryIdsSV();
-      for (int i = 0; i < length; i++) {
-        setDictIdForGroupKeys(groupByResultHolder, groupKeysArray[i], 
dictionary, dictIds[i]);
+      if (nullBitmap != null && !nullBitmap.isEmpty()) {
+        for (int i = 0; i < length; i++) {
+          if (!nullBitmap.contains(i)) {
+            setDictIdForGroupKeys(groupByResultHolder, groupKeysArray[i], 
dictionary, dictIds[i]);
+          }
+        }
+      } else {
+        for (int i = 0; i < length; i++) {
+          setDictIdForGroupKeys(groupByResultHolder, groupKeysArray[i], 
dictionary, dictIds[i]);
+        }
       }
       return;
     }
@@ -444,38 +619,92 @@ public abstract class 
BaseDistinctAggregateAggregationFunction<T extends Compara
     switch (storedType) {
       case INT:
         int[] intValues = blockValSet.getIntValuesSV();
-        for (int i = 0; i < length; i++) {
-          setValueForGroupKeys(groupByResultHolder, groupKeysArray[i], 
intValues[i]);
+        nullBitmap = blockValSet.getNullBitmap();
+        if (nullBitmap != null && !nullBitmap.isEmpty()) {
+          for (int i = 0; i < length; i++) {
+            if (!nullBitmap.contains(i)) {
+              setValueForGroupKeys(groupByResultHolder, groupKeysArray[i], 
intValues[i]);
+            }
+          }
+        } else {
+          for (int i = 0; i < length; i++) {
+            setValueForGroupKeys(groupByResultHolder, groupKeysArray[i], 
intValues[i]);
+          }
         }
         break;
       case LONG:
         long[] longValues = blockValSet.getLongValuesSV();
-        for (int i = 0; i < length; i++) {
-          setValueForGroupKeys(groupByResultHolder, groupKeysArray[i], 
longValues[i]);
+        nullBitmap = blockValSet.getNullBitmap();
+        if (nullBitmap != null && !nullBitmap.isEmpty()) {
+          for (int i = 0; i < length; i++) {
+            if (!nullBitmap.contains(i)) {
+              setValueForGroupKeys(groupByResultHolder, groupKeysArray[i], 
longValues[i]);
+            }
+          }
+        } else {
+          for (int i = 0; i < length; i++) {
+            setValueForGroupKeys(groupByResultHolder, groupKeysArray[i], 
longValues[i]);
+          }
         }
         break;
       case FLOAT:
         float[] floatValues = blockValSet.getFloatValuesSV();
-        for (int i = 0; i < length; i++) {
-          setValueForGroupKeys(groupByResultHolder, groupKeysArray[i], 
floatValues[i]);
+        nullBitmap = blockValSet.getNullBitmap();
+        if (nullBitmap != null && !nullBitmap.isEmpty()) {
+          for (int i = 0; i < length; i++) {
+            if (!nullBitmap.contains(i)) {
+              setValueForGroupKeys(groupByResultHolder, groupKeysArray[i], 
floatValues[i]);
+            }
+          }
+        } else {
+          for (int i = 0; i < length; i++) {
+            setValueForGroupKeys(groupByResultHolder, groupKeysArray[i], 
floatValues[i]);
+          }
         }
         break;
       case DOUBLE:
         double[] doubleValues = blockValSet.getDoubleValuesSV();
-        for (int i = 0; i < length; i++) {
-          setValueForGroupKeys(groupByResultHolder, groupKeysArray[i], 
doubleValues[i]);
+        nullBitmap = blockValSet.getNullBitmap();
+        if (nullBitmap != null && !nullBitmap.isEmpty()) {
+          for (int i = 0; i < length; i++) {
+            if (!nullBitmap.contains(i)) {
+              setValueForGroupKeys(groupByResultHolder, groupKeysArray[i], 
doubleValues[i]);
+            }
+          }
+        } else {
+          for (int i = 0; i < length; i++) {
+            setValueForGroupKeys(groupByResultHolder, groupKeysArray[i], 
doubleValues[i]);
+          }
         }
         break;
       case STRING:
         String[] stringValues = blockValSet.getStringValuesSV();
-        for (int i = 0; i < length; i++) {
-          setValueForGroupKeys(groupByResultHolder, groupKeysArray[i], 
stringValues[i]);
+        nullBitmap = blockValSet.getNullBitmap();
+        if (nullBitmap != null && !nullBitmap.isEmpty()) {
+          for (int i = 0; i < length; i++) {
+            if (!nullBitmap.contains(i)) {
+              setValueForGroupKeys(groupByResultHolder, groupKeysArray[i], 
stringValues[i]);
+            }
+          }
+        } else {
+          for (int i = 0; i < length; i++) {
+            setValueForGroupKeys(groupByResultHolder, groupKeysArray[i], 
stringValues[i]);
+          }
         }
         break;
       case BYTES:
         byte[][] bytesValues = blockValSet.getBytesValuesSV();
-        for (int i = 0; i < length; i++) {
-          setValueForGroupKeys(groupByResultHolder, groupKeysArray[i], new 
ByteArray(bytesValues[i]));
+        nullBitmap = blockValSet.getNullBitmap();
+        if (nullBitmap != null && !nullBitmap.isEmpty()) {
+          for (int i = 0; i < length; i++) {
+            if (!nullBitmap.contains(i)) {
+              setValueForGroupKeys(groupByResultHolder, groupKeysArray[i], new 
ByteArray(bytesValues[i]));
+            }
+          }
+        } else {
+          for (int i = 0; i < length; i++) {
+            setValueForGroupKeys(groupByResultHolder, groupKeysArray[i], new 
ByteArray(bytesValues[i]));
+          }
         }
         break;
       default:
diff --git 
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctAvgAggregationFunction.java
 
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctAvgAggregationFunction.java
index bca958957f..f3d45eed27 100644
--- 
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctAvgAggregationFunction.java
+++ 
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctAvgAggregationFunction.java
@@ -33,8 +33,8 @@ import org.apache.pinot.segment.spi.AggregationFunctionType;
  */
 public class DistinctAvgAggregationFunction extends 
BaseDistinctAggregateAggregationFunction<Double> {
 
-  public DistinctAvgAggregationFunction(ExpressionContext expression) {
-    super(expression, AggregationFunctionType.DISTINCTAVG);
+  public DistinctAvgAggregationFunction(ExpressionContext expression, boolean 
nullHandlingEnabled) {
+    super(expression, AggregationFunctionType.DISTINCTAVG, 
nullHandlingEnabled);
   }
 
   @Override
diff --git 
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctAvgMVAggregationFunction.java
 
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctAvgMVAggregationFunction.java
index 30a8e00492..5f7f04e8ee 100644
--- 
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctAvgMVAggregationFunction.java
+++ 
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctAvgMVAggregationFunction.java
@@ -34,7 +34,7 @@ import org.apache.pinot.segment.spi.AggregationFunctionType;
 public class DistinctAvgMVAggregationFunction extends 
BaseDistinctAggregateAggregationFunction<Double> {
 
   public DistinctAvgMVAggregationFunction(ExpressionContext expression) {
-    super(expression, AggregationFunctionType.DISTINCTAVGMV);
+    super(expression, AggregationFunctionType.DISTINCTAVGMV, false);
   }
 
   @Override
diff --git 
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctCountAggregationFunction.java
 
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctCountAggregationFunction.java
index aec983b8df..a31779ab6b 100644
--- 
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctCountAggregationFunction.java
+++ 
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctCountAggregationFunction.java
@@ -33,8 +33,8 @@ import org.apache.pinot.segment.spi.AggregationFunctionType;
  */
 public class DistinctCountAggregationFunction extends 
BaseDistinctAggregateAggregationFunction<Integer> {
 
-  public DistinctCountAggregationFunction(ExpressionContext expression) {
-    super(expression, AggregationFunctionType.DISTINCTCOUNT);
+  public DistinctCountAggregationFunction(ExpressionContext expression, 
boolean nullHandlingEnabled) {
+    super(expression, AggregationFunctionType.DISTINCTCOUNT, 
nullHandlingEnabled);
   }
 
   @Override
diff --git 
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctCountMVAggregationFunction.java
 
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctCountMVAggregationFunction.java
index d8d257b400..cd03a0071f 100644
--- 
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctCountMVAggregationFunction.java
+++ 
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctCountMVAggregationFunction.java
@@ -34,7 +34,7 @@ import org.apache.pinot.segment.spi.AggregationFunctionType;
 public class DistinctCountMVAggregationFunction extends 
BaseDistinctAggregateAggregationFunction<Integer> {
 
   public DistinctCountMVAggregationFunction(ExpressionContext expression) {
-    super(expression, AggregationFunctionType.DISTINCTCOUNTMV);
+    super(expression, AggregationFunctionType.DISTINCTCOUNTMV, false);
   }
 
 
diff --git 
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctSumAggregationFunction.java
 
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctSumAggregationFunction.java
index f3a6c55805..c0ee5907dd 100644
--- 
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctSumAggregationFunction.java
+++ 
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctSumAggregationFunction.java
@@ -33,8 +33,8 @@ import org.apache.pinot.segment.spi.AggregationFunctionType;
  */
 public class DistinctSumAggregationFunction extends 
BaseDistinctAggregateAggregationFunction<Double> {
 
-  public DistinctSumAggregationFunction(ExpressionContext expression) {
-    super(expression, AggregationFunctionType.DISTINCTSUM);
+  public DistinctSumAggregationFunction(ExpressionContext expression, boolean 
nullHandlingEnabled) {
+    super(expression, AggregationFunctionType.DISTINCTSUM, 
nullHandlingEnabled);
   }
 
   @Override
diff --git 
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctSumMVAggregationFunction.java
 
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctSumMVAggregationFunction.java
index 3ffd0acd16..78b3d2cf09 100644
--- 
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctSumMVAggregationFunction.java
+++ 
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctSumMVAggregationFunction.java
@@ -34,7 +34,7 @@ import org.apache.pinot.segment.spi.AggregationFunctionType;
 public class DistinctSumMVAggregationFunction extends 
BaseDistinctAggregateAggregationFunction<Double> {
 
   public DistinctSumMVAggregationFunction(ExpressionContext expression) {
-    super(expression, AggregationFunctionType.DISTINCTSUMMV);
+    super(expression, AggregationFunctionType.DISTINCTSUMMV, false);
   }
 
   @Override
diff --git 
a/pinot-core/src/test/java/org/apache/pinot/queries/AllNullQueriesTest.java 
b/pinot-core/src/test/java/org/apache/pinot/queries/AllNullQueriesTest.java
index f345e99f58..ed8a5a716d 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/AllNullQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/AllNullQueriesTest.java
@@ -560,7 +560,7 @@ public class AllNullQueriesTest extends BaseQueriesTest {
           } else if (dataType == DataType.LONG) {
             assertEquals(((Double) row[1]).longValue(), Long.MIN_VALUE);
           }
-          assertEquals(row[2], 1);
+          assertEquals(row[2], 0);
         }
       }
       {
diff --git 
a/pinot-core/src/test/java/org/apache/pinot/queries/NullHandlingEnabledQueriesTest.java
 
b/pinot-core/src/test/java/org/apache/pinot/queries/NullHandlingEnabledQueriesTest.java
index 33eb0b0069..b53bfa17fa 100644
--- 
a/pinot-core/src/test/java/org/apache/pinot/queries/NullHandlingEnabledQueriesTest.java
+++ 
b/pinot-core/src/test/java/org/apache/pinot/queries/NullHandlingEnabledQueriesTest.java
@@ -479,6 +479,213 @@ public class NullHandlingEnabledQueriesTest extends 
BaseQueriesTest {
     assertNotNull(resultTable.getRows().get(0)[0]);
   }
 
+  @Test(dataProvider = "NumberTypes")
+  public void testDistinctCountDictNumberTypes(FieldSpec.DataType dataType)
+      throws Exception {
+    initializeRows();
+    insertRow(null);
+    insertRow(1);
+    TableConfig tableConfig = new 
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).build();
+    Schema schema = new 
Schema.SchemaBuilder().addSingleValueDimension(COLUMN1, dataType).build();
+    setUpSegments(tableConfig, schema);
+    String query = String.format("SELECT DISTINCTCOUNT(%s) FROM testTable", 
COLUMN1);
+
+    BrokerResponseNative brokerResponse = getBrokerResponse(query, 
QUERY_OPTIONS);
+
+    ResultTable resultTable = brokerResponse.getResultTable();
+    assertEquals(resultTable.getRows().size(), 1);
+    assertEquals(resultTable.getRows().get(0)[0], 1);
+  }
+
+  @Test(dataProvider = "NumberTypes")
+  public void testDistinctCountNonDictNumberTypes(FieldSpec.DataType dataType)
+      throws Exception {
+    initializeRows();
+    insertRow(null);
+    insertRow(1);
+    TableConfig tableConfig = new 
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME)
+        .setNoDictionaryColumns(Collections.singletonList(COLUMN1)).build();
+    Schema schema = new 
Schema.SchemaBuilder().addSingleValueDimension(COLUMN1, dataType).build();
+    setUpSegments(tableConfig, schema);
+    String query = String.format("SELECT DISTINCTCOUNT(%s) FROM testTable", 
COLUMN1);
+
+    BrokerResponseNative brokerResponse = getBrokerResponse(query, 
QUERY_OPTIONS);
+
+    ResultTable resultTable = brokerResponse.getResultTable();
+    assertEquals(resultTable.getRows().size(), 1);
+    assertEquals(resultTable.getRows().get(0)[0], 1);
+  }
+
+  @Test(dataProvider = "NumberTypes")
+  public void testGroupByDistinctCountDictNumberTypes(FieldSpec.DataType 
dataType)
+      throws Exception {
+    initializeRows();
+    insertRowWithTwoColumns(null, "key");
+    insertRowWithTwoColumns(1, "key");
+    TableConfig tableConfig = new 
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).build();
+    Schema schema = new 
Schema.SchemaBuilder().addSingleValueDimension(COLUMN1, dataType)
+        .addSingleValueDimension(COLUMN2, FieldSpec.DataType.STRING).build();
+    setUpSegments(tableConfig, schema);
+    String query = String.format("SELECT DISTINCTCOUNT(%s), %s FROM testTable 
GROUP BY %s", COLUMN1, COLUMN2, COLUMN2);
+
+    BrokerResponseNative brokerResponse = getBrokerResponse(query, 
QUERY_OPTIONS);
+
+    ResultTable resultTable = brokerResponse.getResultTable();
+    assertEquals(resultTable.getRows().size(), 1);
+    assertArrayEquals(resultTable.getRows().get(0), new Object[]{1, "key"});
+  }
+
+  @Test(dataProvider = "NumberTypes")
+  public void testGroupByDistinctCountNonDictNumberTypes(FieldSpec.DataType 
dataType)
+      throws Exception {
+    initializeRows();
+    insertRowWithTwoColumns(null, "key");
+    insertRowWithTwoColumns(1, "key");
+    TableConfig tableConfig = new 
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME)
+        .setNoDictionaryColumns(Collections.singletonList(COLUMN1)).build();
+    Schema schema = new 
Schema.SchemaBuilder().addSingleValueDimension(COLUMN1, dataType)
+        .addSingleValueDimension(COLUMN2, FieldSpec.DataType.STRING).build();
+    setUpSegments(tableConfig, schema);
+    String query = String.format("SELECT DISTINCTCOUNT(%s), %s FROM testTable 
GROUP BY %s", COLUMN1, COLUMN2, COLUMN2);
+
+    BrokerResponseNative brokerResponse = getBrokerResponse(query, 
QUERY_OPTIONS);
+
+    ResultTable resultTable = brokerResponse.getResultTable();
+    assertEquals(resultTable.getRows().size(), 1);
+    assertArrayEquals(resultTable.getRows().get(0), new Object[]{1, "key"});
+  }
+
+  @Test(dataProvider = "NumberTypes")
+  public void testGroupByMvDistinctCountNonDictNumberTypes(FieldSpec.DataType 
dataType)
+      throws Exception {
+    initializeRows();
+    insertRowWithTwoColumns(null, new String[]{"key1", "key2"});
+    insertRowWithTwoColumns(1, new String[]{"key1", "key2"});
+    insertRowWithTwoColumns(2, new String[]{"key2"});
+    TableConfig tableConfig = new 
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME)
+        .setNoDictionaryColumns(Collections.singletonList(COLUMN1)).build();
+    Schema schema = new 
Schema.SchemaBuilder().addSingleValueDimension(COLUMN1, dataType)
+        .addMultiValueDimension(COLUMN2, FieldSpec.DataType.STRING).build();
+    setUpSegments(tableConfig, schema);
+    String query =
+        String.format("SELECT DISTINCTCOUNT(%s), %s FROM testTable GROUP BY %s 
ORDER BY %s", COLUMN1, COLUMN2, COLUMN2,
+            COLUMN2);
+
+    BrokerResponseNative brokerResponse = getBrokerResponse(query, 
QUERY_OPTIONS);
+
+    ResultTable resultTable = brokerResponse.getResultTable();
+    assertEquals(resultTable.getRows().get(0), new Object[]{1, "key1"});
+    assertEquals(resultTable.getRows().get(1), new Object[]{2, "key2"});
+  }
+
+  @Test(dataProvider = "NumberTypes")
+  public void testGroupByMvDistinctCountDictNumberTypes(FieldSpec.DataType 
dataType)
+      throws Exception {
+    initializeRows();
+    insertRowWithTwoColumns(null, new String[]{"key1", "key2"});
+    insertRowWithTwoColumns(1, new String[]{"key1", "key2"});
+    insertRowWithTwoColumns(2, new String[]{"key2"});
+    TableConfig tableConfig = new 
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).build();
+    Schema schema = new 
Schema.SchemaBuilder().addSingleValueDimension(COLUMN1, dataType)
+        .addMultiValueDimension(COLUMN2, FieldSpec.DataType.STRING).build();
+    setUpSegments(tableConfig, schema);
+    String query =
+        String.format("SELECT DISTINCTCOUNT(%s), %s FROM testTable GROUP BY %s 
ORDER BY %s", COLUMN1, COLUMN2, COLUMN2,
+            COLUMN2);
+
+    BrokerResponseNative brokerResponse = getBrokerResponse(query, 
QUERY_OPTIONS);
+
+    ResultTable resultTable = brokerResponse.getResultTable();
+    assertEquals(resultTable.getRows().get(0), new Object[]{1, "key1"});
+    assertEquals(resultTable.getRows().get(1), new Object[]{2, "key2"});
+  }
+
+  @DataProvider(name = "DistinctCountObjectTypes")
+  public static Object[][] getDistinctAggregationObjectTypes() {
+    return new Object[][]{
+        {FieldSpec.DataType.STRING, "a"}, {
+        FieldSpec.DataType.BYTES, "a string".getBytes()
+    }
+    };
+  }
+
+  @Test(dataProvider = "DistinctCountObjectTypes")
+  public void testObjectDistinctCountObjectTypes(FieldSpec.DataType dataType, 
Object value)
+      throws Exception {
+    initializeRows();
+    insertRow(null);
+    insertRow(value);
+    TableConfig tableConfig = new 
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).build();
+    Schema schema = new 
Schema.SchemaBuilder().addSingleValueDimension(COLUMN1, dataType).build();
+    setUpSegments(tableConfig, schema);
+    String query = String.format("SELECT DISTINCTCOUNT(%s) FROM testTable", 
COLUMN1);
+
+    BrokerResponseNative brokerResponse = getBrokerResponse(query, 
QUERY_OPTIONS);
+
+    ResultTable resultTable = brokerResponse.getResultTable();
+    assertEquals(resultTable.getRows().size(), 1);
+    assertEquals(resultTable.getRows().get(0)[0], 1);
+  }
+
+  @Test(dataProvider = "DistinctCountObjectTypes")
+  public void testGroupByDistinctCountObjectTypes(FieldSpec.DataType dataType, 
Object value)
+      throws Exception {
+    initializeRows();
+    insertRowWithTwoColumns(null, "key");
+    insertRowWithTwoColumns(value, "key");
+    TableConfig tableConfig = new 
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).build();
+    Schema schema = new 
Schema.SchemaBuilder().addSingleValueDimension(COLUMN1, dataType)
+        .addSingleValueDimension(COLUMN2, FieldSpec.DataType.STRING).build();
+    setUpSegments(tableConfig, schema);
+    String query = String.format("SELECT DISTINCTCOUNT(%s), %s FROM testTable 
GROUP BY %s", COLUMN1, COLUMN2, COLUMN2);
+
+    BrokerResponseNative brokerResponse = getBrokerResponse(query, 
QUERY_OPTIONS);
+
+    ResultTable resultTable = brokerResponse.getResultTable();
+    assertEquals(resultTable.getRows().size(), 1);
+    assertArrayEquals(resultTable.getRows().get(0), new Object[]{1, "key"});
+  }
+
+  @Test
+  public void testDistinctSum()
+      throws Exception {
+    initializeRows();
+    insertRow(null);
+    insertRow(1);
+    insertRow(2);
+    insertRow(2);
+    TableConfig tableConfig = new 
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).build();
+    Schema schema = new 
Schema.SchemaBuilder().addSingleValueDimension(COLUMN1, 
FieldSpec.DataType.INT).build();
+    setUpSegments(tableConfig, schema);
+    String query = String.format("SELECT DISTINCTSUM(%s) FROM testTable", 
COLUMN1);
+
+    BrokerResponseNative brokerResponse = getBrokerResponse(query, 
QUERY_OPTIONS);
+
+    ResultTable resultTable = brokerResponse.getResultTable();
+    assertEquals(resultTable.getRows().size(), 1);
+    assertEquals(resultTable.getRows().get(0)[0], (double) 3);
+  }
+
+  @Test
+  public void testDistinctAvg()
+      throws Exception {
+    initializeRows();
+    insertRow(null);
+    insertRow(1);
+    insertRow(2);
+    insertRow(2);
+    TableConfig tableConfig = new 
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).build();
+    Schema schema = new 
Schema.SchemaBuilder().addSingleValueDimension(COLUMN1, 
FieldSpec.DataType.INT).build();
+    setUpSegments(tableConfig, schema);
+    String query = String.format("SELECT DISTINCTAVG(%s) FROM testTable", 
COLUMN1);
+
+    BrokerResponseNative brokerResponse = getBrokerResponse(query, 
QUERY_OPTIONS);
+
+    ResultTable resultTable = brokerResponse.getResultTable();
+    assertEquals(resultTable.getRows().size(), 1);
+    assertEquals(resultTable.getRows().get(0)[0], 1.5);
+  }
+
   @Test
   public void testTransformBlockValSetGetNullBitmap()
       throws Exception {


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]


Reply via email to