This is an automated email from the ASF dual-hosted git repository.
abhishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 9f9faeec81 object[] handling for DimensionHandlers for arrays (#12552)
9f9faeec81 is described below
commit 9f9faeec816fee7bc8936dc3cb4aa1b91450e6c3
Author: Karan Kumar <[email protected]>
AuthorDate: Wed May 25 15:24:18 2022 +0530
object[] handling for DimensionHandlers for arrays (#12552)
Description
Fixes a bug when running q's like
SELECT cntarray,
Count(*)
FROM (SELECT dim1,
dim2,
Array_agg(cnt) AS cntarray
FROM (SELECT dim1,
dim2,
dim3,
Count(*) AS cnt
FROM foo
GROUP BY 1,
2,
3)
GROUP BY 1,
2)
GROUP BY 1
This generates an error:
org.apache.druid.java.util.common.ISE: Unable to convert type
[Ljava.lang.Object; to org.apache.druid.segment.data.ComparableList
at
org.apache.druid.segment.DimensionHandlerUtils.convertToList(DimensionHandlerUtils.java:405)
~[druid-xx]
Because it's an array of numbers it looks like it does the convertToList
call, which looks like:
@Nullable
public static ComparableList convertToList(Object obj)
{
if (obj == null) {
return null;
}
if (obj instanceof List) {
return new ComparableList((List) obj);
}
if (obj instanceof ComparableList) {
return (ComparableList) obj;
}
throw new ISE("Unable to convert type %s to %s",
obj.getClass().getName(), ComparableList.class.getName());
}
I.e. it doesn't know about arrays. Added the array handling as part of this
PR.
---
.../apache/druid/query/groupby/GroupByQuery.java | 4 +-
.../epinephelinae/RowBasedGrouperHelper.java | 13 +-
.../query/groupby/orderby/DefaultLimitSpec.java | 5 +-
.../druid/segment/DimensionHandlerUtils.java | 82 ++++++--
.../query/groupby/GroupByQueryRunnerTest.java | 143 +++++++++++++-
.../druid/segment/DimensionHandlerUtilsTest.java | 176 ++++++++++++++++-
.../java/org/apache/druid/segment/TestHelper.java | 6 +-
.../druid/sql/calcite/CalciteArraysQueryTest.java | 218 ++++++++++++++++++++-
8 files changed, 609 insertions(+), 38 deletions(-)
diff --git
a/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java
b/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java
index 3f0b9a163a..0493145cf3 100644
--- a/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java
+++ b/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java
@@ -787,8 +787,8 @@ public class GroupByQuery extends BaseQuery<ResultRow>
dimCompare =
Comparators.<Comparable>naturalNullsFirst().compare(lhsArr, rhsArr);
} else if (dimensionType.equals(ColumnType.LONG_ARRAY)
|| dimensionType.equals(ColumnType.DOUBLE_ARRAY)) {
- final ComparableList lhsArr =
DimensionHandlerUtils.convertToList(lhsObj);
- final ComparableList rhsArr =
DimensionHandlerUtils.convertToList(rhsObj);
+ final ComparableList lhsArr =
DimensionHandlerUtils.convertToList(lhsObj,
dimensionType.getElementType().getType());
+ final ComparableList rhsArr =
DimensionHandlerUtils.convertToList(rhsObj,
dimensionType.getElementType().getType());
dimCompare =
Comparators.<Comparable>naturalNullsFirst().compare(lhsArr, rhsArr);
} else {
dimCompare = comparator.compare((String) lhsObj, (String) rhsObj);
diff --git
a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java
b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java
index 3381a9192e..bdf930f3e2 100644
---
a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java
+++
b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java
@@ -760,7 +760,8 @@ public class RowBasedGrouperHelper
case DOUBLE:
return
(InputRawSupplierColumnSelectorStrategy<ColumnValueSelector>)
columnSelector ->
- () ->
DimensionHandlerUtils.convertToList(columnSelector.getObject());
+ () ->
DimensionHandlerUtils.convertToList(columnSelector.getObject(),
+
capabilities.getElementType().getType());
default:
throw new IAE(
"Cannot create query type helper from invalid type [%s]",
@@ -1049,8 +1050,10 @@ public class RowBasedGrouperHelper
cmp = Comparators.<Comparable>naturalNullsFirst().compare(lhs, rhs);
} else if (fieldTypes.get(i - dimStart).equals(ColumnType.LONG_ARRAY)
|| fieldTypes.get(i -
dimStart).equals(ColumnType.DOUBLE_ARRAY)) {
- final ComparableList lhs =
DimensionHandlerUtils.convertToList(key1.getKey()[i]);
- final ComparableList rhs =
DimensionHandlerUtils.convertToList(key2.getKey()[i]);
+ final ComparableList lhs =
DimensionHandlerUtils.convertToList(key1.getKey()[i],
+
fieldTypes.get(i - dimStart).getElementType().getType());
+ final ComparableList rhs =
DimensionHandlerUtils.convertToList(key2.getKey()[i],
+
fieldTypes.get(i - dimStart).getElementType().getType());
cmp = Comparators.<Comparable>naturalNullsFirst().compare(lhs, rhs);
} else {
cmp = Comparators.<Comparable>naturalNullsFirst().compare(
@@ -1128,8 +1131,8 @@ public class RowBasedGrouperHelper
cmp = ComparableList.compareWithComparator(
comparator,
- DimensionHandlerUtils.convertToList(lhs),
- DimensionHandlerUtils.convertToList(rhs)
+ DimensionHandlerUtils.convertToList(lhs,
fieldType.getElementType().getType()),
+ DimensionHandlerUtils.convertToList(rhs,
fieldType.getElementType().getType())
);
} else {
diff --git
a/processing/src/main/java/org/apache/druid/query/groupby/orderby/DefaultLimitSpec.java
b/processing/src/main/java/org/apache/druid/query/groupby/orderby/DefaultLimitSpec.java
index 5597dcf597..c52c14bae7 100644
---
a/processing/src/main/java/org/apache/druid/query/groupby/orderby/DefaultLimitSpec.java
+++
b/processing/src/main/java/org/apache/druid/query/groupby/orderby/DefaultLimitSpec.java
@@ -440,11 +440,12 @@ public class DefaultLimitSpec implements LimitSpec
{
Comparator arrayComparator = null;
if (columnType.isArray()) {
+ final ValueType elementType = columnType.getElementType().getType();
if (columnType.getElementType().isNumeric()) {
arrayComparator = (Comparator<Object>) (o1, o2) ->
ComparableList.compareWithComparator(
comparator,
- DimensionHandlerUtils.convertToList(o1),
- DimensionHandlerUtils.convertToList(o2)
+ DimensionHandlerUtils.convertToList(o1, elementType),
+ DimensionHandlerUtils.convertToList(o2, elementType)
);
} else if (columnType.getElementType().equals(ColumnType.STRING)) {
arrayComparator = (Comparator<Object>) (o1, o2) ->
ComparableStringArray.compareWithComparator(
diff --git
a/processing/src/main/java/org/apache/druid/segment/DimensionHandlerUtils.java
b/processing/src/main/java/org/apache/druid/segment/DimensionHandlerUtils.java
index 4d05ecb089..309e14b3e2 100644
---
a/processing/src/main/java/org/apache/druid/segment/DimensionHandlerUtils.java
+++
b/processing/src/main/java/org/apache/druid/segment/DimensionHandlerUtils.java
@@ -42,6 +42,7 @@ import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.data.ComparableList;
import org.apache.druid.segment.data.ComparableStringArray;
+import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.math.BigDecimal;
import java.util.ArrayList;
@@ -381,8 +382,12 @@ public final class DimensionHandlerUtils
switch (type.getElementType().getType()) {
case STRING:
return convertToComparableStringArray(obj);
- default:
- return convertToList(obj);
+ case LONG:
+ return convertToListWithObjectFunction(obj,
DimensionHandlerUtils::convertObjectToLong);
+ case FLOAT:
+ return convertToListWithObjectFunction(obj,
DimensionHandlerUtils::convertObjectToFloat);
+ case DOUBLE:
+ return convertToListWithObjectFunction(obj,
DimensionHandlerUtils::convertObjectToDouble);
}
default:
@@ -391,18 +396,57 @@ public final class DimensionHandlerUtils
}
@Nullable
- public static ComparableList convertToList(Object obj)
+ public static ComparableList convertToList(Object obj, ValueType elementType)
+ {
+ switch (elementType) {
+ case LONG:
+ return convertToListWithObjectFunction(obj,
DimensionHandlerUtils::convertObjectToLong);
+ case FLOAT:
+ return convertToListWithObjectFunction(obj,
DimensionHandlerUtils::convertObjectToFloat);
+ case DOUBLE:
+ return convertToListWithObjectFunction(obj,
DimensionHandlerUtils::convertObjectToDouble);
+ }
+ throw new ISE(
+ "Unable to convert object of type[%s] to [%s]",
+ obj.getClass().getName(),
+ ComparableList.class.getName()
+ );
+ }
+
+
+ private static <T> ComparableList convertToListWithObjectFunction(Object
obj, Function<Object, T> convertFunction)
{
if (obj == null) {
return null;
}
if (obj instanceof List) {
- return new ComparableList((List) obj);
+ return convertToComparableList((List) obj, convertFunction);
}
if (obj instanceof ComparableList) {
- return (ComparableList) obj;
+ return convertToComparableList(((ComparableList) obj).getDelegate(),
convertFunction);
+ }
+ if (obj instanceof Object[]) {
+ final List<T> delegateList = new ArrayList<>();
+ for (Object eachObj : (Object[]) obj) {
+ delegateList.add(convertFunction.apply(eachObj));
+ }
+ return new ComparableList(delegateList);
+ }
+ throw new ISE(
+ "Unable to convert object of type[%s] to [%s]",
+ obj.getClass().getName(),
+ ComparableList.class.getName()
+ );
+ }
+
+ @Nonnull
+ private static <T> ComparableList convertToComparableList(List obj,
Function<Object, T> convertFunction)
+ {
+ final List<T> delegateList = new ArrayList<>();
+ for (Object eachObj : obj) {
+ delegateList.add(convertFunction.apply(eachObj));
}
- throw new ISE("Unable to convert type %s to %s", obj.getClass().getName(),
ComparableList.class.getName());
+ return new ComparableList(delegateList);
}
@@ -415,19 +459,27 @@ public final class DimensionHandlerUtils
if (obj instanceof ComparableStringArray) {
return (ComparableStringArray) obj;
}
- if (obj instanceof String[]) {
- return ComparableStringArray.of((String[]) obj);
- }
// Jackson converts the serialized array into a list. Converting it back
to a string array
if (obj instanceof List) {
- return ComparableStringArray.of((String[]) ((List) obj).toArray(new
String[0]));
+ String[] delegate = new String[((List) obj).size()];
+ for (int i = 0; i < delegate.length; i++) {
+ delegate[i] = convertObjectToString(((List) obj).get(i));
+ }
+ return ComparableStringArray.of(delegate);
}
- Objects[] objects = (Objects[]) obj;
- String[] delegate = new String[objects.length];
- for (int i = 0; i < objects.length; i++) {
- delegate[i] = convertObjectToString(objects[i]);
+ if (obj instanceof Object[]) {
+ Object[] objects = (Object[]) obj;
+ String[] delegate = new String[objects.length];
+ for (int i = 0; i < objects.length; i++) {
+ delegate[i] = convertObjectToString(objects[i]);
+ }
+ return ComparableStringArray.of(delegate);
}
- return ComparableStringArray.of(delegate);
+ throw new ISE(
+ "Unable to convert object of type[%s] to [%s]",
+ obj.getClass().getName(),
+ ComparableStringArray.class.getName()
+ );
}
public static int compareObjectsAsType(
diff --git
a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java
b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java
index 1511750bca..5426f48302 100644
---
a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java
+++
b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java
@@ -2055,6 +2055,141 @@ public class GroupByQueryRunnerTest extends
InitializedNullHandlingTest
.build();
+ List<ResultRow> expectedResults = Arrays.asList(
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(78L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(97L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(109L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(110L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(112L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(113L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(114L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(118L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(119L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(120L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(121L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(126L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(135L)), "rows", 2L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(144L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(147L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(158L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(166L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(1049L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(1144L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(1193L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(1234L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(1314L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(1321L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(1447L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(1522L)), "rows", 1L)
+ );
+
+ Iterable<ResultRow> results =
GroupByQueryRunnerTestHelper.runQuery(factory, runner, outer);
+ TestHelper.assertExpectedObjects(expectedResults, results,
"long-groupby-arrays");
+ }
+
+ @Test
+ public void testGroupByWithLongArraysDesc()
+ {
+ if
(config.getDefaultStrategy().equals(GroupByStrategySelector.STRATEGY_V1)) {
+ expectedException.expect(UnsupportedOperationException.class);
+ expectedException.expectMessage("GroupBy v1 only supports dimensions
with an outputType of STRING");
+ }
+ cannotVectorize();
+ GroupByQuery outer = makeQueryBuilder()
+ .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
+ .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
+ .setVirtualColumns(new ExpressionVirtualColumn(
+ "v0",
+ "array(index)",
+ ColumnType.LONG_ARRAY,
+ ExprMacroTable.nil()
+ ))
+ .setDimensions(
+ new DefaultDimensionSpec("v0", "alias_outer",
ColumnType.LONG_ARRAY)
+ )
+ .setLimitSpec(new DefaultLimitSpec(
+ ImmutableList.of(new OrderByColumnSpec(
+ "alias_outer",
+ OrderByColumnSpec.Direction.DESCENDING,
+ StringComparators.NUMERIC
+ )),
+ Integer.MAX_VALUE - 1
+ ))
+ .setAggregatorSpecs(
+ QueryRunnerTestHelper.ROWS_COUNT
+ )
+ .setGranularity(QueryRunnerTestHelper.ALL_GRAN)
+ .build();
+
+
+ List<ResultRow> expectedResults = Arrays.asList(
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(78L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(97L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(109L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(110L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(112L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(113L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(114L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(118L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(119L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(120L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(121L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(126L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(135L)), "rows", 2L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(144L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(147L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(158L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(166L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(1049L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(1144L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(1193L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(1234L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(1314L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(1321L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(1447L)), "rows", 1L),
+ makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(1522L)), "rows", 1L)
+ );
+ // reversing list
+ Collections.reverse(expectedResults);
+ Iterable<ResultRow> results =
GroupByQueryRunnerTestHelper.runQuery(factory, runner, outer);
+ TestHelper.assertExpectedObjects(expectedResults, results,
"long-groupby-arrays");
+ }
+
+ @Test
+ public void testGroupByWithDoubleArrays()
+ {
+ if
(config.getDefaultStrategy().equals(GroupByStrategySelector.STRATEGY_V1)) {
+ expectedException.expect(UnsupportedOperationException.class);
+ expectedException.expectMessage("GroupBy v1 only supports dimensions
with an outputType of STRING");
+ }
+ cannotVectorize();
+ GroupByQuery outer = makeQueryBuilder()
+ .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
+ .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
+ .setVirtualColumns(new ExpressionVirtualColumn(
+ "v0",
+ "array(index)",
+ ColumnType.DOUBLE_ARRAY,
+ ExprMacroTable.nil()
+ ))
+ .setDimensions(
+ new DefaultDimensionSpec("v0", "alias_outer",
ColumnType.DOUBLE_ARRAY)
+ )
+ .setLimitSpec(new DefaultLimitSpec(
+ ImmutableList.of(new OrderByColumnSpec(
+ "alias_outer",
+ OrderByColumnSpec.Direction.ASCENDING,
+ StringComparators.NUMERIC
+ )),
+ Integer.MAX_VALUE - 1
+ ))
+ .setAggregatorSpecs(
+ QueryRunnerTestHelper.ROWS_COUNT
+ )
+ .setGranularity(QueryRunnerTestHelper.ALL_GRAN)
+ .build();
+
+
List<ResultRow> expectedResults = Arrays.asList(
makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(78.622547)), "rows", 1L),
makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(97.387433)), "rows", 1L),
@@ -2083,13 +2218,13 @@ public class GroupByQueryRunnerTest extends
InitializedNullHandlingTest
makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(1447.34116)), "rows", 1L),
makeRow(outer, "2011-04-01", "alias_outer", new
ComparableList(ImmutableList.of(1522.043733)), "rows", 1L)
);
-
Iterable<ResultRow> results =
GroupByQueryRunnerTestHelper.runQuery(factory, runner, outer);
TestHelper.assertExpectedObjects(expectedResults, results,
"long-groupby-arrays");
}
+
@Test
- public void testGroupByWithLongArraysDesc()
+ public void testGroupByWithDoubleArraysDesc()
{
if
(config.getDefaultStrategy().equals(GroupByStrategySelector.STRATEGY_V1)) {
expectedException.expect(UnsupportedOperationException.class);
@@ -2102,11 +2237,11 @@ public class GroupByQueryRunnerTest extends
InitializedNullHandlingTest
.setVirtualColumns(new ExpressionVirtualColumn(
"v0",
"array(index)",
- ColumnType.LONG_ARRAY,
+ ColumnType.DOUBLE_ARRAY,
ExprMacroTable.nil()
))
.setDimensions(
- new DefaultDimensionSpec("v0", "alias_outer",
ColumnType.LONG_ARRAY)
+ new DefaultDimensionSpec("v0", "alias_outer",
ColumnType.DOUBLE_ARRAY)
)
.setLimitSpec(new DefaultLimitSpec(
ImmutableList.of(new OrderByColumnSpec(
diff --git
a/processing/src/test/java/org/apache/druid/segment/DimensionHandlerUtilsTest.java
b/processing/src/test/java/org/apache/druid/segment/DimensionHandlerUtilsTest.java
index 088783e89f..53b598a0cd 100644
---
a/processing/src/test/java/org/apache/druid/segment/DimensionHandlerUtilsTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/DimensionHandlerUtilsTest.java
@@ -19,6 +19,7 @@
package org.apache.druid.segment;
+import com.google.common.collect.ImmutableList;
import org.apache.druid.data.input.impl.DimensionSchema;
import org.apache.druid.data.input.impl.DoubleDimensionSchema;
import org.apache.druid.data.input.impl.FloatDimensionSchema;
@@ -28,6 +29,9 @@ import org.apache.druid.java.util.common.ISE;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.column.ValueType;
+import org.apache.druid.segment.data.ComparableList;
+import org.apache.druid.segment.data.ComparableStringArray;
import org.apache.druid.testing.InitializedNullHandlingTest;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -44,12 +48,19 @@ public class DimensionHandlerUtilsTest extends
InitializedNullHandlingTest
@Rule
public ExpectedException expectedException = ExpectedException.none();
+ private static final ComparableList<Long> LONG_COMPARABLE_LIST = new
ComparableList<>(ImmutableList.of(1L, 2L));
+ private static final ComparableList<Double> DOUBLE_COMPARABLE_LIST = new
ComparableList<>(ImmutableList.of(1.0, 2.0));
+ private static final ComparableList<Float> FLOAT_COMPARABLE_LIST = new
ComparableList<>(ImmutableList.of(1F, 2F));
+ private static final ComparableStringArray COMPARABLE_STRING_ARRAY =
ComparableStringArray.of("1", "2");
+ private static final ComparableStringArray COMPARABLE_STRING_ARRAY_DECIMAL =
ComparableStringArray.of("1.0", "2.0");
+
@BeforeClass
public static void setupTests()
{
DimensionHandlerUtils.registerDimensionHandlerProvider(
TYPE,
- d -> new DoubleDimensionHandler(d) {
+ d -> new DoubleDimensionHandler(d)
+ {
@Override
public DimensionSchema getDimensionSchema(ColumnCapabilities
capabilities)
{
@@ -145,6 +156,169 @@ public class DimensionHandlerUtilsTest extends
InitializedNullHandlingTest
Assert.assertTrue(handler.getDimensionSchema(capabilities) instanceof
LongDimensionSchema);
}
+ @Test
+ public void testComparableLongList()
+ {
+ Assert.assertEquals(null, DimensionHandlerUtils.convertToList(null,
ValueType.LONG));
+ Assert.assertEquals(
+ LONG_COMPARABLE_LIST,
+ DimensionHandlerUtils.convertToList(ImmutableList.of(1L, 2L),
ValueType.LONG)
+ );
+ Assert.assertEquals(
+ LONG_COMPARABLE_LIST,
+ DimensionHandlerUtils.convertToList(
+ new ComparableList(ImmutableList.of(1L, 2L)),
+ ValueType.LONG
+ )
+ );
+
+ assertArrayCases(LONG_COMPARABLE_LIST, ValueType.LONG);
+
+ Assert.assertThrows(
+ "Unable to convert object of type[Long] to [ComparableList]",
+ ISE.class,
+ () -> DimensionHandlerUtils.convertToList(1L, ValueType.LONG)
+ );
+
+ Assert.assertThrows(
+ "Unable to convert object of type[Long] to [ComparableList]",
+ ISE.class,
+ () -> DimensionHandlerUtils.convertToList(1L, ValueType.ARRAY)
+ );
+
+ Assert.assertThrows(
+ "Unable to convert object of type[Long] to [ComparableList]",
+ ISE.class,
+ () -> DimensionHandlerUtils.convertToList(1L, ValueType.STRING)
+ );
+ }
+
+ @Test
+ public void testComparableFloatList()
+ {
+ Assert.assertEquals(null, DimensionHandlerUtils.convertToList(null,
ValueType.FLOAT));
+ Assert.assertEquals(
+ FLOAT_COMPARABLE_LIST,
+ DimensionHandlerUtils.convertToList(ImmutableList.of(1.0F, 2.0F),
ValueType.FLOAT)
+ );
+ Assert.assertEquals(
+ FLOAT_COMPARABLE_LIST,
+ DimensionHandlerUtils.convertToList(
+ new ComparableList(ImmutableList.of(1.0F, 2.0F)),
+ ValueType.FLOAT
+ )
+ );
+
+ assertArrayCases(FLOAT_COMPARABLE_LIST, ValueType.FLOAT);
+
+ Assert.assertThrows(
+ "Unable to convert object of type[Float] to [ComparableList]",
+ ISE.class,
+ () -> DimensionHandlerUtils.convertToList(1.0F, ValueType.FLOAT)
+ );
+
+ Assert.assertThrows(
+ "Unable to convert object of type[Float] to [ComparableList]",
+ ISE.class,
+ () -> DimensionHandlerUtils.convertToList(1.0F, ValueType.ARRAY)
+ );
+
+ Assert.assertThrows(
+ "Unable to convert object of type[Float] to [ComparableList]",
+ ISE.class,
+ () -> DimensionHandlerUtils.convertToList(1.0F, ValueType.STRING)
+ );
+ }
+
+ @Test
+ public void testComparableDoubleList()
+ {
+ Assert.assertEquals(null, DimensionHandlerUtils.convertToList(null,
ValueType.DOUBLE));
+ Assert.assertEquals(
+ DOUBLE_COMPARABLE_LIST,
+ DimensionHandlerUtils.convertToList(ImmutableList.of(1.0D, 2.0D),
ValueType.DOUBLE)
+ );
+ Assert.assertEquals(
+ DOUBLE_COMPARABLE_LIST,
+ DimensionHandlerUtils.convertToList(
+ new ComparableList(ImmutableList.of(1.0D, 2.0D)),
+ ValueType.DOUBLE
+ )
+ );
+
+ assertArrayCases(DOUBLE_COMPARABLE_LIST, ValueType.DOUBLE);
+
+ Assert.assertThrows(
+ "Unable to convert object of type[Double] to [ComparableList]",
+ ISE.class,
+ () -> DimensionHandlerUtils.convertToList(1.0D, ValueType.DOUBLE)
+ );
+
+ Assert.assertThrows(
+ "Unable to convert object of type[Double] to [ComparableList]",
+ ISE.class,
+ () -> DimensionHandlerUtils.convertToList(1.0D, ValueType.ARRAY)
+ );
+
+ Assert.assertThrows(
+ "Unable to convert object of type[Double] to [ComparableList]",
+ ISE.class,
+ () -> DimensionHandlerUtils.convertToList(1.0D, ValueType.STRING)
+ );
+ }
+
+ @Test
+ public void testComparableStringArrayList()
+ {
+ Assert.assertEquals(null,
DimensionHandlerUtils.convertToComparableStringArray(null));
+ Assert.assertEquals(
+ COMPARABLE_STRING_ARRAY,
+
DimensionHandlerUtils.convertToComparableStringArray(ImmutableList.of("1", "2"))
+ );
+
+ Assert.assertEquals(
+ COMPARABLE_STRING_ARRAY,
+ DimensionHandlerUtils.convertToComparableStringArray(new Object[]{1L,
2L})
+ );
+ Assert.assertEquals(
+ COMPARABLE_STRING_ARRAY,
+ DimensionHandlerUtils.convertToComparableStringArray(new Long[]{1L,
2L})
+ );
+ Assert.assertEquals(
+ COMPARABLE_STRING_ARRAY_DECIMAL,
+ DimensionHandlerUtils.convertToComparableStringArray(new
String[]{"1.0", "2.0"})
+ );
+ Assert.assertEquals(
+ COMPARABLE_STRING_ARRAY_DECIMAL,
+ DimensionHandlerUtils.convertToComparableStringArray(new Double[]{1.0,
2.0})
+ );
+ Assert.assertEquals(
+ COMPARABLE_STRING_ARRAY_DECIMAL,
+ DimensionHandlerUtils.convertToComparableStringArray(new Float[]{1F,
2F})
+ );
+
+ Assert.assertThrows(
+ "Unable to convert object of type[String] to
[ComparablComparableStringArray]",
+ ISE.class,
+ () -> DimensionHandlerUtils.convertToComparableStringArray("1")
+ );
+ }
+
+ private static void assertArrayCases(ComparableList expectedComparableList,
ValueType elementType)
+ {
+ Assert.assertEquals(expectedComparableList,
DimensionHandlerUtils.convertToList(new Object[]{1L, 2L}, elementType));
+ Assert.assertEquals(expectedComparableList,
DimensionHandlerUtils.convertToList(new Long[]{1L, 2L}, elementType));
+ Assert.assertEquals(
+ expectedComparableList,
+ DimensionHandlerUtils.convertToList(new String[]{"1.0", "2.0"},
elementType)
+ );
+ Assert.assertEquals(
+ expectedComparableList,
+ DimensionHandlerUtils.convertToList(new Double[]{1.0, 2.0},
elementType)
+ );
+ Assert.assertEquals(expectedComparableList,
DimensionHandlerUtils.convertToList(new Float[]{1F, 2F}, elementType));
+ }
+
private static class TestDimensionSchema extends DimensionSchema
{
diff --git a/processing/src/test/java/org/apache/druid/segment/TestHelper.java
b/processing/src/test/java/org/apache/druid/segment/TestHelper.java
index 8a913a5790..4ee90243dd 100644
--- a/processing/src/test/java/org/apache/druid/segment/TestHelper.java
+++ b/processing/src/test/java/org/apache/druid/segment/TestHelper.java
@@ -416,9 +416,9 @@ public class TestHelper
ExprEval.coerceListToArray((List) actualValue, true).rhs
);
} else if (expectedValue instanceof ComparableList && actualValue
instanceof List) {
- Assert.assertEquals(
- ((ComparableList) expectedValue).getDelegate(),
- (List) actualValue
+ Assert.assertArrayEquals(
+ ((ComparableList) expectedValue).getDelegate().toArray(new
Object[0]),
+ ExprEval.coerceListToArray((List) actualValue, true).rhs
);
} else {
Assert.assertEquals(
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java
index d18a6c1676..fd9915d67d 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java
@@ -1069,15 +1069,15 @@ public class CalciteArraysQueryTest extends
BaseCalciteQueryTest
.build()
),
useDefault ? ImmutableList.of(
- new Object[]{ImmutableList.of(0.0), 4L},
- new Object[]{ImmutableList.of(0.10000000149011612), 1L},
- new Object[]{ImmutableList.of(1.0), 1L}
+ new Object[]{ImmutableList.of(0.0F), 4L},
+ new Object[]{ImmutableList.of(0.10000000149011612F), 1L},
+ new Object[]{ImmutableList.of(1.0F), 1L}
) :
ImmutableList.of(
new Object[]{Collections.singletonList(null), 3L},
- new Object[]{ImmutableList.of(0.0), 1L},
- new Object[]{ImmutableList.of(0.10000000149011612), 1L},
- new Object[]{ImmutableList.of(1.0), 1L}
+ new Object[]{ImmutableList.of(0.0F), 1L},
+ new Object[]{ImmutableList.of(0.10000000149011612F), 1L},
+ new Object[]{ImmutableList.of(1.0F), 1L}
)
);
}
@@ -2177,6 +2177,212 @@ public class CalciteArraysQueryTest extends
BaseCalciteQueryTest
);
}
+ @Test
+ public void testArrayAggGroupByArrayAggOfLongsFromSubquery() throws Exception
+ {
+ requireMergeBuffers(3);
+ cannotVectorize();
+ testQuery(
+ "select cntarray, count(*) from ( select dim1, dim2, ARRAY_AGG(cnt) as
cntarray from ( select dim1, dim2, dim3, count(*) as cnt from foo group by 1,
2, 3 ) group by 1, 2 ) group by 1",
+ QUERY_CONTEXT_NO_STRINGIFY_ARRAY,
+ ImmutableList.of(
+ GroupByQuery.builder()
+ .setDataSource(new QueryDataSource(
+ GroupByQuery.builder()
+ .setDataSource(new QueryDataSource(
+ GroupByQuery.builder()
+ .setDataSource(new
TableDataSource(CalciteTests.DATASOURCE1))
+
.setQuerySegmentSpec(querySegmentSpec(Filtration.eternity()))
+
.setGranularity(Granularities.ALL)
+
.setInterval(querySegmentSpec(Filtration.eternity()))
+
.setContext(QUERY_CONTEXT_NO_STRINGIFY_ARRAY)
+ .setDimensions(
+ new
DefaultDimensionSpec("dim1", "d0"),
+ new
DefaultDimensionSpec("dim2", "d1"),
+ new
DefaultDimensionSpec("dim3", "d2"
+ )
+ )
+ .setAggregatorSpecs(
+ new
CountAggregatorFactory("a0"))
+ .build()))
+ .setQuerySegmentSpec(
+
querySegmentSpec(Filtration.eternity()))
+ .setGranularity(Granularities.ALL)
+ .setDimensions(
+ new DefaultDimensionSpec(
+ "d0",
+ "_d0"
+ ),
+ new DefaultDimensionSpec(
+ "d1",
+ "_d1"
+ )
+ )
+ .setAggregatorSpecs(new
ExpressionLambdaAggregatorFactory(
+ "_a0",
+ ImmutableSet.of("a0"),
+ "__acc",
+ "ARRAY<LONG>[]",
+ "ARRAY<LONG>[]",
+ true,
+ true,
+ false,
+ "array_append(\"__acc\", \"a0\")",
+ "array_concat(\"__acc\", \"_a0\")",
+ null,
+ null,
+ new HumanReadableBytes(1024),
+ ExprMacroTable.nil()
+ ))
+ .build()))
+
.setQuerySegmentSpec(querySegmentSpec(Filtration.eternity()))
+ .setGranularity(Granularities.ALL)
+ .setInterval(querySegmentSpec(Filtration.eternity()))
+ .setContext(QUERY_CONTEXT_NO_STRINGIFY_ARRAY)
+ .setDimensions(new DefaultDimensionSpec("_a0", "d0",
ColumnType.LONG_ARRAY))
+ .setAggregatorSpecs(new CountAggregatorFactory("a0"))
+ .build()
+ ),
+ ImmutableList.of(
+ new Object[]{ImmutableList.of(1L), 4L},
+ new Object[]{ImmutableList.of(1L, 1L), 2L}
+ )
+ );
+ }
+
+ @Test
+ public void testArrayAggGroupByArrayAggOfStringsFromSubquery() throws
Exception
+ {
+ requireMergeBuffers(3);
+ cannotVectorize();
+ testQuery(
+ "select cntarray, count(*) from ( select dim1, dim2, ARRAY_AGG(cnt) as
cntarray from ( select dim1, dim2, dim3, cast( count(*) as VARCHAR ) as cnt
from foo group by 1, 2, 3 ) group by 1, 2 ) group by 1",
+ QUERY_CONTEXT_NO_STRINGIFY_ARRAY,
+ ImmutableList.of(
+ GroupByQuery.builder()
+ .setDataSource(new QueryDataSource(
+ GroupByQuery.builder()
+ .setDataSource(new QueryDataSource(
+ GroupByQuery.builder()
+ .setDataSource(new
TableDataSource(CalciteTests.DATASOURCE1))
+
.setQuerySegmentSpec(querySegmentSpec(Filtration.eternity()))
+
.setGranularity(Granularities.ALL)
+
.setInterval(querySegmentSpec(Filtration.eternity()))
+
.setContext(QUERY_CONTEXT_NO_STRINGIFY_ARRAY)
+ .setDimensions(
+ new
DefaultDimensionSpec("dim1", "d0"),
+ new
DefaultDimensionSpec("dim2", "d1"),
+ new
DefaultDimensionSpec("dim3", "d2"
+ )
+ )
+ .setAggregatorSpecs(
+ new
CountAggregatorFactory("a0"))
+ .build()))
+
.setQuerySegmentSpec(querySegmentSpec(Filtration.eternity()))
+ .setGranularity(Granularities.ALL)
+ .setDimensions(
+ new DefaultDimensionSpec("d0",
"_d0"),
+ new DefaultDimensionSpec("d1",
"_d1")
+ )
+ .setAggregatorSpecs(new
ExpressionLambdaAggregatorFactory(
+ "_a0",
+ ImmutableSet.of("a0"),
+ "__acc",
+ "ARRAY<STRING>[]",
+ "ARRAY<STRING>[]",
+ true,
+ true,
+ false,
+ "array_append(\"__acc\", \"a0\")",
+ "array_concat(\"__acc\", \"_a0\")",
+ null,
+ null,
+ new HumanReadableBytes(1024),
+ ExprMacroTable.nil()
+ ))
+ .build()))
+
.setQuerySegmentSpec(querySegmentSpec(Filtration.eternity()))
+ .setGranularity(Granularities.ALL)
+ .setInterval(querySegmentSpec(Filtration.eternity()))
+ .setContext(QUERY_CONTEXT_NO_STRINGIFY_ARRAY)
+ .setDimensions(new DefaultDimensionSpec("_a0", "d0",
ColumnType.STRING_ARRAY))
+ .setAggregatorSpecs(new CountAggregatorFactory("a0"))
+ .build()
+ ),
+ ImmutableList.of(
+ new Object[]{ImmutableList.of("1"), 4L},
+ new Object[]{ImmutableList.of("1", "1"), 2L}
+ )
+ );
+ }
+
+ @Test
+ public void testArrayAggGroupByArrayAggOfDoubleFromSubquery() throws
Exception
+ {
+ requireMergeBuffers(3);
+ cannotVectorize();
+ testQuery(
+ "select cntarray, count(*) from ( select dim1, dim2, ARRAY_AGG(cnt) as
cntarray from ( select dim1, dim2, dim3, cast( count(*) as DOUBLE ) as cnt from
foo group by 1, 2, 3 ) group by 1, 2 ) group by 1",
+ QUERY_CONTEXT_NO_STRINGIFY_ARRAY,
+ ImmutableList.of(
+ GroupByQuery
+ .builder()
+ .setDataSource(new QueryDataSource(
+ GroupByQuery.builder()
+ .setDataSource(new QueryDataSource(
+ GroupByQuery.builder()
+ .setDataSource(new
TableDataSource(CalciteTests.DATASOURCE1))
+
.setQuerySegmentSpec(querySegmentSpec(Filtration.eternity()))
+
.setGranularity(Granularities.ALL)
+
.setInterval(querySegmentSpec(Filtration.eternity()))
+
.setContext(QUERY_CONTEXT_NO_STRINGIFY_ARRAY)
+ .setDimensions(
+ new
DefaultDimensionSpec("dim1", "d0"),
+ new
DefaultDimensionSpec("dim2", "d1"),
+ new
DefaultDimensionSpec("dim3", "d2"
+ )
+ )
+ .setAggregatorSpecs(
+ new
CountAggregatorFactory("a0"))
+ .build()))
+
.setQuerySegmentSpec(querySegmentSpec(Filtration.eternity()))
+ .setGranularity(Granularities.ALL)
+ .setDimensions(
+ new DefaultDimensionSpec("d0", "_d0"),
+ new DefaultDimensionSpec("d1", "_d1")
+ )
+ .setAggregatorSpecs(new
ExpressionLambdaAggregatorFactory(
+ "_a0",
+ ImmutableSet.of("a0"),
+ "__acc",
+ "ARRAY<DOUBLE>[]",
+ "ARRAY<DOUBLE>[]",
+ true,
+ true,
+ false,
+ "array_append(\"__acc\", \"a0\")",
+ "array_concat(\"__acc\", \"_a0\")",
+ null,
+ null,
+ new HumanReadableBytes(1024),
+ ExprMacroTable.nil()
+ ))
+ .build()))
+ .setQuerySegmentSpec(querySegmentSpec(Filtration.eternity()))
+ .setGranularity(Granularities.ALL)
+ .setInterval(querySegmentSpec(Filtration.eternity()))
+ .setContext(QUERY_CONTEXT_NO_STRINGIFY_ARRAY)
+ .setDimensions(new DefaultDimensionSpec("_a0", "d0",
ColumnType.DOUBLE_ARRAY))
+ .setAggregatorSpecs(new CountAggregatorFactory("a0"))
+ .build()
+ ),
+ ImmutableList.of(
+ new Object[]{ImmutableList.of(1.0), 4L},
+ new Object[]{ImmutableList.of(1.0, 1.0), 2L}
+ )
+ );
+ }
+
@Test
public void testArrayAggArrayContainsSubquery() throws Exception
{
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
