This is an automated email from the ASF dual-hosted git repository. volodymyr pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git
commit 7badf0f12a59c75a02e5ac3eee9a49c40c01426f Author: Bohdan Kazydub <[email protected]> AuthorDate: Thu Nov 7 19:12:26 2019 +0200 DRILL-7440: Failure during loading of RepeatedCount functions closes #1894 --- .../exec/hive/complex_types/TestHiveArrays.java | 22 +++++++++++++ .../src/main/codegen/templates/TypeHelper.java | 38 +++++++++++++++++++++- .../apache/drill/exec/expr/fn/DrillFuncHolder.java | 28 +++++----------- .../exec/expr/holders/RepeatedDictHolder.java | 3 +- 4 files changed, 69 insertions(+), 22 deletions(-) diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/complex_types/TestHiveArrays.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/complex_types/TestHiveArrays.java index 4731d4d..955b800 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/complex_types/TestHiveArrays.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/complex_types/TestHiveArrays.java @@ -1673,6 +1673,28 @@ public class TestHiveArrays extends ClusterTest { } @Test + public void mapArrayRepeatedCount() throws Exception { + testBuilder() + .sqlQuery("SELECT rid, REPEATED_COUNT(arr_n_0) rc FROM hive.map_array") + .unOrdered() + .baselineColumns("rid", "rc") + .baselineValues(1, 3) + .baselineValues(2, 2) + .baselineValues(3, 1) + .go(); + } + + @Test + public void mapArrayCount() throws Exception { + testBuilder() + .sqlQuery("SELECT COUNT(arr_n_0) cnt FROM hive.map_array") + .unOrdered() + .baselineColumns("cnt") + .baselineValues(3L) + .go(); + } + + @Test public void unionArray() throws Exception { testBuilder() .sqlQuery("SELECT rid, un_arr FROM hive.union_array") diff --git a/exec/java-exec/src/main/codegen/templates/TypeHelper.java b/exec/java-exec/src/main/codegen/templates/TypeHelper.java index 73152ce..0e1a2dc 100644 --- a/exec/java-exec/src/main/codegen/templates/TypeHelper.java +++ b/exec/java-exec/src/main/codegen/templates/TypeHelper.java @@ -72,7 +72,13 @@ public class TypeHelper extends BasicTypeHelper { case UNION: return model._ref(UnionHolder.class); case DICT: - return model._ref(DictHolder.class); + switch (mode) { + case REQUIRED: + case OPTIONAL: + return model._ref(DictHolder.class); + case REPEATED: + return model._ref(RepeatedDictHolder.class); + } case MAP: case LIST: return model._ref(ComplexHolder.class); @@ -100,4 +106,34 @@ public class TypeHelper extends BasicTypeHelper { throw new UnsupportedOperationException(buildErrorMessage("get holder type", type, mode)); } + public static JType getComplexHolderType(JCodeModel model, MinorType type, DataMode mode) { + switch (type) { + case DICT: + switch (mode) { + case REQUIRED: + case OPTIONAL: + return model._ref(DictHolder.class); + case REPEATED: + return model._ref(RepeatedDictHolder.class); + } + case MAP: + switch (mode) { + case REQUIRED: + case OPTIONAL: + return model._ref(MapHolder.class); + case REPEATED: + return model._ref(RepeatedMapHolder.class); + } + case LIST: + switch (mode) { + case REQUIRED: + case OPTIONAL: + return model._ref(ListHolder.class); + case REPEATED: + return model._ref(RepeatedListHolder.class); + } + default: + throw new IllegalArgumentException("Complex type expected. Found: " + type); + } + } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java index 7f4acc0..4d758f8 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java @@ -26,7 +26,6 @@ import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.FieldReference; import org.apache.drill.common.expression.FunctionHolderExpression; import org.apache.drill.common.expression.LogicalExpression; -import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.common.types.Types; @@ -39,10 +38,6 @@ import org.apache.drill.exec.expr.DrillFuncHolderExpr; import org.apache.drill.exec.expr.TypeHelper; import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; import org.apache.drill.exec.expr.fn.output.OutputWidthCalculator; -import org.apache.drill.exec.expr.holders.ListHolder; -import org.apache.drill.exec.expr.holders.MapHolder; -import org.apache.drill.exec.expr.holders.RepeatedListHolder; -import org.apache.drill.exec.expr.holders.RepeatedMapHolder; import org.apache.drill.exec.expr.holders.ValueHolder; import org.apache.drill.exec.ops.UdfUtilities; import org.apache.drill.exec.vector.complex.reader.FieldReader; @@ -234,11 +229,10 @@ public abstract class DrillFuncHolder extends AbstractFuncHolder { JInvocation reader = JExpr._new(singularReaderClass).arg(inputVariable.getHolder()); declare(sub, parameter, fieldReadClass, reader, i); } else if (!parameter.isFieldReader() && inputVariable.isReader() && Types.isComplex(parameter.getType())) { - // For complex data-types (repeated maps/lists) the input to the aggregate will be a FieldReader. However, aggregate + // For complex data-types (repeated maps/lists/dicts) the input to the aggregate will be a FieldReader. However, aggregate // functions like ANY_VALUE, will assume the input to be a RepeatedMapHolder etc. Generate boilerplate code, to map // from FieldReader to respective Holder. - if (parameter.getType().getMinorType() == MinorType.MAP - || parameter.getType().getMinorType() == MinorType.LIST) { + if (Types.isComplex(parameter.getType())) { JType holderClass = getParamClass(g.getModel(), parameter, inputVariable.getHolder().type()); JAssignmentTarget holderVar = declare(sub, parameter, holderClass, JExpr._new(holderClass), i); sub.assign(holderVar.ref("reader"), inputVariable.getHolder()); @@ -304,19 +298,13 @@ public abstract class DrillFuncHolder extends AbstractFuncHolder { private JType getParamClass(JCodeModel model, ValueReference parameter, JType defaultType) { if (parameter.isFieldReader()) { return model._ref(FieldReader.class); - } else if (parameter.getType().getMinorType() == MinorType.MAP) { - if (parameter.getType().getMode() == TypeProtos.DataMode.REPEATED) { - return model._ref(RepeatedMapHolder.class); - } else { - return model._ref(MapHolder.class); - } - } else if (parameter.getType().getMinorType() == MinorType.LIST) { - if (parameter.getType().getMode() == TypeProtos.DataMode.REPEATED) { - return model._ref(RepeatedListHolder.class); - } else { - return model._ref(ListHolder.class); - } } + + if (Types.isComplex(parameter.getType())) { + MajorType type = parameter.getType(); + return TypeHelper.getComplexHolderType(model, type.getMinorType(), type.getMode()); + } + return defaultType; } diff --git a/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedDictHolder.java b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedDictHolder.java index adb604c..12a34be 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedDictHolder.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedDictHolder.java @@ -18,11 +18,12 @@ package org.apache.drill.exec.expr.holders; import org.apache.drill.common.types.TypeProtos; +import org.apache.drill.common.types.Types; import org.apache.drill.exec.vector.complex.RepeatedDictVector; public final class RepeatedDictHolder extends RepeatedValueHolder { - public TypeProtos.MajorType TYPE = RepeatedDictVector.TYPE; + public static final TypeProtos.MajorType TYPE = Types.repeated(TypeProtos.MinorType.DICT); public RepeatedDictVector vector; }
