http://git-wip-us.apache.org/repos/asf/hive/blob/edc53cc0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterDecimal64ColumnInList.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterDecimal64ColumnInList.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterDecimal64ColumnInList.java new file mode 100644 index 0000000..a75cdbf --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterDecimal64ColumnInList.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor.Descriptor; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; + +/** + * Evaluate IN filter on a batch for a vector of longs. + */ +public class FilterDecimal64ColumnInList extends FilterLongColumnInList { + + private static final long serialVersionUID = 1L; + + public FilterDecimal64ColumnInList() { + super(); + } + + /** + * After construction you must call setInListValues() to add the values to the IN set. + */ + public FilterDecimal64ColumnInList(int colNum) { + super(colNum); + } + + @Override + public Descriptor getDescriptor() { + + // This VectorExpression (IN) is a special case, so don't return a descriptor. + return null; + } + + @Override + public String vectorExpressionParameters() { + DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) inputTypeInfos[0]; + final int scale = decimalTypeInfo.scale(); + HiveDecimalWritable writable = new HiveDecimalWritable(); + StringBuilder sb = new StringBuilder(); + sb.append(getColumnParamString(0, inputCol)); + sb.append(", values ["); + for (long value : inListValues) { + writable.deserialize64(value, scale); + sb.append(", decimal64Val "); + sb.append(value); + sb.append(", decimalVal "); + sb.append(writable.toString()); + } + sb.append("]"); + return sb.toString(); + } +}
http://git-wip-us.apache.org/repos/asf/hive/blob/edc53cc0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterDecimal64ColumnNotBetween.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterDecimal64ColumnNotBetween.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterDecimal64ColumnNotBetween.java new file mode 100644 index 0000000..13d5c1a --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterDecimal64ColumnNotBetween.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColumnNotBetween; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; + +public class FilterDecimal64ColumnNotBetween extends FilterLongColumnNotBetween { + + private static final long serialVersionUID = 1L; + + public FilterDecimal64ColumnNotBetween(int colNum, long leftValue, long rightValue) { + super(colNum, leftValue, rightValue); + } + + public FilterDecimal64ColumnNotBetween() { + super(); + } + + @Override + public String vectorExpressionParameters() { + DecimalTypeInfo decimalTypeInfo1 = (DecimalTypeInfo) inputTypeInfos[1]; + HiveDecimalWritable writable1 = new HiveDecimalWritable(); + writable1.deserialize64(leftValue, decimalTypeInfo1.scale()); + + DecimalTypeInfo decimalTypeInfo2 = (DecimalTypeInfo) inputTypeInfos[2]; + HiveDecimalWritable writable2 = new HiveDecimalWritable(); + writable2.deserialize64(rightValue, decimalTypeInfo2.scale()); + return + getColumnParamString(0, colNum) + + ", decimal64LeftVal " + leftValue + ", decimalLeftVal " + writable1.toString() + + ", decimal64RightVal " + rightValue + ", decimalRightVal " + writable2.toString(); + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.FILTER) + .setNumArguments(3) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.DECIMAL_64, + VectorExpressionDescriptor.ArgumentType.DECIMAL_64, + VectorExpressionDescriptor.ArgumentType.DECIMAL_64) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.SCALAR, + VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/edc53cc0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterLongColumnInList.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterLongColumnInList.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterLongColumnInList.java index 312a388..7306bbf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterLongColumnInList.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterLongColumnInList.java @@ -36,8 +36,9 @@ import java.util.regex.Pattern; public class FilterLongColumnInList extends VectorExpression implements ILongInExpr { private static final long serialVersionUID = 1L; - private final int inputCol; - private long[] inListValues; + + protected final int inputCol; + protected long[] inListValues; // Transient members initialized by transientInit method. http://git-wip-us.apache.org/repos/asf/hive/blob/edc53cc0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColumnInList.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColumnInList.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColumnInList.java index 8469882..d519141 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColumnInList.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColumnInList.java @@ -33,8 +33,8 @@ public class LongColumnInList extends VectorExpression implements ILongInExpr { private static final long serialVersionUID = 1L; - private int colNum; - private long[] inListValues; + protected int colNum; + protected long[] inListValues; // The set object containing the IN list. This is optimized for lookup // of the data type of the column. http://git-wip-us.apache.org/repos/asf/hive/blob/edc53cc0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringColumnInList.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringColumnInList.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringColumnInList.java index 55c2586..9328eb4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringColumnInList.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringColumnInList.java @@ -170,6 +170,11 @@ public class StringColumnInList extends VectorExpression implements IStringInExp @Override public String vectorExpressionParameters() { - return getColumnParamString(0, inputCol) + ", values " + Arrays.toString(inListValues); + StringBuilder sb = new StringBuilder(); + sb.append("col "); + sb.append(inputCol); + sb.append(", values "); + sb.append(displayArrayOfUtf8ByteArrays(inListValues)); + return sb.toString(); } } http://git-wip-us.apache.org/repos/asf/hive/blob/edc53cc0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateLong.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateLong.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateLong.java deleted file mode 100644 index 8e5f9da..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateLong.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions; - -/** - * Vectorized version of TO_DATE(TIMESTAMP)/TO_DATE(DATE). - * As TO_DATE() now returns DATE type, this should be the same behavior as the DATE cast operator. - */ -public class VectorUDFDateLong extends CastLongToDate { - private static final long serialVersionUID = 1L; - - public VectorUDFDateLong() { - super(); - } - - public VectorUDFDateLong(int inputColumn, int outputColumnNum) { - super(inputColumn, outputColumnNum); - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/edc53cc0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFArgDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFArgDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFArgDesc.java index 69a2bef..af35ee6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFArgDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFArgDesc.java @@ -19,6 +19,8 @@ package org.apache.hadoop.hive.ql.exec.vector.udf; import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -27,10 +29,20 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.StandardConstantStructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; /** * Descriptor for function argument. @@ -73,13 +85,55 @@ public class VectorUDFArgDesc implements Serializable { public void prepareConstant() { final Object writableValue; if (constExpr != null) { - PrimitiveCategory pc = ((PrimitiveTypeInfo) constExpr.getTypeInfo()) - .getPrimitiveCategory(); - - // Convert from Java to Writable - writableValue = PrimitiveObjectInspectorFactory - .getPrimitiveJavaObjectInspector(pc).getPrimitiveWritableObject( - constExpr.getValue()); + Object constantValue = constExpr.getValue(); + TypeInfo typeInfo = constExpr.getTypeInfo(); + ObjectInspector objectInspector = + TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(typeInfo); + Category category = typeInfo.getCategory(); + switch (category) { + case PRIMITIVE: + { + PrimitiveCategory pc = + ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory(); + + // Convert from Java to Writable + AbstractPrimitiveJavaObjectInspector primitiveJavaObjectInspector = + PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(pc); + writableValue = + primitiveJavaObjectInspector.getPrimitiveWritableObject(constantValue); + } + break; + case STRUCT: + { + if (constantValue.getClass().isArray()) { + constantValue = java.util.Arrays.asList((Object[]) constantValue); + } + + StructObjectInspector structObjectInspector = + (StructObjectInspector) objectInspector; + List<? extends StructField> fields = structObjectInspector.getAllStructFieldRefs(); + List<String> fieldNames = new ArrayList<String>(fields.size()); + List<ObjectInspector> fieldObjectInspectors = + new ArrayList<ObjectInspector>(fields.size()); + for (StructField f : fields) { + fieldNames.add(f.getFieldName()); + fieldObjectInspectors.add( + ObjectInspectorUtils.getStandardObjectInspector( + f.getFieldObjectInspector(), ObjectInspectorCopyOption.WRITABLE)); + } + + StandardConstantStructObjectInspector constantStructObjectInspector = + ObjectInspectorFactory.getStandardConstantStructObjectInspector( + fieldNames, + fieldObjectInspectors, + (List<?>) constantValue); + writableValue = + constantStructObjectInspector.getWritableConstantValue(); + } + break; + default: + throw new RuntimeException("Unexpected category " + category); + } } else { writableValue = null; } http://git-wip-us.apache.org/repos/asf/hive/blob/edc53cc0/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDate.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDate.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDate.java index f5c4eb5..3c39af7e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDate.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDate.java @@ -23,7 +23,6 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFDateLong; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFDateString; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFDateTimestamp; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -48,7 +47,7 @@ import org.apache.hive.common.util.DateParser; extended = "Example:\n " + " > SELECT _FUNC_('2009-07-30 04:17:52') FROM src LIMIT 1;\n" + " '2009-07-30'") -@VectorizedExpressions({VectorUDFDateString.class, VectorUDFDateLong.class, VectorUDFDateTimestamp.class}) +@VectorizedExpressions({VectorUDFDateString.class, VectorUDFDateTimestamp.class}) public class GenericUDFDate extends GenericUDF { private transient TimestampConverter timestampConverter; private transient Converter textConverter; http://git-wip-us.apache.org/repos/asf/hive/blob/edc53cc0/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDate.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDate.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDate.java index b53ddcb..c309ffa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDate.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDate.java @@ -21,7 +21,6 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; -import org.apache.hadoop.hive.ql.exec.vector.expressions.CastLongToDate; import org.apache.hadoop.hive.ql.exec.vector.expressions.CastStringToDate; import org.apache.hadoop.hive.ql.exec.vector.expressions.CastTimestampToDate; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -42,7 +41,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectIn + "Example:\n " + " > SELECT CAST('2009-01-01' AS DATE) FROM src LIMIT 1;\n" + " '2009-01-01'") -@VectorizedExpressions({CastStringToDate.class, CastLongToDate.class, CastTimestampToDate.class}) +@VectorizedExpressions({CastStringToDate.class, CastTimestampToDate.class}) public class GenericUDFToDate extends GenericUDF { private transient PrimitiveObjectInspector argumentOI; http://git-wip-us.apache.org/repos/asf/hive/blob/edc53cc0/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorRandomRowSource.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorRandomRowSource.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorRandomRowSource.java index b6ae7d2..dfbf9d4 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorRandomRowSource.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorRandomRowSource.java @@ -118,6 +118,8 @@ public class VectorRandomRowSource { private List<ObjectInspector> primitiveObjectInspectorList; + private List<String> columnNames; + private StructObjectInspector rowStructObjectInspector; private List<GenerationSpec> generationSpecList; @@ -159,20 +161,24 @@ public class VectorRandomRowSource { OMIT_GENERATION, STRING_FAMILY, STRING_FAMILY_OTHER_TYPE_VALUE, - TIMESTAMP_MILLISECONDS + TIMESTAMP_MILLISECONDS, + VALUE_LIST } private final GenerationKind generationKind; private final TypeInfo typeInfo; private final TypeInfo sourceTypeInfo; private final StringGenerationOption stringGenerationOption; + private final List<Object> valueList; private GenerationSpec(GenerationKind generationKind, TypeInfo typeInfo, - TypeInfo sourceTypeInfo, StringGenerationOption stringGenerationOption) { + TypeInfo sourceTypeInfo, StringGenerationOption stringGenerationOption, + List<Object> valueList) { this.generationKind = generationKind; this.typeInfo = typeInfo; this.sourceTypeInfo = sourceTypeInfo; this.stringGenerationOption = stringGenerationOption; + this.valueList = valueList; } public GenerationKind getGenerationKind() { @@ -191,31 +197,40 @@ public class VectorRandomRowSource { return stringGenerationOption; } + public List<Object> getValueList() { + return valueList; + } + public static GenerationSpec createSameType(TypeInfo typeInfo) { return new GenerationSpec( - GenerationKind.SAME_TYPE, typeInfo, null, null); + GenerationKind.SAME_TYPE, typeInfo, null, null, null); } public static GenerationSpec createOmitGeneration(TypeInfo typeInfo) { return new GenerationSpec( - GenerationKind.OMIT_GENERATION, typeInfo, null, null); + GenerationKind.OMIT_GENERATION, typeInfo, null, null, null); } public static GenerationSpec createStringFamily(TypeInfo typeInfo, StringGenerationOption stringGenerationOption) { return new GenerationSpec( - GenerationKind.STRING_FAMILY, typeInfo, null, stringGenerationOption); + GenerationKind.STRING_FAMILY, typeInfo, null, stringGenerationOption, null); } public static GenerationSpec createStringFamilyOtherTypeValue(TypeInfo typeInfo, TypeInfo otherTypeTypeInfo) { return new GenerationSpec( - GenerationKind.STRING_FAMILY_OTHER_TYPE_VALUE, typeInfo, otherTypeTypeInfo, null); + GenerationKind.STRING_FAMILY_OTHER_TYPE_VALUE, typeInfo, otherTypeTypeInfo, null, null); } public static GenerationSpec createTimestampMilliseconds(TypeInfo typeInfo) { return new GenerationSpec( - GenerationKind.TIMESTAMP_MILLISECONDS, typeInfo, null, null); + GenerationKind.TIMESTAMP_MILLISECONDS, typeInfo, null, null, null); + } + + public static GenerationSpec createValueList(TypeInfo typeInfo, List<Object> valueList) { + return new GenerationSpec( + GenerationKind.VALUE_LIST, typeInfo, null, null, valueList); } } @@ -243,6 +258,10 @@ public class VectorRandomRowSource { return primitiveTypeInfos; } + public List<String> columnNames() { + return columnNames; + } + public StructObjectInspector rowStructObjectInspector() { return rowStructObjectInspector; } @@ -342,7 +361,7 @@ public class VectorRandomRowSource { "map" }; - private static String getRandomTypeName(Random random, SupportedTypes supportedTypes, + public static String getRandomTypeName(Random random, SupportedTypes supportedTypes, Set<String> allowedTypeNameSet) { String typeName = null; @@ -370,7 +389,7 @@ public class VectorRandomRowSource { return getDecoratedTypeName(random, typeName, null, null, 0, 1); } - private static String getDecoratedTypeName(Random random, String typeName, + public static String getDecoratedTypeName(Random random, String typeName, SupportedTypes supportedTypes, Set<String> allowedTypeNameSet, int depth, int maxDepth) { depth++; @@ -421,7 +440,7 @@ public class VectorRandomRowSource { if (i > 0) { sb.append(","); } - sb.append("col"); + sb.append("field"); sb.append(i); sb.append(":"); sb.append(fieldTypeName); @@ -549,7 +568,7 @@ public class VectorRandomRowSource { allTypes = false; onlyOne = false; } else if (allowedTypeNameSet != null) { - columnCount = 1 + r.nextInt(20); + columnCount = 1 + r.nextInt(allowedTypeNameSet.size()); allTypes = false; onlyOne = false; } else { @@ -586,9 +605,9 @@ public class VectorRandomRowSource { primitiveCategories = new PrimitiveCategory[columnCount]; primitiveTypeInfos = new PrimitiveTypeInfo[columnCount]; primitiveObjectInspectorList = new ArrayList<ObjectInspector>(columnCount); - List<String> columnNames = new ArrayList<String>(columnCount); + columnNames = new ArrayList<String>(columnCount); for (int c = 0; c < columnCount; c++) { - columnNames.add(String.format("col%d", c)); + columnNames.add(String.format("col%d", c + 1)); final String typeName; DataTypePhysicalVariation dataTypePhysicalVariation = DataTypePhysicalVariation.NONE; @@ -902,6 +921,13 @@ public class VectorRandomRowSource { object = longWritable; } break; + case VALUE_LIST: + { + List<Object> valueList = generationSpec.getValueList(); + final int valueCount = valueList.size(); + object = valueList.get(r.nextInt(valueCount)); + } + break; default: throw new RuntimeException("Unexpected generationKind " + generationKind); } @@ -1180,6 +1206,42 @@ public class VectorRandomRowSource { } } + public static Object getWritableObject(TypeInfo typeInfo, + ObjectInspector objectInspector, Object object) { + + final Category category = typeInfo.getCategory(); + switch (category) { + case PRIMITIVE: + return + getWritablePrimitiveObject( + (PrimitiveTypeInfo) typeInfo, + objectInspector, DataTypePhysicalVariation.NONE, object); + case STRUCT: + { + final StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; + final StandardStructObjectInspector structInspector = + (StandardStructObjectInspector) objectInspector; + final List<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos(); + final int size = fieldTypeInfos.size(); + final List<? extends StructField> structFields = + structInspector.getAllStructFieldRefs(); + + List<Object> input = (ArrayList<Object>) object; + List<Object> result = new ArrayList<Object>(size); + for (int i = 0; i < size; i++) { + final StructField structField = structFields.get(i); + final TypeInfo fieldTypeInfo = fieldTypeInfos.get(i); + result.add( + getWritableObject( + fieldTypeInfo, structField.getFieldObjectInspector(), input.get(i))); + } + return result; + } + default: + throw new RuntimeException("Unexpected category " + category); + } + } + public static Object getNonWritablePrimitiveObject(Object object, TypeInfo typeInfo, ObjectInspector objectInspector) { @@ -1290,41 +1352,91 @@ public class VectorRandomRowSource { } } + public static Object getNonWritableObject(Object object, TypeInfo typeInfo, + ObjectInspector objectInspector) { + final Category category = typeInfo.getCategory(); + switch (category) { + case PRIMITIVE: + return getNonWritablePrimitiveObject(object, typeInfo, objectInspector); + case STRUCT: + { + final StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; + final StandardStructObjectInspector structInspector = + (StandardStructObjectInspector) objectInspector; + final List<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos(); + final int size = fieldTypeInfos.size(); + final List<? extends StructField> structFields = + structInspector.getAllStructFieldRefs(); + + List<Object> input = (ArrayList<Object>) object; + List<Object> result = new ArrayList<Object>(size); + for (int i = 0; i < size; i++) { + final StructField structField = structFields.get(i); + final TypeInfo fieldTypeInfo = fieldTypeInfos.get(i); + result.add( + getNonWritableObject(input.get(i), fieldTypeInfo, + structField.getFieldObjectInspector())); + } + return result; + } + default: + throw new RuntimeException("Unexpected category " + category); + } + } + public Object randomWritable(int column) { return randomWritable( - typeInfos[column], objectInspectorList.get(column), dataTypePhysicalVariations[column], + r, typeInfos[column], objectInspectorList.get(column), dataTypePhysicalVariations[column], allowNull); } public Object randomWritable(TypeInfo typeInfo, ObjectInspector objectInspector) { - return randomWritable(typeInfo, objectInspector, DataTypePhysicalVariation.NONE, allowNull); + return randomWritable(r, typeInfo, objectInspector, DataTypePhysicalVariation.NONE, allowNull); } public Object randomWritable(TypeInfo typeInfo, ObjectInspector objectInspector, boolean allowNull) { - return randomWritable(typeInfo, objectInspector, DataTypePhysicalVariation.NONE, allowNull); + return randomWritable(r, typeInfo, objectInspector, DataTypePhysicalVariation.NONE, allowNull); } public Object randomWritable(TypeInfo typeInfo, ObjectInspector objectInspector, DataTypePhysicalVariation dataTypePhysicalVariation, boolean allowNull) { + return randomWritable(r, typeInfo, objectInspector, dataTypePhysicalVariation, allowNull); + } + + public static Object randomWritable(Random random, TypeInfo typeInfo, + ObjectInspector objectInspector) { + return randomWritable( + random, typeInfo, objectInspector, DataTypePhysicalVariation.NONE, false); + } + + public static Object randomWritable(Random random, TypeInfo typeInfo, + ObjectInspector objectInspector, boolean allowNull) { + return randomWritable( + random, typeInfo, objectInspector, DataTypePhysicalVariation.NONE, allowNull); + } + + public static Object randomWritable(Random random, TypeInfo typeInfo, + ObjectInspector objectInspector, DataTypePhysicalVariation dataTypePhysicalVariation, + boolean allowNull) { switch (typeInfo.getCategory()) { case PRIMITIVE: { - if (allowNull && r.nextInt(20) == 0) { + if (allowNull && random.nextInt(20) == 0) { return null; } - final Object object = randomPrimitiveObject(r, (PrimitiveTypeInfo) typeInfo); + final Object object = randomPrimitiveObject(random, (PrimitiveTypeInfo) typeInfo); return getWritablePrimitiveObject( (PrimitiveTypeInfo) typeInfo, objectInspector, dataTypePhysicalVariation, object); } case LIST: { - if (allowNull && r.nextInt(20) == 0) { + if (allowNull && random.nextInt(20) == 0) { return null; } // Always generate a list with at least 1 value? - final int elementCount = 1 + r.nextInt(100); + final int elementCount = 1 + random.nextInt(100); final StandardListObjectInspector listObjectInspector = (StandardListObjectInspector) objectInspector; final ObjectInspector elementObjectInspector = @@ -1345,7 +1457,8 @@ public class VectorRandomRowSource { } final Object listObj = listObjectInspector.create(elementCount); for (int i = 0; i < elementCount; i++) { - final Object ele = randomWritable(elementTypeInfo, elementObjectInspector, allowNull); + final Object ele = randomWritable( + random, elementTypeInfo, elementObjectInspector, allowNull); // UNDONE: For now, a 1-element list with a null element is a null list... if (ele == null && elementCount == 1) { return null; @@ -1382,10 +1495,10 @@ public class VectorRandomRowSource { } case MAP: { - if (allowNull && r.nextInt(20) == 0) { + if (allowNull && random.nextInt(20) == 0) { return null; } - final int keyPairCount = r.nextInt(100); + final int keyPairCount = random.nextInt(100); final StandardMapObjectInspector mapObjectInspector = (StandardMapObjectInspector) objectInspector; final ObjectInspector keyObjectInspector = @@ -1400,15 +1513,15 @@ public class VectorRandomRowSource { valueObjectInspector); final Object mapObj = mapObjectInspector.create(); for (int i = 0; i < keyPairCount; i++) { - Object key = randomWritable(keyTypeInfo, keyObjectInspector); - Object value = randomWritable(valueTypeInfo, valueObjectInspector); + Object key = randomWritable(random, keyTypeInfo, keyObjectInspector); + Object value = randomWritable(random, valueTypeInfo, valueObjectInspector); mapObjectInspector.put(mapObj, key, value); } return mapObj; } case STRUCT: { - if (allowNull && r.nextInt(20) == 0) { + if (allowNull && random.nextInt(20) == 0) { return null; } final StandardStructObjectInspector structObjectInspector = @@ -1423,7 +1536,7 @@ public class VectorRandomRowSource { final TypeInfo fieldTypeInfo = TypeInfoUtils.getTypeInfoFromObjectInspector( fieldObjectInspector); - final Object fieldObj = randomWritable(fieldTypeInfo, fieldObjectInspector); + final Object fieldObj = randomWritable(random, fieldTypeInfo, fieldObjectInspector); structObjectInspector.setStructFieldData(structObj, fieldRef, fieldObj); } return structObj; @@ -1434,13 +1547,13 @@ public class VectorRandomRowSource { (StandardUnionObjectInspector) objectInspector; final List<ObjectInspector> objectInspectorList = unionObjectInspector.getObjectInspectors(); final int unionCount = objectInspectorList.size(); - final byte tag = (byte) r.nextInt(unionCount); + final byte tag = (byte) random.nextInt(unionCount); final ObjectInspector fieldObjectInspector = objectInspectorList.get(tag); final TypeInfo fieldTypeInfo = TypeInfoUtils.getTypeInfoFromObjectInspector( fieldObjectInspector); - final Object fieldObj = randomWritable(fieldTypeInfo, fieldObjectInspector, false); + final Object fieldObj = randomWritable(random, fieldTypeInfo, fieldObjectInspector, false); if (fieldObj == null) { throw new RuntimeException(); } http://git-wip-us.apache.org/repos/asf/hive/blob/edc53cc0/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmetic.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmetic.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmetic.java index a09daf3..1b61071 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmetic.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmetic.java @@ -89,7 +89,7 @@ public class TestVectorArithmetic { public TestVectorArithmetic() { // Arithmetic operations rely on getting conf from SessionState, need to initialize here. SessionState ss = new SessionState(new HiveConf()); - ss.getConf().setVar(HiveConf.ConfVars.HIVE_COMPAT, "latest"); + ss.getConf().setVar(HiveConf.ConfVars.HIVE_COMPAT, "default"); SessionState.setCurrentSessionState(ss); } @@ -364,7 +364,7 @@ public class TestVectorArithmetic { new ArrayList<DataTypePhysicalVariation>(); List<String> columns = new ArrayList<String>(); - int columnNum = 0; + int columnNum = 1; ExprNodeDesc col1Expr; Object scalar1Object = null; http://git-wip-us.apache.org/repos/asf/hive/blob/edc53cc0/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorBetweenIn.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorBetweenIn.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorBetweenIn.java new file mode 100644 index 0000000..3f1a137 --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorBetweenIn.java @@ -0,0 +1,1014 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; + +import org.apache.hadoop.hive.common.type.DataTypePhysicalVariation; +import org.apache.hadoop.hive.common.type.HiveChar; +import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; +import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory; +import org.apache.hadoop.hive.ql.exec.FunctionInfo; +import org.apache.hadoop.hive.ql.exec.FunctionRegistry; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.ql.exec.vector.VectorExtractRow; +import org.apache.hadoop.hive.ql.exec.vector.VectorRandomBatchSource; +import org.apache.hadoop.hive.ql.exec.vector.VectorRandomRowSource; +import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx; +import org.apache.hadoop.hive.ql.exec.vector.VectorRandomRowSource.GenerationSpec; +import org.apache.hadoop.hive.ql.exec.vector.VectorRandomRowSource.SupportedTypes; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IdentityExpression; +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.expressions.TestVectorArithmetic.ColumnScalarMode; +import org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFAdaptor; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.VirtualColumn; +import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBetween; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIn; +import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.io.HiveCharWritable; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; +import org.apache.hadoop.io.BooleanWritable; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.WritableComparator; +import org.apache.hadoop.io.WritableComparable; + +import junit.framework.Assert; + +import org.junit.Ignore; +import org.junit.Test; + +public class TestVectorBetweenIn { + + @Test + public void testTinyInt() throws Exception { + Random random = new Random(5371); + + doBetweenIn(random, "tinyint"); + } + + @Test + public void testSmallInt() throws Exception { + Random random = new Random(2772); + + doBetweenIn(random, "smallint"); + } + + @Test + public void testInt() throws Exception { + Random random = new Random(12882); + + doBetweenIn(random, "int"); + } + + @Test + public void testBigInt() throws Exception { + Random random = new Random(12882); + + doBetweenIn(random, "bigint"); + } + + @Test + public void testString() throws Exception { + Random random = new Random(12882); + + doBetweenIn(random, "string"); + } + + @Test + public void testTimestamp() throws Exception { + Random random = new Random(12882); + + doBetweenIn(random, "timestamp"); + } + + @Test + public void testDate() throws Exception { + Random random = new Random(12882); + + doBetweenIn(random, "date"); + } + + @Test + public void testFloat() throws Exception { + Random random = new Random(7322); + + doBetweenIn(random, "float"); + } + + @Test + public void testDouble() throws Exception { + Random random = new Random(12882); + + doBetweenIn(random, "double"); + } + + @Test + public void testChar() throws Exception { + Random random = new Random(12882); + + doBetweenIn(random, "char(10)"); + } + + @Test + public void testVarchar() throws Exception { + Random random = new Random(12882); + + doBetweenIn(random, "varchar(15)"); + } + + @Test + public void testDecimal() throws Exception { + Random random = new Random(9300); + + doDecimalTests(random, /* tryDecimal64 */ false); + } + + @Test + public void testDecimal64() throws Exception { + Random random = new Random(9300); + + doDecimalTests(random, /* tryDecimal64 */ true); + } + + @Test + public void testStruct() throws Exception { + Random random = new Random(9300); + + doStructTests(random); + } + + public enum BetweenInTestMode { + ROW_MODE, + ADAPTOR, + VECTOR_EXPRESSION; + + static final int count = values().length; + } + + public enum BetweenInVariation { + FILTER_BETWEEN, + FILTER_NOT_BETWEEN, + PROJECTION_BETWEEN, + PROJECTION_NOT_BETWEEN, + FILTER_IN, + PROJECTION_IN; + + static final int count = values().length; + + final boolean isFilter; + BetweenInVariation() { + isFilter = name().startsWith("FILTER"); + } + } + + private static TypeInfo[] decimalTypeInfos = new TypeInfo[] { + new DecimalTypeInfo(38, 18), + new DecimalTypeInfo(25, 2), + new DecimalTypeInfo(19, 4), + new DecimalTypeInfo(18, 10), + new DecimalTypeInfo(17, 3), + new DecimalTypeInfo(12, 2), + new DecimalTypeInfo(7, 1) + }; + + private void doDecimalTests(Random random, boolean tryDecimal64) + throws Exception { + for (TypeInfo typeInfo : decimalTypeInfos) { + doBetweenIn( + random, typeInfo.getTypeName(), tryDecimal64); + } + } + + private void doBetweenIn(Random random, String typeName) + throws Exception { + doBetweenIn(random, typeName, /* tryDecimal64 */ false); + } + + private static final BetweenInVariation[] structInVarations = + new BetweenInVariation[] { BetweenInVariation.FILTER_IN, BetweenInVariation.PROJECTION_IN }; + + private void doStructTests(Random random) throws Exception { + + String typeName = "struct"; + + // These are the only type supported for STRUCT IN by the VectorizationContext class. + Set<String> allowedTypeNameSet = new HashSet<String>(); + allowedTypeNameSet.add("int"); + allowedTypeNameSet.add("bigint"); + allowedTypeNameSet.add("double"); + allowedTypeNameSet.add("string"); + + // Only STRUCT type IN currently supported. + for (BetweenInVariation betweenInVariation : structInVarations) { + + for (int i = 0; i < 4; i++) { + typeName = + VectorRandomRowSource.getDecoratedTypeName( + random, typeName, SupportedTypes.ALL, allowedTypeNameSet, + /* depth */ 0, /* maxDepth */ 1); + + doBetweenStructInVariation( + random, typeName, betweenInVariation); + } + } + } + + private void doBetweenIn(Random random, String typeName, boolean tryDecimal64) + throws Exception { + + int subVariation; + for (BetweenInVariation betweenInVariation : BetweenInVariation.values()) { + subVariation = 0; + while (true) { + if (!doBetweenInVariation( + random, typeName, tryDecimal64, betweenInVariation, subVariation)) { + break; + } + subVariation++; + } + } + } + + private boolean checkDecimal64(boolean tryDecimal64, TypeInfo typeInfo) { + if (!tryDecimal64 || !(typeInfo instanceof DecimalTypeInfo)) { + return false; + } + DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo; + boolean result = HiveDecimalWritable.isPrecisionDecimal64(decimalTypeInfo.getPrecision()); + return result; + } + + private void removeValue(List<Object> valueList, Object value) { + valueList.remove(value); + } + + private boolean needsValidDataTypeData(TypeInfo typeInfo) { + if (!(typeInfo instanceof PrimitiveTypeInfo)) { + return false; + } + PrimitiveCategory primitiveCategory = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory(); + if (primitiveCategory == PrimitiveCategory.STRING || + primitiveCategory == PrimitiveCategory.CHAR || + primitiveCategory == PrimitiveCategory.VARCHAR || + primitiveCategory == PrimitiveCategory.BINARY) { + return false; + } + return true; + } + + private boolean doBetweenInVariation(Random random, String typeName, + boolean tryDecimal64, BetweenInVariation betweenInVariation, int subVariation) + throws Exception { + + TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeName); + + boolean isDecimal64 = checkDecimal64(tryDecimal64, typeInfo); + DataTypePhysicalVariation dataTypePhysicalVariation = + (isDecimal64 ? DataTypePhysicalVariation.DECIMAL_64 : DataTypePhysicalVariation.NONE); + final int decimal64Scale = + (isDecimal64 ? ((DecimalTypeInfo) typeInfo).getScale() : 0); + + //---------------------------------------------------------------------------------------------- + + ObjectInspector objectInspector = + TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo( + typeInfo); + + final int valueCount = 10 + random.nextInt(10); + List<Object> valueList = new ArrayList<Object>(valueCount); + for (int i = 0; i < valueCount; i++) { + valueList.add( + VectorRandomRowSource.randomWritable( + random, typeInfo, objectInspector, dataTypePhysicalVariation, /* allowNull */ false)); + } + + final boolean isBetween = + (betweenInVariation == BetweenInVariation.FILTER_BETWEEN || + betweenInVariation == BetweenInVariation.FILTER_NOT_BETWEEN || + betweenInVariation == BetweenInVariation.PROJECTION_BETWEEN || + betweenInVariation == BetweenInVariation.PROJECTION_NOT_BETWEEN); + + List<Object> compareList = new ArrayList<Object>(); + + List<Object> sortedList = new ArrayList<Object>(valueCount); + sortedList.addAll(valueList); + + Object object = valueList.get(0); + WritableComparator writableComparator = + WritableComparator.get((Class<? extends WritableComparable>) object.getClass()); + sortedList.sort(writableComparator); + + final boolean isInvert; + if (isBetween) { + + // FILTER_BETWEEN + // FILTER_NOT_BETWEEN + // PROJECTION_BETWEEN + // PROJECTION_NOT_BETWEEN + isInvert = + (betweenInVariation == BetweenInVariation.FILTER_NOT_BETWEEN || + betweenInVariation == BetweenInVariation.PROJECTION_NOT_BETWEEN); + switch (subVariation) { + case 0: + // Range covers all values exactly. + compareList.add(sortedList.get(0)); + compareList.add(sortedList.get(valueCount - 1)); + break; + case 1: + // Exclude the first and last sorted. + compareList.add(sortedList.get(1)); + compareList.add(sortedList.get(valueCount - 2)); + break; + case 2: + // Only last 2 sorted. + compareList.add(sortedList.get(valueCount - 2)); + compareList.add(sortedList.get(valueCount - 1)); + break; + case 3: + case 4: + case 5: + case 6: + { + // Choose 2 adjacent in the middle. + Object min = sortedList.get(5); + Object max = sortedList.get(6); + compareList.add(min); + compareList.add(max); + if (subVariation == 4) { + removeValue(valueList, min); + } else if (subVariation == 5) { + removeValue(valueList, max); + } else if (subVariation == 6) { + removeValue(valueList, min); + removeValue(valueList, max); + } + } + break; + default: + return false; + } + } else { + + // FILTER_IN. + // PROJECTION_IN. + isInvert = false; + switch (subVariation) { + case 0: + // All values. + compareList.addAll(valueList); + break; + case 1: + // Don't include the first and last sorted. + for (int i = 1; i < valueCount - 1; i++) { + compareList.add(valueList.get(i)); + } + break; + case 2: + // The even ones. + for (int i = 2; i < valueCount; i += 2) { + compareList.add(valueList.get(i)); + } + break; + case 3: + { + // Choose 2 adjacent in the middle. + Object min = sortedList.get(5); + Object max = sortedList.get(6); + compareList.add(min); + compareList.add(max); + if (subVariation == 4) { + removeValue(valueList, min); + } else if (subVariation == 5) { + removeValue(valueList, max); + } else if (subVariation == 6) { + removeValue(valueList, min); + removeValue(valueList, max); + } + } + break; + default: + return false; + } + } + + //---------------------------------------------------------------------------------------------- + + GenerationSpec generationSpec = GenerationSpec.createValueList(typeInfo, valueList); + + List<GenerationSpec> generationSpecList = new ArrayList<GenerationSpec>(); + List<DataTypePhysicalVariation> explicitDataTypePhysicalVariationList = + new ArrayList<DataTypePhysicalVariation>(); + generationSpecList.add(generationSpec); + explicitDataTypePhysicalVariationList.add(dataTypePhysicalVariation); + + VectorRandomRowSource rowSource = new VectorRandomRowSource(); + + rowSource.initGenerationSpecSchema( + random, generationSpecList, /* maxComplexDepth */ 0, /* allowNull */ true, + explicitDataTypePhysicalVariationList); + + List<String> columns = new ArrayList<String>(); + String col1Name = rowSource.columnNames().get(0); + columns.add(col1Name); + final ExprNodeDesc col1Expr = new ExprNodeColumnDesc(typeInfo, col1Name, "table", false); + + List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>(); + if (isBetween) { + children.add(new ExprNodeConstantDesc(new Boolean(isInvert))); + } + children.add(col1Expr); + for (Object compareObject : compareList) { + ExprNodeConstantDesc constDesc = + new ExprNodeConstantDesc( + typeInfo, + VectorRandomRowSource.getNonWritableObject( + compareObject, typeInfo, objectInspector)); + children.add(constDesc); + } + + String[] columnNames = columns.toArray(new String[0]); + + Object[][] randomRows = rowSource.randomRows(100000); + + VectorRandomBatchSource batchSource = + VectorRandomBatchSource.createInterestingBatches( + random, + rowSource, + randomRows, + null); + + final GenericUDF udf; + final ObjectInspector outputObjectInspector; + if (isBetween) { + + udf = new GenericUDFBetween(); + + // First argument is boolean invert. Arguments 1..3 are inspectors for range limits... + ObjectInspector[] argumentOIs = new ObjectInspector[4]; + argumentOIs[0] = PrimitiveObjectInspectorFactory.writableBooleanObjectInspector; + argumentOIs[1] = objectInspector; + argumentOIs[2] = objectInspector; + argumentOIs[3] = objectInspector; + outputObjectInspector = udf.initialize(argumentOIs); + } else { + final int compareCount = compareList.size(); + udf = new GenericUDFIn(); + ObjectInspector[] argumentOIs = new ObjectInspector[compareCount]; + ConstantObjectInspector constantObjectInspector = + (ConstantObjectInspector) children.get(1).getWritableObjectInspector(); + for (int i = 0; i < compareCount; i++) { + argumentOIs[i] = constantObjectInspector; + } + outputObjectInspector = udf.initialize(argumentOIs); + } + + TypeInfo outputTypeInfo = TypeInfoUtils.getTypeInfoFromObjectInspector(outputObjectInspector); + + ExprNodeGenericFuncDesc exprDesc = + new ExprNodeGenericFuncDesc( + TypeInfoFactory.booleanTypeInfo, udf, children); + + return executeTestModesAndVerify( + typeInfo, betweenInVariation, compareList, columns, columnNames, children, + udf, exprDesc, + randomRows, rowSource, batchSource, outputTypeInfo, + /* skipAdaptor */ false); + } + + private boolean doBetweenStructInVariation(Random random, String structTypeName, + BetweenInVariation betweenInVariation) + throws Exception { + + StructTypeInfo structTypeInfo = + (StructTypeInfo) TypeInfoUtils.getTypeInfoFromTypeString(structTypeName); + + ObjectInspector structObjectInspector = + TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo( + structTypeInfo); + + final int valueCount = 10 + random.nextInt(10); + List<Object> valueList = new ArrayList<Object>(valueCount); + for (int i = 0; i < valueCount; i++) { + valueList.add( + VectorRandomRowSource.randomWritable( + random, structTypeInfo, structObjectInspector, DataTypePhysicalVariation.NONE, + /* allowNull */ false)); + } + + final boolean isInvert = false; + + // No convenient WritableComparator / WritableComparable available for STRUCT. + List<Object> compareList = new ArrayList<Object>(); + + Set<Integer> includedSet = new HashSet<Integer>(); + final int chooseLimit = 4 + random.nextInt(valueCount/2); + int chooseCount = 0; + while (chooseCount < chooseLimit) { + final int index = random.nextInt(valueCount); + if (includedSet.contains(index)) { + continue; + } + includedSet.add(index); + compareList.add(valueList.get(index)); + chooseCount++; + } + + //---------------------------------------------------------------------------------------------- + + GenerationSpec structGenerationSpec = GenerationSpec.createValueList(structTypeInfo, valueList); + + List<GenerationSpec> structGenerationSpecList = new ArrayList<GenerationSpec>(); + List<DataTypePhysicalVariation> structExplicitDataTypePhysicalVariationList = + new ArrayList<DataTypePhysicalVariation>(); + structGenerationSpecList.add(structGenerationSpec); + structExplicitDataTypePhysicalVariationList.add(DataTypePhysicalVariation.NONE); + + VectorRandomRowSource structRowSource = new VectorRandomRowSource(); + + structRowSource.initGenerationSpecSchema( + random, structGenerationSpecList, /* maxComplexDepth */ 0, /* allowNull */ true, + structExplicitDataTypePhysicalVariationList); + + Object[][] structRandomRows = structRowSource.randomRows(100000); + + // --------------------------------------------------------------------------------------------- + + List<GenerationSpec> generationSpecList = new ArrayList<GenerationSpec>(); + List<DataTypePhysicalVariation> explicitDataTypePhysicalVariationList = + new ArrayList<DataTypePhysicalVariation>(); + + List<TypeInfo> fieldTypeInfoList = structTypeInfo.getAllStructFieldTypeInfos(); + final int fieldCount = fieldTypeInfoList.size(); + for (int i = 0; i < fieldCount; i++) { + GenerationSpec generationSpec = GenerationSpec.createOmitGeneration(fieldTypeInfoList.get(i)); + generationSpecList.add(generationSpec); + explicitDataTypePhysicalVariationList.add(DataTypePhysicalVariation.NONE); + } + + VectorRandomRowSource rowSource = new VectorRandomRowSource(); + + rowSource.initGenerationSpecSchema( + random, generationSpecList, /* maxComplexDepth */ 0, /* allowNull */ true, + explicitDataTypePhysicalVariationList); + + Object[][] randomRows = rowSource.randomRows(100000); + + final int rowCount = randomRows.length; + for (int r = 0; r < rowCount; r++) { + List<Object> fieldValueList = (ArrayList) structRandomRows[r][0]; + for (int f = 0; f < fieldCount; f++) { + randomRows[r][f] = fieldValueList.get(f); + } + } + + // --------------------------------------------------------------------------------------------- + + // Currently, STRUCT IN vectorization assumes a GenericUDFStruct. + + List<ObjectInspector> structUdfObjectInspectorList = new ArrayList<ObjectInspector>(); + List<ExprNodeDesc> structUdfChildren = new ArrayList<ExprNodeDesc>(fieldCount); + List<String> rowColumnNameList = rowSource.columnNames(); + for (int i = 0; i < fieldCount; i++) { + TypeInfo fieldTypeInfo = fieldTypeInfoList.get(i); + ExprNodeColumnDesc fieldExpr = + new ExprNodeColumnDesc( + fieldTypeInfo, rowColumnNameList.get(i), "table", false); + structUdfChildren.add(fieldExpr); + ObjectInspector fieldObjectInspector = + VectorRandomRowSource.getObjectInspector(fieldTypeInfo, DataTypePhysicalVariation.NONE); + structUdfObjectInspectorList.add(fieldObjectInspector); + } + StandardStructObjectInspector structUdfObjectInspector = + ObjectInspectorFactory. + getStandardStructObjectInspector(rowColumnNameList, structUdfObjectInspectorList); + String structUdfTypeName = structUdfObjectInspector.getTypeName(); + TypeInfo structUdfTypeInfo = TypeInfoUtils.getTypeInfoFromTypeString(structUdfTypeName); + + String structFuncText = "struct"; + FunctionInfo fi = FunctionRegistry.getFunctionInfo(structFuncText); + GenericUDF genericUDF = fi.getGenericUDF(); + ExprNodeDesc col1Expr = + new ExprNodeGenericFuncDesc( + structUdfObjectInspector, genericUDF, structFuncText, structUdfChildren); + + // --------------------------------------------------------------------------------------------- + + List<String> columns = new ArrayList<String>(); + + List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>(); + children.add(col1Expr); + for (int i = 0; i < compareList.size(); i++) { + Object compareObject = compareList.get(i); + ExprNodeConstantDesc constDesc = + new ExprNodeConstantDesc( + structUdfTypeInfo, + VectorRandomRowSource.getNonWritableObject( + compareObject, structUdfTypeInfo, structUdfObjectInspector)); + children.add(constDesc); + } + + for (int i = 0; i < fieldCount; i++) { + columns.add(rowColumnNameList.get(i)); + } + + String[] columnNames = columns.toArray(new String[0]); + + VectorRandomBatchSource batchSource = + VectorRandomBatchSource.createInterestingBatches( + random, + rowSource, + randomRows, + null); + + // --------------------------------------------------------------------------------------------- + + final GenericUDF udf = new GenericUDFIn(); + final int compareCount = compareList.size(); + ObjectInspector[] argumentOIs = new ObjectInspector[compareCount]; + for (int i = 0; i < compareCount; i++) { + argumentOIs[i] = structUdfObjectInspector; + } + final ObjectInspector outputObjectInspector = udf.initialize(argumentOIs); + + TypeInfo outputTypeInfo = TypeInfoUtils.getTypeInfoFromObjectInspector(outputObjectInspector); + + ExprNodeGenericFuncDesc exprDesc = + new ExprNodeGenericFuncDesc( + TypeInfoFactory.booleanTypeInfo, udf, children); + + return executeTestModesAndVerify( + structUdfTypeInfo, betweenInVariation, compareList, columns, columnNames, children, + udf, exprDesc, + randomRows, rowSource, batchSource, outputTypeInfo, + /* skipAdaptor */ true); + } + + private boolean executeTestModesAndVerify(TypeInfo typeInfo, + BetweenInVariation betweenInVariation, List<Object> compareList, + List<String> columns, String[] columnNames, List<ExprNodeDesc> children, + GenericUDF udf, ExprNodeGenericFuncDesc exprDesc, + Object[][] randomRows, + VectorRandomRowSource rowSource, VectorRandomBatchSource batchSource, + TypeInfo outputTypeInfo, boolean skipAdaptor) + throws Exception { + + final int rowCount = randomRows.length; + Object[][] resultObjectsArray = new Object[BetweenInTestMode.count][]; + for (int i = 0; i < BetweenInTestMode.count; i++) { + + Object[] resultObjects = new Object[rowCount]; + resultObjectsArray[i] = resultObjects; + + BetweenInTestMode betweenInTestMode = BetweenInTestMode.values()[i]; + switch (betweenInTestMode) { + case ROW_MODE: + if (!doRowCastTest( + typeInfo, + betweenInVariation, + compareList, + columns, + children, + udf, exprDesc, + randomRows, + rowSource.rowStructObjectInspector(), + resultObjects)) { + return false; + } + break; + case ADAPTOR: + if (skipAdaptor) { + continue; + } + case VECTOR_EXPRESSION: + if (!doVectorCastTest( + typeInfo, + betweenInVariation, + compareList, + columns, + columnNames, + rowSource.typeInfos(), + rowSource.dataTypePhysicalVariations(), + children, + udf, exprDesc, + betweenInTestMode, + batchSource, + exprDesc.getWritableObjectInspector(), + outputTypeInfo, + resultObjects)) { + return false; + } + break; + default: + throw new RuntimeException("Unexpected IF statement test mode " + betweenInTestMode); + } + } + + for (int i = 0; i < rowCount; i++) { + // Row-mode is the expected value. + Object expectedResult = resultObjectsArray[0][i]; + + for (int v = 1; v < BetweenInTestMode.count; v++) { + BetweenInTestMode betweenInTestMode = BetweenInTestMode.values()[v]; + if (skipAdaptor) { + continue; + } + Object vectorResult = resultObjectsArray[v][i]; + if (betweenInVariation.isFilter && + expectedResult == null && + vectorResult != null) { + // This is OK. + boolean vectorBoolean = ((BooleanWritable) vectorResult).get(); + if (vectorBoolean) { + Assert.fail( + "Row " + i + + " typeName " + typeInfo.getTypeName() + + " outputTypeName " + outputTypeInfo.getTypeName() + + " " + betweenInVariation + + " " + betweenInTestMode + + " result is NOT NULL and true" + + " does not match row-mode expected result is NULL which means false here" + + " row values " + Arrays.toString(randomRows[i]) + + " exprDesc " + exprDesc.toString()); + } + } else if (expectedResult == null || vectorResult == null) { + if (expectedResult != null || vectorResult != null) { + Assert.fail( + "Row " + i + + " sourceTypeName " + typeInfo.getTypeName() + + " " + betweenInVariation + + " " + betweenInTestMode + + " result is NULL " + (vectorResult == null ? "YES" : "NO result " + vectorResult.toString()) + + " does not match row-mode expected result is NULL " + + (expectedResult == null ? "YES" : "NO result " + expectedResult.toString()) + + " row values " + Arrays.toString(randomRows[i]) + + " exprDesc " + exprDesc.toString()); + } + } else { + + if (!expectedResult.equals(vectorResult)) { + Assert.fail( + "Row " + i + + " sourceTypeName " + typeInfo.getTypeName() + + " " + betweenInVariation + + " " + betweenInTestMode + + " result " + vectorResult.toString() + + " (" + vectorResult.getClass().getSimpleName() + ")" + + " does not match row-mode expected result " + expectedResult.toString() + + " (" + expectedResult.getClass().getSimpleName() + ")" + + " row values " + Arrays.toString(randomRows[i]) + + " exprDesc " + exprDesc.toString()); + } + } + } + } + return true; + } + + private boolean doRowCastTest(TypeInfo typeInfo, + BetweenInVariation betweenInVariation, List<Object> compareList, + List<String> columns, List<ExprNodeDesc> children, + GenericUDF udf, ExprNodeGenericFuncDesc exprDesc, + Object[][] randomRows, + ObjectInspector rowInspector, Object[] resultObjects) + throws Exception { + + /* + System.out.println( + "*DEBUG* typeInfo " + typeInfo.toString() + + " targetTypeInfo " + targetTypeInfo + + " betweenInTestMode ROW_MODE" + + " exprDesc " + exprDesc.toString()); + */ + + HiveConf hiveConf = new HiveConf(); + ExprNodeEvaluator evaluator = + ExprNodeEvaluatorFactory.get(exprDesc, hiveConf); + + evaluator.initialize(rowInspector); + + final int rowCount = randomRows.length; + for (int i = 0; i < rowCount; i++) { + Object[] row = randomRows[i]; + Object result = evaluator.evaluate(row); + Object copyResult = + ObjectInspectorUtils.copyToStandardObject( + result, PrimitiveObjectInspectorFactory.writableBooleanObjectInspector, + ObjectInspectorCopyOption.WRITABLE); + resultObjects[i] = copyResult; + } + + return true; + } + + private void extractResultObjects(VectorizedRowBatch batch, int rowIndex, + VectorExtractRow resultVectorExtractRow, Object[] scrqtchRow, + ObjectInspector objectInspector, Object[] resultObjects) { + + boolean selectedInUse = batch.selectedInUse; + int[] selected = batch.selected; + for (int logicalIndex = 0; logicalIndex < batch.size; logicalIndex++) { + final int batchIndex = (selectedInUse ? selected[logicalIndex] : logicalIndex); + resultVectorExtractRow.extractRow(batch, batchIndex, scrqtchRow); + + Object copyResult = + ObjectInspectorUtils.copyToStandardObject( + scrqtchRow[0], objectInspector, ObjectInspectorCopyOption.WRITABLE); + resultObjects[rowIndex++] = copyResult; + } + } + + private boolean doVectorCastTest(TypeInfo typeInfo, + BetweenInVariation betweenInVariation, List<Object> compareList, + List<String> columns, String[] columnNames, + TypeInfo[] typeInfos, DataTypePhysicalVariation[] dataTypePhysicalVariations, + List<ExprNodeDesc> children, + GenericUDF udf, ExprNodeGenericFuncDesc exprDesc, + BetweenInTestMode betweenInTestMode, + VectorRandomBatchSource batchSource, + ObjectInspector objectInspector, + TypeInfo outputTypeInfo, Object[] resultObjects) + throws Exception { + + HiveConf hiveConf = new HiveConf(); + if (betweenInTestMode == BetweenInTestMode.ADAPTOR) { + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_TEST_VECTOR_ADAPTOR_OVERRIDE, true); + } + + final boolean isFilter = betweenInVariation.isFilter; + + VectorizationContext vectorizationContext = + new VectorizationContext( + "name", + columns, + Arrays.asList(typeInfos), + Arrays.asList(dataTypePhysicalVariations), + hiveConf); + VectorExpression vectorExpression = + vectorizationContext.getVectorExpression(exprDesc, + (isFilter ? + VectorExpressionDescriptor.Mode.FILTER : + VectorExpressionDescriptor.Mode.PROJECTION)); + vectorExpression.transientInit(); + + if (betweenInTestMode == BetweenInTestMode.VECTOR_EXPRESSION && + vectorExpression instanceof VectorUDFAdaptor) { + System.out.println( + "*NO NATIVE VECTOR EXPRESSION* typeInfo " + typeInfo.toString() + + " betweenInTestMode " + betweenInTestMode + + " betweenInVariation " + betweenInVariation + + " vectorExpression " + vectorExpression.toString()); + } + + // System.out.println("*VECTOR EXPRESSION* " + vectorExpression.getClass().getSimpleName()); + + /* + System.out.println( + "*DEBUG* typeInfo " + typeInfo.toString() + + " betweenInTestMode " + betweenInTestMode + + " betweenInVariation " + betweenInVariation + + " vectorExpression " + vectorExpression.toString()); + */ + + VectorRandomRowSource rowSource = batchSource.getRowSource(); + VectorizedRowBatchCtx batchContext = + new VectorizedRowBatchCtx( + columnNames, + rowSource.typeInfos(), + rowSource.dataTypePhysicalVariations(), + /* dataColumnNums */ null, + /* partitionColumnCount */ 0, + /* virtualColumnCount */ 0, + /* neededVirtualColumns */ null, + vectorizationContext.getScratchColumnTypeNames(), + vectorizationContext.getScratchDataTypePhysicalVariations()); + + VectorizedRowBatch batch = batchContext.createVectorizedRowBatch(); + + VectorExtractRow resultVectorExtractRow = null; + Object[] scrqtchRow = null; + if (!isFilter) { + resultVectorExtractRow = new VectorExtractRow(); + final int outputColumnNum = vectorExpression.getOutputColumnNum(); + resultVectorExtractRow.init( + new TypeInfo[] { outputTypeInfo }, new int[] { outputColumnNum }); + scrqtchRow = new Object[1]; + } + + boolean copySelectedInUse = false; + int[] copySelected = new int[VectorizedRowBatch.DEFAULT_SIZE]; + + batchSource.resetBatchIteration(); + int rowIndex = 0; + while (true) { + if (!batchSource.fillNextBatch(batch)) { + break; + } + final int originalBatchSize = batch.size; + if (isFilter) { + copySelectedInUse = batch.selectedInUse; + if (batch.selectedInUse) { + System.arraycopy(batch.selected, 0, copySelected, 0, originalBatchSize); + } + } + + // In filter mode, the batch size can be made smaller. + vectorExpression.evaluate(batch); + + if (!isFilter) { + extractResultObjects(batch, rowIndex, resultVectorExtractRow, scrqtchRow, + objectInspector, resultObjects); + } else { + final int currentBatchSize = batch.size; + if (copySelectedInUse && batch.selectedInUse) { + int selectIndex = 0; + for (int i = 0; i < originalBatchSize; i++) { + final int originalBatchIndex = copySelected[i]; + final boolean booleanResult; + if (selectIndex < currentBatchSize && batch.selected[selectIndex] == originalBatchIndex) { + booleanResult = true; + selectIndex++; + } else { + booleanResult = false; + } + resultObjects[rowIndex + i] = new BooleanWritable(booleanResult); + } + } else if (batch.selectedInUse) { + int selectIndex = 0; + for (int i = 0; i < originalBatchSize; i++) { + final boolean booleanResult; + if (selectIndex < currentBatchSize && batch.selected[selectIndex] == i) { + booleanResult = true; + selectIndex++; + } else { + booleanResult = false; + } + resultObjects[rowIndex + i] = new BooleanWritable(booleanResult); + } + } else if (currentBatchSize == 0) { + // Whole batch got zapped. + for (int i = 0; i < originalBatchSize; i++) { + resultObjects[rowIndex + i] = new BooleanWritable(false); + } + } else { + // Every row kept. + for (int i = 0; i < originalBatchSize; i++) { + resultObjects[rowIndex + i] = new BooleanWritable(true); + } + } + } + + rowIndex += originalBatchSize; + } + + return true; + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/edc53cc0/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorCastStatement.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorCastStatement.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorCastStatement.java index d4d8ef7..cc1415a 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorCastStatement.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorCastStatement.java @@ -189,9 +189,14 @@ public class TestVectorCastStatement { for (PrimitiveCategory targetPrimitiveCategory : PrimitiveCategory.values()) { + if (targetPrimitiveCategory == PrimitiveCategory.INTERVAL_YEAR_MONTH || + targetPrimitiveCategory == PrimitiveCategory.INTERVAL_DAY_TIME) { + if (primitiveCategory != PrimitiveCategory.STRING) { + continue; + } + } + if (targetPrimitiveCategory == PrimitiveCategory.VOID || - targetPrimitiveCategory == PrimitiveCategory.INTERVAL_YEAR_MONTH || - targetPrimitiveCategory == PrimitiveCategory.INTERVAL_DAY_TIME || targetPrimitiveCategory == PrimitiveCategory.TIMESTAMPLOCALTZ || targetPrimitiveCategory == PrimitiveCategory.UNKNOWN) { continue; @@ -273,7 +278,8 @@ public class TestVectorCastStatement { } List<GenerationSpec> generationSpecList = new ArrayList<GenerationSpec>(); - List<DataTypePhysicalVariation> explicitDataTypePhysicalVariationList = new ArrayList<DataTypePhysicalVariation>(); + List<DataTypePhysicalVariation> explicitDataTypePhysicalVariationList = + new ArrayList<DataTypePhysicalVariation>(); generationSpecList.add(generationSpec); explicitDataTypePhysicalVariationList.add(dataTypePhysicalVariation); @@ -284,8 +290,8 @@ public class TestVectorCastStatement { explicitDataTypePhysicalVariationList); List<String> columns = new ArrayList<String>(); - columns.add("col0"); - ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(typeInfo, "col0", "table", false); + columns.add("col1"); + ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(typeInfo, "col1", "table", false); List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>(); children.add(col1Expr); @@ -443,7 +449,12 @@ public class TestVectorCastStatement { int[] selected = batch.selected; for (int logicalIndex = 0; logicalIndex < batch.size; logicalIndex++) { final int batchIndex = (selectedInUse ? selected[logicalIndex] : logicalIndex); + + try { resultVectorExtractRow.extractRow(batch, batchIndex, scrqtchRow); + } catch (Exception e) { + System.out.println("here"); + } // UNDONE: Need to copy the object. resultObjects[rowIndex++] = scrqtchRow[0]; http://git-wip-us.apache.org/repos/asf/hive/blob/edc53cc0/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateAddSub.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateAddSub.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateAddSub.java index 4dc01be..68c14c8 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateAddSub.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateAddSub.java @@ -181,7 +181,7 @@ public class TestVectorDateAddSub { new ArrayList<DataTypePhysicalVariation>(); List<String> columns = new ArrayList<String>(); - int columnNum = 0; + int columnNum = 1; ExprNodeDesc col1Expr; if (columnScalarMode == ColumnScalarMode.COLUMN_COLUMN || columnScalarMode == ColumnScalarMode.COLUMN_SCALAR) { @@ -253,8 +253,8 @@ public class TestVectorDateAddSub { // Fixup numbers to limit the range to 0 ... N-1. for (int i = 0; i < randomRows.length; i++) { Object[] row = randomRows[i]; - if (row[columnNum - 1] != null) { - row[columnNum - 1] = + if (row[columnNum - 2] != null) { + row[columnNum - 2] = smallerRange( random, integerPrimitiveCategory, /* wantWritable */ true); } http://git-wip-us.apache.org/repos/asf/hive/blob/edc53cc0/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateDiff.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateDiff.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateDiff.java index c5c5c72..0da9d8c 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateDiff.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateDiff.java @@ -179,7 +179,7 @@ public class TestVectorDateDiff { new ArrayList<DataTypePhysicalVariation>(); List<String> columns = new ArrayList<String>(); - int columnNum = 0; + int columnNum = 1; ExprNodeDesc col1Expr; if (columnScalarMode == ColumnScalarMode.COLUMN_COLUMN || columnScalarMode == ColumnScalarMode.COLUMN_SCALAR) { http://git-wip-us.apache.org/repos/asf/hive/blob/edc53cc0/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterCompare.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterCompare.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterCompare.java index 1ff11ec..ba9eaca 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterCompare.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterCompare.java @@ -329,7 +329,7 @@ public class TestVectorFilterCompare { new ArrayList<DataTypePhysicalVariation>(); List<String> columns = new ArrayList<String>(); - int columnNum = 0; + int columnNum = 1; ExprNodeDesc col1Expr; Object scalar1Object = null; http://git-wip-us.apache.org/repos/asf/hive/blob/edc53cc0/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorGenericDateExpressions.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorGenericDateExpressions.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorGenericDateExpressions.java index e7884b2..9d57aec 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorGenericDateExpressions.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorGenericDateExpressions.java @@ -657,7 +657,7 @@ public class TestVectorGenericDateExpressions { } else if (colType == PrimitiveCategory.TIMESTAMP) { udf = new VectorUDFDateTimestamp(0, 1); } else { - udf = new VectorUDFDateLong(0, 1); + throw new RuntimeException("Unexpected column type " + colType); } udf.setInputTypeInfos(new TypeInfo[] {primitiveCategoryToTypeInfo(colType)}); @@ -684,6 +684,9 @@ public class TestVectorGenericDateExpressions { @Test public void testDate() throws HiveException { for (PrimitiveCategory colType : dateTimestampStringTypes) { + if (colType == PrimitiveCategory.DATE) { + continue; + } LongColumnVector date = newRandomLongColumnVector(10000, size); LongColumnVector output = new LongColumnVector(size); @@ -722,7 +725,7 @@ public class TestVectorGenericDateExpressions { } else if (colType == PrimitiveCategory.TIMESTAMP) { udf = new CastTimestampToDate(0, 1); } else { - udf = new CastLongToDate(0, 1); + throw new RuntimeException("Unexpected column type " + colType); } udf.setInputTypeInfos(new TypeInfo[] {primitiveCategoryToTypeInfo(colType)}); udf.transientInit(); http://git-wip-us.apache.org/repos/asf/hive/blob/edc53cc0/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorIfStatement.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorIfStatement.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorIfStatement.java index 58e32ca..666d26c 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorIfStatement.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorIfStatement.java @@ -247,10 +247,10 @@ public class TestVectorIfStatement { explicitDataTypePhysicalVariationList); List<String> columns = new ArrayList<String>(); - columns.add("col0"); // The boolean predicate. + columns.add("col1"); // The boolean predicate. - ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(Boolean.class, "col0", "table", false); - int columnNum = 1; + ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(Boolean.class, "col1", "table", false); + int columnNum = 2; ExprNodeDesc col2Expr; if (columnScalarMode == ColumnScalarMode.COLUMN_COLUMN || columnScalarMode == ColumnScalarMode.COLUMN_SCALAR) { http://git-wip-us.apache.org/repos/asf/hive/blob/edc53cc0/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorNegative.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorNegative.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorNegative.java index 54c085b..ea39848 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorNegative.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorNegative.java @@ -195,7 +195,7 @@ public class TestVectorNegative { new ArrayList<DataTypePhysicalVariation>(); List<String> columns = new ArrayList<String>(); - int columnNum = 0; + int columnNum = 1; generationSpecList.add( GenerationSpec.createSameType(typeInfo));
