Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java Fri Sep 5 15:59:47 2014 @@ -48,6 +48,7 @@ import org.apache.hadoop.hive.ql.exec.ve import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.TruthValue; @@ -908,10 +909,10 @@ class RecordReaderImpl implements Record } private static class BinaryTreeReader extends TreeReader{ - private InStream stream; - private IntegerReader lengths = null; + protected InStream stream; + protected IntegerReader lengths = null; - private final LongColumnVector scratchlcv; + protected final LongColumnVector scratchlcv; BinaryTreeReader(Path path, int columnId, Configuration conf) { super(path, columnId, conf); @@ -983,7 +984,7 @@ class RecordReaderImpl implements Record // Read present/isNull stream super.nextVector(result, batchSize); - BytesColumnVectorUtil.setRefToOrcByteArrays(stream, lengths, scratchlcv, result, batchSize); + BytesColumnVectorUtil.readOrcByteArrays(stream, lengths, scratchlcv, result, batchSize); return result; } @@ -1376,12 +1377,13 @@ class RecordReaderImpl implements Record } } + // This class collects together very similar methods for reading an ORC vector of byte arrays and + // creating the BytesColumnVector. + // private static class BytesColumnVectorUtil { - // This method has the common code for reading in bytes into a BytesColumnVector. - // It is used by the BINARY, STRING, CHAR, VARCHAR types. - public static void setRefToOrcByteArrays(InStream stream, IntegerReader lengths, LongColumnVector scratchlcv, - BytesColumnVector result, long batchSize) throws IOException { + private static byte[] commonReadByteArrays(InStream stream, IntegerReader lengths, LongColumnVector scratchlcv, + BytesColumnVector result, long batchSize) throws IOException { // Read lengths scratchlcv.isNull = result.isNull; // Notice we are replacing the isNull vector here... lengths.nextVector(scratchlcv, batchSize); @@ -1409,11 +1411,20 @@ class RecordReaderImpl implements Record } len -= bytesRead; offset += bytesRead; - } + } + + return allBytes; + } + + // This method has the common code for reading in bytes into a BytesColumnVector. + public static void readOrcByteArrays(InStream stream, IntegerReader lengths, LongColumnVector scratchlcv, + BytesColumnVector result, long batchSize) throws IOException { + + byte[] allBytes = commonReadByteArrays(stream, lengths, scratchlcv, result, batchSize); // Too expensive to figure out 'repeating' by comparisons. result.isRepeating = false; - offset = 0; + int offset = 0; if (!scratchlcv.isRepeating) { for (int i = 0; i < batchSize; i++) { if (!scratchlcv.isNull[i]) { @@ -1518,7 +1529,7 @@ class RecordReaderImpl implements Record // Read present/isNull stream super.nextVector(result, batchSize); - BytesColumnVectorUtil.setRefToOrcByteArrays(stream, lengths, scratchlcv, result, batchSize); + BytesColumnVectorUtil.readOrcByteArrays(stream, lengths, scratchlcv, result, batchSize); return result; } @@ -1734,6 +1745,42 @@ class RecordReaderImpl implements Record result.enforceMaxLength(maxLength); return result; } + + @Override + Object nextVector(Object previousVector, long batchSize) throws IOException { + // Get the vector of strings from StringTreeReader, then make a 2nd pass to + // adjust down the length (right trim and truncate) if necessary. + BytesColumnVector result = (BytesColumnVector) super.nextVector(previousVector, batchSize); + + int adjustedDownLen; + if (result.isRepeating) { + if (result.noNulls || !result.isNull[0]) { + adjustedDownLen = StringExpr.rightTrimAndTruncate(result.vector[0], result.start[0], result.length[0], maxLength); + if (adjustedDownLen < result.length[0]) { + result.setRef(0, result.vector[0], result.start[0], adjustedDownLen); + } + } + } else { + if (result.noNulls){ + for (int i = 0; i < batchSize; i++) { + adjustedDownLen = StringExpr.rightTrimAndTruncate(result.vector[i], result.start[i], result.length[i], maxLength); + if (adjustedDownLen < result.length[i]) { + result.setRef(i, result.vector[i], result.start[i], adjustedDownLen); + } + } + } else { + for (int i = 0; i < batchSize; i++) { + if (!result.isNull[i]) { + adjustedDownLen = StringExpr.rightTrimAndTruncate(result.vector[i], result.start[i], result.length[i], maxLength); + if (adjustedDownLen < result.length[i]) { + result.setRef(i, result.vector[i], result.start[i], adjustedDownLen); + } + } + } + } + } + return result; + } } private static class VarcharTreeReader extends StringTreeReader { @@ -1762,6 +1809,42 @@ class RecordReaderImpl implements Record result.enforceMaxLength(maxLength); return result; } + + @Override + Object nextVector(Object previousVector, long batchSize) throws IOException { + // Get the vector of strings from StringTreeReader, then make a 2nd pass to + // adjust down the length (truncate) if necessary. + BytesColumnVector result = (BytesColumnVector) super.nextVector(previousVector, batchSize); + + int adjustedDownLen; + if (result.isRepeating) { + if (result.noNulls || !result.isNull[0]) { + adjustedDownLen = StringExpr.truncate(result.vector[0], result.start[0], result.length[0], maxLength); + if (adjustedDownLen < result.length[0]) { + result.setRef(0, result.vector[0], result.start[0], adjustedDownLen); + } + } + } else { + if (result.noNulls){ + for (int i = 0; i < batchSize; i++) { + adjustedDownLen = StringExpr.truncate(result.vector[i], result.start[i], result.length[i], maxLength); + if (adjustedDownLen < result.length[i]) { + result.setRef(i, result.vector[i], result.start[i], adjustedDownLen); + } + } + } else { + for (int i = 0; i < batchSize; i++) { + if (!result.isNull[i]) { + adjustedDownLen = StringExpr.truncate(result.vector[i], result.start[i], result.length[i], maxLength); + if (adjustedDownLen < result.length[i]) { + result.setRef(i, result.vector[i], result.start[i], adjustedDownLen); + } + } + } + } + } + return result; + } } private static class StructTreeReader extends TreeReader {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java Fri Sep 5 15:59:47 2014 @@ -156,6 +156,10 @@ public class Vectorizer implements Physi // The regex matches only the "decimal" prefix of the type. patternBuilder.append("|decimal.*"); + // CHAR and VARCHAR types can be specified with maximum length. + patternBuilder.append("|char.*"); + patternBuilder.append("|varchar.*"); + supportedDataTypesPattern = Pattern.compile(patternBuilder.toString()); supportedGenericUDFs.add(GenericUDFOPPlus.class); @@ -248,6 +252,8 @@ public class Vectorizer implements Physi supportedGenericUDFs.add(GenericUDFTimestamp.class); supportedGenericUDFs.add(GenericUDFToDecimal.class); supportedGenericUDFs.add(GenericUDFToDate.class); + supportedGenericUDFs.add(GenericUDFToChar.class); + supportedGenericUDFs.add(GenericUDFToVarchar.class); // For conditional expressions supportedGenericUDFs.add(GenericUDFIf.class); Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java Fri Sep 5 15:59:47 2014 @@ -25,6 +25,8 @@ import org.apache.hadoop.hive.ql.exec.ve import org.apache.hadoop.hive.ql.exec.vector.expressions.CastDecimalToBoolean; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.CastDoubleToBooleanViaDoubleToLong; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.CastLongToBooleanViaLongToLong; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.CastDateToBooleanViaLongToLong; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.CastTimestampToBooleanViaLongToLong; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; @@ -43,6 +45,7 @@ import org.apache.hadoop.io.Text; * */ @VectorizedExpressions({CastLongToBooleanViaLongToLong.class, + CastDateToBooleanViaLongToLong.class, CastTimestampToBooleanViaLongToLong.class, CastDoubleToBooleanViaDoubleToLong.class, CastDecimalToBoolean.class}) public class UDFToBoolean extends UDF { private final BooleanWritable booleanWritable = new BooleanWritable(); Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFConcat.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFConcat.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFConcat.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFConcat.java Fri Sep 5 15:59:47 2014 @@ -23,9 +23,13 @@ import org.apache.hadoop.hive.common.typ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; -import org.apache.hadoop.hive.ql.exec.vector.expressions.StringConcatColCol; -import org.apache.hadoop.hive.ql.exec.vector.expressions.StringConcatColScalar; -import org.apache.hadoop.hive.ql.exec.vector.expressions.StringConcatScalarCol; +import org.apache.hadoop.hive.ql.exec.vector.expressions.StringGroupConcatColCol; +import org.apache.hadoop.hive.ql.exec.vector.expressions.StringGroupColConcatStringScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.StringGroupColConcatCharScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.StringGroupColConcatVarCharScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.StringScalarConcatStringGroupCol; +import org.apache.hadoop.hive.ql.exec.vector.expressions.CharScalarConcatStringGroupCol; +import org.apache.hadoop.hive.ql.exec.vector.expressions.VarCharScalarConcatStringGroupCol; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; @@ -49,8 +53,11 @@ extended = "Returns NULL if any argument + "Example:\n" + " > SELECT _FUNC_('abc', 'def') FROM src LIMIT 1;\n" + " 'abcdef'") -@VectorizedExpressions({StringConcatColCol.class, StringConcatColScalar.class, - StringConcatScalarCol.class}) +@VectorizedExpressions({StringGroupConcatColCol.class, + StringGroupColConcatStringScalar.class, + StringGroupColConcatCharScalar.class, StringGroupColConcatVarCharScalar.class, + StringScalarConcatStringGroupCol.class, + CharScalarConcatStringGroupCol.class, VarCharScalarConcatStringGroupCol.class}) public class GenericUDFConcat extends GenericUDF { private transient ObjectInspector[] argumentOIs; private transient StringConverter[] stringConverters; Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java Fri Sep 5 15:59:47 2014 @@ -41,10 +41,20 @@ import org.apache.hadoop.hive.ql.exec.ve import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDoubleScalarDoubleScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDoubleScalarLongScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongScalarDoubleScalar; -import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringColumnStringColumn; -import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringColumnStringScalar; -import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringScalarStringColumn; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringGroupColumnStringGroupColumn; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringGroupColumnStringScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringGroupColumnCharScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringGroupColumnVarCharScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringScalarStringGroupColumn; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprCharScalarStringGroupColumn; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprVarCharScalarStringGroupColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringScalarStringScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringScalarCharScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringScalarVarCharScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprCharScalarStringScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprVarCharScalarStringScalar; + + /** * IF(expr1,expr2,expr3) <br> @@ -60,8 +70,14 @@ import org.apache.hadoop.hive.ql.exec.ve IfExprLongScalarDoubleColumn.class, IfExprDoubleScalarLongColumn.class, IfExprLongScalarLongScalar.class, IfExprDoubleScalarDoubleScalar.class, IfExprLongScalarDoubleScalar.class, IfExprDoubleScalarLongScalar.class, - IfExprStringColumnStringColumn.class, IfExprStringColumnStringScalar.class, - IfExprStringScalarStringColumn.class, IfExprStringScalarStringScalar.class + IfExprStringGroupColumnStringGroupColumn.class, + IfExprStringGroupColumnStringScalar.class, + IfExprStringGroupColumnCharScalar.class, IfExprStringGroupColumnVarCharScalar.class, + IfExprStringScalarStringGroupColumn.class, + IfExprCharScalarStringGroupColumn.class, IfExprVarCharScalarStringGroupColumn.class, + IfExprStringScalarStringScalar.class, + IfExprStringScalarCharScalar.class, IfExprStringScalarVarCharScalar.class, + IfExprCharScalarStringScalar.class, IfExprVarCharScalarStringScalar.class, }) public class GenericUDFIf extends GenericUDF { private transient ObjectInspector[] argumentOIs; Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java Fri Sep 5 15:59:47 2014 @@ -34,9 +34,14 @@ import org.apache.hadoop.hive.serde2.obj DoubleColEqualLongScalar.class, DoubleColEqualDoubleScalar.class, LongScalarEqualLongColumn.class, LongScalarEqualDoubleColumn.class, DoubleScalarEqualLongColumn.class, DoubleScalarEqualDoubleColumn.class, - StringColEqualStringColumn.class, StringColEqualStringScalar.class, - StringScalarEqualStringColumn.class, FilterStringColEqualStringColumn.class, - FilterStringColEqualStringScalar.class, FilterStringScalarEqualStringColumn.class, + StringGroupColEqualStringGroupColumn.class, FilterStringGroupColEqualStringGroupColumn.class, + StringGroupColEqualStringScalar.class, + StringGroupColEqualVarCharScalar.class, StringGroupColEqualCharScalar.class, + StringScalarEqualStringGroupColumn.class, + VarCharScalarEqualStringGroupColumn.class, CharScalarEqualStringGroupColumn.class, + FilterStringGroupColEqualStringScalar.class, FilterStringScalarEqualStringGroupColumn.class, + FilterStringGroupColEqualVarCharScalar.class, FilterVarCharScalarEqualStringGroupColumn.class, + FilterStringGroupColEqualCharScalar.class, FilterCharScalarEqualStringGroupColumn.class, FilterLongColEqualLongColumn.class, FilterLongColEqualDoubleColumn.class, FilterDoubleColEqualLongColumn.class, FilterDoubleColEqualDoubleColumn.class, FilterLongColEqualLongScalar.class, FilterLongColEqualDoubleScalar.class, Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java Fri Sep 5 15:59:47 2014 @@ -35,9 +35,14 @@ import org.apache.hadoop.io.Text; DoubleColGreaterEqualLongScalar.class, DoubleColGreaterEqualDoubleScalar.class, LongScalarGreaterEqualLongColumn.class, LongScalarGreaterEqualDoubleColumn.class, DoubleScalarGreaterEqualLongColumn.class, DoubleScalarGreaterEqualDoubleColumn.class, - StringColGreaterEqualStringColumn.class, StringColGreaterEqualStringScalar.class, - StringScalarGreaterEqualStringColumn.class, FilterStringColGreaterEqualStringColumn.class, - FilterStringColGreaterEqualStringScalar.class, FilterStringScalarGreaterEqualStringColumn.class, + StringGroupColGreaterEqualStringGroupColumn.class, FilterStringGroupColGreaterEqualStringGroupColumn.class, + StringGroupColGreaterEqualStringScalar.class, + StringGroupColGreaterEqualVarCharScalar.class, StringGroupColGreaterEqualCharScalar.class, + StringScalarGreaterEqualStringGroupColumn.class, + VarCharScalarGreaterEqualStringGroupColumn.class, CharScalarGreaterEqualStringGroupColumn.class, + FilterStringGroupColGreaterEqualStringScalar.class, FilterStringScalarGreaterEqualStringGroupColumn.class, + FilterStringGroupColGreaterEqualVarCharScalar.class, FilterVarCharScalarGreaterEqualStringGroupColumn.class, + FilterStringGroupColGreaterEqualCharScalar.class, FilterCharScalarGreaterEqualStringGroupColumn.class, FilterLongColGreaterEqualLongColumn.class, FilterLongColGreaterEqualDoubleColumn.class, FilterDoubleColGreaterEqualLongColumn.class, FilterDoubleColGreaterEqualDoubleColumn.class, FilterLongColGreaterEqualLongScalar.class, FilterLongColGreaterEqualDoubleScalar.class, Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java Fri Sep 5 15:59:47 2014 @@ -35,9 +35,14 @@ import org.apache.hadoop.io.Text; DoubleColLessEqualLongScalar.class, DoubleColLessEqualDoubleScalar.class, LongScalarLessEqualLongColumn.class, LongScalarLessEqualDoubleColumn.class, DoubleScalarLessEqualLongColumn.class, DoubleScalarLessEqualDoubleColumn.class, - StringColLessEqualStringColumn.class, StringColLessEqualStringScalar.class, - StringScalarLessEqualStringColumn.class, FilterStringColLessEqualStringColumn.class, - FilterStringColLessEqualStringScalar.class, FilterStringScalarLessEqualStringColumn.class, + StringGroupColLessEqualStringGroupColumn.class, FilterStringGroupColLessEqualStringGroupColumn.class, + StringGroupColLessEqualStringScalar.class, + StringGroupColLessEqualVarCharScalar.class, StringGroupColLessEqualCharScalar.class, + StringScalarLessEqualStringGroupColumn.class, + VarCharScalarLessEqualStringGroupColumn.class, CharScalarLessEqualStringGroupColumn.class, + FilterStringGroupColLessEqualStringScalar.class, FilterStringScalarLessEqualStringGroupColumn.class, + FilterStringGroupColLessEqualVarCharScalar.class, FilterVarCharScalarLessEqualStringGroupColumn.class, + FilterStringGroupColLessEqualCharScalar.class, FilterCharScalarLessEqualStringGroupColumn.class, FilterLongColLessEqualLongColumn.class, FilterLongColLessEqualDoubleColumn.class, FilterDoubleColLessEqualLongColumn.class, FilterDoubleColLessEqualDoubleColumn.class, FilterLongColLessEqualLongScalar.class, FilterLongColLessEqualDoubleScalar.class, Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java Fri Sep 5 15:59:47 2014 @@ -35,9 +35,14 @@ import org.apache.hadoop.io.Text; DoubleColGreaterLongScalar.class, DoubleColGreaterDoubleScalar.class, LongScalarGreaterLongColumn.class, LongScalarGreaterDoubleColumn.class, DoubleScalarGreaterLongColumn.class, DoubleScalarGreaterDoubleColumn.class, - StringColGreaterStringColumn.class, StringColGreaterStringScalar.class, - StringScalarGreaterStringColumn.class, FilterStringColGreaterStringColumn.class, - FilterStringColGreaterStringScalar.class, FilterStringScalarGreaterStringColumn.class, + StringGroupColGreaterStringGroupColumn.class, FilterStringGroupColGreaterStringGroupColumn.class, + StringGroupColGreaterStringScalar.class, + StringGroupColGreaterVarCharScalar.class, StringGroupColGreaterCharScalar.class, + StringScalarGreaterStringGroupColumn.class, + VarCharScalarGreaterStringGroupColumn.class, CharScalarGreaterStringGroupColumn.class, + FilterStringGroupColGreaterStringScalar.class, FilterStringScalarGreaterStringGroupColumn.class, + FilterStringGroupColGreaterVarCharScalar.class, FilterVarCharScalarGreaterStringGroupColumn.class, + FilterStringGroupColGreaterCharScalar.class, FilterCharScalarGreaterStringGroupColumn.class, FilterLongColGreaterLongColumn.class, FilterLongColGreaterDoubleColumn.class, FilterDoubleColGreaterLongColumn.class, FilterDoubleColGreaterDoubleColumn.class, FilterLongColGreaterLongScalar.class, FilterLongColGreaterDoubleScalar.class, Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java Fri Sep 5 15:59:47 2014 @@ -35,9 +35,14 @@ import org.apache.hadoop.io.Text; DoubleColLessLongScalar.class, DoubleColLessDoubleScalar.class, LongScalarLessLongColumn.class, LongScalarLessDoubleColumn.class, DoubleScalarLessLongColumn.class, DoubleScalarLessDoubleColumn.class, - StringColLessStringColumn.class, StringColLessStringScalar.class, - StringScalarLessStringColumn.class, FilterStringColLessStringColumn.class, - FilterStringColLessStringScalar.class, FilterStringScalarLessStringColumn.class, + StringGroupColLessStringGroupColumn.class, FilterStringGroupColLessStringGroupColumn.class, + StringGroupColLessStringScalar.class, + StringGroupColLessVarCharScalar.class, StringGroupColLessCharScalar.class, + StringScalarLessStringGroupColumn.class, + VarCharScalarLessStringGroupColumn.class, CharScalarLessStringGroupColumn.class, + FilterStringGroupColLessStringScalar.class, FilterStringScalarLessStringGroupColumn.class, + FilterStringGroupColLessVarCharScalar.class, FilterVarCharScalarLessStringGroupColumn.class, + FilterStringGroupColLessCharScalar.class, FilterCharScalarLessStringGroupColumn.class, FilterLongColLessLongColumn.class, FilterLongColLessDoubleColumn.class, FilterDoubleColLessLongColumn.class, FilterDoubleColLessDoubleColumn.class, FilterLongColLessLongScalar.class, FilterLongColLessDoubleScalar.class, Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java Fri Sep 5 15:59:47 2014 @@ -34,9 +34,14 @@ import org.apache.hadoop.hive.serde2.obj DoubleColNotEqualLongScalar.class, DoubleColNotEqualDoubleScalar.class, LongScalarNotEqualLongColumn.class, LongScalarNotEqualDoubleColumn.class, DoubleScalarNotEqualLongColumn.class, DoubleScalarNotEqualDoubleColumn.class, - StringColNotEqualStringColumn.class, StringColNotEqualStringScalar.class, - StringScalarNotEqualStringColumn.class, FilterStringColNotEqualStringColumn.class, - FilterStringColNotEqualStringScalar.class, FilterStringScalarNotEqualStringColumn.class, + StringGroupColNotEqualStringGroupColumn.class, FilterStringGroupColNotEqualStringGroupColumn.class, + StringGroupColNotEqualStringScalar.class, + StringGroupColNotEqualVarCharScalar.class, StringGroupColNotEqualCharScalar.class, + StringScalarNotEqualStringGroupColumn.class, + VarCharScalarNotEqualStringGroupColumn.class, CharScalarNotEqualStringGroupColumn.class, + FilterStringGroupColNotEqualStringScalar.class, FilterStringScalarNotEqualStringGroupColumn.class, + FilterStringGroupColNotEqualVarCharScalar.class, FilterVarCharScalarNotEqualStringGroupColumn.class, + FilterStringGroupColNotEqualCharScalar.class, FilterCharScalarNotEqualStringGroupColumn.class, FilterLongColNotEqualLongColumn.class, FilterLongColNotEqualDoubleColumn.class, FilterDoubleColNotEqualLongColumn.class, FilterDoubleColNotEqualDoubleColumn.class, FilterLongColNotEqualLongScalar.class, FilterLongColNotEqualDoubleScalar.class, Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java (original) +++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java Fri Sep 5 15:59:47 2014 @@ -29,7 +29,9 @@ import java.util.Map; import junit.framework.Assert; +import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.ql.exec.vector.expressions.ColAndCol; import org.apache.hadoop.hive.ql.exec.vector.expressions.ColOrCol; import org.apache.hadoop.hive.ql.exec.vector.expressions.DoubleColumnInList; @@ -38,10 +40,14 @@ import org.apache.hadoop.hive.ql.exec.ve import org.apache.hadoop.hive.ql.exec.vector.expressions.FuncLogWithBaseDoubleToDouble; import org.apache.hadoop.hive.ql.exec.vector.expressions.FuncLogWithBaseLongToDouble; import org.apache.hadoop.hive.ql.exec.vector.expressions.FuncPowerDoubleToDouble; -import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringColumnStringColumn; -import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringColumnStringScalar; -import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringScalarStringColumn; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprCharScalarStringGroupColumn; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringGroupColumnCharScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringGroupColumnStringGroupColumn; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringGroupColumnStringScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringGroupColumnVarCharScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringScalarStringGroupColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringScalarStringScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprVarCharScalarStringGroupColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.IsNotNull; import org.apache.hadoop.hive.ql.exec.vector.expressions.IsNull; import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColumnInList; @@ -80,10 +86,14 @@ import org.apache.hadoop.hive.ql.exec.ve import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColumnBetween; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColumnNotBetween; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongScalarGreaterLongColumn; -import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColGreaterStringColumn; -import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColGreaterStringScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringGroupColGreaterStringGroupColumn; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringGroupColGreaterStringScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColumnBetween; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColumnNotBetween; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterVarCharColumnBetween; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterVarCharColumnNotBetween; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterCharColumnBetween; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterCharColumnNotBetween; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncLnDoubleToDouble; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncRoundDoubleToDouble; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncSinDoubleToDouble; @@ -128,6 +138,8 @@ import org.apache.hadoop.hive.ql.udf.gen import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToDecimal; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToUnixTimeStamp; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFTimestamp; +import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.junit.Test; @@ -139,39 +151,39 @@ public class TestVectorizationContext { VectorUDFUnixTimeStampLong v1 = new VectorUDFUnixTimeStampLong(); VectorExpressionDescriptor.Builder builder1 = new VectorExpressionDescriptor.Builder(); VectorExpressionDescriptor.Descriptor d1 = builder1.setMode(VectorExpressionDescriptor.Mode.PROJECTION) - .setNumArguments(1).setArgumentTypes(VectorExpressionDescriptor.ArgumentType.LONG) + .setNumArguments(1).setArgumentTypes(VectorExpressionDescriptor.ArgumentType.INT_DATETIME_FAMILY) .setInputExpressionTypes(VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); - Assert.assertEquals(d1, v1.getDescriptor()); + assertTrue(d1.matches(v1.getDescriptor())); VectorExpressionDescriptor.Builder builder2 = new VectorExpressionDescriptor.Builder(); VectorExpressionDescriptor.Descriptor d2 = builder2.setMode(VectorExpressionDescriptor.Mode.FILTER) - .setNumArguments(2).setArgumentTypes(VectorExpressionDescriptor.ArgumentType.LONG, - VectorExpressionDescriptor.ArgumentType.DOUBLE).setInputExpressionTypes( + .setNumArguments(2).setArgumentTypes(VectorExpressionDescriptor.ArgumentType.INT_FAMILY, + VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY).setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN, VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); FilterLongColLessDoubleScalar v2 = new FilterLongColLessDoubleScalar(); - Assert.assertEquals(d2, v2.getDescriptor()); + assertTrue(d2.matches(v2.getDescriptor())); VectorExpressionDescriptor.Builder builder3 = new VectorExpressionDescriptor.Builder(); VectorExpressionDescriptor.Descriptor d3 = builder3.setMode(VectorExpressionDescriptor.Mode.PROJECTION) .setNumArguments(1).setArgumentTypes(VectorExpressionDescriptor.ArgumentType.STRING) .setInputExpressionTypes(VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); StringLower v3 = new StringLower(); - Assert.assertEquals(d3, v3.getDescriptor()); + assertTrue(d3.matches(v3.getDescriptor())); VectorExpressionDescriptor.Builder builder4 = new VectorExpressionDescriptor.Builder(); VectorExpressionDescriptor.Descriptor d4 = builder4.setMode(VectorExpressionDescriptor.Mode.PROJECTION) - .setNumArguments(1).setArgumentTypes(VectorExpressionDescriptor.ArgumentType.ANY) + .setNumArguments(1).setArgumentTypes(VectorExpressionDescriptor.ArgumentType.ALL_FAMILY) .setInputExpressionTypes(VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); StringUpper v4 = new StringUpper(); - Assert.assertEquals(d4, v4.getDescriptor()); + assertTrue(d4.matches(v4.getDescriptor())); VectorExpressionDescriptor.Builder builder5 = new VectorExpressionDescriptor.Builder(); VectorExpressionDescriptor.Descriptor d5 = builder5.setMode(VectorExpressionDescriptor.Mode.PROJECTION) .setNumArguments(1).setArgumentTypes(VectorExpressionDescriptor.ArgumentType.STRING) .setInputExpressionTypes(VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); IsNull v5 = new IsNull(); - Assert.assertEquals(d5, v5.getDescriptor()); + assertTrue(d5.matches(v5.getDescriptor())); } @Test @@ -289,11 +301,12 @@ public class TestVectorizationContext { VectorExpression ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); - assertTrue(ve instanceof FilterStringColGreaterStringScalar); + assertTrue(ve instanceof FilterStringGroupColGreaterStringScalar); } @Test public void testFilterStringColCompareStringColumnExpressions() throws HiveException { + // Strings test ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(String.class, "col1", "table", false); ExprNodeColumnDesc col2Expr = new ExprNodeColumnDesc(String.class, "col2", "table", false); @@ -313,7 +326,97 @@ public class TestVectorizationContext { VectorExpression ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); - assertTrue(ve instanceof FilterStringColGreaterStringColumn); + assertTrue(ve instanceof FilterStringGroupColGreaterStringGroupColumn); + + // 2 CHAR test + CharTypeInfo charTypeInfo = new CharTypeInfo(10); + col1Expr = new ExprNodeColumnDesc(charTypeInfo, "col1", "table", false); + col2Expr = new ExprNodeColumnDesc(charTypeInfo, "col2", "table", false); + + udf = new GenericUDFOPGreaterThan(); + exprDesc = new ExprNodeGenericFuncDesc(); + exprDesc.setGenericUDF(udf); + children1 = new ArrayList<ExprNodeDesc>(2); + children1.add(col1Expr); + children1.add(col2Expr); + exprDesc.setChildren(children1); + + vc = new VectorizationContext(columnMap, 2); + + ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); + + assertTrue(ve instanceof FilterStringGroupColGreaterStringGroupColumn); + + // 2 VARCHAR test + VarcharTypeInfo varcharTypeInfo = new VarcharTypeInfo(10); + col1Expr = new ExprNodeColumnDesc(varcharTypeInfo, "col1", "table", false); + col2Expr = new ExprNodeColumnDesc(varcharTypeInfo, "col2", "table", false); + + udf = new GenericUDFOPGreaterThan(); + exprDesc = new ExprNodeGenericFuncDesc(); + exprDesc.setGenericUDF(udf); + children1 = new ArrayList<ExprNodeDesc>(2); + children1.add(col1Expr); + children1.add(col2Expr); + exprDesc.setChildren(children1); + + vc = new VectorizationContext(columnMap, 2); + + ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); + + assertTrue(ve instanceof FilterStringGroupColGreaterStringGroupColumn); + + // Some mix tests (STRING, CHAR), (VARCHAR, CHAR), (VARCHAR, STRING)... + col1Expr = new ExprNodeColumnDesc(String.class, "col1", "table", false); + col2Expr = new ExprNodeColumnDesc(charTypeInfo, "col2", "table", false); + + udf = new GenericUDFOPGreaterThan(); + exprDesc = new ExprNodeGenericFuncDesc(); + exprDesc.setGenericUDF(udf); + children1 = new ArrayList<ExprNodeDesc>(2); + children1.add(col1Expr); + children1.add(col2Expr); + exprDesc.setChildren(children1); + + vc = new VectorizationContext(columnMap, 2); + + ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); + + assertTrue(ve instanceof FilterStringGroupColGreaterStringGroupColumn); + + col1Expr = new ExprNodeColumnDesc(varcharTypeInfo, "col1", "table", false); + col2Expr = new ExprNodeColumnDesc(charTypeInfo, "col2", "table", false); + + udf = new GenericUDFOPGreaterThan(); + exprDesc = new ExprNodeGenericFuncDesc(); + exprDesc.setGenericUDF(udf); + children1 = new ArrayList<ExprNodeDesc>(2); + children1.add(col1Expr); + children1.add(col2Expr); + exprDesc.setChildren(children1); + + vc = new VectorizationContext(columnMap, 2); + + ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); + + assertTrue(ve instanceof FilterStringGroupColGreaterStringGroupColumn); + + col1Expr = new ExprNodeColumnDesc(varcharTypeInfo, "col1", "table", false); + col2Expr = new ExprNodeColumnDesc(String.class, "col2", "table", false); + + udf = new GenericUDFOPGreaterThan(); + exprDesc = new ExprNodeGenericFuncDesc(); + exprDesc.setGenericUDF(udf); + children1 = new ArrayList<ExprNodeDesc>(2); + children1.add(col1Expr); + children1.add(col2Expr); + exprDesc.setChildren(children1); + + vc = new VectorizationContext(columnMap, 2); + + ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); + + assertTrue(ve instanceof FilterStringGroupColGreaterStringGroupColumn); } @Test @@ -908,6 +1011,7 @@ public class TestVectorizationContext { @Test public void testBetweenFilters() throws HiveException { + // string tests ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(String.class, "col1", "table", false); ExprNodeConstantDesc constDesc = new ExprNodeConstantDesc("Alpha"); ExprNodeConstantDesc constDesc2 = new ExprNodeConstantDesc("Bravo"); @@ -934,6 +1038,56 @@ public class TestVectorizationContext { ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); assertTrue(ve instanceof FilterStringColumnNotBetween); + // CHAR tests + CharTypeInfo charTypeInfo = new CharTypeInfo(10); + col1Expr = new ExprNodeColumnDesc(charTypeInfo, "col1", "table", false); + constDesc = new ExprNodeConstantDesc(charTypeInfo, new HiveChar("Alpha", 10)); + constDesc2 = new ExprNodeConstantDesc(charTypeInfo, new HiveChar("Bravo", 10)); + + // CHAR BETWEEN + udf = new GenericUDFBetween(); + children1 = new ArrayList<ExprNodeDesc>(); + children1.add(new ExprNodeConstantDesc(new Boolean(false))); // no NOT keyword + children1.add(col1Expr); + children1.add(constDesc); + children1.add(constDesc2); + exprDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, + children1); + + vc = new VectorizationContext(columnMap, 2); + ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); + assertTrue(ve instanceof FilterCharColumnBetween); + + // CHAR NOT BETWEEN + children1.set(0, new ExprNodeConstantDesc(new Boolean(true))); // has NOT keyword + ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); + assertTrue(ve instanceof FilterCharColumnNotBetween); + + // VARCHAR tests + VarcharTypeInfo varcharTypeInfo = new VarcharTypeInfo(10); + col1Expr = new ExprNodeColumnDesc(varcharTypeInfo, "col1", "table", false); + constDesc = new ExprNodeConstantDesc(varcharTypeInfo, new HiveVarchar("Alpha", 10)); + constDesc2 = new ExprNodeConstantDesc(varcharTypeInfo, new HiveVarchar("Bravo", 10)); + + // VARCHAR BETWEEN + udf = new GenericUDFBetween(); + children1 = new ArrayList<ExprNodeDesc>(); + children1.add(new ExprNodeConstantDesc(new Boolean(false))); // no NOT keyword + children1.add(col1Expr); + children1.add(constDesc); + children1.add(constDesc2); + exprDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, + children1); + + vc = new VectorizationContext(columnMap, 2); + ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); + assertTrue(ve instanceof FilterVarCharColumnBetween); + + // VARCHAR NOT BETWEEN + children1.set(0, new ExprNodeConstantDesc(new Boolean(true))); // has NOT keyword + ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); + assertTrue(ve instanceof FilterVarCharColumnNotBetween); + // long BETWEEN children1.set(0, new ExprNodeConstantDesc(new Boolean(false))); children1.set(1, new ExprNodeColumnDesc(Long.class, "col1", "table", false)); @@ -1173,12 +1327,12 @@ public class TestVectorizationContext { children1.set(1, col2Expr); children1.set(2, col3Expr); ve = vc.getVectorExpression(exprDesc); - assertTrue(ve instanceof IfExprStringColumnStringColumn); + assertTrue(ve instanceof IfExprStringGroupColumnStringGroupColumn); // column/scalar children1.set(2, constDesc3); ve = vc.getVectorExpression(exprDesc); - assertTrue(ve instanceof IfExprStringColumnStringScalar); + assertTrue(ve instanceof IfExprStringGroupColumnStringScalar); // scalar/scalar children1.set(1, constDesc2); @@ -1188,7 +1342,62 @@ public class TestVectorizationContext { // scalar/column children1.set(2, col3Expr); ve = vc.getVectorExpression(exprDesc); - assertTrue(ve instanceof IfExprStringScalarStringColumn); - } + assertTrue(ve instanceof IfExprStringScalarStringGroupColumn); + + // test for CHAR type + CharTypeInfo charTypeInfo = new CharTypeInfo(10); + constDesc2 = new ExprNodeConstantDesc(charTypeInfo, new HiveChar("Alpha", 10)); + constDesc3 = new ExprNodeConstantDesc(charTypeInfo, new HiveChar("Bravo", 10)); + col2Expr = new ExprNodeColumnDesc(charTypeInfo, "col2", "table", false); + col3Expr = new ExprNodeColumnDesc(charTypeInfo, "col3", "table", false); + + // column/column + children1.set(1, col2Expr); + children1.set(2, col3Expr); + ve = vc.getVectorExpression(exprDesc); + assertTrue(ve instanceof IfExprStringGroupColumnStringGroupColumn); + + // column/scalar + children1.set(2, constDesc3); + ve = vc.getVectorExpression(exprDesc); + assertTrue(ve instanceof IfExprStringGroupColumnCharScalar); + + // scalar/scalar + children1.set(1, constDesc2); +// ve = vc.getVectorExpression(exprDesc); +// assertTrue(ve instanceof IfExprCharScalarCharScalar); + + // scalar/column + children1.set(2, col3Expr); + ve = vc.getVectorExpression(exprDesc); + assertTrue(ve instanceof IfExprCharScalarStringGroupColumn); + + // test for VARCHAR type + VarcharTypeInfo varcharTypeInfo = new VarcharTypeInfo(10); + constDesc2 = new ExprNodeConstantDesc(varcharTypeInfo, new HiveVarchar("Alpha", 10)); + constDesc3 = new ExprNodeConstantDesc(varcharTypeInfo, new HiveVarchar("Bravo", 10)); + col2Expr = new ExprNodeColumnDesc(varcharTypeInfo, "col2", "table", false); + col3Expr = new ExprNodeColumnDesc(varcharTypeInfo, "col3", "table", false); + + // column/column + children1.set(1, col2Expr); + children1.set(2, col3Expr); + ve = vc.getVectorExpression(exprDesc); + assertTrue(ve instanceof IfExprStringGroupColumnStringGroupColumn); + + // column/scalar + children1.set(2, constDesc3); + ve = vc.getVectorExpression(exprDesc); + assertTrue(ve instanceof IfExprStringGroupColumnVarCharScalar); + // scalar/scalar + children1.set(1, constDesc2); +// ve = vc.getVectorExpression(exprDesc); +// assertTrue(ve instanceof IfExprVarCharScalarVarCharScalar); + + // scalar/column + children1.set(2, col3Expr); + ve = vc.getVectorExpression(exprDesc); + assertTrue(ve instanceof IfExprVarCharScalarStringGroupColumn); + } } Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorConditionalExpressions.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorConditionalExpressions.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorConditionalExpressions.java (original) +++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorConditionalExpressions.java Fri Sep 5 15:59:47 2014 @@ -32,10 +32,10 @@ import org.apache.hadoop.hive.ql.exec.ve import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDoubleScalarDoubleScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDoubleScalarDoubleColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDoubleColumnDoubleScalar; -import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringColumnStringColumn; -import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringColumnStringScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringGroupColumnStringGroupColumn; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringGroupColumnStringScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringScalarStringGroupColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringScalarStringScalar; -import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringScalarStringColumn; import org.junit.Test; @@ -388,7 +388,7 @@ public class TestVectorConditionalExpres @Test public void testIfExprStringColumnStringColumn() { VectorizedRowBatch batch = getBatch1Long3BytesVectors(); - VectorExpression expr = new IfExprStringColumnStringColumn(0, 1, 2, 3); + VectorExpression expr = new IfExprStringGroupColumnStringGroupColumn(0, 1, 2, 3); BytesColumnVector r = (BytesColumnVector) batch.cols[3]; expr.evaluate(batch); assertTrue(getString(r, 0).equals("arg3_0")); @@ -474,7 +474,7 @@ public class TestVectorConditionalExpres public void testIfExprStringColumnStringScalar() { VectorizedRowBatch batch = getBatch1Long3BytesVectors(); byte[] scalar = getUTF8Bytes("scalar"); - VectorExpression expr = new IfExprStringColumnStringScalar(0, 1, scalar, 3); + VectorExpression expr = new IfExprStringGroupColumnStringScalar(0, 1, scalar, 3); BytesColumnVector r = (BytesColumnVector) batch.cols[3]; expr.evaluate(batch); assertTrue(getString(r, 0).equals("scalar")); @@ -498,7 +498,7 @@ public class TestVectorConditionalExpres public void testIfExprStringScalarStringColumn() { VectorizedRowBatch batch = getBatch1Long3BytesVectors(); byte[] scalar = getUTF8Bytes("scalar"); - VectorExpression expr = new IfExprStringScalarStringColumn(0,scalar, 2, 3); + VectorExpression expr = new IfExprStringScalarStringGroupColumn(0,scalar, 2, 3); BytesColumnVector r = (BytesColumnVector) batch.cols[3]; expr.evaluate(batch); assertTrue(getString(r, 0).equals("arg3_0")); Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorGenericDateExpressions.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorGenericDateExpressions.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorGenericDateExpressions.java (original) +++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorGenericDateExpressions.java Fri Sep 5 15:59:47 2014 @@ -27,7 +27,6 @@ import org.apache.hadoop.hive.serde2.io. import org.junit.Assert; import org.junit.Test; - import java.io.UnsupportedEncodingException; import java.sql.Date; import java.text.SimpleDateFormat; @@ -136,9 +135,12 @@ public class TestVectorGenericDateExpres return toTimestamp(date); case STRING: + case CHAR: + case VARCHAR: return toString(date); + default: + throw new Error("Unsupported input type " + type.name()); } - return null; } private void testDateAddColScalar(VectorExpression.Type colType1, boolean isPositive) { @@ -214,8 +216,12 @@ public class TestVectorGenericDateExpres udf = new VectorUDFDateAddScalarCol(toTimestamp(scalar1), 0, 1); break; case STRING: + case CHAR: + case VARCHAR: udf = new VectorUDFDateAddScalarCol(toString(scalar1), 0, 1); break; + default: + throw new Error("Invalid input type: " + colType1.name()); } } else { switch (colType1) { @@ -226,8 +232,12 @@ public class TestVectorGenericDateExpres udf = new VectorUDFDateSubScalarCol(toTimestamp(scalar1), 0, 1); break; case STRING: + case CHAR: + case VARCHAR: udf = new VectorUDFDateSubScalarCol(toString(scalar1), 0, 1); break; + default: + throw new Error("Invalid input type: " + colType1.name()); } } udf.setInputTypes(colType1, VectorExpression.Type.OTHER); @@ -694,7 +704,9 @@ public class TestVectorGenericDateExpres private void validateToDate(VectorizedRowBatch batch, VectorExpression.Type colType, LongColumnVector date) { VectorExpression udf; - if (colType == VectorExpression.Type.STRING) { + if (colType == VectorExpression.Type.STRING || + colType == VectorExpression.Type.CHAR || + colType == VectorExpression.Type.VARCHAR) { udf = new CastStringToDate(0, 1); } else { udf = new CastLongToDate(0, 1);
