Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/PosModDoubleToDouble.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/PosModDoubleToDouble.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/PosModDoubleToDouble.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/PosModDoubleToDouble.java Fri Sep 5 15:59:47 2014 @@ -61,8 +61,8 @@ public class PosModDoubleToDouble extend VectorExpressionDescriptor.Mode.PROJECTION) .setNumArguments(2) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.LONG, - VectorExpressionDescriptor.ArgumentType.DOUBLE) + VectorExpressionDescriptor.ArgumentType.INT_FAMILY, + VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN, VectorExpressionDescriptor.InputExpressionType.SCALAR).build();
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/PosModLongToLong.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/PosModLongToLong.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/PosModLongToLong.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/PosModLongToLong.java Fri Sep 5 15:59:47 2014 @@ -61,8 +61,8 @@ public class PosModLongToLong extends Ma VectorExpressionDescriptor.Mode.PROJECTION) .setNumArguments(2) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.LONG, - VectorExpressionDescriptor.ArgumentType.LONG) + VectorExpressionDescriptor.ArgumentType.INT_FAMILY, + VectorExpressionDescriptor.ArgumentType.INT_FAMILY) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN, VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/RoundWithNumDigitsDoubleToDouble.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/RoundWithNumDigitsDoubleToDouble.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/RoundWithNumDigitsDoubleToDouble.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/RoundWithNumDigitsDoubleToDouble.java Fri Sep 5 15:59:47 2014 @@ -64,8 +64,8 @@ public class RoundWithNumDigitsDoubleToD b.setMode(VectorExpressionDescriptor.Mode.PROJECTION) .setNumArguments(2) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.DOUBLE, - VectorExpressionDescriptor.ArgumentType.LONG) + VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, + VectorExpressionDescriptor.ArgumentType.INT_FAMILY) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN, VectorExpressionDescriptor.InputExpressionType.SCALAR); Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/SelectColumnIsFalse.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/SelectColumnIsFalse.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/SelectColumnIsFalse.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/SelectColumnIsFalse.java Fri Sep 5 15:59:47 2014 @@ -144,7 +144,7 @@ public class SelectColumnIsFalse extends b.setMode(VectorExpressionDescriptor.Mode.FILTER) .setNumArguments(1) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.LONG) + VectorExpressionDescriptor.ArgumentType.INT_FAMILY) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN); return b.build(); Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/SelectColumnIsNotNull.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/SelectColumnIsNotNull.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/SelectColumnIsNotNull.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/SelectColumnIsNotNull.java Fri Sep 5 15:59:47 2014 @@ -113,7 +113,7 @@ public class SelectColumnIsNotNull exten b.setMode(VectorExpressionDescriptor.Mode.FILTER) .setNumArguments(1) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.ANY) + VectorExpressionDescriptor.ArgumentType.ALL_FAMILY) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN); return b.build(); Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/SelectColumnIsNull.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/SelectColumnIsNull.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/SelectColumnIsNull.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/SelectColumnIsNull.java Fri Sep 5 15:59:47 2014 @@ -111,7 +111,7 @@ public class SelectColumnIsNull extends b.setMode(VectorExpressionDescriptor.Mode.FILTER) .setNumArguments(1) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.ANY) + VectorExpressionDescriptor.ArgumentType.ALL_FAMILY) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN); return b.build(); Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/SelectColumnIsTrue.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/SelectColumnIsTrue.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/SelectColumnIsTrue.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/SelectColumnIsTrue.java Fri Sep 5 15:59:47 2014 @@ -143,7 +143,7 @@ public class SelectColumnIsTrue extends b.setMode(VectorExpressionDescriptor.Mode.FILTER) .setNumArguments(1) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.LONG) + VectorExpressionDescriptor.ArgumentType.INT_FAMILY) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN); return b.build(); Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringExpr.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringExpr.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringExpr.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringExpr.java Fri Sep 5 15:59:47 2014 @@ -18,6 +18,10 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; +import java.util.Arrays; + +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; + /** * String expression evaluation helper functions. */ @@ -32,6 +36,7 @@ public class StringExpr { */ public static int compare(byte[] arg1, int start1, int len1, byte[] arg2, int start2, int len2) { for (int i = 0; i < len1 && i < len2; i++) { + // Note the "& 0xff" is just a way to convert unsigned bytes to signed integer. int b1 = arg1[i + start1] & 0xff; int b2 = arg2[i + start2] & 0xff; if (b1 != b2) { @@ -40,4 +45,259 @@ public class StringExpr { } return len1 - len2; } + + public static int characterCount(byte[] bytes) { + int end = bytes.length; + + // count characters + int j = 0; + int charCount = 0; + while(j < end) { + // UTF-8 continuation bytes have 2 high bits equal to 0x80. + if ((bytes[j] & 0xc0) != 0x80) { + ++charCount; + } + j++; + } + return charCount; + } + + public static int characterCount(byte[] bytes, int start, int length) { + int end = start + length; + + // count characters + int j = start; + int charCount = 0; + while(j < end) { + // UTF-8 continuation bytes have 2 high bits equal to 0x80. + if ((bytes[j] & 0xc0) != 0x80) { + ++charCount; + } + j++; + } + return charCount; + } + + // A setVal with the same function signature as rightTrim, leftTrim, truncate, etc, below. + // Useful for class generation via templates. + public static void assign(BytesColumnVector outV, int i, byte[] bytes, int start, int length) { + // set output vector + outV.setVal(i, bytes, start, length); + } + + /* + * Right trim a slice of a byte array and return the new byte length. + */ + public static int rightTrim(byte[] bytes, int start, int length) { + // skip trailing blank characters + int j = start + length - 1; + while(j >= start && bytes[j] == 0x20) { + j--; + } + + return (j - start) + 1; + } + + /* + * Right trim a slice of a byte array and place the result into element i of a vector. + */ + public static void rightTrim(BytesColumnVector outV, int i, byte[] bytes, int start, int length) { + // skip trailing blank characters + int j = start + length - 1; + while(j >= start && bytes[j] == 0x20) { + j--; + } + + // set output vector + outV.setVal(i, bytes, start, (j - start) + 1); + } + + /* + * Truncate a slice of a byte array to a maximum number of characters and + * return the new byte length. + */ + public static int truncate(byte[] bytes, int start, int length, int maxLength) { + int end = start + length; + + // count characters forward + int j = start; + int charCount = 0; + while(j < end) { + // UTF-8 continuation bytes have 2 high bits equal to 0x80. + if ((bytes[j] & 0xc0) != 0x80) { + if (charCount == maxLength) { + break; + } + ++charCount; + } + j++; + } + return (j - start); + } + + /* + * Truncate a slice of a byte array to a maximum number of characters and + * place the result into element i of a vector. + */ + public static void truncate(BytesColumnVector outV, int i, byte[] bytes, int start, int length, int maxLength) { + int end = start + length; + + // count characters forward + int j = start; + int charCount = 0; + while(j < end) { + // UTF-8 continuation bytes have 2 high bits equal to 0x80. + if ((bytes[j] & 0xc0) != 0x80) { + if (charCount == maxLength) { + break; + } + ++charCount; + } + j++; + } + + // set output vector + outV.setVal(i, bytes, start, (j - start)); + } + + /* + * Truncate a byte array to a maximum number of characters and + * return a byte array with only truncated bytes. + */ + public static byte[] truncateScalar(byte[] bytes, int maxLength) { + int end = bytes.length; + + // count characters forward + int j = 0; + int charCount = 0; + while(j < end) { + // UTF-8 continuation bytes have 2 high bits equal to 0x80. + if ((bytes[j] & 0xc0) != 0x80) { + if (charCount == maxLength) { + break; + } + ++charCount; + } + j++; + } + if (j == end) { + return bytes; + } else { + return Arrays.copyOf(bytes, j); + } + } + + /* + * Right trim and truncate a slice of a byte array to a maximum number of characters and + * return the new byte length. + */ + public static int rightTrimAndTruncate(byte[] bytes, int start, int length, int maxLength) { + int end = start + length; + + // count characters forward and watch for final run of pads + int j = start; + int charCount = 0; + int padRunStart = -1; + while(j < end) { + // UTF-8 continuation bytes have 2 high bits equal to 0x80. + if ((bytes[j] & 0xc0) != 0x80) { + if (charCount == maxLength) { + break; + } + if (bytes[j] == 0x20) { + if (padRunStart == -1) { + padRunStart = j; + } + } else { + padRunStart = -1; + } + ++charCount; + } else { + padRunStart = -1; + } + j++; + } + if (padRunStart != -1) { + return (padRunStart - start); + } else { + return (j - start); + } + } + + /* + * Right trim and truncate a slice of a byte array to a maximum number of characters and + * place the result into element i of a vector. + */ + public static void rightTrimAndTruncate(BytesColumnVector outV, int i, byte[] bytes, int start, int length, int maxLength) { + int end = start + length; + + // count characters forward and watch for final run of pads + int j = start; + int charCount = 0; + int padRunStart = -1; + while(j < end) { + // UTF-8 continuation bytes have 2 high bits equal to 0x80. + if ((bytes[j] & 0xc0) != 0x80) { + if (charCount == maxLength) { + break; + } + if (bytes[j] == 0x20) { + if (padRunStart == -1) { + padRunStart = j; + } + } else { + padRunStart = -1; + } + ++charCount; + } else { + padRunStart = -1; + } + j++; + } + // set output vector + if (padRunStart != -1) { + outV.setVal(i, bytes, start, (padRunStart - start)); + } else { + outV.setVal(i, bytes, start, (j - start) ); + } + } + + /* + * Right trim and truncate a byte array to a maximum number of characters and + * return a byte array with only the trimmed and truncated bytes. + */ + public static byte[] rightTrimAndTruncateScalar(byte[] bytes, int maxLength) { + int end = bytes.length; + + // count characters forward and watch for final run of pads + int j = 0; + int charCount = 0; + int padRunStart = -1; + while(j < end) { + // UTF-8 continuation bytes have 2 high bits equal to 0x80. + if ((bytes[j] & 0xc0) != 0x80) { + if (charCount == maxLength) { + break; + } + if (bytes[j] == 0x20) { + if (padRunStart == -1) { + padRunStart = j; + } + } else { + padRunStart = -1; + } + ++charCount; + } else { + padRunStart = -1; + } + j++; + } + if (padRunStart != -1) { + return Arrays.copyOf(bytes, padRunStart); + } else if (j == end) { + return bytes; + } else { + return Arrays.copyOf(bytes, j); + } + } } Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringGroupColConcatCharScalar.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringGroupColConcatCharScalar.java?rev=1622729&view=auto ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringGroupColConcatCharScalar.java (added) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringGroupColConcatCharScalar.java Fri Sep 5 15:59:47 2014 @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.hadoop.hive.common.type.HiveChar; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; + +/** + * Vectorized instruction to concatenate a string column to a scalar and put + * the result in an output column. + */ +public class StringGroupColConcatCharScalar extends StringGroupColConcatStringScalar { + private static final long serialVersionUID = 1L; + + public StringGroupColConcatCharScalar(int colNum, HiveChar value, int outputColumn) { + super(colNum, value.getStrippedValue().getBytes(), outputColumn); + } + + public StringGroupColConcatCharScalar() { + super(); + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(2) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.STRING_FAMILY, + VectorExpressionDescriptor.ArgumentType.CHAR) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); + } +} Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringGroupColConcatStringScalar.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringGroupColConcatStringScalar.java?rev=1622729&view=auto ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringGroupColConcatStringScalar.java (added) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringGroupColConcatStringScalar.java Fri Sep 5 15:59:47 2014 @@ -0,0 +1,164 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; + +/** + * Vectorized instruction to concatenate a string column to a scalar and put + * the result in an output column. + */ +public class StringGroupColConcatStringScalar extends VectorExpression { + private static final long serialVersionUID = 1L; + private int colNum; + private int outputColumn; + private byte[] value; + + public StringGroupColConcatStringScalar(int colNum, byte[] value, int outputColumn) { + this(); + this.colNum = colNum; + this.outputColumn = outputColumn; + this.value = value; + } + + public StringGroupColConcatStringScalar() { + super(); + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + BytesColumnVector inputColVector = (BytesColumnVector) batch.cols[colNum]; + BytesColumnVector outV = (BytesColumnVector) batch.cols[outputColumn]; + int[] sel = batch.selected; + int n = batch.size; + byte[][] vector = inputColVector.vector; + int[] start = inputColVector.start; + int[] length = inputColVector.length; + + if (n == 0) { + + // Nothing to do + return; + } + + // initialize output vector buffer to receive data + outV.initBuffer(); + + if (inputColVector.noNulls) { + outV.noNulls = true; + if (inputColVector.isRepeating) { + outV.isRepeating = true; + outV.setConcat(0, vector[0], start[0], length[0], value, 0, value.length); + } else if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outV.setConcat(i, vector[i], start[i], length[i], value, 0, value.length); + } + outV.isRepeating = false; + } else { + for(int i = 0; i != n; i++) { + outV.setConcat(i, vector[i], start[i], length[i], value, 0, value.length); + } + outV.isRepeating = false; + } + } else { + + /* + * Handle case with nulls. Don't do function if the value is null, to save time, + * because calling the function can be expensive. + */ + outV.noNulls = false; + if (inputColVector.isRepeating) { + outV.isRepeating = true; + outV.isNull[0] = inputColVector.isNull[0]; + if (!inputColVector.isNull[0]) { + outV.setConcat(0, vector[0], start[0], length[0], value, 0, value.length); + } + } else if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (!inputColVector.isNull[i]) { + outV.setConcat(i, vector[i], start[i], length[i], value, 0, value.length); + } + outV.isNull[i] = inputColVector.isNull[i]; + } + outV.isRepeating = false; + } else { + for(int i = 0; i != n; i++) { + if (!inputColVector.isNull[i]) { + outV.setConcat(i, vector[i], start[i], length[i], value, 0, value.length); + } + outV.isNull[i] = inputColVector.isNull[i]; + } + outV.isRepeating = false; + } + } + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return "StringGroup"; + } + + public int getColNum() { + return colNum; + } + + public void setColNum(int colNum) { + this.colNum = colNum; + } + + public byte[] getValue() { + return value; + } + + public void setValue(byte[] value) { + this.value = value; + } + + public void setOutputColumn(int outputColumn) { + this.outputColumn = outputColumn; + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(2) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.STRING_FAMILY, + VectorExpressionDescriptor.ArgumentType.STRING) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); + } +} Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringGroupColConcatVarCharScalar.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringGroupColConcatVarCharScalar.java?rev=1622729&view=auto ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringGroupColConcatVarCharScalar.java (added) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringGroupColConcatVarCharScalar.java Fri Sep 5 15:59:47 2014 @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; + +/** + * Vectorized instruction to concatenate a string column to a scalar and put + * the result in an output column. + */ +public class StringGroupColConcatVarCharScalar extends StringGroupColConcatStringScalar { + private static final long serialVersionUID = 1L; + + public StringGroupColConcatVarCharScalar(int colNum, HiveVarchar value, int outputColumn) { + super(colNum, value.getValue().getBytes(), outputColumn); + } + + public StringGroupColConcatVarCharScalar() { + super(); + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(2) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.STRING_FAMILY, + VectorExpressionDescriptor.ArgumentType.VARCHAR) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); + } +} Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringGroupConcatColCol.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringGroupConcatColCol.java?rev=1622729&view=auto ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringGroupConcatColCol.java (added) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringGroupConcatColCol.java Fri Sep 5 15:59:47 2014 @@ -0,0 +1,455 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; + +/** + * Vectorized instruction to concatenate two string columns and put + * the output in a third column. + */ +public class StringGroupConcatColCol extends VectorExpression { + private static final long serialVersionUID = 1L; + private int colNum1; + private int colNum2; + private int outputColumn; + + public StringGroupConcatColCol(int colNum1, int colNum2, int outputColumn) { + this(); + this.colNum1 = colNum1; + this.colNum2 = colNum2; + this.outputColumn = outputColumn; + } + + public StringGroupConcatColCol() { + super(); + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + BytesColumnVector inV1 = (BytesColumnVector) batch.cols[colNum1]; + BytesColumnVector inV2 = (BytesColumnVector) batch.cols[colNum2]; + BytesColumnVector outV = (BytesColumnVector) batch.cols[outputColumn]; + int[] sel = batch.selected; + int n = batch.size; + byte[][] vector1 = inV1.vector; + byte[][] vector2 = inV2.vector; + int[] len1 = inV1.length; + int[] len2 = inV2.length; + int[] start1 = inV1.start; + int[] start2 = inV2.start; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + // prepare output buffer to accept results + outV.initBuffer(); + + /* Handle default case for isRepeating setting for output. This will be set to true + * later in the special cases where that is necessary. + */ + outV.isRepeating = false; + + if (inV1.noNulls && !inV2.noNulls) { + + // propagate nulls + + /* We'll assume that there *may* be nulls in the input if !noNulls is true + * for an input vector. This is to be more forgiving of errors in loading + * the vectors. A properly-written vectorized iterator will make sure that + * isNull[0] is set if !noNulls and isRepeating are true for the vector. + */ + outV.noNulls = false; + if (inV2.isRepeating) { + if (inV2.isNull[0]) { + + // Output will also be repeating and null + outV.isNull[0] = true; + outV.isRepeating = true; + + //return as no further processing is needed + return; + } + } else { + propagateNulls(batch.selectedInUse, n, sel, inV2, outV); + } + + // perform data operation + if (inV1.isRepeating && inV2.isRepeating) { + + /* All must be selected otherwise size would be zero. + * Repeating property will not change. + */ + if (!inV2.isNull[0]) { + outV.setConcat(0, vector1[0], start1[0], len1[0], vector2[0], start2[0], len2[0]); + } + outV.isRepeating = true; + } else if (inV1.isRepeating) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (!inV2.isNull[i]) { + outV.setConcat(i, vector1[0], start1[0], len1[0], vector2[i], start2[i], len2[i]); + } + } + } else { + for(int i = 0; i != n; i++) { + if (!inV2.isNull[0]) { + outV.setConcat(i, vector1[0], start1[0], len1[0], vector2[i], start2[i], len2[i]); + } + } + } + } else if (inV2.isRepeating) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (!inV2.isNull[i]) { + outV.setConcat(i, vector1[i], start1[i], len1[i], vector2[0], start2[0], len2[0]); + } + } + } else { + for(int i = 0; i != n; i++) { + if (!inV2.isNull[i]) { + outV.setConcat(i, vector1[i], start1[i], len1[i], vector2[0], start2[0], len2[0]); + } + } + } + } else { + if (batch.selectedInUse) { + for(int j=0; j != n; j++) { + int i = sel[j]; + if (!inV2.isNull[i]) { + outV.setConcat(i, vector1[i], start1[i], len1[i], vector2[i], start2[i], len2[i]); + } + } + } else { + for(int i = 0; i != n; i++) { + if (!inV2.isNull[i]) { + outV.setConcat(i, vector1[i], start1[i], len1[i], vector2[i], start2[i], len2[i]); + } + } + } + } + } else if (!inV1.noNulls && inV2.noNulls) { + + // propagate nulls + outV.noNulls = false; + if (inV1.isRepeating) { + + //Output will also be repeating and null + outV.isRepeating = true; + outV.isNull[0] = true; + + //return as no further processing is needed + return; + } else { + propagateNulls(batch.selectedInUse, n, sel, inV1, outV); + } + + // perform data operation + if (inV1.isRepeating && inV2.isRepeating) { + //All must be selected otherwise size would be zero + //Repeating property will not change. + if (!inV1.isNull[0]) { + outV.setConcat(0, vector1[0], start1[0], len1[0], vector2[0], start2[0], len2[0]); + } + outV.isRepeating = true; + } else if (inV1.isRepeating) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (!inV1.isNull[0]) { + outV.setConcat(i, vector1[0], start1[0], len1[0], vector2[i], start2[i], len2[i]); + } + } + } else { + for(int i = 0; i != n; i++) { + if (!inV1.isNull[0]) { + outV.setConcat(i, vector1[0], start1[0], len1[0], vector2[i], start2[i], len2[i]); + } + } + } + } else if (inV2.isRepeating) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (!inV1.isNull[i]) { + outV.setConcat(i, vector1[i], start1[i], len1[i], vector2[0], start2[0], len2[0]); + } + } + } else { + for(int i = 0; i != n; i++) { + if (!inV1.isNull[i]) { + outV.setConcat(i, vector1[i], start1[i], len1[i], vector2[0], start2[0], len2[0]); + } + } + } + } else { + if (batch.selectedInUse) { + for(int j=0; j != n; j++) { + int i = sel[j]; + if (!inV1.isNull[i]) { + outV.setConcat(i, vector1[i], start1[i], len1[i], vector2[i], start2[i], len2[i]); + } + } + } else { + for(int i = 0; i != n; i++) { + if (!inV1.isNull[i]) { + outV.setConcat(i, vector1[i], start1[i], len1[i], vector2[i], start2[i], len2[i]); + } + } + } + } + } else if (!inV1.noNulls && !inV2.noNulls) { + + // propagate nulls + outV.noNulls = false; + if (inV1.isRepeating && inV2.isRepeating) { + outV.isNull[0] = inV1.isNull[0] || inV2.isNull[0]; + + //Output will also be repeating + outV.isRepeating = true; + + // return if output is null because no additional work is needed + if (outV.isNull[0]) { + return; + } + } else if (inV1.isRepeating) { + if (inV1.isNull[0]) { // then all output will be null + outV.isRepeating = true; + outV.isNull[0] = true; + return; + } else { + outV.isRepeating = false; + propagateNulls(batch.selectedInUse, n, sel, inV2, outV); + } + } else if (inV2.isRepeating) { + if (inV2.isNull[0]) { + outV.isRepeating = true; + outV.isNull[0] = true; + return; + } else { + outV.isRepeating = false; + propagateNulls(batch.selectedInUse, n, sel, inV1, outV); + } + } else { + propagateNullsCombine(batch.selectedInUse, n, sel, inV1, inV2, outV); + } + + // perform data operation + if (inV1.isRepeating && inV2.isRepeating) { + + // All must be selected otherwise size would be zero. Repeating property will not change. + if (!inV1.isNull[0] && !inV2.isNull[0]) { + outV.setConcat(0, vector1[0], start1[0], len1[0], vector2[0], start2[0], len2[0]); + } + outV.isRepeating = true; + } else if (inV1.isRepeating) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (!inV1.isNull[0] && !inV2.isNull[i]) { + outV.setConcat(i, vector1[0], start1[0], len1[0], vector2[i], start2[i], len2[i]); + } + } + } else { + for(int i = 0; i != n; i++) { + if (!inV1.isNull[0] && !inV2.isNull[i]) { + outV.setConcat(i, vector1[0], start1[0], len1[0], vector2[i], start2[i], len2[i]); + } + } + } + } else if (inV2.isRepeating) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (!inV1.isNull[i] && !inV2.isNull[0]) { + outV.setConcat(i, vector1[i], start1[i], len1[i], vector2[0], start2[0], len2[0]); + } + } + } else { + for(int i = 0; i != n; i++) { + if (!inV1.isNull[i] && !inV2.isNull[0]) { + outV.setConcat(i, vector1[i], start1[i], len1[i], vector2[0], start2[0], len2[0]); + } + } + } + } else { + if (batch.selectedInUse) { + for(int j=0; j != n; j++) { + int i = sel[j]; + if (!inV1.isNull[i] && !inV2.isNull[i]) { + outV.setConcat(i, vector1[i], start1[i], len1[i], vector2[i], start2[i], len2[i]); + } + } + } else { + for(int i = 0; i != n; i++) { + if (!inV1.isNull[i] && !inV2.isNull[i]) { + outV.setConcat(i, vector1[i], start1[i], len1[i], vector2[i], start2[i], len2[i]); + } + } + } + } + } else { // there are no nulls in either input vector + + // propagate null information + outV.noNulls = true; + + // perform data operation + if (inV1.isRepeating && inV2.isRepeating) { + + // All must be selected otherwise size would be zero. Repeating property will not change. + outV.setConcat(0, vector1[0], start1[0], len1[0], vector2[0], start2[0], len2[0]); + outV.isRepeating = true; + } else if (inV1.isRepeating) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outV.setConcat(i, vector1[0], start1[0], len1[0], vector2[i], start2[i], len2[i]); + } + } else { + for(int i = 0; i != n; i++) { + outV.setConcat(i, vector1[0], start1[0], len1[0], vector2[i], start2[i], len2[i]); + } + } + } else if (inV2.isRepeating) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outV.setConcat(i, vector1[i], start1[i], len1[i], vector2[0], start2[0], len2[0]); + } + } else { + for(int i = 0; i != n; i++) { + outV.setConcat(i, vector1[i], start1[i], len1[i], vector2[0], start2[0], len2[0]); + } + } + } else { + if (batch.selectedInUse) { + for(int j=0; j != n; j++) { + int i = sel[j]; + outV.setConcat(i, vector1[i], start1[i], len1[i], vector2[i], start2[i], len2[i]); + } + } else { + for(int i = 0; i != n; i++) { + outV.setConcat(i, vector1[i], start1[i], len1[i], vector2[i], start2[i], len2[i]); + } + } + } + } + } + + /** + * Propagate the logic OR of null vectors from two inputs to output. + * + * @param selectedInUse true/false flag to tell if sel[] is in use + * @param n number of qualifying rows + * @param sel selected value position array + * @param inV1 input vector 1 + * @param inV2 input vector 2 + * @param outV output vector + */ + private static void propagateNullsCombine(boolean selectedInUse, int n, int[] sel, + ColumnVector inV1, ColumnVector inV2, BytesColumnVector outV) { + if (selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outV.isNull[i] = inV1.isNull[i] || inV2.isNull[i]; + } + } else { + for(int i = 0; i != n; i++) { + outV.isNull[i] = inV1.isNull[i] || inV2.isNull[i]; + } + } + } + + /** + * Propagate nulls from input vector inV to output vector outV. + * + * @param selectedInUse true/false flag to tell if sel[] is in use + * @param sel selected value position array + * @param n number of qualifying rows + * @param inV input vector + * @param outV ouput vector + */ + private static void propagateNulls(boolean selectedInUse, int n, int[] sel, ColumnVector inV, + ColumnVector outV) { + if (selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outV.isNull[i] = inV.isNull[i]; + } + } else { + System.arraycopy(inV.isNull, 0, outV.isNull, 0, n); + } + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return "StringGroup"; + } + + public int getColNum1() { + return colNum1; + } + + public void setColNum1(int colNum1) { + this.colNum1 = colNum1; + } + + public int getColNum2() { + return colNum2; + } + + public void setColNum2(int colNum2) { + this.colNum2 = colNum2; + } + + public void setOutputColumn(int outputColumn) { + this.outputColumn = outputColumn; + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(2) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.STRING_FAMILY, + VectorExpressionDescriptor.ArgumentType.STRING_FAMILY) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); + } +} Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringLength.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringLength.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringLength.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringLength.java Fri Sep 5 15:59:47 2014 @@ -162,7 +162,7 @@ public class StringLength extends Vector b.setMode(VectorExpressionDescriptor.Mode.PROJECTION) .setNumArguments(1) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.STRING) + VectorExpressionDescriptor.ArgumentType.STRING_FAMILY) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN); return b.build(); Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringScalarConcatStringGroupCol.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringScalarConcatStringGroupCol.java?rev=1622729&view=auto ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringScalarConcatStringGroupCol.java (added) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringScalarConcatStringGroupCol.java Fri Sep 5 15:59:47 2014 @@ -0,0 +1,164 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; + +/** + * Vectorized instruction to concatenate a scalar to a string column and put + * the result in an output column. + */ +public class StringScalarConcatStringGroupCol extends VectorExpression { + private static final long serialVersionUID = 1L; + private int colNum; + private int outputColumn; + private byte[] value; + + public StringScalarConcatStringGroupCol(byte[] value, int colNum, int outputColumn) { + this(); + this.colNum = colNum; + this.outputColumn = outputColumn; + this.value = value; + } + + public StringScalarConcatStringGroupCol() { + super(); + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + BytesColumnVector inputColVector = (BytesColumnVector) batch.cols[colNum]; + BytesColumnVector outV = (BytesColumnVector) batch.cols[outputColumn]; + int[] sel = batch.selected; + int n = batch.size; + byte[][] vector = inputColVector.vector; + int[] start = inputColVector.start; + int[] length = inputColVector.length; + + if (n == 0) { + + // Nothing to do + return; + } + + // initialize output vector buffer to receive data + outV.initBuffer(); + + if (inputColVector.noNulls) { + outV.noNulls = true; + if (inputColVector.isRepeating) { + outV.isRepeating = true; + outV.setConcat(0, value, 0, value.length, vector[0], start[0], length[0]); + } else if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outV.setConcat(i, value, 0, value.length, vector[i], start[i], length[i]); + } + outV.isRepeating = false; + } else { + for(int i = 0; i != n; i++) { + outV.setConcat(i, value, 0, value.length, vector[i], start[i], length[i]); + } + outV.isRepeating = false; + } + } else { + + /* + * Handle case with nulls. Don't do function if the value is null, to save time, + * because calling the function can be expensive. + */ + outV.noNulls = false; + if (inputColVector.isRepeating) { + outV.isRepeating = true; + outV.isNull[0] = inputColVector.isNull[0]; + if (!inputColVector.isNull[0]) { + outV.setConcat(0, value, 0, value.length, vector[0], start[0], length[0]); + } + } else if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (!inputColVector.isNull[i]) { + outV.setConcat(i, value, 0, value.length, vector[i], start[i], length[i]); + } + outV.isNull[i] = inputColVector.isNull[i]; + } + outV.isRepeating = false; + } else { + for(int i = 0; i != n; i++) { + if (!inputColVector.isNull[i]) { + outV.setConcat(i, value, 0, value.length, vector[i], start[i], length[i]); + } + outV.isNull[i] = inputColVector.isNull[i]; + } + outV.isRepeating = false; + } + } + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return "StringGroup"; + } + + public int getColNum() { + return colNum; + } + + public void setColNum(int colNum) { + this.colNum = colNum; + } + + public byte[] getValue() { + return value; + } + + public void setValue(byte[] value) { + this.value = value; + } + + public void setOutputColumn(int outputColumn) { + this.outputColumn = outputColumn; + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(2) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.STRING, + VectorExpressionDescriptor.ArgumentType.STRING_FAMILY) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.SCALAR, + VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); + } +} Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringSubstrColStart.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringSubstrColStart.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringSubstrColStart.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringSubstrColStart.java Fri Sep 5 15:59:47 2014 @@ -250,8 +250,8 @@ public class StringSubstrColStart extend b.setMode(VectorExpressionDescriptor.Mode.PROJECTION) .setNumArguments(2) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.STRING, - VectorExpressionDescriptor.ArgumentType.LONG) + VectorExpressionDescriptor.ArgumentType.STRING_FAMILY, + VectorExpressionDescriptor.ArgumentType.INT_FAMILY) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN, VectorExpressionDescriptor.InputExpressionType.SCALAR); Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringSubstrColStartLen.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringSubstrColStartLen.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringSubstrColStartLen.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringSubstrColStartLen.java Fri Sep 5 15:59:47 2014 @@ -277,9 +277,9 @@ public class StringSubstrColStartLen ext b.setMode(VectorExpressionDescriptor.Mode.PROJECTION) .setNumArguments(3) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.STRING, - VectorExpressionDescriptor.ArgumentType.LONG, - VectorExpressionDescriptor.ArgumentType.LONG) + VectorExpressionDescriptor.ArgumentType.STRING_FAMILY, + VectorExpressionDescriptor.ArgumentType.INT_FAMILY, + VectorExpressionDescriptor.ArgumentType.INT_FAMILY) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN, VectorExpressionDescriptor.InputExpressionType.SCALAR, Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringUnaryUDF.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringUnaryUDF.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringUnaryUDF.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringUnaryUDF.java Fri Sep 5 15:59:47 2014 @@ -200,7 +200,7 @@ public class StringUnaryUDF extends Vect b.setMode(VectorExpressionDescriptor.Mode.PROJECTION) .setNumArguments(1) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.STRING) + VectorExpressionDescriptor.ArgumentType.STRING_FAMILY) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN); return b.build(); Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringUnaryUDFDirect.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringUnaryUDFDirect.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringUnaryUDFDirect.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringUnaryUDFDirect.java Fri Sep 5 15:59:47 2014 @@ -142,7 +142,7 @@ abstract public class StringUnaryUDFDire b.setMode(VectorExpressionDescriptor.Mode.PROJECTION) .setNumArguments(1) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.STRING) + VectorExpressionDescriptor.ArgumentType.STRING_FAMILY) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN); return b.build(); Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/TruncStringOutput.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/TruncStringOutput.java?rev=1622729&view=auto ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/TruncStringOutput.java (added) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/TruncStringOutput.java Fri Sep 5 15:59:47 2014 @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +public interface TruncStringOutput { + abstract public int getMaxLength(); + + abstract public void setMaxLength(int maxLength); +} \ No newline at end of file Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VarCharScalarConcatStringGroupCol.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VarCharScalarConcatStringGroupCol.java?rev=1622729&view=auto ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VarCharScalarConcatStringGroupCol.java (added) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VarCharScalarConcatStringGroupCol.java Fri Sep 5 15:59:47 2014 @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; + +/** + * Vectorized instruction to concatenate a scalar to a string column and put + * the result in an output column. + */ +public class VarCharScalarConcatStringGroupCol extends StringScalarConcatStringGroupCol { + private static final long serialVersionUID = 1L; + + public VarCharScalarConcatStringGroupCol(HiveVarchar value, int colNum, int outputColumn) { + super(value.getValue().getBytes(), colNum, outputColumn); + } + + public VarCharScalarConcatStringGroupCol() { + super(); + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(2) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.VARCHAR, + VectorExpressionDescriptor.ArgumentType.STRING_FAMILY) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.SCALAR, + VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); + } +} Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpression.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpression.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpression.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpression.java Fri Sep 5 15:59:47 2014 @@ -30,16 +30,22 @@ import org.apache.hadoop.hive.ql.exec.ve */ public abstract class VectorExpression implements Serializable { public enum Type { - STRING, TIMESTAMP, DATE, OTHER; + STRING, CHAR, VARCHAR, TIMESTAMP, DATE, LONG, DOUBLE, DECIMAL, OTHER; private static Map<String, Type> types = ImmutableMap.<String, Type>builder() .put("string", STRING) + .put("char", CHAR) + .put("varchar", VARCHAR) .put("timestamp", TIMESTAMP) .put("date", DATE) + .put("long", LONG) + .put("double", DOUBLE) + .put("decimal", DECIMAL) .build(); public static Type getValue(String name) { - if (types.containsKey(name.toLowerCase())) { - return types.get(name); + String nameLower = name.toLowerCase(); + if (types.containsKey(nameLower)) { + return types.get(nameLower); } return OTHER; } Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java Fri Sep 5 15:59:47 2014 @@ -28,6 +28,7 @@ import java.util.List; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.common.type.Decimal128; +import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.ql.exec.vector.*; @@ -46,6 +47,7 @@ import org.apache.hadoop.hive.serde2.obj import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableDateObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableDoubleObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableFloatObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveCharObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveVarcharObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableIntObjectInspector; @@ -403,6 +405,9 @@ public final class VectorExpressionWrite case STRING: return genVectorExpressionWritableString( (SettableStringObjectInspector) fieldObjInspector); + case CHAR: + return genVectorExpressionWritableChar( + (SettableHiveCharObjectInspector) fieldObjInspector); case VARCHAR: return genVectorExpressionWritableVarchar( (SettableHiveVarcharObjectInspector) fieldObjInspector); @@ -558,6 +563,46 @@ public final class VectorExpressionWrite }.init(fieldObjInspector); } + private static VectorExpressionWriter genVectorExpressionWritableChar( + SettableHiveCharObjectInspector fieldObjInspector) throws HiveException { + return new VectorExpressionWriterBytes() { + private Object obj; + private Text text; + + public VectorExpressionWriter init(SettableHiveCharObjectInspector objInspector) + throws HiveException { + super.init(objInspector); + this.text = new Text(); + this.obj = initValue(null); + return this; + } + + @Override + public Object writeValue(byte[] value, int start, int length) throws HiveException { + text.set(value, start, length); + ((SettableHiveCharObjectInspector) this.objectInspector).set(this.obj, text.toString()); + return this.obj; + } + + @Override + public Object setValue(Object field, byte[] value, int start, int length) + throws HiveException { + if (null == field) { + field = initValue(null); + } + text.set(value, start, length); + ((SettableHiveCharObjectInspector) this.objectInspector).set(field, text.toString()); + return field; + } + + @Override + public Object initValue(Object ignored) { + return ((SettableHiveCharObjectInspector) this.objectInspector) + .create(new HiveChar(StringUtils.EMPTY, -1)); + } + }.init(fieldObjInspector); + } + private static VectorExpressionWriter genVectorExpressionWritableVarchar( SettableHiveVarcharObjectInspector fieldObjInspector) throws HiveException { return new VectorExpressionWriterBytes() { Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColCol.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColCol.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColCol.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColCol.java Fri Sep 5 15:59:47 2014 @@ -127,6 +127,8 @@ public class VectorUDFDateAddColCol exte break; case STRING: + case CHAR: + case VARCHAR: // Now disregard null in second pass. if ((inputColVector1.isRepeating) && (inputColVector2.isRepeating)) { // All must be selected otherwise size would be zero @@ -144,6 +146,8 @@ public class VectorUDFDateAddColCol exte } } break; + default: + throw new Error("Unsupported input type " + inputTypes[0].name()); } } @@ -235,8 +239,8 @@ public class VectorUDFDateAddColCol exte b.setMode(VectorExpressionDescriptor.Mode.PROJECTION) .setNumArguments(2) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.ANY, - VectorExpressionDescriptor.ArgumentType.LONG) + VectorExpressionDescriptor.ArgumentType.STRING_DATETIME_FAMILY, + VectorExpressionDescriptor.ArgumentType.INT_FAMILY) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN, VectorExpressionDescriptor.InputExpressionType.COLUMN); Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColScalar.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColScalar.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColScalar.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColScalar.java Fri Sep 5 15:59:47 2014 @@ -165,6 +165,8 @@ public class VectorUDFDateAddColScalar e break; case STRING: + case CHAR: + case VARCHAR: if (inputCol.noNulls) { outV.noNulls = true; if (batch.selectedInUse) { @@ -199,6 +201,8 @@ public class VectorUDFDateAddColScalar e } } break; + default: + throw new Error("Unsupported input type " + inputTypes[0].name()); } } @@ -286,8 +290,8 @@ public class VectorUDFDateAddColScalar e b.setMode(VectorExpressionDescriptor.Mode.PROJECTION) .setNumArguments(2) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.ANY, - VectorExpressionDescriptor.ArgumentType.LONG) + VectorExpressionDescriptor.ArgumentType.STRING_DATETIME_FAMILY, + VectorExpressionDescriptor.ArgumentType.INT_FAMILY) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN, VectorExpressionDescriptor.InputExpressionType.SCALAR); Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddScalarCol.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddScalarCol.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddScalarCol.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddScalarCol.java Fri Sep 5 15:59:47 2014 @@ -84,6 +84,8 @@ public class VectorUDFDateAddScalarCol e break; case STRING: + case CHAR: + case VARCHAR: try { baseDate = formatter.parse(new String(stringValue, "UTF-8")); break; @@ -101,6 +103,8 @@ public class VectorUDFDateAddScalarCol e } return; } + default: + throw new Error("Unsupported input type " + inputTypes[0].name()); } if(batch.size == 0) { @@ -213,8 +217,8 @@ public class VectorUDFDateAddScalarCol e b.setMode(VectorExpressionDescriptor.Mode.PROJECTION) .setNumArguments(2) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.ANY, - VectorExpressionDescriptor.ArgumentType.LONG) + VectorExpressionDescriptor.ArgumentType.STRING_DATETIME_FAMILY, + VectorExpressionDescriptor.ArgumentType.INT_FAMILY) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.SCALAR, VectorExpressionDescriptor.InputExpressionType.COLUMN); Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColCol.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColCol.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColCol.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColCol.java Fri Sep 5 15:59:47 2014 @@ -191,12 +191,14 @@ public class VectorUDFDateDiffColCol ext return dateVector; case STRING: + case CHAR: + case VARCHAR: BytesColumnVector bcv = (BytesColumnVector) inputColVector; copySelected(bcv, batch.selectedInUse, batch.selected, batch.size, dateVector); return dateVector; + default: + throw new Error("Unsupported input type " + colType.name()); } - - return null; } // Copy the current object contents into the output. Only copy selected entries, @@ -314,8 +316,8 @@ public class VectorUDFDateDiffColCol ext b.setMode(VectorExpressionDescriptor.Mode.PROJECTION) .setNumArguments(2) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.ANY, - VectorExpressionDescriptor.ArgumentType.ANY) + VectorExpressionDescriptor.ArgumentType.STRING_DATETIME_FAMILY, + VectorExpressionDescriptor.ArgumentType.STRING_DATETIME_FAMILY) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN, VectorExpressionDescriptor.InputExpressionType.COLUMN); Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColScalar.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColScalar.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColScalar.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColScalar.java Fri Sep 5 15:59:47 2014 @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; +import org.apache.hadoop.hive.metastore.parser.ExpressionTree.Operator; import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; @@ -90,6 +91,8 @@ public class VectorUDFDateDiffColScalar break; case STRING: + case CHAR: + case VARCHAR: try { date.setTime(formatter.parse(new String(stringValue, "UTF-8")).getTime()); baseDate = DateWritable.dateToDays(date); @@ -108,6 +111,8 @@ public class VectorUDFDateDiffColScalar } return; } + default: + throw new Error("Invalid input type #1: " + inputTypes[1].name()); } switch (inputTypes[0]) { @@ -184,6 +189,8 @@ public class VectorUDFDateDiffColScalar break; case STRING: + case CHAR: + case VARCHAR: if (inputCol.noNulls) { outV.noNulls = true; if (batch.selectedInUse) { @@ -218,6 +225,8 @@ public class VectorUDFDateDiffColScalar } } break; + default: + throw new Error("Invalid input type #0: " + inputTypes[0].name()); } } @@ -287,8 +296,8 @@ public class VectorUDFDateDiffColScalar b.setMode(VectorExpressionDescriptor.Mode.PROJECTION) .setNumArguments(2) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.ANY, - VectorExpressionDescriptor.ArgumentType.ANY) + VectorExpressionDescriptor.ArgumentType.STRING_DATETIME_FAMILY, + VectorExpressionDescriptor.ArgumentType.STRING_DATETIME_FAMILY) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN, VectorExpressionDescriptor.InputExpressionType.SCALAR); Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffScalarCol.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffScalarCol.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffScalarCol.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffScalarCol.java Fri Sep 5 15:59:47 2014 @@ -90,6 +90,8 @@ public class VectorUDFDateDiffScalarCol break; case STRING: + case CHAR: + case VARCHAR: try { date.setTime(formatter.parse(new String(stringValue, "UTF-8")).getTime()); baseDate = DateWritable.dateToDays(date); @@ -108,6 +110,8 @@ public class VectorUDFDateDiffScalarCol } return; } + default: + throw new Error("Unsupported input type " + inputTypes[0].name()); } switch (inputTypes[1]) { @@ -184,6 +188,8 @@ public class VectorUDFDateDiffScalarCol break; case STRING: + case CHAR: + case VARCHAR: if (inputCol.noNulls) { outV.noNulls = true; if (batch.selectedInUse) { @@ -218,6 +224,8 @@ public class VectorUDFDateDiffScalarCol } } break; + default: + throw new Error("Unsupported input type " + inputTypes[1].name()); } } @@ -287,8 +295,8 @@ public class VectorUDFDateDiffScalarCol b.setMode(VectorExpressionDescriptor.Mode.PROJECTION) .setNumArguments(2) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.ANY, - VectorExpressionDescriptor.ArgumentType.ANY) + VectorExpressionDescriptor.ArgumentType.STRING_DATETIME_FAMILY, + VectorExpressionDescriptor.ArgumentType.STRING_DATETIME_FAMILY) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.SCALAR, VectorExpressionDescriptor.InputExpressionType.COLUMN); Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateLong.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateLong.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateLong.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateLong.java Fri Sep 5 15:59:47 2014 @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.serde2.io.DateWritable; import java.io.UnsupportedEncodingException; @@ -49,6 +50,8 @@ public class VectorUDFDateLong extends L case TIMESTAMP: date.setTime(vector[i] / 1000000); break; + default: + throw new Error("Unsupported input type " + inputTypes[0].name()); } try { byte[] bytes = formatter.format(date).getBytes("UTF-8"); @@ -58,4 +61,16 @@ public class VectorUDFDateLong extends L outV.isNull[i] = true; } } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + VectorExpressionDescriptor.Builder b = new VectorExpressionDescriptor.Builder(); + b.setMode(VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(1) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.DATETIME_FAMILY) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN); + return b.build(); + } } Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldLong.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldLong.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldLong.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldLong.java Fri Sep 5 15:59:47 2014 @@ -179,6 +179,8 @@ public abstract class VectorUDFTimestamp } } break; + default: + throw new Error("Unsupported input type " + inputTypes[0].name()); } } @@ -218,7 +220,7 @@ public abstract class VectorUDFTimestamp b.setMode(VectorExpressionDescriptor.Mode.PROJECTION) .setNumArguments(1) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.LONG) + VectorExpressionDescriptor.ArgumentType.DATETIME_FAMILY) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN); return b.build(); Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldString.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldString.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldString.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldString.java Fri Sep 5 15:59:47 2014 @@ -181,7 +181,7 @@ public abstract class VectorUDFTimestamp b.setMode(VectorExpressionDescriptor.Mode.PROJECTION) .setNumArguments(1) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.STRING) + VectorExpressionDescriptor.ArgumentType.STRING_FAMILY) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN); return b.build(); Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFAdaptor.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFAdaptor.java?rev=1622729&r1=1622728&r2=1622729&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFAdaptor.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFAdaptor.java Fri Sep 5 15:59:47 2014 @@ -23,6 +23,7 @@ import java.sql.Timestamp; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.exec.vector.*; +import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriterFactory; @@ -30,8 +31,12 @@ import org.apache.hadoop.hive.ql.metadat import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.HiveCharWritable; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.*; +import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; import org.apache.hadoop.io.Text; /** @@ -215,6 +220,35 @@ public class VectorUDFAdaptor extends Ve t = ((WritableStringObjectInspector) outputOI).getPrimitiveWritableObject(value); } bv.setVal(i, t.getBytes(), 0, t.getLength()); + } else if (outputOI instanceof WritableHiveCharObjectInspector) { + WritableHiveCharObjectInspector writableHiveCharObjectOI = (WritableHiveCharObjectInspector) outputOI; + int maxLength = ((CharTypeInfo) writableHiveCharObjectOI.getTypeInfo()).getLength(); + BytesColumnVector bv = (BytesColumnVector) colVec; + + HiveCharWritable hiveCharWritable; + if (value instanceof HiveCharWritable) { + hiveCharWritable = ((HiveCharWritable) value); + } else { + hiveCharWritable = writableHiveCharObjectOI.getPrimitiveWritableObject(value); + } + Text t = hiveCharWritable.getTextValue(); + + // In vector mode, we stored CHAR as unpadded. + StringExpr.rightTrimAndTruncate(bv, i, t.getBytes(), 0, t.getLength(), maxLength); + } else if (outputOI instanceof WritableHiveVarcharObjectInspector) { + WritableHiveVarcharObjectInspector writableHiveVarcharObjectOI = (WritableHiveVarcharObjectInspector) outputOI; + int maxLength = ((VarcharTypeInfo) writableHiveVarcharObjectOI.getTypeInfo()).getLength(); + BytesColumnVector bv = (BytesColumnVector) colVec; + + HiveVarcharWritable hiveVarcharWritable; + if (value instanceof HiveVarcharWritable) { + hiveVarcharWritable = ((HiveVarcharWritable) value); + } else { + hiveVarcharWritable = writableHiveVarcharObjectOI.getPrimitiveWritableObject(value); + } + Text t = hiveVarcharWritable.getTextValue(); + + StringExpr.truncate(bv, i, t.getBytes(), 0, t.getLength(), maxLength); } else if (outputOI instanceof WritableIntObjectInspector) { LongColumnVector lv = (LongColumnVector) colVec; if (value instanceof Integer) {
