Author: hashutosh
Date: Thu Nov 21 02:20:49 2013
New Revision: 1544002
URL: http://svn.apache.org/r1544002
Log:
HIVE-5845 : CTAS failed on vectorized code path (Remus Rusanu via Ashutosh
Chauhan)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcSerde.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorExpressionWriters.java
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java?rev=1544002&r1=1544001&r2=1544002&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java
Thu Nov 21 02:20:49 2013
@@ -33,4 +33,6 @@ public interface VectorExpressionWriter
Object writeValue(long value) throws HiveException;
Object writeValue(double value) throws HiveException;
Object writeValue(byte[] value, int start, int length) throws HiveException;
+ Object setValue(Object row, ColumnVector column, int columnRow) throws
HiveException;
+ Object initValue(Object ost) throws HiveException;
}
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java?rev=1544002&r1=1544001&r2=1544002&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
Thu Nov 21 02:20:49 2013
@@ -20,8 +20,12 @@ package org.apache.hadoop.hive.ql.exec.v
import java.sql.Timestamp;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
@@ -35,16 +39,39 @@ import org.apache.hadoop.hive.serde2.io.
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import
org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBinaryObjectInspector;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBooleanObjectInspector;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableByteObjectInspector;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableDateObjectInspector;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableDoubleObjectInspector;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableFloatObjectInspector;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveDecimalObjectInspector;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveVarcharObjectInspector;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableIntObjectInspector;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableLongObjectInspector;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveVarcharObjectInspector;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableTimestampObjectInspector;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableShortObjectInspector;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableStringObjectInspector;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableTimestampObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo;
import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
/**
* VectorExpressionWritableFactory helper class for generating
VectorExpressionWritable objects.
@@ -67,20 +94,11 @@ public final class VectorExpressionWrite
return objectInspector;
}
- public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws
HiveException {
- objectInspector = nodeDesc.getWritableObjectInspector();
- if (null == objectInspector) {
- objectInspector = TypeInfoUtils
-
.getStandardWritableObjectInspectorFromTypeInfo(nodeDesc.getTypeInfo());
- }
- if (null == objectInspector) {
- throw new HiveException(String.format(
- "Failed to initialize VectorExpressionWriter for expr: %s",
- nodeDesc.getExprString()));
- }
+ public VectorExpressionWriter init(ObjectInspector objectInspector) throws
HiveException {
+ this.objectInspector = objectInspector;
return this;
}
-
+
/**
* The base implementation must be overridden by the Long specialization
*/
@@ -88,6 +106,13 @@ public final class VectorExpressionWrite
public Object writeValue(long value) throws HiveException {
throw new HiveException("Internal error: should not reach here");
}
+
+ /**
+ * The base implementation must be overridden by the Long specialization
+ */
+ public Object setValue(Object field, long value) throws HiveException {
+ throw new HiveException("Internal error: should not reach here");
+ }
/**
* The base implementation must be overridden by the Double specialization
@@ -98,12 +123,26 @@ public final class VectorExpressionWrite
}
/**
+ * The base implementation must be overridden by the Double specialization
+ */
+ public Object setValue(Object field, double value) throws HiveException {
+ throw new HiveException("Internal error: should not reach here");
+ }
+
+ /**
* The base implementation must be overridden by the Bytes specialization
*/
@Override
public Object writeValue(byte[] value, int start, int length) throws
HiveException {
throw new HiveException("Internal error: should not reach here");
}
+
+ /**
+ * The base implementation must be overridden by the Bytes specialization
+ */
+ public Object setValue(Object field, byte[] value, int start, int length)
throws HiveException {
+ throw new HiveException("Internal error: should not reach here");
+ }
}
/**
@@ -133,7 +172,29 @@ public final class VectorExpressionWrite
"Incorrect null/repeating: row:%d noNulls:%b isRepeating:%b
isNull[row]:%b isNull[0]:%b",
row, lcv.noNulls, lcv.isRepeating, lcv.isNull[row], lcv.isNull[0]));
}
- }
+
+ @Override
+ public Object setValue(Object field, ColumnVector column, int row) throws
HiveException {
+ LongColumnVector lcv = (LongColumnVector) column;
+ if (lcv.noNulls && !lcv.isRepeating) {
+ return setValue(field, lcv.vector[row]);
+ } else if (lcv.noNulls && lcv.isRepeating) {
+ return setValue(field, lcv.vector[0]);
+ } else if (!lcv.noNulls && !lcv.isRepeating && !lcv.isNull[row]) {
+ return setValue(field, lcv.vector[row]);
+ } else if (!lcv.noNulls && !lcv.isRepeating && lcv.isNull[row]) {
+ return null;
+ } else if (!lcv.noNulls && lcv.isRepeating && !lcv.isNull[0]) {
+ return setValue(field, lcv.vector[0]);
+ } else if (!lcv.noNulls && lcv.isRepeating && lcv.isNull[0]) {
+ return null;
+ }
+ throw new HiveException(
+ String.format(
+ "Incorrect null/repeating: row:%d noNulls:%b isRepeating:%b
isNull[row]:%b isNull[0]:%b",
+ row, lcv.noNulls, lcv.isRepeating, lcv.isNull[row], lcv.isNull[0]));
+ }
+ }
/**
* Specialized writer for DoubleColumnVector. Will throw cast exception
@@ -161,6 +222,28 @@ public final class VectorExpressionWrite
"Incorrect null/repeating: row:%d noNulls:%b isRepeating:%b
isNull[row]:%b isNull[0]:%b",
row, dcv.noNulls, dcv.isRepeating, dcv.isNull[row], dcv.isNull[0]));
}
+
+ @Override
+ public Object setValue(Object field, ColumnVector column, int row) throws
HiveException {
+ DoubleColumnVector dcv = (DoubleColumnVector) column;
+ if (dcv.noNulls && !dcv.isRepeating) {
+ return setValue(field, dcv.vector[row]);
+ } else if (dcv.noNulls && dcv.isRepeating) {
+ return setValue(field, dcv.vector[0]);
+ } else if (!dcv.noNulls && !dcv.isRepeating && !dcv.isNull[row]) {
+ return setValue(field, dcv.vector[row]);
+ } else if (!dcv.noNulls && !dcv.isRepeating && dcv.isNull[row]) {
+ return null;
+ } else if (!dcv.noNulls && dcv.isRepeating && !dcv.isNull[0]) {
+ return setValue(field, dcv.vector[0]);
+ } else if (!dcv.noNulls && dcv.isRepeating && dcv.isNull[0]) {
+ return null;
+ }
+ throw new HiveException(
+ String.format(
+ "Incorrect null/repeating: row:%d noNulls:%b isRepeating:%b
isNull[row]:%b isNull[0]:%b",
+ row, dcv.noNulls, dcv.isRepeating, dcv.isNull[row], dcv.isNull[0]));
+ }
}
/**
@@ -189,176 +272,528 @@ public final class VectorExpressionWrite
"Incorrect null/repeating: row:%d noNulls:%b isRepeating:%b
isNull[row]:%b isNull[0]:%b",
row, bcv.noNulls, bcv.isRepeating, bcv.isNull[row], bcv.isNull[0]));
}
+
+ @Override
+ public Object setValue(Object field, ColumnVector column, int row) throws
HiveException {
+ BytesColumnVector bcv = (BytesColumnVector) column;
+ if (bcv.noNulls && !bcv.isRepeating) {
+ return setValue(field, bcv.vector[row], bcv.start[row],
bcv.length[row]);
+ } else if (bcv.noNulls && bcv.isRepeating) {
+ return setValue(field, bcv.vector[0], bcv.start[0], bcv.length[0]);
+ } else if (!bcv.noNulls && !bcv.isRepeating && !bcv.isNull[row]) {
+ return setValue(field, bcv.vector[row], bcv.start[row],
bcv.length[row]);
+ } else if (!bcv.noNulls && !bcv.isRepeating && bcv.isNull[row]) {
+ return null;
+ } else if (!bcv.noNulls && bcv.isRepeating && !bcv.isNull[0]) {
+ return setValue(field, bcv.vector[0], bcv.start[0], bcv.length[0]);
+ } else if (!bcv.noNulls && bcv.isRepeating && bcv.isNull[0]) {
+ return null;
+ }
+ throw new HiveException(
+ String.format(
+ "Incorrect null/repeating: row:%d noNulls:%b isRepeating:%b
isNull[row]:%b isNull[0]:%b",
+ row, bcv.noNulls, bcv.isRepeating, bcv.isNull[row], bcv.isNull[0]));
+ }
}
- /**
- * Compiles the appropriate vector expression writer based on an expression
info (ExprNodeDesc)
- */
- public static VectorExpressionWriter
genVectorExpressionWritable(ExprNodeDesc nodeDesc)
- throws HiveException {
- String nodeType = nodeDesc.getTypeString();
- if (nodeType.equalsIgnoreCase("tinyint")) {
- return new VectorExpressionWriterLong()
- {
- private ByteWritable writable;
-
- @Override
- public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws
HiveException {
- super.init(nodeDesc);
- writable = new ByteWritable();
- return this;
- }
-
- @Override
- public Object writeValue(long value) {
- writable.set((byte) value);
- return writable;
- }
- }.init(nodeDesc);
- } else if (nodeType.equalsIgnoreCase("smallint")) {
- return new VectorExpressionWriterLong()
- {
- private ShortWritable writable;
- @Override
- public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws
HiveException {
- super.init(nodeDesc);
- writable = new ShortWritable();
- return this;
- }
-
- @Override
- public Object writeValue(long value) {
- writable.set((short) value);
- return writable;
- }
- }.init(nodeDesc);
- } else if (nodeType.equalsIgnoreCase("int")) {
- return new VectorExpressionWriterLong()
- {
- private IntWritable writable;
- @Override
- public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws
HiveException {
- super.init(nodeDesc);
- writable = new IntWritable();
- return this;
- }
-
- @Override
- public Object writeValue(long value) {
- writable.set((int) value);
- return writable;
- }
- }.init(nodeDesc);
- } else if (nodeType.equalsIgnoreCase("bigint")) {
- return new VectorExpressionWriterLong()
- {
- private LongWritable writable;
- @Override
- public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws
HiveException {
- super.init(nodeDesc);
- writable = new LongWritable();
- return this;
- }
-
- @Override
- public Object writeValue(long value) {
- writable.set(value);
- return writable;
- }
- }.init(nodeDesc);
- } else if (nodeType.equalsIgnoreCase("boolean")) {
- return new VectorExpressionWriterLong()
- {
- private BooleanWritable writable;
- @Override
- public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws
HiveException {
- super.init(nodeDesc);
- writable = new BooleanWritable();
- return this;
- }
-
- @Override
- public Object writeValue(long value) {
- writable.set(value != 0 ? true : false);
- return writable;
- }
- }.init(nodeDesc);
- } else if (nodeType.equalsIgnoreCase("timestamp")) {
- return new VectorExpressionWriterLong()
- {
- private TimestampWritable writable;
- private Timestamp timestamp;
- @Override
- public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws
HiveException {
- super.init(nodeDesc);
- writable = new TimestampWritable();
- timestamp = new Timestamp(0);
- return this;
- }
-
- @Override
- public Object writeValue(long value) {
- TimestampUtils.assignTimeInNanoSec(value, timestamp);
- writable.set(timestamp);
- return writable;
- }
- }.init(nodeDesc);
- } else if (nodeType.equalsIgnoreCase("string")) {
- return new VectorExpressionWriterBytes()
- {
- private Text writable;
- @Override
- public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws
HiveException {
- super.init(nodeDesc);
- writable = new Text();
- return this;
- }
-
- @Override
- public Object writeValue(byte[] value, int start, int length) throws
HiveException {
- writable.set(value, start, length);
- return writable;
- }
- }.init(nodeDesc);
- } else if (nodeType.equalsIgnoreCase("float")) {
- return new VectorExpressionWriterDouble()
- {
- private FloatWritable writable;
- @Override
- public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws
HiveException {
- super.init(nodeDesc);
- writable = new FloatWritable();
- return this;
- }
-
- @Override
- public Object writeValue(double value) {
- writable.set((float)value);
- return writable;
- }
- }.init(nodeDesc);
- } else if (nodeType.equalsIgnoreCase("double")) {
- return new VectorExpressionWriterDouble()
- {
- private DoubleWritable writable;
- @Override
- public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws
HiveException {
- super.init(nodeDesc);
- writable = new DoubleWritable();
- return this;
- }
-
- @Override
- public Object writeValue(double value) {
- writable.set(value);
- return writable;
- }
- }.init(nodeDesc);
- }
-
- throw new HiveException(String.format(
- "Unimplemented genVectorExpressionWritable type: %s for expression:
%s",
- nodeType, nodeDesc));
+ /**
+ * Compiles the appropriate vector expression writer based on an
expression info (ExprNodeDesc)
+ */
+ public static VectorExpressionWriter
genVectorExpressionWritable(ExprNodeDesc nodeDesc)
+ throws HiveException {
+ String nodeType = nodeDesc.getTypeString();
+ ObjectInspector objectInspector = nodeDesc.getWritableObjectInspector();
+ if (null == objectInspector) {
+ objectInspector = TypeInfoUtils
+
.getStandardWritableObjectInspectorFromTypeInfo(nodeDesc.getTypeInfo());
+ }
+ if (null == objectInspector) {
+ throw new HiveException(String.format(
+ "Failed to initialize VectorExpressionWriter for expr: %s",
+ nodeDesc.getExprString()));
+ }
+ return genVectorExpressionWritable(objectInspector);
+ }
+
+ /**
+ * Compiles the appropriate vector expression writer based on an
expression info (ExprNodeDesc)
+ */
+ public static VectorExpressionWriter genVectorExpressionWritable(
+ ObjectInspector fieldObjInspector) throws HiveException {
+
+ switch (fieldObjInspector.getCategory()) {
+ case PRIMITIVE:
+ switch (((PrimitiveObjectInspector)
fieldObjInspector).getPrimitiveCategory()) {
+ case FLOAT:
+ return genVectorExpressionWritableFloat(
+ (SettableFloatObjectInspector) fieldObjInspector);
+ case DOUBLE:
+ return genVectorExpressionWritableDouble(
+ (SettableDoubleObjectInspector) fieldObjInspector);
+ case BOOLEAN:
+ return genVectorExpressionWritableBoolean(
+ (SettableBooleanObjectInspector) fieldObjInspector);
+ case BYTE:
+ return genVectorExpressionWritableByte(
+ (SettableByteObjectInspector) fieldObjInspector);
+ case SHORT:
+ return genVectorExpressionWritableShort(
+ (SettableShortObjectInspector) fieldObjInspector);
+ case INT:
+ return genVectorExpressionWritableInt(
+ (SettableIntObjectInspector) fieldObjInspector);
+ case LONG:
+ return genVectorExpressionWritableLong(
+ (SettableLongObjectInspector) fieldObjInspector);
+ case BINARY:
+ return genVectorExpressionWritableBinary(
+ (SettableBinaryObjectInspector) fieldObjInspector);
+ case STRING:
+ return genVectorExpressionWritableString(
+ (SettableStringObjectInspector) fieldObjInspector);
+ case VARCHAR:
+ return genVectorExpressionWritableVarchar(
+ (SettableHiveVarcharObjectInspector) fieldObjInspector);
+ case TIMESTAMP:
+ return genVectorExpressionWritableTimestamp(
+ (SettableTimestampObjectInspector) fieldObjInspector);
+ case DATE:
+ return genVectorExpressionWritableDate(
+ (SettableDateObjectInspector) fieldObjInspector);
+ case DECIMAL:
+ return genVectorExpressionWritableDecimal(
+ (SettableHiveDecimalObjectInspector) fieldObjInspector);
+ default:
+ throw new IllegalArgumentException("Unknown primitive type: " +
+ ((PrimitiveObjectInspector)
fieldObjInspector).getPrimitiveCategory());
+ }
+
+ case STRUCT:
+ case UNION:
+ case MAP:
+ case LIST:
+ throw new IllegalArgumentException("Unsupported complex type: " +
+ fieldObjInspector.getCategory());
+ default:
+ throw new IllegalArgumentException("Unknown type " +
+ fieldObjInspector.getCategory());
+ }
+ }
+
+ private static VectorExpressionWriter genVectorExpressionWritableDecimal(
+ SettableHiveDecimalObjectInspector fieldObjInspector) throws
HiveException {
+
+ // We should never reach this, the compile validation should guard us
+ throw new HiveException("DECIMAL primitive type not supported in
vectorization.");
+ }
+
+ private static VectorExpressionWriter genVectorExpressionWritableDate(
+ SettableDateObjectInspector fieldObjInspector) throws HiveException {
+ // We should never reach this, the compile validation should guard us
+ throw new HiveException("DATE primitive type not supported in
vectorization.");
+ }
+
+ private static VectorExpressionWriter genVectorExpressionWritableTimestamp(
+ SettableTimestampObjectInspector fieldObjInspector) throws
HiveException {
+ return new VectorExpressionWriterLong() {
+ private Object obj;
+ private Timestamp ts;
+
+ public VectorExpressionWriter init(SettableTimestampObjectInspector
objInspector)
+ throws HiveException {
+ super.init(objInspector);
+ ts = new Timestamp(0);
+ obj = initValue(null);
+ return this;
+ }
+
+ @Override
+ public Object writeValue(long value) {
+ TimestampUtils.assignTimeInNanoSec(value, ts);
+ ((SettableTimestampObjectInspector) this.objectInspector).set(obj, ts);
+ return obj;
+ }
+
+ @Override
+ public Object setValue(Object field, long value) {
+ if (null == field) {
+ field = initValue(null);
+ }
+ TimestampUtils.assignTimeInNanoSec(value, ts);
+ ((SettableTimestampObjectInspector) this.objectInspector).set(field,
ts);
+ return field;
+ }
+
+ @Override
+ public Object initValue(Object ignored) {
+ return ((SettableTimestampObjectInspector)
this.objectInspector).create(new Timestamp(0));
+ }
+ }.init(fieldObjInspector);
+ }
+
+ private static VectorExpressionWriter genVectorExpressionWritableVarchar(
+ SettableHiveVarcharObjectInspector fieldObjInspector) throws
HiveException {
+ return new VectorExpressionWriterBytes() {
+ private Object obj;
+ private Text text;
+
+ public VectorExpressionWriter init(SettableHiveVarcharObjectInspector
objInspector)
+ throws HiveException {
+ super.init(objInspector);
+ this.text = new Text();
+ this.obj = initValue(null);
+ return this;
+ }
+
+ @Override
+ public Object writeValue(byte[] value, int start, int length) throws
HiveException {
+ text.set(value, start, length);
+ ((SettableHiveVarcharObjectInspector)
this.objectInspector).set(this.obj, text.toString());
+ return this.obj;
+ }
+
+ @Override
+ public Object setValue(Object field, byte[] value, int start, int
length)
+ throws HiveException {
+ if (null == field) {
+ field = initValue(null);
+ }
+ text.set(value, start, length);
+ ((SettableHiveVarcharObjectInspector) this.objectInspector).set(field,
text.toString());
+ return field;
+ }
+
+ @Override
+ public Object initValue(Object ignored) {
+ return ((SettableHiveVarcharObjectInspector) this.objectInspector)
+ .create(new HiveVarchar(StringUtils.EMPTY, -1));
+ }
+ }.init(fieldObjInspector);
+ }
+
+ private static VectorExpressionWriter genVectorExpressionWritableString(
+ SettableStringObjectInspector fieldObjInspector) throws HiveException {
+ return new VectorExpressionWriterBytes() {
+ private Object obj;
+ private Text text;
+
+ public VectorExpressionWriter init(SettableStringObjectInspector
objInspector)
+ throws HiveException {
+ super.init(objInspector);
+ this.text = new Text();
+ this.obj = initValue(null);
+ return this;
+ }
+
+ @Override
+ public Object writeValue(byte[] value, int start, int length) throws
HiveException {
+ this.text.set(value, start, length);
+ ((SettableStringObjectInspector) this.objectInspector).set(this.obj,
this.text.toString());
+ return this.obj;
+ }
+
+ @Override
+ public Object setValue(Object field, byte[] value, int start, int
length)
+ throws HiveException {
+ if (null == field) {
+ field = initValue(null);
+ }
+ this.text.set(value, start, length);
+ ((SettableStringObjectInspector) this.objectInspector).set(field,
this.text.toString());
+ return field;
+ }
+
+ @Override
+ public Object initValue(Object ignored) {
+ return ((SettableStringObjectInspector)
this.objectInspector).create(StringUtils.EMPTY);
+ }
+ }.init(fieldObjInspector);
+ }
+
+ private static VectorExpressionWriter genVectorExpressionWritableBinary(
+ SettableBinaryObjectInspector fieldObjInspector) throws HiveException {
+ return new VectorExpressionWriterBytes() {
+ private Object obj;
+ private byte[] bytes;
+
+ public VectorExpressionWriter init(SettableBinaryObjectInspector
objInspector)
+ throws HiveException {
+ super.init(objInspector);
+ this.bytes = ArrayUtils.EMPTY_BYTE_ARRAY;
+ this.obj = initValue(null);
+ return this;
+ }
+
+ @Override
+ public Object writeValue(byte[] value, int start, int length) throws
HiveException {
+ bytes = Arrays.copyOfRange(value, start, start + length);
+ ((SettableBinaryObjectInspector) this.objectInspector).set(this.obj,
bytes);
+ return this.obj;
+ }
+
+ @Override
+ public Object setValue(Object field, byte[] value, int start, int
length) throws HiveException {
+ if (null == field) {
+ field = initValue(null);
+ }
+ bytes = Arrays.copyOfRange(value, start, start + length);
+ ((SettableBinaryObjectInspector) this.objectInspector).set(field,
bytes);
+ return field;
+ }
+
+ @Override
+ public Object initValue(Object ignored) {
+ return ((SettableBinaryObjectInspector) this.objectInspector)
+ .create(ArrayUtils.EMPTY_BYTE_ARRAY);
+ }
+ }.init(fieldObjInspector);
+ }
+
+ private static VectorExpressionWriter genVectorExpressionWritableLong(
+ SettableLongObjectInspector fieldObjInspector) throws HiveException {
+ return new VectorExpressionWriterLong() {
+ private Object obj;
+
+ public VectorExpressionWriter init(SettableLongObjectInspector
objInspector)
+ throws HiveException {
+ super.init(objInspector);
+ this.obj = initValue(null);
+ return this;
+ }
+
+ @Override
+ public Object writeValue(long value) throws HiveException {
+ ((SettableLongObjectInspector) this.objectInspector).set(this.obj,
value);
+ return this.obj;
+ }
+
+ @Override
+ public Object setValue(Object field, long value) throws HiveException {
+ if (null == field) {
+ field = initValue(null);
+ }
+ ((SettableLongObjectInspector) this.objectInspector).set(field, value);
+ return field;
+ }
+
+ @Override
+ public Object initValue(Object ignored) {
+ return ((SettableLongObjectInspector) this.objectInspector)
+ .create(0L);
+ }
+ }.init(fieldObjInspector);
+ }
+
+ private static VectorExpressionWriter genVectorExpressionWritableInt(
+ SettableIntObjectInspector fieldObjInspector) throws HiveException {
+ return new VectorExpressionWriterLong() {
+ private Object obj;
+
+ public VectorExpressionWriter init(SettableIntObjectInspector
objInspector)
+ throws HiveException {
+ super.init(objInspector);
+ this.obj = initValue(null);
+ return this;
+ }
+
+ @Override
+ public Object writeValue(long value) throws HiveException {
+ ((SettableIntObjectInspector) this.objectInspector).set(this.obj,
(int) value);
+ return this.obj;
+ }
+
+ @Override
+ public Object setValue(Object field, long value) throws HiveException {
+ if (null == field) {
+ field = initValue(null);
+ }
+ ((SettableIntObjectInspector) this.objectInspector).set(field, (int)
value);
+ return field;
+ }
+
+ @Override
+ public Object initValue(Object ignored) {
+ return ((SettableIntObjectInspector) this.objectInspector)
+ .create(0);
+ }
+ }.init(fieldObjInspector);
+ }
+
+ private static VectorExpressionWriter genVectorExpressionWritableShort(
+ SettableShortObjectInspector fieldObjInspector) throws HiveException {
+ return new VectorExpressionWriterLong() {
+ private Object obj;
+
+ public VectorExpressionWriter init(SettableShortObjectInspector
objInspector)
+ throws HiveException {
+ super.init(objInspector);
+ this.obj = initValue(null);
+ return this;
+ }
+
+ @Override
+ public Object writeValue(long value) throws HiveException {
+ ((SettableShortObjectInspector) this.objectInspector).set(this.obj,
(short) value);
+ return this.obj;
+ }
+
+ @Override
+ public Object setValue(Object field, long value) throws HiveException {
+ if (null == field) {
+ field = initValue(null);
+ }
+ ((SettableShortObjectInspector) this.objectInspector).set(field,
(short) value);
+ return field;
+ }
+
+ @Override
+ public Object initValue(Object ignored) {
+ return ((SettableShortObjectInspector) this.objectInspector)
+ .create((short) 0);
+ }
+ }.init(fieldObjInspector);
+ }
+
+ private static VectorExpressionWriter genVectorExpressionWritableByte(
+ SettableByteObjectInspector fieldObjInspector) throws HiveException {
+ return new VectorExpressionWriterLong() {
+ private Object obj;
+
+ public VectorExpressionWriter init(SettableByteObjectInspector
objInspector)
+ throws HiveException {
+ super.init(objInspector);
+ this.obj = initValue(null);
+ return this;
+ }
+
+ @Override
+ public Object writeValue(long value) throws HiveException {
+ ((SettableByteObjectInspector) this.objectInspector).set(this.obj,
(byte) value);
+ return this.obj;
+ }
+
+ @Override
+ public Object setValue(Object field, long value) throws HiveException {
+ if (null == field) {
+ field = initValue(null);
+ }
+ ((SettableByteObjectInspector) this.objectInspector).set(field, (byte)
value);
+ return field;
+ }
+
+ @Override
+ public Object initValue(Object ignored) {
+ return ((SettableByteObjectInspector) this.objectInspector)
+ .create((byte) 0);
+ }
+ }.init(fieldObjInspector);
+ }
+
+ private static VectorExpressionWriter genVectorExpressionWritableBoolean(
+ SettableBooleanObjectInspector fieldObjInspector) throws HiveException {
+ return new VectorExpressionWriterLong() {
+ private Object obj;
+
+ public VectorExpressionWriter init(SettableBooleanObjectInspector
objInspector)
+ throws HiveException {
+ super.init(objInspector);
+ this.obj = initValue(null);
+ return this;
+ }
+
+ @Override
+ public Object writeValue(long value) throws HiveException {
+ ((SettableBooleanObjectInspector) this.objectInspector).set(this.obj,
+ value == 0 ? false : true);
+ return this.obj;
+ }
+
+ @Override
+ public Object setValue(Object field, long value) throws HiveException {
+ if (null == field) {
+ field = initValue(null);
+ }
+ ((SettableBooleanObjectInspector) this.objectInspector).set(field,
+ value == 0 ? false : true);
+ return field;
+ }
+
+ @Override
+ public Object initValue(Object ignored) {
+ return ((SettableBooleanObjectInspector) this.objectInspector)
+ .create(false);
+ }
+ }.init(fieldObjInspector);
+ }
+
+ private static VectorExpressionWriter genVectorExpressionWritableDouble(
+ SettableDoubleObjectInspector fieldObjInspector) throws HiveException {
+ return new VectorExpressionWriterDouble() {
+ private Object obj;
+
+ public VectorExpressionWriter init(SettableDoubleObjectInspector
objInspector)
+ throws HiveException {
+ super.init(objInspector);
+ this.obj = initValue(null);
+ return this;
+ }
+
+ @Override
+ public Object writeValue(double value) throws HiveException {
+ ((SettableDoubleObjectInspector) this.objectInspector).set(this.obj,
value);
+ return this.obj;
+ }
+
+ @Override
+ public Object setValue(Object field, double value) throws HiveException {
+ if (null == field) {
+ field = initValue(null);
+ }
+ ((SettableDoubleObjectInspector) this.objectInspector).set(field,
value);
+ return field;
+ }
+
+ @Override
+ public Object initValue(Object ignored) {
+ return ((SettableDoubleObjectInspector) this.objectInspector)
+ .create(0f);
+ }
+ }.init(fieldObjInspector);
+ }
+
+ private static VectorExpressionWriter genVectorExpressionWritableFloat(
+ SettableFloatObjectInspector fieldObjInspector) throws HiveException {
+ return new VectorExpressionWriterDouble() {
+ private Object obj;
+
+ public VectorExpressionWriter init(SettableFloatObjectInspector
objInspector)
+ throws HiveException {
+ super.init(objInspector);
+ this.obj = initValue(null);
+ return this;
+ }
+
+ @Override
+ public Object writeValue(double value) throws HiveException {
+ ((SettableFloatObjectInspector) this.objectInspector).set(this.obj,
(float) value);
+ return this.obj;
+ }
+
+ @Override
+ public Object setValue(Object field, double value) throws HiveException {
+ if (null == field) {
+ field = initValue(null);
+ }
+ ((SettableFloatObjectInspector) this.objectInspector).set(field,
(float) value);
+ return field;
+ }
+
+ @Override
+ public Object initValue(Object ignored) {
+ return ((SettableFloatObjectInspector) this.objectInspector)
+ .create(0f);
+ }
+ }.init(fieldObjInspector);
}
/**
@@ -432,25 +867,87 @@ public final class VectorExpressionWrite
*/
public static VectorExpressionWriter[]
getExpressionWriters(StructObjectInspector objInspector)
throws HiveException {
- List<ExprNodeDesc> outputNodeDescs = new ArrayList<ExprNodeDesc>();
- for (StructField fieldRef : (objInspector)
- .getAllStructFieldRefs()) {
- String typeName = fieldRef.getFieldObjectInspector().getTypeName();
- TypeInfo ti = TypeInfoFactory.getPrimitiveTypeInfo(typeName);
- outputNodeDescs.add(new ExprNodeDesc(ti) {
- private static final long serialVersionUID = 1L;
-
- @Override
- public ExprNodeDesc clone() { /* Not needed */
- return null;
- }
-
- @Override
- public boolean isSame(Object o) { /* Not needed */
- return false;
- }
- });
+
+ if (objInspector.isSettable()) {
+ return getSettableExpressionWriters((SettableStructObjectInspector)
objInspector);
+ }
+
+ List<? extends StructField> allFieldRefs =
objInspector.getAllStructFieldRefs();
+
+ VectorExpressionWriter[] expressionWriters = new
VectorExpressionWriter[allFieldRefs.size()];
+
+ for(int i=0; i<expressionWriters.length; ++i) {
+ expressionWriters[i] =
genVectorExpressionWritable(allFieldRefs.get(i).getFieldObjectInspector());
+ }
+
+ return expressionWriters;
+ }
+
+ public static VectorExpressionWriter[] getSettableExpressionWriters(
+ SettableStructObjectInspector objInspector) throws HiveException {
+ List<? extends StructField> fieldsRef =
objInspector.getAllStructFieldRefs();
+ VectorExpressionWriter[] writers = new
VectorExpressionWriter[fieldsRef.size()];
+ for(int i=0; i<writers.length; ++i) {
+ StructField fieldRef = fieldsRef.get(i);
+ VectorExpressionWriter baseWriter = genVectorExpressionWritable(
+ fieldRef.getFieldObjectInspector());
+ writers[i] = genVectorExpressionWritable(objInspector, fieldRef,
baseWriter);
+ }
+ return writers;
+
+ }
+
+ /**
+ * VectorExpressionWriterSetter helper for vector expression writers that
use
+ * settable ObjectInspector fields to assign the values.
+ * This is used by the OrcStruct serialization (eg. CREATE TABLE ... AS ...)
+ */
+ private static class VectorExpressionWriterSetter extends
VectorExpressionWriterBase {
+ private SettableStructObjectInspector settableObjInspector;
+ private StructField fieldRef;
+ private VectorExpressionWriter baseWriter;
+
+ public VectorExpressionWriterSetter init(
+ SettableStructObjectInspector objInspector,
+ StructField fieldRef,
+ VectorExpressionWriter baseWriter) {
+ this.fieldRef = fieldRef;
+ this.settableObjInspector = objInspector;
+ this.objectInspector = fieldRef.getFieldObjectInspector();
+ this.baseWriter = baseWriter;
+ return this;
+ }
+
+ @Override
+ public Object writeValue(ColumnVector column, int row)
+ throws HiveException {
+ throw new HiveException("Should never reach here");
}
- return VectorExpressionWriterFactory.getExpressionWriters(outputNodeDescs);
+
+ @Override
+ public Object setValue(Object row, ColumnVector column, int columnRow)
+ throws HiveException {
+
+ // NULLs are handled by each individual base writer setter
+ // We could handle NULLs centrally here but that would result in
spurious allocs
+
+ Object fieldValue = this.settableObjInspector.getStructFieldData(row,
fieldRef);
+ fieldValue = baseWriter.setValue(fieldValue, column, columnRow);
+ return this.settableObjInspector.setStructFieldData(row, fieldRef,
fieldValue);
+ }
+
+ @Override
+ public Object initValue(Object struct) throws HiveException {
+ Object initValue = this.baseWriter.initValue(null);
+ this.settableObjInspector.setStructFieldData(struct, fieldRef,
initValue);
+ return struct;
+ }
+ }
+
+ private static VectorExpressionWriter genVectorExpressionWritable(
+ SettableStructObjectInspector objInspector,
+ StructField fieldRef,
+ VectorExpressionWriter baseWriter) throws HiveException {
+ return new VectorExpressionWriterSetter().init(objInspector, fieldRef,
baseWriter);
}
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java?rev=1544002&r1=1544001&r2=1544002&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java Thu
Nov 21 02:20:49 2013
@@ -143,9 +143,9 @@ public class OrcSerde implements SerDe,
public Writable serializeVector(VectorizedRowBatch vrg, ObjectInspector
objInspector)
throws SerDeException {
if (vos == null) {
- vos = new VectorizedOrcSerde(objInspector);
+ vos = new VectorizedOrcSerde(getObjectInspector());
}
- return vos.serialize(vrg, objInspector);
+ return vos.serialize(vrg, getObjectInspector());
}
@Override
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcSerde.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcSerde.java?rev=1544002&r1=1544001&r2=1544002&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcSerde.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcSerde.java
Thu Nov 21 02:20:49 2013
@@ -69,13 +69,11 @@ public class VectorizedOrcSerde extends
}
for (int p = 0; p < batch.projectionSize; p++) {
int k = batch.projectedColumns[p];
- Writable w;
if (batch.cols[k].isRepeating) {
- w = (Writable) valueWriters[p].writeValue(batch.cols[k], 0);
+ valueWriters[p].setValue(ost, batch.cols[k], 0);
} else {
- w = (Writable) valueWriters[p].writeValue(batch.cols[k], index);
+ valueWriters[p].setValue(ost, batch.cols[k], index);
}
- ost.setFieldValue(k, w);
}
OrcSerdeRow row = (OrcSerdeRow) orcRowArray[i];
row.realRow = ost;
Modified:
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorExpressionWriters.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorExpressionWriters.java?rev=1544002&r1=1544001&r2=1544002&view=diff
==============================================================================
---
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorExpressionWriters.java
(original)
+++
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorExpressionWriters.java
Thu Nov 21 02:20:49 2013
@@ -20,10 +20,12 @@ package org.apache.hadoop.hive.ql.exec.v
import java.sql.Timestamp;
+import java.util.ArrayList;
import java.util.Random;
import junit.framework.Assert;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
@@ -34,11 +36,18 @@ import org.apache.hadoop.hive.ql.plan.Ex
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import
org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.io.BooleanWritable;
+import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
@@ -73,6 +82,11 @@ public class TestVectorExpressionWriters
private Writable getWritableValue(TypeInfo ti, byte[] value) {
if (ti.equals(TypeInfoFactory.stringTypeInfo)) {
return new Text(value);
+ } else if (ti.equals(TypeInfoFactory.varcharTypeInfo)) {
+ return new HiveVarcharWritable(
+ new HiveVarchar(new Text(value).toString(), -1));
+ } else if (ti.equals(TypeInfoFactory.binaryTypeInfo)) {
+ return new BytesWritable(value);
}
return null;
}
@@ -112,6 +126,25 @@ public class TestVectorExpressionWriters
}
}
}
+
+ private void testSetterDouble(TypeInfo type) throws HiveException {
+ DoubleColumnVector dcv =
VectorizedRowGroupGenUtil.generateDoubleColumnVector(true, false,
+ this.vectorSize, new Random(10));
+ dcv.isNull[2] = true;
+ Object[] values = new Object[this.vectorSize];
+
+ VectorExpressionWriter vew = getWriter(type);
+ for (int i = 0; i < vectorSize; i++) {
+ values[i] = vew.initValue(null);
+ values[i] = vew.setValue(values[i], dcv, i);
+ if (values[i] != null) {
+ Writable expected = getWritableValue(type, dcv.vector[i]);
+ Assert.assertEquals(expected, values[i]);
+ } else {
+ Assert.assertTrue(dcv.isNull[i]);
+ }
+ }
+ }
private void testWriterLong(TypeInfo type) throws HiveException {
LongColumnVector lcv =
VectorizedRowGroupGenUtil.generateLongColumnVector(true, false,
@@ -135,8 +168,89 @@ public class TestVectorExpressionWriters
}
}
}
+
+ private void testSetterLong(TypeInfo type) throws HiveException {
+ LongColumnVector lcv =
VectorizedRowGroupGenUtil.generateLongColumnVector(true, false,
+ vectorSize, new Random(10));
+ lcv.isNull[3] = true;
- private void testWriterBytes(TypeInfo type) throws HiveException {
+ Object[] values = new Object[this.vectorSize];
+
+ VectorExpressionWriter vew = getWriter(type);
+ for (int i = 0; i < vectorSize; i++) {
+ values[i] = vew.initValue(null);
+ values[i] = vew.setValue(values[i], lcv, i);
+ if (values[i] != null) {
+ Writable expected = getWritableValue(type, lcv.vector[i]);
+ if (expected instanceof TimestampWritable) {
+ TimestampWritable t1 = (TimestampWritable) expected;
+ TimestampWritable t2 = (TimestampWritable) values[i];
+ Assert.assertTrue(t1.getNanos() == t2.getNanos());
+ Assert.assertTrue(t1.getSeconds() == t2.getSeconds());
+ continue;
+ }
+ Assert.assertEquals(expected, values[i]);
+ } else {
+ Assert.assertTrue(lcv.isNull[i]);
+ }
+ }
+ }
+
+ private StructObjectInspector genStructOI() {
+ ArrayList<String> fieldNames1 = new ArrayList<String>();
+ fieldNames1.add("theInt");
+ fieldNames1.add("theBool");
+ ArrayList<ObjectInspector> fieldObjectInspectors1 = new
ArrayList<ObjectInspector>();
+ fieldObjectInspectors1
+ .add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
+ fieldObjectInspectors1
+ .add(PrimitiveObjectInspectorFactory.writableBooleanObjectInspector);
+ return ObjectInspectorFactory
+ .getStandardStructObjectInspector(fieldNames1, fieldObjectInspectors1);
+ }
+
+ private void testStructLong(TypeInfo type) throws HiveException {
+ LongColumnVector icv =
VectorizedRowGroupGenUtil.generateLongColumnVector(true, false,
+ vectorSize, new Random(10));
+ icv.isNull[3] = true;
+
+ LongColumnVector bcv =
VectorizedRowGroupGenUtil.generateLongColumnVector(true, false,
+ vectorSize, new Random(10));
+ bcv.isNull[2] = true;
+
+ ArrayList<Object>[] values = (ArrayList<Object>[]) new
ArrayList[this.vectorSize];
+
+ StructObjectInspector soi = genStructOI();
+
+ VectorExpressionWriter[] vew =
VectorExpressionWriterFactory.getExpressionWriters(soi);
+
+ for (int i = 0; i < vectorSize; i++) {
+ values[i] = new ArrayList<Object>(2);
+ values[i].add(null);
+ values[i].add(null);
+
+ vew[0].setValue(values[i], icv, i);
+ vew[1].setValue(values[i], bcv, i);
+
+ Object theInt = values[i].get(0);
+ if (theInt == null) {
+ Assert.assertTrue(icv.isNull[i]);
+ } else {
+ IntWritable w = (IntWritable) theInt;
+ Assert.assertEquals((int) icv.vector[i], w.get());
+ }
+
+ Object theBool = values[i].get(1);
+ if (theBool == null) {
+ Assert.assertTrue(bcv.isNull[i]);
+ } else {
+ BooleanWritable w = (BooleanWritable) theBool;
+ Assert.assertEquals(bcv.vector[i] == 0 ? false : true, w.get());
+ }
+ }
+ }
+
+ private void testWriterText(TypeInfo type) throws HiveException {
Text t1 = new Text("alpha");
Text t2 = new Text("beta");
BytesColumnVector bcv = new BytesColumnVector(vectorSize);
@@ -160,36 +274,100 @@ public class TestVectorExpressionWriters
}
}
}
-
+
+ private void testSetterText(TypeInfo type) throws HiveException {
+ Text t1 = new Text("alpha");
+ Text t2 = new Text("beta");
+ BytesColumnVector bcv = new BytesColumnVector(vectorSize);
+ bcv.noNulls = false;
+ bcv.initBuffer();
+ bcv.setVal(0, t1.getBytes(), 0, t1.getLength());
+ bcv.isNull[1] = true;
+ bcv.setVal(2, t2.getBytes(), 0, t2.getLength());
+ bcv.isNull[3] = true;
+ bcv.setVal(4, t1.getBytes(), 0, t1.getLength());
+
+ Object[] values = new Object[this.vectorSize];
+ VectorExpressionWriter vew = getWriter(type);
+ for (int i = 0; i < vectorSize; i++) {
+ values[i] = vew.initValue(null);
+ Writable w = (Writable) vew.setValue(values[i], bcv, i);
+ if (w != null) {
+ byte [] val = new byte[bcv.length[i]];
+ System.arraycopy(bcv.vector[i], bcv.start[i], val, 0, bcv.length[i]);
+ Writable expected = getWritableValue(type, val);
+ Assert.assertEquals(expected, w);
+ } else {
+ Assert.assertTrue(bcv.isNull[i]);
+ }
+ }
+ }
+
@Test
public void testVectorExpressionWriterDouble() throws HiveException {
testWriterDouble(TypeInfoFactory.doubleTypeInfo);
}
@Test
+ public void testVectorExpressionSetterDouble() throws HiveException {
+ testSetterDouble(TypeInfoFactory.doubleTypeInfo);
+ }
+
+ @Test
public void testVectorExpressionWriterFloat() throws HiveException {
testWriterDouble(TypeInfoFactory.floatTypeInfo);
}
@Test
+ public void testVectorExpressionSetterFloat() throws HiveException {
+ testSetterDouble(TypeInfoFactory.floatTypeInfo);
+ }
+
+ @Test
public void testVectorExpressionWriterLong() throws HiveException {
testWriterLong(TypeInfoFactory.longTypeInfo);
}
-
+
+ @Test
+ public void testVectorExpressionSetterLong() throws HiveException {
+ testSetterLong(TypeInfoFactory.longTypeInfo);
+ }
+
+ @Test
+ public void testVectorExpressionStructLong() throws HiveException {
+ testStructLong(TypeInfoFactory.longTypeInfo);
+ }
+
@Test
public void testVectorExpressionWriterInt() throws HiveException {
testWriterLong(TypeInfoFactory.intTypeInfo);
}
@Test
+ public void testVectorExpressionSetterInt() throws HiveException {
+ testSetterLong(TypeInfoFactory.intTypeInfo);
+ }
+
+ @Test
public void testVectorExpressionWriterShort() throws HiveException {
testWriterLong(TypeInfoFactory.shortTypeInfo);
}
@Test
+ public void testVectorExpressionSetterShort() throws HiveException {
+ testSetterLong(TypeInfoFactory.shortTypeInfo);
+ }
+
+
+ @Test
public void testVectorExpressionWriterBoolean() throws HiveException {
testWriterLong(TypeInfoFactory.booleanTypeInfo);
}
+
+ @Test
+ public void testVectorExpressionSetterBoolean() throws HiveException {
+ testSetterLong(TypeInfoFactory.booleanTypeInfo);
+ }
@Test
public void testVectorExpressionWriterTimestamp() throws HiveException {
@@ -197,12 +375,47 @@ public class TestVectorExpressionWriters
}
@Test
- public void testVectorExpressionWriterBye() throws HiveException {
+ public void testVectorExpressionSetterTimestamp() throws HiveException {
+ testSetterLong(TypeInfoFactory.timestampTypeInfo);
+ }
+
+ @Test
+ public void testVectorExpressionWriterByte() throws HiveException {
testWriterLong(TypeInfoFactory.byteTypeInfo);
}
+
+ @Test
+ public void testVectorExpressionSetterByte() throws HiveException {
+ testSetterLong(TypeInfoFactory.byteTypeInfo);
+ }
@Test
- public void testVectorExpressionWriterBytes() throws HiveException {
- testWriterBytes(TypeInfoFactory.stringTypeInfo);
+ public void testVectorExpressionWriterString() throws HiveException {
+ testWriterText(TypeInfoFactory.stringTypeInfo);
+ }
+
+ @Test
+ public void testVectorExpressionSetterString() throws HiveException {
+ testSetterText(TypeInfoFactory.stringTypeInfo);
+ }
+
+ @Test
+ public void testVectorExpressionWriterVarchar() throws HiveException {
+ testWriterText(TypeInfoFactory.varcharTypeInfo);
+ }
+
+ @Test
+ public void testVectorExpressionSetterVarchar() throws HiveException {
+ testSetterText(TypeInfoFactory.varcharTypeInfo);
+ }
+
+ @Test
+ public void testVectorExpressionWriterBinary() throws HiveException {
+ testWriterText(TypeInfoFactory.binaryTypeInfo);
+ }
+
+ @Test
+ public void testVectorExpressionSetterBinary() throws HiveException {
+ testSetterText(TypeInfoFactory.binaryTypeInfo);
}
}
