Modified: hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java?rev=1501145&r1=1501144&r2=1501145&view=diff ============================================================================== --- hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java (original) +++ hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java Tue Jul 9 09:07:35 2013 @@ -35,12 +35,12 @@ import org.apache.hadoop.hive.serde2.io. import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions; import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveWritableObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; @@ -516,17 +516,29 @@ public final class ObjectInspectorUtils ObjectInspector keyOI = mapOI.getMapKeyObjectInspector(); ObjectInspector valueOI = mapOI.getMapValueObjectInspector(); Map<?, ?> map = mapOI.getMap(o); - for (Map.Entry entry : map.entrySet()) { + for (Map.Entry<?,?> entry : map.entrySet()) { r += hashCode(entry.getKey(), keyOI) ^ hashCode(entry.getValue(), valueOI); } return r; } case STRUCT: + int r = 0; + StructObjectInspector structOI = (StructObjectInspector)objIns; + List<? extends StructField> fields = structOI.getAllStructFieldRefs(); + for (StructField field : fields) { + r = 31 * r + hashCode(structOI.getStructFieldData(o, field), + field.getFieldObjectInspector()); + } + return r; + case UNION: + UnionObjectInspector uOI = (UnionObjectInspector)objIns; + byte tag = uOI.getTag(o); + return hashCode(uOI.getField(o), uOI.getObjectInspectors().get(tag)); + default: - throw new RuntimeException( - "Hash code on complex types not supported yet."); + throw new RuntimeException("Unknown type: "+ objIns.getTypeName()); } }
Modified: hive/branches/vectorization/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java?rev=1501145&r1=1501144&r2=1501145&view=diff ============================================================================== --- hive/branches/vectorization/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java (original) +++ hive/branches/vectorization/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java Tue Jul 9 09:07:35 2013 @@ -19,13 +19,21 @@ package org.apache.hadoop.hive.serde2.la import java.util.ArrayList; import java.util.Arrays; +import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; import junit.framework.TestCase; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.io.IntWritable; @@ -37,6 +45,10 @@ import org.apache.hadoop.io.Text; */ public class TestLazyArrayMapStruct extends TestCase { + // nesting level limits + static final int EXTENDED_LEVEL_THRESHOLD = 24; + static final int DEFAULT_LEVEL_THRESHOLD = 8; + /** * Test the LazyArray class. */ @@ -325,4 +337,287 @@ public class TestLazyArrayMapStruct exte throw e; } } + + /** + * Test the LazyArray class with multiple levels of nesting + */ + public void testLazyArrayNested() throws Throwable { + for(int i = 2; i < EXTENDED_LEVEL_THRESHOLD; i++ ){ + testNestedinArrayAtLevelExtended(i, ObjectInspector.Category.LIST); + } + } + + /** + * Test the LazyArray class with multiple levels of nesting + */ + public void testLazyArrayNestedExceedLimit() throws Throwable { + checkExtendedLimitExceeded(EXTENDED_LEVEL_THRESHOLD, ObjectInspector.Category.LIST); + } + + private void checkExtendedLimitExceeded(int maxLevel, Category type) { + boolean foundException = false; + try { + testNestedinArrayAtLevelExtended(maxLevel, type); + }catch (SerDeException serdeEx){ + foundException = true; + } + assertTrue("Got exception for exceeding nesting limit" , foundException); + } + + /** + * Test the LazyArray class with multiple levels of nesting, when nesting + * levels are not extended + */ + public void testLazyArrayNestedExceedLimitNotExtended() throws Throwable { + checkNotExtendedLimitExceeded(DEFAULT_LEVEL_THRESHOLD, + ObjectInspector.Category.LIST); + } + + /** + * Test the LazyMap class with multiple levels of nesting, when nesting + * levels are not extended + */ + public void testLazyMapNestedExceedLimitNotExtended() throws Throwable { + checkNotExtendedLimitExceeded(DEFAULT_LEVEL_THRESHOLD-1, + ObjectInspector.Category.MAP); + } + + /** + * Test the LazyMap class with multiple levels of nesting, when nesting + * levels are not extended + */ + public void testLazyStructNestedExceedLimitNotExtended() throws Throwable { + checkNotExtendedLimitExceeded(DEFAULT_LEVEL_THRESHOLD, + ObjectInspector.Category.STRUCT); + } + + /** + * Test the LazyMap class with multiple levels of nesting, when nesting + * levels are not extended + */ + public void testLazyUnionNestedExceedLimitNotExtended() throws Throwable { + checkNotExtendedLimitExceeded(DEFAULT_LEVEL_THRESHOLD, + ObjectInspector.Category.UNION); + } + + private void checkNotExtendedLimitExceeded(int maxLevel, Category type) { + boolean foundException = false; + try { + testNestedinArrayAtLevel(maxLevel, type, new Properties()); + }catch (SerDeException serdeEx){ + foundException = true; + } + assertTrue("Expected exception for exceeding nesting limit" , foundException); + } + + /** + * Test the LazyMap class with multiple levels of nesting + */ + public void testLazyMapNested() throws Throwable { + //map max nesting level is one less because it uses an additional separator + for(int i = 2; i < EXTENDED_LEVEL_THRESHOLD - 1; i++ ){ + testNestedinArrayAtLevelExtended(i, ObjectInspector.Category.MAP); + } + } + + /** + * Test the LazyMap class with multiple levels of nesting + */ + public void testLazyMapNestedExceedLimit() throws Throwable { + //map max nesting level is one less because it uses an additional separator + checkExtendedLimitExceeded(EXTENDED_LEVEL_THRESHOLD - 1, ObjectInspector.Category.MAP); + } + + /** + * Test the LazyUnion class with multiple levels of nesting + */ + public void testLazyUnionNested() throws Throwable { + for(int i = 2; i < EXTENDED_LEVEL_THRESHOLD; i++ ){ + testNestedinArrayAtLevelExtended(i, ObjectInspector.Category.UNION); + } + } + + /** + * Test the LazyUnion class with multiple levels of nesting + */ + public void testLazyUnionNestedExceedLimit() throws Throwable { + checkExtendedLimitExceeded(EXTENDED_LEVEL_THRESHOLD, ObjectInspector.Category.UNION); + } + + /** + * Test the LazyStruct class with multiple levels of nesting + */ + public void testLazyStructNested() throws Throwable { + for(int i = 2; i < EXTENDED_LEVEL_THRESHOLD; i++ ){ + testNestedinArrayAtLevelExtended(i, ObjectInspector.Category.STRUCT); + } + } + + /** + * Verify the serialized format for given type dtype, when it is nested in an + * array with nestingLevel levels. with extended nesting enabled. + * @param nestingLevel + * @param dtype + * @throws SerDeException + */ + private void testNestedinArrayAtLevelExtended(int nestingLevel, + ObjectInspector.Category dtype) throws SerDeException { + Properties tableProp = new Properties(); + tableProp.setProperty(LazySimpleSerDe.SERIALIZATION_EXTEND_NESTING_LEVELS, "true"); + testNestedinArrayAtLevel(nestingLevel, dtype, tableProp); + } + + /** + * Test the LazyStruct class with multiple levels of nesting + */ + public void testLazyStructNestedExceedLimit() throws Throwable { + checkExtendedLimitExceeded(EXTENDED_LEVEL_THRESHOLD, ObjectInspector.Category.STRUCT); + } + + /** + * @param nestingLevel + * @param dtype + * @param tableProp + * @throws SerDeException + */ + private void testNestedinArrayAtLevel(int nestingLevel, + ObjectInspector.Category dtype, Properties tableProp) throws SerDeException { + + //create type with nestingLevel levels of nesting + //set inner schema for dtype + String inSchema = null; + switch(dtype){ + case LIST: + inSchema = "array<tinyint>"; + break; + case MAP: + inSchema = "map<string,int>"; + break; + case STRUCT: + inSchema = "struct<s:string,i:tinyint>"; + break; + case UNION: + inSchema = "uniontype<string,tinyint>"; + break; + default : + fail("type not supported by test case"); + } + + StringBuilder schema = new StringBuilder(inSchema); + for(int i = 0; i < nestingLevel - 1; i++){ + schema.insert(0, "array<"); + schema.append(">"); + } + System.err.println("Testing nesting level " + nestingLevel + + ". Using schema " + schema); + + + // Create the SerDe + LazySimpleSerDe serDe = new LazySimpleSerDe(); + Configuration conf = new Configuration(); + tableProp.setProperty("columns", "narray"); + tableProp.setProperty("columns.types", schema.toString()); + serDe.initialize(conf, tableProp); + + //create the serialized string for type + byte[] separators = serDe.serdeParams.getSeparators(); + System.err.println("Using separator " + (char)separators[nestingLevel]); + byte [] serializedRow = null; + switch(dtype){ + case LIST: + serializedRow = new byte[] {'8',separators[nestingLevel],'9'}; + break; + case MAP: + byte kvSep = separators[nestingLevel+1]; + byte kvPairSep = separators[nestingLevel]; + serializedRow = new byte[] {'1', kvSep, '1', kvPairSep, '2', kvSep, '2'}; + break; + case STRUCT: + serializedRow = new byte[] {'8',separators[nestingLevel],'9'}; + break; + case UNION: + serializedRow = new byte[] {'0',separators[nestingLevel],'9'}; + break; + default : + fail("type not supported by test case"); + } + + + //create LazyStruct with serialized string with expected separators + StructObjectInspector oi = (StructObjectInspector) serDe + .getObjectInspector(); + LazyStruct struct = (LazyStruct) LazyFactory.createLazyObject(oi); + + TestLazyPrimitive.initLazyObject(struct, serializedRow, 0, serializedRow.length); + + + //Get fields out of the lazy struct and check if they match expected + // results + //Get first level array + LazyArray array = (LazyArray) struct.getField(0); + + //Peel off the n-1 levels to get to the underlying array + for(int i = 0; i < nestingLevel - 2; i++){ + array = (LazyArray) array.getListElementObject(0); + } + + //verify the serialized format for dtype + switch(dtype){ + case LIST: + LazyArray array1 = (LazyArray) array.getListElementObject(0); + //check elements of the innermost array + assertEquals(2, array1.getListLength()); + assertEquals(new ByteWritable((byte) 8), ((LazyByte) array1 + .getListElementObject(0)).getWritableObject()); + assertEquals(new ByteWritable((byte) 9), ((LazyByte) array1 + .getListElementObject(1)).getWritableObject()); + break; + + case MAP: + LazyMap lazyMap = (LazyMap) array.getListElementObject(0); + Map map = lazyMap.getMap(); + System.err.println(map); + assertEquals(2, map.size()); + Iterator<Map.Entry<LazyString, LazyInteger>> it = map.entrySet().iterator(); + + Entry<LazyString, LazyInteger> e1 = it.next(); + assertEquals(e1.getKey().getWritableObject(), new Text(new byte[]{'1'}) ); + assertEquals(e1.getValue().getWritableObject(), new IntWritable(1) ); + + Entry<LazyString, LazyInteger> e2 = it.next(); + assertEquals(e2.getKey().getWritableObject(), new Text(new byte[]{'2'}) ); + assertEquals(e2.getValue().getWritableObject(), new IntWritable(2) ); + break; + + case STRUCT: + LazyStruct innerStruct = (LazyStruct) array.getListElementObject(0); + //check elements of the innermost struct + assertEquals(2, innerStruct.getFieldsAsList().size()); + assertEquals(new Text(new byte[]{'8'}), + ((LazyString) innerStruct.getField(0)).getWritableObject()); + assertEquals(new ByteWritable((byte) 9), + ((LazyByte) innerStruct.getField(1)).getWritableObject()); + break; + + case UNION: + LazyUnion lazyUnion = (LazyUnion) array.getListElementObject(0); + //check elements of the innermost union + assertEquals(new Text(new byte[]{'9'}), + ((LazyString)lazyUnion.getField()).getWritableObject()); + break; + + + default : + fail("type not supported by test case"); + } + + //test serialization + Text serializedText = + (Text) serDe.serialize(struct.getObject(), serDe.getObjectInspector()); + org.junit.Assert.assertArrayEquals(serializedRow, serializedText.getBytes()); + + } + + + } Modified: hive/branches/vectorization/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyPrimitive.java URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyPrimitive.java?rev=1501145&r1=1501144&r2=1501145&view=diff ============================================================================== --- hive/branches/vectorization/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyPrimitive.java (original) +++ hive/branches/vectorization/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyPrimitive.java Tue Jul 9 09:07:35 2013 @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hive.serde2.lazy; +import java.sql.Timestamp; + import junit.framework.TestCase; import org.apache.hadoop.hive.serde2.ByteStream; @@ -388,6 +390,24 @@ public class TestLazyPrimitive extends T assertEquals(new BytesWritable(new byte[] {}), ba.getWritableObject()); } + public void testLazyTimestamp() throws Throwable { + LazyTimestamp t = new LazyTimestamp(LazyPrimitiveObjectInspectorFactory.LAZY_TIMESTAMP_OBJECT_INSPECTOR); + String nullDate = "NULL"; + byte[] nullBytes = nullDate.getBytes(); + initLazyObject(t, nullBytes, 0, nullBytes.length); + assertEquals(true, t.isNull); + String sampleDate = "2013-02-12 21:04:58"; + byte[] good2013 = sampleDate.getBytes(); + initLazyObject(t, good2013, 0, good2013.length); + assertEquals(false, t.isNull); + assertEquals(Timestamp.valueOf(sampleDate), + t.getWritableObject().getTimestamp()); + String badDate = "2013-02-12 21:04:XX"; + byte[] bad2013 = badDate.getBytes(); + initLazyObject(t, bad2013, 0, bad2013.length); + assertEquals(true, t.isNull); + } + public void testLazyIntegerWrite() throws Throwable { try { ByteStream.Output out = new ByteStream.Output(); Modified: hive/branches/vectorization/service/if/TCLIService.thrift URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/if/TCLIService.thrift?rev=1501145&r1=1501144&r2=1501145&view=diff ============================================================================== --- hive/branches/vectorization/service/if/TCLIService.thrift (original) +++ hive/branches/vectorization/service/if/TCLIService.thrift Tue Jul 9 09:07:35 2013 @@ -57,21 +57,23 @@ enum TTypeId { STRUCT_TYPE, UNION_TYPE, USER_DEFINED_TYPE, - DECIMAL_TYPE + DECIMAL_TYPE, + NULL_TYPE, } const set<TTypeId> PRIMITIVE_TYPES = [ - TTypeId.BOOLEAN_TYPE - TTypeId.TINYINT_TYPE - TTypeId.SMALLINT_TYPE - TTypeId.INT_TYPE - TTypeId.BIGINT_TYPE - TTypeId.FLOAT_TYPE - TTypeId.DOUBLE_TYPE - TTypeId.STRING_TYPE - TTypeId.TIMESTAMP_TYPE + TTypeId.BOOLEAN_TYPE, + TTypeId.TINYINT_TYPE, + TTypeId.SMALLINT_TYPE, + TTypeId.INT_TYPE, + TTypeId.BIGINT_TYPE, + TTypeId.FLOAT_TYPE, + TTypeId.DOUBLE_TYPE, + TTypeId.STRING_TYPE, + TTypeId.TIMESTAMP_TYPE, TTypeId.BINARY_TYPE, - TTypeId.DECIMAL_TYPE + TTypeId.DECIMAL_TYPE, + TTypeId.NULL_TYPE ] const set<TTypeId> COMPLEX_TYPES = [ @@ -101,8 +103,9 @@ const map<TTypeId,string> TYPE_NAMES = { TTypeId.ARRAY_TYPE: "ARRAY", TTypeId.MAP_TYPE: "MAP", TTypeId.STRUCT_TYPE: "STRUCT", - TTypeId.UNION_TYPE: "UNIONTYPE" - TTypeId.DECIMAL_TYPE: "DECIMAL" + TTypeId.UNION_TYPE: "UNIONTYPE", + TTypeId.DECIMAL_TYPE: "DECIMAL", + TTypeId.NULL_TYPE: "NULL" } // Thrift does not support recursively defined types or forward declarations, @@ -285,7 +288,7 @@ union TColumnValue { 4: TI32Value i32Val // INT 5: TI64Value i64Val // BIGINT, TIMESTAMP 6: TDoubleValue doubleVal // FLOAT, DOUBLE - 7: TStringValue stringVal // STRING, LIST, MAP, STRUCT, UNIONTYPE, BINARY, DECIMAL + 7: TStringValue stringVal // STRING, LIST, MAP, STRUCT, UNIONTYPE, BINARY, DECIMAL, NULL } // Represents a row in a rowset. Modified: hive/branches/vectorization/service/src/gen/thrift/gen-cpp/TCLIService_constants.cpp URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/gen/thrift/gen-cpp/TCLIService_constants.cpp?rev=1501145&r1=1501144&r2=1501145&view=diff ============================================================================== --- hive/branches/vectorization/service/src/gen/thrift/gen-cpp/TCLIService_constants.cpp (original) +++ hive/branches/vectorization/service/src/gen/thrift/gen-cpp/TCLIService_constants.cpp Tue Jul 9 09:07:35 2013 @@ -22,6 +22,7 @@ TCLIServiceConstants::TCLIServiceConstan PRIMITIVE_TYPES.insert((TTypeId::type)8); PRIMITIVE_TYPES.insert((TTypeId::type)9); PRIMITIVE_TYPES.insert((TTypeId::type)15); + PRIMITIVE_TYPES.insert((TTypeId::type)16); COMPLEX_TYPES.insert((TTypeId::type)10); COMPLEX_TYPES.insert((TTypeId::type)11); @@ -47,6 +48,7 @@ TCLIServiceConstants::TCLIServiceConstan TYPE_NAMES.insert(std::make_pair((TTypeId::type)12, "STRUCT")); TYPE_NAMES.insert(std::make_pair((TTypeId::type)13, "UNIONTYPE")); TYPE_NAMES.insert(std::make_pair((TTypeId::type)15, "DECIMAL")); + TYPE_NAMES.insert(std::make_pair((TTypeId::type)16, "NULL")); } Modified: hive/branches/vectorization/service/src/gen/thrift/gen-cpp/TCLIService_types.cpp URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/gen/thrift/gen-cpp/TCLIService_types.cpp?rev=1501145&r1=1501144&r2=1501145&view=diff ============================================================================== --- hive/branches/vectorization/service/src/gen/thrift/gen-cpp/TCLIService_types.cpp (original) +++ hive/branches/vectorization/service/src/gen/thrift/gen-cpp/TCLIService_types.cpp Tue Jul 9 09:07:35 2013 @@ -34,7 +34,8 @@ int _kTTypeIdValues[] = { TTypeId::STRUCT_TYPE, TTypeId::UNION_TYPE, TTypeId::USER_DEFINED_TYPE, - TTypeId::DECIMAL_TYPE + TTypeId::DECIMAL_TYPE, + TTypeId::NULL_TYPE }; const char* _kTTypeIdNames[] = { "BOOLEAN_TYPE", @@ -52,9 +53,10 @@ const char* _kTTypeIdNames[] = { "STRUCT_TYPE", "UNION_TYPE", "USER_DEFINED_TYPE", - "DECIMAL_TYPE" + "DECIMAL_TYPE", + "NULL_TYPE" }; -const std::map<int, const char*> _TTypeId_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(16, _kTTypeIdValues, _kTTypeIdNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL)); +const std::map<int, const char*> _TTypeId_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(17, _kTTypeIdValues, _kTTypeIdNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL)); int _kTStatusCodeValues[] = { TStatusCode::SUCCESS_STATUS, Modified: hive/branches/vectorization/service/src/gen/thrift/gen-cpp/TCLIService_types.h URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/gen/thrift/gen-cpp/TCLIService_types.h?rev=1501145&r1=1501144&r2=1501145&view=diff ============================================================================== --- hive/branches/vectorization/service/src/gen/thrift/gen-cpp/TCLIService_types.h (original) +++ hive/branches/vectorization/service/src/gen/thrift/gen-cpp/TCLIService_types.h Tue Jul 9 09:07:35 2013 @@ -41,7 +41,8 @@ struct TTypeId { STRUCT_TYPE = 12, UNION_TYPE = 13, USER_DEFINED_TYPE = 14, - DECIMAL_TYPE = 15 + DECIMAL_TYPE = 15, + NULL_TYPE = 16 }; }; Modified: hive/branches/vectorization/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TCLIServiceConstants.java URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TCLIServiceConstants.java?rev=1501145&r1=1501144&r2=1501145&view=diff ============================================================================== --- hive/branches/vectorization/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TCLIServiceConstants.java (original) +++ hive/branches/vectorization/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TCLIServiceConstants.java Tue Jul 9 09:07:35 2013 @@ -46,6 +46,7 @@ public class TCLIServiceConstants { PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.TIMESTAMP_TYPE); PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.BINARY_TYPE); PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.DECIMAL_TYPE); + PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.NULL_TYPE); } public static final Set<TTypeId> COMPLEX_TYPES = new HashSet<TTypeId>(); @@ -80,6 +81,7 @@ public class TCLIServiceConstants { TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.STRUCT_TYPE, "STRUCT"); TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.UNION_TYPE, "UNIONTYPE"); TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.DECIMAL_TYPE, "DECIMAL"); + TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.NULL_TYPE, "NULL"); } } Modified: hive/branches/vectorization/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeId.java URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeId.java?rev=1501145&r1=1501144&r2=1501145&view=diff ============================================================================== --- hive/branches/vectorization/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeId.java (original) +++ hive/branches/vectorization/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeId.java Tue Jul 9 09:07:35 2013 @@ -27,7 +27,8 @@ public enum TTypeId implements org.apach STRUCT_TYPE(12), UNION_TYPE(13), USER_DEFINED_TYPE(14), - DECIMAL_TYPE(15); + DECIMAL_TYPE(15), + NULL_TYPE(16); private final int value; @@ -80,6 +81,8 @@ public enum TTypeId implements org.apach return USER_DEFINED_TYPE; case 15: return DECIMAL_TYPE; + case 16: + return NULL_TYPE; default: return null; } Modified: hive/branches/vectorization/service/src/gen/thrift/gen-py/TCLIService/constants.py URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/gen/thrift/gen-py/TCLIService/constants.py?rev=1501145&r1=1501144&r2=1501145&view=diff ============================================================================== --- hive/branches/vectorization/service/src/gen/thrift/gen-py/TCLIService/constants.py (original) +++ hive/branches/vectorization/service/src/gen/thrift/gen-py/TCLIService/constants.py Tue Jul 9 09:07:35 2013 @@ -21,6 +21,7 @@ PRIMITIVE_TYPES = set([ 8, 9, 15, + 16, ]) COMPLEX_TYPES = set([ 10, @@ -49,4 +50,5 @@ TYPE_NAMES = { 12 : "STRUCT", 13 : "UNIONTYPE", 15 : "DECIMAL", + 16 : "NULL", } Modified: hive/branches/vectorization/service/src/gen/thrift/gen-py/TCLIService/ttypes.py URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/gen/thrift/gen-py/TCLIService/ttypes.py?rev=1501145&r1=1501144&r2=1501145&view=diff ============================================================================== --- hive/branches/vectorization/service/src/gen/thrift/gen-py/TCLIService/ttypes.py (original) +++ hive/branches/vectorization/service/src/gen/thrift/gen-py/TCLIService/ttypes.py Tue Jul 9 09:07:35 2013 @@ -44,6 +44,7 @@ class TTypeId: UNION_TYPE = 13 USER_DEFINED_TYPE = 14 DECIMAL_TYPE = 15 + NULL_TYPE = 16 _VALUES_TO_NAMES = { 0: "BOOLEAN_TYPE", @@ -62,6 +63,7 @@ class TTypeId: 13: "UNION_TYPE", 14: "USER_DEFINED_TYPE", 15: "DECIMAL_TYPE", + 16: "NULL_TYPE", } _NAMES_TO_VALUES = { @@ -81,6 +83,7 @@ class TTypeId: "UNION_TYPE": 13, "USER_DEFINED_TYPE": 14, "DECIMAL_TYPE": 15, + "NULL_TYPE": 16, } class TStatusCode: Modified: hive/branches/vectorization/service/src/gen/thrift/gen-rb/t_c_l_i_service_constants.rb URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/gen/thrift/gen-rb/t_c_l_i_service_constants.rb?rev=1501145&r1=1501144&r2=1501145&view=diff ============================================================================== --- hive/branches/vectorization/service/src/gen/thrift/gen-rb/t_c_l_i_service_constants.rb (original) +++ hive/branches/vectorization/service/src/gen/thrift/gen-rb/t_c_l_i_service_constants.rb Tue Jul 9 09:07:35 2013 @@ -19,6 +19,7 @@ PRIMITIVE_TYPES = Set.new([ 8, 9, 15, + 16, ]) COMPLEX_TYPES = Set.new([ @@ -50,5 +51,6 @@ TYPE_NAMES = { 12 => %q"STRUCT", 13 => %q"UNIONTYPE", 15 => %q"DECIMAL", + 16 => %q"NULL", } Modified: hive/branches/vectorization/service/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb?rev=1501145&r1=1501144&r2=1501145&view=diff ============================================================================== --- hive/branches/vectorization/service/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb (original) +++ hive/branches/vectorization/service/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb Tue Jul 9 09:07:35 2013 @@ -29,8 +29,9 @@ module TTypeId UNION_TYPE = 13 USER_DEFINED_TYPE = 14 DECIMAL_TYPE = 15 - VALUE_MAP = {0 => "BOOLEAN_TYPE", 1 => "TINYINT_TYPE", 2 => "SMALLINT_TYPE", 3 => "INT_TYPE", 4 => "BIGINT_TYPE", 5 => "FLOAT_TYPE", 6 => "DOUBLE_TYPE", 7 => "STRING_TYPE", 8 => "TIMESTAMP_TYPE", 9 => "BINARY_TYPE", 10 => "ARRAY_TYPE", 11 => "MAP_TYPE", 12 => "STRUCT_TYPE", 13 => "UNION_TYPE", 14 => "USER_DEFINED_TYPE", 15 => "DECIMAL_TYPE"} - VALID_VALUES = Set.new([BOOLEAN_TYPE, TINYINT_TYPE, SMALLINT_TYPE, INT_TYPE, BIGINT_TYPE, FLOAT_TYPE, DOUBLE_TYPE, STRING_TYPE, TIMESTAMP_TYPE, BINARY_TYPE, ARRAY_TYPE, MAP_TYPE, STRUCT_TYPE, UNION_TYPE, USER_DEFINED_TYPE, DECIMAL_TYPE]).freeze + NULL_TYPE = 16 + VALUE_MAP = {0 => "BOOLEAN_TYPE", 1 => "TINYINT_TYPE", 2 => "SMALLINT_TYPE", 3 => "INT_TYPE", 4 => "BIGINT_TYPE", 5 => "FLOAT_TYPE", 6 => "DOUBLE_TYPE", 7 => "STRING_TYPE", 8 => "TIMESTAMP_TYPE", 9 => "BINARY_TYPE", 10 => "ARRAY_TYPE", 11 => "MAP_TYPE", 12 => "STRUCT_TYPE", 13 => "UNION_TYPE", 14 => "USER_DEFINED_TYPE", 15 => "DECIMAL_TYPE", 16 => "NULL_TYPE"} + VALID_VALUES = Set.new([BOOLEAN_TYPE, TINYINT_TYPE, SMALLINT_TYPE, INT_TYPE, BIGINT_TYPE, FLOAT_TYPE, DOUBLE_TYPE, STRING_TYPE, TIMESTAMP_TYPE, BINARY_TYPE, ARRAY_TYPE, MAP_TYPE, STRUCT_TYPE, UNION_TYPE, USER_DEFINED_TYPE, DECIMAL_TYPE, NULL_TYPE]).freeze end module TStatusCode Modified: hive/branches/vectorization/service/src/java/org/apache/hive/service/auth/LdapAuthenticationProviderImpl.java URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/java/org/apache/hive/service/auth/LdapAuthenticationProviderImpl.java?rev=1501145&r1=1501144&r2=1501145&view=diff ============================================================================== --- hive/branches/vectorization/service/src/java/org/apache/hive/service/auth/LdapAuthenticationProviderImpl.java (original) +++ hive/branches/vectorization/service/src/java/org/apache/hive/service/auth/LdapAuthenticationProviderImpl.java Tue Jul 9 09:07:35 2013 @@ -29,13 +29,15 @@ import org.apache.hadoop.hive.conf.HiveC public class LdapAuthenticationProviderImpl implements PasswdAuthenticationProvider { - String ldapURL; - String baseDN; + private final String ldapURL; + private final String baseDN; + private final String ldapDomain; LdapAuthenticationProviderImpl () { HiveConf conf = new HiveConf(); this.ldapURL = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_URL); this.baseDN = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_BASEDN); + this.ldapDomain = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_DOMAIN); } @Override @@ -46,6 +48,12 @@ public class LdapAuthenticationProviderI env.put(Context.INITIAL_CONTEXT_FACTORY, "com.sun.jndi.ldap.LdapCtxFactory"); env.put(Context.PROVIDER_URL, ldapURL); + // If the domain is supplied, then append it. LDAP providers like Active Directory + // use a fully qualified user name like [email protected]. + if (ldapDomain != null) { + user = user + "@" + ldapDomain; + } + // setup the security principal String bindDN; if (baseDN != null) { Modified: hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/ColumnValue.java URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/ColumnValue.java?rev=1501145&r1=1501144&r2=1501145&view=diff ============================================================================== --- hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/ColumnValue.java (original) +++ hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/ColumnValue.java Tue Jul 9 09:07:35 2013 @@ -36,6 +36,12 @@ import org.apache.hive.service.cli.thrif */ public class ColumnValue { + public static final TColumnValue NULL = new TColumnValue(); + + static { + NULL.setStringVal(new TStringValue()); + } + // TODO: replace this with a non-Thrift implementation private final TColumnValue tColumnValue; Modified: hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/Row.java URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/Row.java?rev=1501145&r1=1501144&r2=1501145&view=diff ============================================================================== --- hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/Row.java (original) +++ hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/Row.java Tue Jul 9 09:07:35 2013 @@ -71,7 +71,11 @@ public class Row { public TRow toTRow() { TRow tRow = new TRow(); for (ColumnValue columnValue : values) { - tRow.addToColVals(columnValue.toTColumnValue()); + if (columnValue != null) { + tRow.addToColVals(columnValue.toTColumnValue()); + } else { + tRow.addToColVals(ColumnValue.NULL); + } } return tRow; } Modified: hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/Type.java URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/Type.java?rev=1501145&r1=1501144&r2=1501145&view=diff ============================================================================== --- hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/Type.java (original) +++ hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/Type.java Tue Jul 9 09:07:35 2013 @@ -27,6 +27,9 @@ import org.apache.hive.service.cli.thrif * */ public enum Type { + NULL_TYPE("VOID", + java.sql.Types.NULL, + TTypeId.NULL_TYPE), BOOLEAN_TYPE("BOOLEAN", java.sql.Types.BOOLEAN, TTypeId.BOOLEAN_TYPE), @@ -77,7 +80,7 @@ public enum Type { java.sql.Types.VARCHAR, TTypeId.STRING_TYPE, true, false), - USER_DEFINED_TYPE(null, + USER_DEFINED_TYPE("USER_DEFINED", java.sql.Types.VARCHAR, TTypeId.STRING_TYPE, true, false); @@ -123,6 +126,9 @@ public enum Type { } public static Type getType(String name) { + if (name == null) { + throw new IllegalArgumentException("Invalid type name: null"); + } for (Type type : values()) { if (name.equalsIgnoreCase(type.name)) { return type; Modified: hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/GetSchemasOperation.java URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/GetSchemasOperation.java?rev=1501145&r1=1501144&r2=1501145&view=diff ============================================================================== --- hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/GetSchemasOperation.java (original) +++ hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/GetSchemasOperation.java Tue Jul 9 09:07:35 2013 @@ -36,7 +36,7 @@ public class GetSchemasOperation extends private final String schemaName; private static final TableSchema RESULT_SET_SCHEMA = new TableSchema() - .addStringColumn("TABLE_SCHEMA", "Schema name.") + .addStringColumn("TABLE_SCHEM", "Schema name.") .addStringColumn("TABLE_CATALOG", "Catalog name."); private RowSet rowSet; Modified: hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java?rev=1501145&r1=1501144&r2=1501145&view=diff ============================================================================== --- hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java (original) +++ hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java Tue Jul 9 09:07:35 2013 @@ -46,7 +46,7 @@ public class GetTablesOperation extends private static final TableSchema RESULT_SET_SCHEMA = new TableSchema() .addStringColumn("TABLE_CAT", "Catalog name. NULL if not applicable.") - .addStringColumn("TABLE_SCHEMA", "Schema name.") + .addStringColumn("TABLE_SCHEM", "Schema name.") .addStringColumn("TABLE_NAME", "Table name.") .addStringColumn("TABLE_TYPE", "The table type, e.g. \"TABLE\", \"VIEW\", etc.") .addStringColumn("REMARKS", "Comments about the table.");
