http://git-wip-us.apache.org/repos/asf/hive/blob/d467e172/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java ---------------------------------------------------------------------- diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java b/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java index b369462..8356a5c 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.serde2.binarysortable; +import java.io.IOException; import java.util.ArrayList; import java.io.EOFException; import java.util.Arrays; @@ -30,13 +31,16 @@ import org.apache.hadoop.hive.serde2.SerdeRandomRowSource; import org.apache.hadoop.hive.serde2.VerifyFast; import org.apache.hadoop.hive.serde2.binarysortable.fast.BinarySortableDeserializeRead; import org.apache.hadoop.hive.serde2.binarysortable.fast.BinarySortableSerializeWrite; +import org.apache.hadoop.hive.serde2.lazy.VerifyLazy; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.objectinspector.UnionObject; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.Writable; import junit.framework.TestCase; +import org.junit.Assert; public class TestBinarySortableFast extends TestCase { @@ -48,11 +52,11 @@ public class TestBinarySortableFast extends TestCase { boolean[] columnSortOrderIsDesc, byte[] columnNullMarker, byte[] columnNotNullMarker, AbstractSerDe serde, StructObjectInspector rowOI, AbstractSerDe serde_fewer, StructObjectInspector writeRowOI, - boolean ascending, PrimitiveTypeInfo[] primitiveTypeInfos, + boolean ascending, TypeInfo[] typeInfos, boolean useIncludeColumns, boolean doWriteFewerColumns, Random r) throws Throwable { int rowCount = rows.length; - int columnCount = primitiveTypeInfos.length; + int columnCount = typeInfos.length; boolean[] columnsToInclude = null; if (useIncludeColumns) { @@ -83,10 +87,7 @@ public class TestBinarySortableFast extends TestCase { int[] perFieldWriteLengths = new int[columnCount]; for (int index = 0; index < writeColumnCount; index++) { - - Writable writable = (Writable) row[index]; - - VerifyFast.serializeWrite(binarySortableSerializeWrite, primitiveTypeInfos[index], writable); + VerifyFast.serializeWrite(binarySortableSerializeWrite, typeInfos[index], row[index]); perFieldWriteLengths[index] = output.getLength(); } perFieldWriteLengthsArray[i] = perFieldWriteLengths; @@ -95,7 +96,8 @@ public class TestBinarySortableFast extends TestCase { bytesWritable.set(output.getData(), 0, output.getLength()); serializeWriteBytes[i] = bytesWritable; if (i > 0) { - int compareResult = serializeWriteBytes[i - 1].compareTo(serializeWriteBytes[i]); + BytesWritable previousBytesWritable = serializeWriteBytes[i - 1]; + int compareResult = previousBytesWritable.compareTo(bytesWritable); if ((compareResult < 0 && !ascending) || (compareResult > 0 && ascending)) { System.out.println("Test failed in " @@ -117,7 +119,7 @@ public class TestBinarySortableFast extends TestCase { Object[] row = rows[i]; BinarySortableDeserializeRead binarySortableDeserializeRead = new BinarySortableDeserializeRead( - primitiveTypeInfos, + typeInfos, /* useExternalBuffer */ false, columnSortOrderIsDesc, columnNullMarker, @@ -132,10 +134,9 @@ public class TestBinarySortableFast extends TestCase { binarySortableDeserializeRead.skipNextField(); } else if (index >= writeColumnCount) { // Should come back a null. - VerifyFast.verifyDeserializeRead(binarySortableDeserializeRead, primitiveTypeInfos[index], null); + VerifyFast.verifyDeserializeRead(binarySortableDeserializeRead, typeInfos[index], null); } else { - Writable writable = (Writable) row[index]; - VerifyFast.verifyDeserializeRead(binarySortableDeserializeRead, primitiveTypeInfos[index], writable); + verifyRead(binarySortableDeserializeRead, typeInfos[index], row[index]); } } if (writeColumnCount == columnCount) { @@ -147,7 +148,7 @@ public class TestBinarySortableFast extends TestCase { */ BinarySortableDeserializeRead binarySortableDeserializeRead2 = new BinarySortableDeserializeRead( - primitiveTypeInfos, + typeInfos, /* useExternalBuffer */ false, columnSortOrderIsDesc, columnNullMarker, @@ -157,22 +158,24 @@ public class TestBinarySortableFast extends TestCase { bytesWritable.getBytes(), 0, bytesWritable.getLength() - 1); // One fewer byte. for (int index = 0; index < writeColumnCount; index++) { - Writable writable = (Writable) row[index]; if (index == writeColumnCount - 1) { boolean threw = false; try { - VerifyFast.verifyDeserializeRead(binarySortableDeserializeRead2, primitiveTypeInfos[index], writable); + verifyRead(binarySortableDeserializeRead2, typeInfos[index], row[index]); } catch (EOFException e) { // debugDetailedReadPositionString = binarySortableDeserializeRead2.getDetailedReadPositionString(); // debugStackTrace = e.getStackTrace(); threw = true; } - TestCase.assertTrue(threw); + + if (!threw && row[index] != null) { + Assert.fail(); + } } else { if (useIncludeColumns && !columnsToInclude[index]) { binarySortableDeserializeRead2.skipNextField(); } else { - VerifyFast.verifyDeserializeRead(binarySortableDeserializeRead2, primitiveTypeInfos[index], writable); + verifyRead(binarySortableDeserializeRead2, typeInfos[index], row[index]); } } } @@ -270,7 +273,7 @@ public class TestBinarySortableFast extends TestCase { "\nSerDe: " + serDeFields.toString() + "\nperFieldWriteLengths " + Arrays.toString(perFieldWriteLengthsArray[i]) + - "\nprimitiveTypeInfos " + Arrays.toString(primitiveTypeInfos) + + "\nprimitiveTypeInfos " + Arrays.toString(typeInfos) + "\nrow " + Arrays.toString(row)); } } @@ -282,7 +285,7 @@ public class TestBinarySortableFast extends TestCase { Object[] row = rows[i]; BinarySortableDeserializeRead binarySortableDeserializeRead = new BinarySortableDeserializeRead( - primitiveTypeInfos, + typeInfos, /* useExternalBuffer */ false, columnSortOrderIsDesc, columnNullMarker, @@ -297,10 +300,9 @@ public class TestBinarySortableFast extends TestCase { binarySortableDeserializeRead.skipNextField(); } else if (index >= writeColumnCount) { // Should come back a null. - VerifyFast.verifyDeserializeRead(binarySortableDeserializeRead, primitiveTypeInfos[index], null); + verifyRead(binarySortableDeserializeRead, typeInfos[index], null); } else { - Writable writable = (Writable) row[index]; - VerifyFast.verifyDeserializeRead(binarySortableDeserializeRead, primitiveTypeInfos[index], writable); + verifyRead(binarySortableDeserializeRead, typeInfos[index], row[index]); } } if (writeColumnCount == columnCount) { @@ -309,11 +311,44 @@ public class TestBinarySortableFast extends TestCase { } } - private void testBinarySortableFastCase(int caseNum, boolean doNonRandomFill, Random r) + private void verifyRead(BinarySortableDeserializeRead binarySortableDeserializeRead, + TypeInfo typeInfo, Object expectedObject) throws IOException { + if (typeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE) { + VerifyFast.verifyDeserializeRead(binarySortableDeserializeRead, typeInfo, expectedObject); + } else { + Object complexFieldObj = VerifyFast.deserializeReadComplexType(binarySortableDeserializeRead, typeInfo); + if (expectedObject == null) { + if (complexFieldObj != null) { + TestCase.fail("Field reports not null but object is null (class " + complexFieldObj.getClass().getName() + + ", " + complexFieldObj.toString() + ")"); + } + } else { + if (complexFieldObj == null) { + // It's hard to distinguish a union with null from a null union. + if (expectedObject instanceof UnionObject) { + UnionObject expectedUnion = (UnionObject) expectedObject; + if (expectedUnion.getObject() == null) { + return; + } + } + TestCase.fail("Field reports null but object is not null (class " + expectedObject.getClass().getName() + + ", " + expectedObject.toString() + ")"); + } + } + if (!VerifyLazy.lazyCompare(typeInfo, complexFieldObj, expectedObject)) { + TestCase.fail("Comparision failed typeInfo " + typeInfo.toString()); + } + } + } + + private void testBinarySortableFastCase( + int caseNum, boolean doNonRandomFill, Random r, SerdeRandomRowSource.SupportedTypes supportedTypes, int depth) throws Throwable { SerdeRandomRowSource source = new SerdeRandomRowSource(); - source.init(r); + + // UNDONE: Until Fast BinarySortable supports complex types -- disable. + source.init(r, supportedTypes, depth); int rowCount = 1000; Object[][] rows = source.randomRows(rowCount); @@ -327,8 +362,8 @@ public class TestBinarySortableFast extends TestCase { StructObjectInspector rowStructObjectInspector = source.rowStructObjectInspector(); - PrimitiveTypeInfo[] primitiveTypeInfos = source.primitiveTypeInfos(); - int columnCount = primitiveTypeInfos.length; + TypeInfo[] typeInfos = source.typeInfos(); + int columnCount = typeInfos.length; int writeColumnCount = columnCount; StructObjectInspector writeRowStructObjectInspector = rowStructObjectInspector; @@ -385,14 +420,14 @@ public class TestBinarySortableFast extends TestCase { columnSortOrderIsDesc, columnNullMarker, columnNotNullMarker, serde_ascending, rowStructObjectInspector, serde_ascending_fewer, writeRowStructObjectInspector, - /* ascending */ true, primitiveTypeInfos, + /* ascending */ true, typeInfos, /* useIncludeColumns */ false, /* doWriteFewerColumns */ false, r); testBinarySortableFast(source, rows, columnSortOrderIsDesc, columnNullMarker, columnNotNullMarker, serde_ascending, rowStructObjectInspector, serde_ascending_fewer, writeRowStructObjectInspector, - /* ascending */ true, primitiveTypeInfos, + /* ascending */ true, typeInfos, /* useIncludeColumns */ true, /* doWriteFewerColumns */ false, r); if (doWriteFewerColumns) { @@ -400,14 +435,14 @@ public class TestBinarySortableFast extends TestCase { columnSortOrderIsDesc, columnNullMarker, columnNotNullMarker, serde_ascending, rowStructObjectInspector, serde_ascending_fewer, writeRowStructObjectInspector, - /* ascending */ true, primitiveTypeInfos, + /* ascending */ true, typeInfos, /* useIncludeColumns */ false, /* doWriteFewerColumns */ true, r); testBinarySortableFast(source, rows, columnSortOrderIsDesc, columnNullMarker, columnNotNullMarker, serde_ascending, rowStructObjectInspector, serde_ascending_fewer, writeRowStructObjectInspector, - /* ascending */ true, primitiveTypeInfos, + /* ascending */ true, typeInfos, /* useIncludeColumns */ true, /* doWriteFewerColumns */ true, r); } @@ -420,14 +455,14 @@ public class TestBinarySortableFast extends TestCase { columnSortOrderIsDesc, columnNullMarker, columnNotNullMarker, serde_descending, rowStructObjectInspector, serde_ascending_fewer, writeRowStructObjectInspector, - /* ascending */ false, primitiveTypeInfos, + /* ascending */ false, typeInfos, /* useIncludeColumns */ false, /* doWriteFewerColumns */ false, r); testBinarySortableFast(source, rows, columnSortOrderIsDesc, columnNullMarker, columnNotNullMarker, serde_descending, rowStructObjectInspector, serde_ascending_fewer, writeRowStructObjectInspector, - /* ascending */ false, primitiveTypeInfos, + /* ascending */ false, typeInfos, /* useIncludeColumns */ true, /* doWriteFewerColumns */ false, r); if (doWriteFewerColumns) { @@ -435,27 +470,27 @@ public class TestBinarySortableFast extends TestCase { columnSortOrderIsDesc, columnNullMarker, columnNotNullMarker, serde_descending, rowStructObjectInspector, serde_descending_fewer, writeRowStructObjectInspector, - /* ascending */ false, primitiveTypeInfos, + /* ascending */ false, typeInfos, /* useIncludeColumns */ false, /* doWriteFewerColumns */ true, r); testBinarySortableFast(source, rows, columnSortOrderIsDesc, columnNullMarker, columnNotNullMarker, serde_descending, rowStructObjectInspector, serde_descending_fewer, writeRowStructObjectInspector, - /* ascending */ false, primitiveTypeInfos, + /* ascending */ false, typeInfos, /* useIncludeColumns */ true, /* doWriteFewerColumns */ true, r); } } - public void testBinarySortableFast() throws Throwable { + public void testBinarySortableFast(SerdeRandomRowSource.SupportedTypes supportedTypes, int depth) throws Throwable { try { Random r = new Random(35790); int caseNum = 0; for (int i = 0; i < 10; i++) { - testBinarySortableFastCase(caseNum, (i % 2 == 0), r); + testBinarySortableFastCase(caseNum, (i % 2 == 0), r, supportedTypes, depth); caseNum++; } @@ -465,6 +500,18 @@ public class TestBinarySortableFast extends TestCase { } } + public void testBinarySortableFastPrimitive() throws Throwable { + testBinarySortableFast(SerdeRandomRowSource.SupportedTypes.PRIMITIVE, 0); + } + + public void testBinarySortableFastComplexDepthOne() throws Throwable { + testBinarySortableFast(SerdeRandomRowSource.SupportedTypes.ALL_EXCEPT_MAP, 1); + } + + public void testBinarySortableFastComplexDepthFour() throws Throwable { + testBinarySortableFast(SerdeRandomRowSource.SupportedTypes.ALL_EXCEPT_MAP, 4); + } + private static String displayBytes(byte[] bytes, int start, int length) { StringBuilder sb = new StringBuilder(); for (int i = start; i < start + length; i++) {
http://git-wip-us.apache.org/repos/asf/hive/blob/d467e172/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java ---------------------------------------------------------------------- diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java index c857b42..6e6a4c2 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hive.serde2.lazy; +import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Properties; import java.util.Random; @@ -33,10 +35,11 @@ import org.apache.hadoop.hive.serde2.lazy.fast.LazySimpleDeserializeRead; import org.apache.hadoop.hive.serde2.lazy.fast.LazySimpleSerializeWrite; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.UnionObject; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; import junit.framework.TestCase; @@ -46,12 +49,12 @@ public class TestLazySimpleFast extends TestCase { SerdeRandomRowSource source, Object[][] rows, LazySimpleSerDe serde, StructObjectInspector rowOI, LazySimpleSerDe serde_fewer, StructObjectInspector writeRowOI, - byte separator, LazySerDeParameters serdeParams, LazySerDeParameters serdeParams_fewer, - PrimitiveTypeInfo[] primitiveTypeInfos, + LazySerDeParameters serdeParams, LazySerDeParameters serdeParams_fewer, + TypeInfo[] typeInfos, boolean useIncludeColumns, boolean doWriteFewerColumns, Random r) throws Throwable { int rowCount = rows.length; - int columnCount = primitiveTypeInfos.length; + int columnCount = typeInfos.length; boolean[] columnsToInclude = null; if (useIncludeColumns) { @@ -62,10 +65,10 @@ public class TestLazySimpleFast extends TestCase { } int writeColumnCount = columnCount; - PrimitiveTypeInfo[] writePrimitiveTypeInfos = primitiveTypeInfos; + TypeInfo[] writeTypeInfos = typeInfos; if (doWriteFewerColumns) { writeColumnCount = writeRowOI.getAllStructFieldRefs().size(); - writePrimitiveTypeInfos = Arrays.copyOf(primitiveTypeInfos, writeColumnCount); + writeTypeInfos = Arrays.copyOf(typeInfos, writeColumnCount); } // Try to serialize @@ -75,16 +78,12 @@ public class TestLazySimpleFast extends TestCase { Output output = new Output(); LazySimpleSerializeWrite lazySimpleSerializeWrite = - new LazySimpleSerializeWrite(columnCount, - separator, serdeParams); + new LazySimpleSerializeWrite(columnCount, serdeParams); lazySimpleSerializeWrite.set(output); for (int index = 0; index < columnCount; index++) { - - Writable writable = (Writable) row[index]; - - VerifyFast.serializeWrite(lazySimpleSerializeWrite, primitiveTypeInfos[index], writable); + VerifyFast.serializeWrite(lazySimpleSerializeWrite, typeInfos[index], row[index]); } BytesWritable bytesWritable = new BytesWritable(); @@ -97,29 +96,24 @@ public class TestLazySimpleFast extends TestCase { Object[] row = rows[i]; LazySimpleDeserializeRead lazySimpleDeserializeRead = new LazySimpleDeserializeRead( - writePrimitiveTypeInfos, + writeTypeInfos, /* useExternalBuffer */ false, - separator, serdeParams); + serdeParams); BytesWritable bytesWritable = serializeWriteBytes[i]; byte[] bytes = bytesWritable.getBytes(); int length = bytesWritable.getLength(); lazySimpleDeserializeRead.set(bytes, 0, length); - char[] chars = new char[length]; - for (int c = 0; c < chars.length; c++) { - chars[c] = (char) (bytes[c] & 0xFF); - } - for (int index = 0; index < columnCount; index++) { if (useIncludeColumns && !columnsToInclude[index]) { lazySimpleDeserializeRead.skipNextField(); } else if (index >= writeColumnCount) { // Should come back a null. - VerifyFast.verifyDeserializeRead(lazySimpleDeserializeRead, primitiveTypeInfos[index], null); + verifyReadNull(lazySimpleDeserializeRead, typeInfos[index]); } else { - Writable writable = (Writable) row[index]; - VerifyFast.verifyDeserializeRead(lazySimpleDeserializeRead, primitiveTypeInfos[index], writable); + Object expectedObject = row[index]; + verifyRead(lazySimpleDeserializeRead, typeInfos[index], expectedObject); } } if (writeColumnCount == columnCount) { @@ -128,28 +122,22 @@ public class TestLazySimpleFast extends TestCase { } // Try to deserialize using SerDe class our Writable row objects created by SerializeWrite. - for (int i = 0; i < rowCount; i++) { - BytesWritable bytesWritable = serializeWriteBytes[i]; + for (int rowIndex = 0; rowIndex < rowCount; rowIndex++) { + BytesWritable bytesWritable = serializeWriteBytes[rowIndex]; LazyStruct lazySimpleStruct = (LazyStruct) serde.deserialize(bytesWritable); - Object[] row = rows[i]; + Object[] row = rows[rowIndex]; for (int index = 0; index < columnCount; index++) { - PrimitiveTypeInfo primitiveTypeInfo = primitiveTypeInfos[index]; - Writable writable = (Writable) row[index]; - LazyPrimitive lazyPrimitive = (LazyPrimitive) lazySimpleStruct.getField(index); - Object object; - if (lazyPrimitive != null) { - object = lazyPrimitive.getWritableObject(); - } else { - object = null; - } - if (writable == null || object == null) { - if (writable != null || object != null) { + TypeInfo typeInfo = typeInfos[index]; + Object expectedObject = row[index]; + Object object = lazySimpleStruct.getField(index); + if (expectedObject == null || object == null) { + if (expectedObject != null || object != null) { fail("SerDe deserialized NULL column mismatch"); } } else { - if (!object.equals(writable)) { + if (!VerifyLazy.lazyCompare(typeInfo, object, expectedObject)) { fail("SerDe deserialized value does not match"); } } @@ -185,9 +173,9 @@ public class TestLazySimpleFast extends TestCase { LazySimpleDeserializeRead lazySimpleDeserializeRead = new LazySimpleDeserializeRead( - writePrimitiveTypeInfos, + writeTypeInfos, /* useExternalBuffer */ false, - separator, serdeParams); + serdeParams); byte[] bytes = serdeBytes[i]; lazySimpleDeserializeRead.set(bytes, 0, bytes.length); @@ -197,10 +185,10 @@ public class TestLazySimpleFast extends TestCase { lazySimpleDeserializeRead.skipNextField(); } else if (index >= writeColumnCount) { // Should come back a null. - VerifyFast.verifyDeserializeRead(lazySimpleDeserializeRead, primitiveTypeInfos[index], null); + verifyReadNull(lazySimpleDeserializeRead, typeInfos[index]); } else { - Writable writable = (Writable) row[index]; - VerifyFast.verifyDeserializeRead(lazySimpleDeserializeRead, primitiveTypeInfos[index], writable); + Object expectedObject = row[index]; + verifyRead(lazySimpleDeserializeRead, typeInfos[index], expectedObject); } } if (writeColumnCount == columnCount) { @@ -209,6 +197,46 @@ public class TestLazySimpleFast extends TestCase { } } + private void verifyReadNull(LazySimpleDeserializeRead lazySimpleDeserializeRead, + TypeInfo typeInfo) throws IOException { + if (typeInfo.getCategory() == Category.PRIMITIVE) { + VerifyFast.verifyDeserializeRead(lazySimpleDeserializeRead, typeInfo, null); + } else { + Object complexFieldObj = VerifyFast.deserializeReadComplexType(lazySimpleDeserializeRead, typeInfo); + if (complexFieldObj != null) { + TestCase.fail("Field report not null but object is null"); + } + } + } + + private void verifyRead(LazySimpleDeserializeRead lazySimpleDeserializeRead, + TypeInfo typeInfo, Object expectedObject) throws IOException { + if (typeInfo.getCategory() == Category.PRIMITIVE) { + VerifyFast.verifyDeserializeRead(lazySimpleDeserializeRead, typeInfo, expectedObject); + } else { + Object complexFieldObj = VerifyFast.deserializeReadComplexType(lazySimpleDeserializeRead, typeInfo); + if (expectedObject == null) { + if (complexFieldObj != null) { + TestCase.fail("Field reports not null but object is null (class " + complexFieldObj.getClass().getName() + ", " + complexFieldObj.toString() + ")"); + } + } else { + if (complexFieldObj == null) { + // It's hard to distinguish a union with null from a null union. + if (expectedObject instanceof UnionObject) { + UnionObject expectedUnion = (UnionObject) expectedObject; + if (expectedUnion.getObject() == null) { + return; + } + } + TestCase.fail("Field reports null but object is not null (class " + expectedObject.getClass().getName() + ", " + expectedObject.toString() + ")"); + } + } + if (!VerifyLazy.lazyCompare(typeInfo, complexFieldObj, expectedObject)) { + TestCase.fail("Comparision failed typeInfo " + typeInfo.toString()); + } + } + } + private byte[] copyBytes(Text serialized) { byte[] result = new byte[serialized.getLength()]; System.arraycopy(serialized.getBytes(), 0, result, 0, serialized.getLength()); @@ -238,19 +266,25 @@ public class TestLazySimpleFast extends TestCase { return serDe; } - private LazySerDeParameters getSerDeParams(String fieldNames, String fieldTypes) throws SerDeException { + private LazySerDeParameters getSerDeParams(String fieldNames, String fieldTypes, + byte[] separators) throws SerDeException { Configuration conf = new Configuration(); Properties tbl = createProperties(fieldNames, fieldTypes); - return new LazySerDeParameters(conf, tbl, LazySimpleSerDe.class.getName()); + LazySerDeParameters lazySerDeParams = new LazySerDeParameters(conf, tbl, LazySimpleSerDe.class.getName()); + for (int i = 0; i < separators.length; i++) { + lazySerDeParams.setSeparator(i, separators[i]); + } + return lazySerDeParams; } - public void testLazySimpleFastCase(int caseNum, boolean doNonRandomFill, Random r) + public void testLazySimpleFastCase( + int caseNum, boolean doNonRandomFill, Random r, SerdeRandomRowSource.SupportedTypes supportedTypes, int depth) throws Throwable { SerdeRandomRowSource source = new SerdeRandomRowSource(); - source.init(r); + source.init(r, supportedTypes, depth); - int rowCount = 1000; + int rowCount = 100; Object[][] rows = source.randomRows(rowCount); if (doNonRandomFill) { @@ -259,8 +293,8 @@ public class TestLazySimpleFast extends TestCase { StructObjectInspector rowStructObjectInspector = source.rowStructObjectInspector(); - PrimitiveTypeInfo[] primitiveTypeInfos = source.primitiveTypeInfos(); - int columnCount = primitiveTypeInfos.length; + TypeInfo[] typeInfos = source.typeInfos(); + int columnCount = typeInfos.length; int writeColumnCount = columnCount; StructObjectInspector writeRowStructObjectInspector = rowStructObjectInspector; @@ -277,8 +311,11 @@ public class TestLazySimpleFast extends TestCase { String fieldNames = ObjectInspectorUtils.getFieldNames(rowStructObjectInspector); String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowStructObjectInspector); + // Use different separator values. + byte[] separators = new byte[] {(byte) 9, (byte) 2, (byte) 3, (byte) 4, (byte) 5, (byte) 6, (byte) 7, (byte) 8}; + LazySimpleSerDe serde = getSerDe(fieldNames, fieldTypes); - LazySerDeParameters serdeParams = getSerDeParams(fieldNames, fieldTypes); + LazySerDeParameters serdeParams = getSerDeParams(fieldNames, fieldTypes, separators); LazySimpleSerDe serde_fewer = null; LazySerDeParameters serdeParams_fewer = null; @@ -287,22 +324,22 @@ public class TestLazySimpleFast extends TestCase { String partialFieldTypes = ObjectInspectorUtils.getFieldTypes(writeRowStructObjectInspector); serde_fewer = getSerDe(fieldNames, fieldTypes); - serdeParams_fewer = getSerDeParams(partialFieldNames, partialFieldTypes); + serdeParams_fewer = getSerDeParams(partialFieldNames, partialFieldTypes, separators); } - byte separator = (byte) '\t'; + testLazySimpleFast( source, rows, serde, rowStructObjectInspector, serde_fewer, writeRowStructObjectInspector, - separator, serdeParams, serdeParams_fewer, primitiveTypeInfos, + serdeParams, serdeParams_fewer, typeInfos, /* useIncludeColumns */ false, /* doWriteFewerColumns */ false, r); testLazySimpleFast( source, rows, serde, rowStructObjectInspector, serde_fewer, writeRowStructObjectInspector, - separator, serdeParams, serdeParams_fewer, primitiveTypeInfos, + serdeParams, serdeParams_fewer, typeInfos, /* useIncludeColumns */ true, /* doWriteFewerColumns */ false, r); if (doWriteFewerColumns) { @@ -310,26 +347,26 @@ public class TestLazySimpleFast extends TestCase { source, rows, serde, rowStructObjectInspector, serde_fewer, writeRowStructObjectInspector, - separator, serdeParams, serdeParams_fewer, primitiveTypeInfos, + serdeParams, serdeParams_fewer, typeInfos, /* useIncludeColumns */ false, /* doWriteFewerColumns */ true, r); testLazySimpleFast( source, rows, serde, rowStructObjectInspector, serde_fewer, writeRowStructObjectInspector, - separator, serdeParams, serdeParams_fewer, primitiveTypeInfos, + serdeParams, serdeParams_fewer, typeInfos, /* useIncludeColumns */ true, /* doWriteFewerColumns */ true, r); } } - public void testLazySimpleFast() throws Throwable { + public void testLazySimpleFast(SerdeRandomRowSource.SupportedTypes supportedTypes, int depth) throws Throwable { try { - Random r = new Random(35790); + Random r = new Random(8322); int caseNum = 0; - for (int i = 0; i < 10; i++) { - testLazySimpleFastCase(caseNum, (i % 2 == 0), r); + for (int i = 0; i < 20; i++) { + testLazySimpleFastCase(caseNum, (i % 2 == 0), r, supportedTypes, depth); caseNum++; } @@ -338,4 +375,16 @@ public class TestLazySimpleFast extends TestCase { throw e; } } + + public void testLazyBinarySimplePrimitive() throws Throwable { + testLazySimpleFast(SerdeRandomRowSource.SupportedTypes.PRIMITIVE, 0); + } + + public void testLazyBinarySimpleComplexDepthOne() throws Throwable { + testLazySimpleFast(SerdeRandomRowSource.SupportedTypes.ALL, 1); + } + + public void testLazyBinarySimpleComplexDepthFour() throws Throwable { + testLazySimpleFast(SerdeRandomRowSource.SupportedTypes.ALL, 4); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/d467e172/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java ---------------------------------------------------------------------- diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java b/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java index e62a80a..13c73be 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hive.serde2.lazybinary; +import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Random; @@ -27,11 +29,14 @@ import org.apache.hadoop.hive.serde2.AbstractSerDe; import org.apache.hadoop.hive.serde2.SerdeRandomRowSource; import org.apache.hadoop.hive.serde2.VerifyFast; import org.apache.hadoop.hive.serde2.binarysortable.MyTestClass; +import org.apache.hadoop.hive.serde2.lazy.VerifyLazy; import org.apache.hadoop.hive.serde2.lazybinary.fast.LazyBinaryDeserializeRead; import org.apache.hadoop.hive.serde2.lazybinary.fast.LazyBinarySerializeWrite; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.objectinspector.UnionObject; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.Writable; @@ -41,11 +46,11 @@ public class TestLazyBinaryFast extends TestCase { SerdeRandomRowSource source, Object[][] rows, AbstractSerDe serde, StructObjectInspector rowOI, AbstractSerDe serde_fewer, StructObjectInspector writeRowOI, - PrimitiveTypeInfo[] primitiveTypeInfos, + TypeInfo[] typeInfos, boolean useIncludeColumns, boolean doWriteFewerColumns, Random r) throws Throwable { int rowCount = rows.length; - int columnCount = primitiveTypeInfos.length; + int columnCount = typeInfos.length; boolean[] columnsToInclude = null; if (useIncludeColumns) { @@ -56,10 +61,10 @@ public class TestLazyBinaryFast extends TestCase { } int writeColumnCount = columnCount; - PrimitiveTypeInfo[] writePrimitiveTypeInfos = primitiveTypeInfos; + TypeInfo[] writeTypeInfos = typeInfos; if (doWriteFewerColumns) { writeColumnCount = writeRowOI.getAllStructFieldRefs().size(); - writePrimitiveTypeInfos = Arrays.copyOf(primitiveTypeInfos, writeColumnCount); + writeTypeInfos = Arrays.copyOf(typeInfos, writeColumnCount); } LazyBinarySerializeWrite lazyBinarySerializeWrite = @@ -73,10 +78,7 @@ public class TestLazyBinaryFast extends TestCase { lazyBinarySerializeWrite.set(output); for (int index = 0; index < writeColumnCount; index++) { - - Writable writable = (Writable) row[index]; - - VerifyFast.serializeWrite(lazyBinarySerializeWrite, primitiveTypeInfos[index], writable); + VerifyFast.serializeWrite(lazyBinarySerializeWrite, typeInfos[index], row[index]); } BytesWritable bytesWritable = new BytesWritable(); @@ -92,7 +94,7 @@ public class TestLazyBinaryFast extends TestCase { // column. LazyBinaryDeserializeRead lazyBinaryDeserializeRead = new LazyBinaryDeserializeRead( - writePrimitiveTypeInfos, + writeTypeInfos, /* useExternalBuffer */ false); BytesWritable bytesWritable = serializeWriteBytes[i]; @@ -103,10 +105,9 @@ public class TestLazyBinaryFast extends TestCase { lazyBinaryDeserializeRead.skipNextField(); } else if (index >= writeColumnCount) { // Should come back a null. - VerifyFast.verifyDeserializeRead(lazyBinaryDeserializeRead, primitiveTypeInfos[index], null); + VerifyFast.verifyDeserializeRead(lazyBinaryDeserializeRead, typeInfos[index], null); } else { - Writable writable = (Writable) row[index]; - VerifyFast.verifyDeserializeRead(lazyBinaryDeserializeRead, primitiveTypeInfos[index], writable); + verifyRead(lazyBinaryDeserializeRead, typeInfos[index], row[index]); } } if (writeColumnCount == columnCount) { @@ -127,15 +128,14 @@ public class TestLazyBinaryFast extends TestCase { Object[] row = rows[i]; for (int index = 0; index < writeColumnCount; index++) { - PrimitiveTypeInfo primitiveTypeInfo = primitiveTypeInfos[index]; - Writable writable = (Writable) row[index]; + TypeInfo typeInfo = typeInfos[index]; Object object = lazyBinaryStruct.getField(index); - if (writable == null || object == null) { - if (writable != null || object != null) { + if (row[index] == null || object == null) { + if (row[index] != null || object != null) { fail("SerDe deserialized NULL column mismatch"); } } else { - if (!object.equals(writable)) { + if (!VerifyLazy.lazyCompare(typeInfo, object, row[index])) { fail("SerDe deserialized value does not match"); } } @@ -172,10 +172,10 @@ public class TestLazyBinaryFast extends TestCase { if (bytes1.length != bytes2.length) { fail("SerializeWrite length " + bytes2.length + " and " + "SerDe serialization length " + bytes1.length + - " do not match (" + Arrays.toString(primitiveTypeInfos) + ")"); + " do not match (" + Arrays.toString(typeInfos) + ")"); } if (!Arrays.equals(bytes1, bytes2)) { - fail("SerializeWrite and SerDe serialization does not match (" + Arrays.toString(primitiveTypeInfos) + ")"); + fail("SerializeWrite and SerDe serialization does not match (" + Arrays.toString(typeInfos) + ")"); } serdeBytes[i] = bytesWritable; } @@ -187,7 +187,7 @@ public class TestLazyBinaryFast extends TestCase { // When doWriteFewerColumns, try to read more fields than exist in buffer. LazyBinaryDeserializeRead lazyBinaryDeserializeRead = new LazyBinaryDeserializeRead( - primitiveTypeInfos, + typeInfos, /* useExternalBuffer */ false); BytesWritable bytesWritable = serdeBytes[i]; @@ -198,10 +198,9 @@ public class TestLazyBinaryFast extends TestCase { lazyBinaryDeserializeRead.skipNextField(); } else if (index >= writeColumnCount) { // Should come back a null. - VerifyFast.verifyDeserializeRead(lazyBinaryDeserializeRead, primitiveTypeInfos[index], null); + VerifyFast.verifyDeserializeRead(lazyBinaryDeserializeRead, typeInfos[index], null); } else { - Writable writable = (Writable) row[index]; - VerifyFast.verifyDeserializeRead(lazyBinaryDeserializeRead, primitiveTypeInfos[index], writable); + verifyRead(lazyBinaryDeserializeRead, typeInfos[index], row[index]); } } if (writeColumnCount == columnCount) { @@ -210,12 +209,45 @@ public class TestLazyBinaryFast extends TestCase { } } - public void testLazyBinaryFastCase(int caseNum, boolean doNonRandomFill, Random r) throws Throwable { + private void verifyRead(LazyBinaryDeserializeRead lazyBinaryDeserializeRead, + TypeInfo typeInfo, Object expectedObject) throws IOException { + if (typeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE) { + VerifyFast.verifyDeserializeRead(lazyBinaryDeserializeRead, typeInfo, expectedObject); + } else { + Object complexFieldObj = VerifyFast.deserializeReadComplexType(lazyBinaryDeserializeRead, typeInfo); + if (expectedObject == null) { + if (complexFieldObj != null) { + TestCase.fail("Field reports not null but object is null (class " + complexFieldObj.getClass().getName() + + ", " + complexFieldObj.toString() + ")"); + } + } else { + if (complexFieldObj == null) { + // It's hard to distinguish a union with null from a null union. + if (expectedObject instanceof UnionObject) { + UnionObject expectedUnion = (UnionObject) expectedObject; + if (expectedUnion.getObject() == null) { + return; + } + } + TestCase.fail("Field reports null but object is not null (class " + expectedObject.getClass().getName() + + ", " + expectedObject.toString() + ")"); + } + } + if (!VerifyLazy.lazyCompare(typeInfo, complexFieldObj, expectedObject)) { + TestCase.fail("Comparision failed typeInfo " + typeInfo.toString()); + } + } + } + + public void testLazyBinaryFastCase( + int caseNum, boolean doNonRandomFill, Random r, SerdeRandomRowSource.SupportedTypes supportedTypes, int depth) + throws Throwable { SerdeRandomRowSource source = new SerdeRandomRowSource(); - source.init(r); - int rowCount = 1000; + source.init(r, supportedTypes, depth); + + int rowCount = 100; Object[][] rows = source.randomRows(rowCount); if (doNonRandomFill) { @@ -224,8 +256,8 @@ public class TestLazyBinaryFast extends TestCase { StructObjectInspector rowStructObjectInspector = source.rowStructObjectInspector(); - PrimitiveTypeInfo[] primitiveTypeInfos = source.primitiveTypeInfos(); - int columnCount = primitiveTypeInfos.length; + TypeInfo[] typeInfos = source.typeInfos(); + int columnCount = typeInfos.length; int writeColumnCount = columnCount; StructObjectInspector writeRowStructObjectInspector = rowStructObjectInspector; @@ -256,14 +288,14 @@ public class TestLazyBinaryFast extends TestCase { source, rows, serde, rowStructObjectInspector, serde_fewer, writeRowStructObjectInspector, - primitiveTypeInfos, + typeInfos, /* useIncludeColumns */ false, /* doWriteFewerColumns */ false, r); testLazyBinaryFast( source, rows, serde, rowStructObjectInspector, serde_fewer, writeRowStructObjectInspector, - primitiveTypeInfos, + typeInfos, /* useIncludeColumns */ true, /* doWriteFewerColumns */ false, r); /* @@ -286,14 +318,13 @@ public class TestLazyBinaryFast extends TestCase { // } } - public void testLazyBinaryFast() throws Throwable { - + private void testLazyBinaryFast(SerdeRandomRowSource.SupportedTypes supportedTypes, int depth) throws Throwable { try { - Random r = new Random(35790); + Random r = new Random(9983); int caseNum = 0; for (int i = 0; i < 10; i++) { - testLazyBinaryFastCase(caseNum, (i % 2 == 0), r); + testLazyBinaryFastCase(caseNum, (i % 2 == 0), r, supportedTypes, depth); caseNum++; } @@ -302,4 +333,16 @@ public class TestLazyBinaryFast extends TestCase { throw e; } } + + public void testLazyBinaryFastPrimitive() throws Throwable { + testLazyBinaryFast(SerdeRandomRowSource.SupportedTypes.PRIMITIVE, 0); + } + + public void testLazyBinaryFastComplexDepthOne() throws Throwable { + testLazyBinaryFast(SerdeRandomRowSource.SupportedTypes.ALL, 1); + } + + public void testLazyBinaryFastComplexDepthFour() throws Throwable { + testLazyBinaryFast(SerdeRandomRowSource.SupportedTypes.ALL, 4); + } } \ No newline at end of file
