Repository: hive Updated Branches: refs/heads/branch-1 f1c1427e6 -> 8d4b600a4
HIVE-12406: HIVE-9500 introduced incompatible change to LazySimpleSerDe public interface (Aihua Xu, reviewed by Szehon Ho) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8d4b600a Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8d4b600a Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8d4b600a Branch: refs/heads/branch-1 Commit: 8d4b600a4f42d54c42e38e54369a747d58222a8e Parents: f1c1427 Author: Aihua Xu <aihu...@apache.org> Authored: Mon Nov 23 14:41:58 2015 -0500 Committer: Aihua Xu <aihu...@apache.org> Committed: Mon Nov 23 15:38:47 2015 -0500 ---------------------------------------------------------------------- .../hive/serde2/lazy/LazySerDeParameters.java | 6 ++- .../hive/serde2/lazy/LazySimpleSerDe.java | 38 +++++++++---- .../hive/serde2/lazy/TestLazySimpleSerDe.java | 56 +++++++++++++++++--- 3 files changed, 82 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/8d4b600a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java index 60d11a2..1e72c72 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java @@ -20,12 +20,12 @@ package org.apache.hadoop.hive.serde2.lazy; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Properties; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.classification.InterfaceAudience.Public; +import org.apache.hadoop.hive.common.classification.InterfaceStability.Stable; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.SerDeException; @@ -40,6 +40,8 @@ import org.apache.hive.common.util.HiveStringUtils; * SerDeParameters. * */ +@Public +@Stable public class LazySerDeParameters implements LazyObjectInspectorParameters { public static final byte[] DefaultSeparators = {(byte) 1, (byte) 2, (byte) 3}; public static final String SERIALIZATION_EXTEND_NESTING_LEVELS http://git-wip-us.apache.org/repos/asf/hive/blob/8d4b600a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java index c373047..5674d15 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java @@ -19,18 +19,16 @@ package org.apache.hadoop.hive.serde2.lazy; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; -import org.apache.commons.lang.ArrayUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.common.classification.InterfaceAudience.Public; +import org.apache.hadoop.hive.common.classification.InterfaceStability.Stable; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.AbstractEncodingAwareSerDe; import org.apache.hadoop.hive.serde2.ByteStream; @@ -38,7 +36,6 @@ import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.SerDeSpec; import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.SerDeUtils; -import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParameters; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParametersImpl; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; @@ -50,14 +47,10 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.io.BinaryComparable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; -import org.apache.hive.common.util.HiveStringUtils; - /** * LazySimpleSerDe can be used to read the same data format as @@ -69,6 +62,8 @@ import org.apache.hive.common.util.HiveStringUtils; * Also LazySimpleSerDe outputs typed columns instead of treating all columns as * String like MetadataTypedColumnsetSerDe. */ +@Public +@Stable @SerDeSpec(schemaProps = { serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES, serdeConstants.FIELD_DELIM, serdeConstants.COLLECTION_DELIM, serdeConstants.MAPKEY_DELIM, @@ -423,4 +418,29 @@ public class LazySimpleSerDe extends AbstractEncodingAwareSerDe { Text text = (Text)blob; return SerDeUtils.transformTextToUTF8(text, this.charset); } + + /** + * This method is deprecated and is only used for backward compatibility. + * Replaced by @see org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters#LazySerDeParameters(Configuration, Properties, String) + */ + @Deprecated + public static SerDeParameters initSerdeParams(Configuration job, + Properties tbl, + String serdeName) throws SerDeException { + return new SerDeParameters(job, tbl, serdeName); + } + + /** + * This class is deprecated and is only used for backward compatibility. Replace by + * @see org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters. + */ + @Deprecated + public static class SerDeParameters extends org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters { + + public SerDeParameters(Configuration job, + Properties tbl, + String serdeName) throws SerDeException { + super(job, tbl, serdeName); + } + } } http://git-wip-us.apache.org/repos/asf/hive/blob/8d4b600a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java ---------------------------------------------------------------------- diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java index b11ce32..391edd4 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.serde2.lazy; import java.io.IOException; import java.util.List; import java.util.Properties; +import java.util.Random; import junit.framework.TestCase; @@ -29,15 +30,22 @@ import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.ByteStream; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.SerDeUtils; +import org.apache.hadoop.hive.serde2.binarysortable.MyTestClass; +import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; +import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; +import org.junit.Test; /** * TestLazySimpleSerDe. @@ -81,8 +89,6 @@ public class TestLazySimpleSerDe extends TestCase { } } - - /** * Test the LazySimpleSerDe class with LastColumnTakesRest option. */ @@ -167,20 +173,56 @@ public class TestLazySimpleSerDe extends TestCase { throw e; } } - - Object serializeAndDeserialize(List<Integer> o1, StructObjectInspector oi1, + + /** + * Tests the deprecated usage of SerDeParameters. + * + */ + @Test + @SuppressWarnings("deprecation") + public void testSerDeParameters() throws SerDeException, IOException { + // Setup + LazySimpleSerDe serDe = new LazySimpleSerDe(); + Configuration conf = new Configuration(); + + MyTestClass row = new MyTestClass(); + ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo(); + row.randomFill(new Random(1234), extraTypeInfo); + + StructObjectInspector rowOI = (StructObjectInspector) ObjectInspectorFactory + .getReflectionObjectInspector(MyTestClass.class, + ObjectInspectorOptions.JAVA); + + String fieldNames = ObjectInspectorUtils.getFieldNames(rowOI); + String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI); + + Properties schema = new Properties(); + schema.setProperty(serdeConstants.LIST_COLUMNS, fieldNames); + schema.setProperty(serdeConstants.LIST_COLUMN_TYPES, fieldTypes); + + SerDeUtils.initializeSerDe(serDe, conf, schema, null); + SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, schema, "testSerdeName"); + + // Test + LazyStruct data = (LazyStruct)serializeAndDeserialize(row, rowOI, serDe, serdeParams); + assertEquals((boolean)row.myBool, ((LazyBoolean)data.getField(0)).getWritableObject().get()); + assertEquals((int)row.myInt, ((LazyInteger)data.getField(3)).getWritableObject().get()); + } + + private Object serializeAndDeserialize(Object row, + StructObjectInspector rowOI, LazySimpleSerDe serde, LazySerDeParameters serdeParams) throws IOException, SerDeException { ByteStream.Output serializeStream = new ByteStream.Output(); - LazySimpleSerDe.serialize(serializeStream, o1, oi1, serdeParams + LazySimpleSerDe.serialize(serializeStream, row, rowOI, serdeParams .getSeparators(), 0, serdeParams.getNullSequence(), serdeParams .isEscaped(), serdeParams.getEscapeChar(), serdeParams .getNeedsEscape()); + Text t = new Text(serializeStream.toByteArray()); return serde.deserialize(t); } - - + private void deserializeAndSerialize(LazySimpleSerDe serDe, Text t, String s, Object[] expectedFieldsData) throws SerDeException { // Get the row structure
