Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java?rev=1656519&r1=1656518&r2=1656519&view=diff ============================================================================== --- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java (original) +++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java Mon Feb 2 17:04:41 2015 @@ -38,6 +38,8 @@ import org.apache.hadoop.hive.serde2.laz import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyHiveVarcharObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyIntObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyLongObjectInspector; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParameters; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParametersImpl; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyPrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyShortObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyStringObjectInspector; @@ -216,6 +218,7 @@ public final class LazyFactory { * @return The ObjectInspector * @throws SerDeException */ + @Deprecated public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, byte[] separator, int separatorIndex, Text nullSequence, boolean escaped, byte escapeChar, ObjectInspectorOptions option) throws SerDeException { @@ -240,6 +243,7 @@ public final class LazyFactory { * @return The ObjectInspector * @throws SerDeException */ + @Deprecated public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, byte[] separator, int separatorIndex, Text nullSequence, boolean escaped, byte escapeChar) throws SerDeException { @@ -261,6 +265,7 @@ public final class LazyFactory { * @return The ObjectInspector * @throws SerDeException */ + @Deprecated public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, byte[] separator, int separatorIndex, Text nullSequence, boolean escaped, byte escapeChar, boolean extendedBooleanLiteral) throws SerDeException { @@ -282,30 +287,51 @@ public final class LazyFactory { * @return The ObjectInspector * @throws SerDeException */ + @Deprecated public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, byte[] separator, int separatorIndex, Text nullSequence, boolean escaped, byte escapeChar, boolean extendedBooleanLiteral, ObjectInspectorOptions option) throws SerDeException { + LazyObjectInspectorParametersImpl lazyParams = new LazyObjectInspectorParametersImpl( + escaped, escapeChar, extendedBooleanLiteral, null, separator, nullSequence); + return createLazyObjectInspector(typeInfo, separatorIndex, lazyParams, option); + } + + /** + * Create a hierarchical ObjectInspector for LazyObject with the given typeInfo. + * + * @param typeInfo The type information for the LazyObject + * @param separator The array of separators for delimiting each level + * @param separatorIndex The current level (for separators). List(array), struct uses 1 level of + * separator, and map uses 2 levels: the first one for delimiting entries, the second one + * for delimiting key and values. + * @param lazyParams Params for lazy types + * @param option the {@link ObjectInspectorOption} + * @return The ObjectInspector + * @throws SerDeException + */ + public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, + int separatorIndex, LazyObjectInspectorParameters lazyParams, + ObjectInspectorOptions option) throws SerDeException { ObjectInspector.Category c = typeInfo.getCategory(); switch (c) { case PRIMITIVE: return LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector( - (PrimitiveTypeInfo) typeInfo, escaped, escapeChar, extendedBooleanLiteral); + (PrimitiveTypeInfo) typeInfo, lazyParams); case MAP: return LazyObjectInspectorFactory.getLazySimpleMapObjectInspector( - createLazyObjectInspector(((MapTypeInfo) typeInfo) - .getMapKeyTypeInfo(), separator, separatorIndex + 2, - nullSequence, escaped, escapeChar, extendedBooleanLiteral, option), createLazyObjectInspector( - ((MapTypeInfo) typeInfo).getMapValueTypeInfo(), separator, - separatorIndex + 2, nullSequence, escaped, escapeChar, extendedBooleanLiteral, option), - LazyUtils.getSeparator(separator, separatorIndex), - LazyUtils.getSeparator(separator, separatorIndex+1), - nullSequence, escaped, escapeChar); + createLazyObjectInspector(((MapTypeInfo) typeInfo).getMapKeyTypeInfo(), + separatorIndex + 2, lazyParams, option), + createLazyObjectInspector(((MapTypeInfo) typeInfo).getMapValueTypeInfo(), + separatorIndex + 2, lazyParams, option), + LazyUtils.getSeparator(lazyParams.getSeparators(), separatorIndex), + LazyUtils.getSeparator(lazyParams.getSeparators(), separatorIndex+1), + lazyParams); case LIST: return LazyObjectInspectorFactory.getLazySimpleListObjectInspector( createLazyObjectInspector(((ListTypeInfo) typeInfo) - .getListElementTypeInfo(), separator, separatorIndex + 1, - nullSequence, escaped, escapeChar, extendedBooleanLiteral, option), LazyUtils.getSeparator(separator, separatorIndex), - nullSequence, escaped, escapeChar); + .getListElementTypeInfo(), separatorIndex + 1, + lazyParams, option), LazyUtils.getSeparator(lazyParams.getSeparators(), separatorIndex), + lazyParams); case STRUCT: StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; List<String> fieldNames = structTypeInfo.getAllStructFieldNames(); @@ -315,25 +341,22 @@ public final class LazyFactory { fieldTypeInfos.size()); for (int i = 0; i < fieldTypeInfos.size(); i++) { fieldObjectInspectors.add(createLazyObjectInspector(fieldTypeInfos - .get(i), separator, separatorIndex + 1, nullSequence, escaped, - escapeChar, extendedBooleanLiteral, option)); + .get(i), separatorIndex + 1, lazyParams, option)); } return LazyObjectInspectorFactory.getLazySimpleStructObjectInspector( - fieldNames, fieldObjectInspectors, - LazyUtils.getSeparator(separator, separatorIndex), - nullSequence, - false, escaped, escapeChar, option); + fieldNames, fieldObjectInspectors, null, + LazyUtils.getSeparator(lazyParams.getSeparators(), separatorIndex), + lazyParams, option); case UNION: UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo; List<ObjectInspector> lazyOIs = new ArrayList<ObjectInspector>(); for (TypeInfo uti : unionTypeInfo.getAllUnionObjectTypeInfos()) { - lazyOIs.add(createLazyObjectInspector(uti, separator, - separatorIndex + 1, nullSequence, escaped, - escapeChar, extendedBooleanLiteral, option)); + lazyOIs.add(createLazyObjectInspector(uti, + separatorIndex + 1, lazyParams, option)); } return LazyObjectInspectorFactory.getLazyUnionObjectInspector(lazyOIs, - LazyUtils.getSeparator(separator, separatorIndex), - nullSequence, escaped, escapeChar); + LazyUtils.getSeparator(lazyParams.getSeparators(), separatorIndex), + lazyParams); } throw new RuntimeException("Hive LazySerDe Internal error."); @@ -350,6 +373,7 @@ public final class LazyFactory { * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text, * boolean, byte) */ + @Deprecated public static ObjectInspector createLazyStructInspector( List<String> columnNames, List<TypeInfo> typeInfos, byte[] separators, Text nullSequence, boolean lastColumnTakesRest, boolean escaped, @@ -370,20 +394,41 @@ public final class LazyFactory { * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text, * boolean, byte) */ + @Deprecated public static ObjectInspector createLazyStructInspector( List<String> columnNames, List<TypeInfo> typeInfos, byte[] separators, Text nullSequence, boolean lastColumnTakesRest, boolean escaped, byte escapeChar, boolean extendedBooleanLiteral) throws SerDeException { + LazyObjectInspectorParametersImpl lazyParams = new LazyObjectInspectorParametersImpl( + escaped, escapeChar, extendedBooleanLiteral, null, separators, nullSequence, lastColumnTakesRest); + return createLazyStructInspector(columnNames, typeInfos, lazyParams); + } + + /** + * Create a hierarchical ObjectInspector for LazyStruct with the given + * columnNames and columnTypeInfos. + * + * @param lastColumnTakesRest + * whether the last column of the struct should take the rest of the + * row if there are extra fields. + * @param lazyParams parameters for the lazy types + * @throws SerDeException + * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text, + * boolean, byte) + */ + public static ObjectInspector createLazyStructInspector( + List<String> columnNames, List<TypeInfo> typeInfos, + LazyObjectInspectorParameters lazyParams) throws SerDeException { ArrayList<ObjectInspector> columnObjectInspectors = new ArrayList<ObjectInspector>( typeInfos.size()); for (int i = 0; i < typeInfos.size(); i++) { columnObjectInspectors.add(LazyFactory.createLazyObjectInspector( - typeInfos.get(i), separators, 1, nullSequence, escaped, escapeChar, - extendedBooleanLiteral)); + typeInfos.get(i), 1, + lazyParams, ObjectInspectorOptions.JAVA)); } return LazyObjectInspectorFactory.getLazySimpleStructObjectInspector( - columnNames, columnObjectInspectors, separators[0], nullSequence, - lastColumnTakesRest, escaped, escapeChar); + columnNames, columnObjectInspectors, null, lazyParams.getSeparators()[0], + lazyParams, ObjectInspectorOptions.JAVA); } /** @@ -394,15 +439,32 @@ public final class LazyFactory { * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text, * boolean, byte) */ + @Deprecated public static ObjectInspector createColumnarStructInspector( List<String> columnNames, List<TypeInfo> columnTypes, byte[] separators, Text nullSequence, boolean escaped, byte escapeChar) throws SerDeException { + LazyObjectInspectorParametersImpl lazyParams = new LazyObjectInspectorParametersImpl( + escaped, escapeChar, false, null, separators, nullSequence); + return createColumnarStructInspector(columnNames, columnTypes, lazyParams); + } + + /** + * Create a hierarchical ObjectInspector for ColumnarStruct with the given + * columnNames and columnTypeInfos. + * @throws SerDeException + * + * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text, + * boolean, byte) + */ + public static ObjectInspector createColumnarStructInspector( + List<String> columnNames, List<TypeInfo> columnTypes, + LazyObjectInspectorParameters lazyParams) throws SerDeException { ArrayList<ObjectInspector> columnObjectInspectors = new ArrayList<ObjectInspector>( columnTypes.size()); for (int i = 0; i < columnTypes.size(); i++) { columnObjectInspectors - .add(LazyFactory.createLazyObjectInspector(columnTypes.get(i), - separators, 1, nullSequence, escaped, escapeChar, false)); + .add(LazyFactory.createLazyObjectInspector(columnTypes.get(i), 1, + lazyParams, ObjectInspectorOptions.JAVA)); } return ObjectInspectorFactory.getColumnarStructObjectInspector(columnNames, columnObjectInspectors);
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java?rev=1656519&r1=1656518&r2=1656519&view=diff ============================================================================== --- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java (original) +++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java Mon Feb 2 17:04:41 2015 @@ -19,6 +19,8 @@ package org.apache.hadoop.hive.serde2.lazy; import java.io.IOException; +import java.nio.charset.Charset; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -36,6 +38,8 @@ import org.apache.hadoop.hive.serde2.Ser import org.apache.hadoop.hive.serde2.SerDeSpec; import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.SerDeUtils; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParameters; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParametersImpl; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -52,6 +56,7 @@ import org.apache.hadoop.hive.serde2.typ import org.apache.hadoop.io.BinaryComparable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; +import org.apache.hive.common.util.HiveStringUtils; /** * LazySimpleSerDe can be used to read the same data format as @@ -127,7 +132,7 @@ public class LazySimpleSerDe extends Abs * SerDeParameters. * */ - public static class SerDeParameters { + public static class SerDeParameters implements LazyObjectInspectorParameters { byte[] separators = DefaultSeparators; String nullString; Text nullSequence; @@ -141,6 +146,10 @@ public class LazySimpleSerDe extends Abs boolean[] needsEscape; boolean extendedBooleanLiteral; + List<String> timestampFormats; + + public SerDeParameters() { + } public List<TypeInfo> getColumnTypes() { return columnTypes; @@ -181,6 +190,14 @@ public class LazySimpleSerDe extends Abs public boolean[] getNeedsEscape() { return needsEscape; } + + public boolean isExtendedBooleanLiteral() { + return extendedBooleanLiteral; + } + + public List<String> getTimestampFormats() { + return timestampFormats; + } } SerDeParameters serdeParams = null; @@ -204,10 +221,7 @@ public class LazySimpleSerDe extends Abs // Create the ObjectInspectors for the fields cachedObjectInspector = LazyFactory.createLazyStructInspector(serdeParams - .getColumnNames(), serdeParams.getColumnTypes(), serdeParams - .getSeparators(), serdeParams.getNullSequence(), serdeParams - .isLastColumnTakesRest(), serdeParams.isEscaped(), serdeParams - .getEscapeChar(), serdeParams.extendedBooleanLiteral); + .getColumnNames(), serdeParams.getColumnTypes(), serdeParams); cachedLazyStruct = (LazyStruct) LazyFactory .createLazyObject(cachedObjectInspector); @@ -216,7 +230,7 @@ public class LazySimpleSerDe extends Abs + serdeParams.columnNames + " columnTypes=" + serdeParams.columnTypes + " separator=" + Arrays.asList(serdeParams.separators) + " nullstring=" + serdeParams.nullString + " lastColumnTakesRest=" - + serdeParams.lastColumnTakesRest); + + serdeParams.lastColumnTakesRest + " timestampFormats=" + serdeParams.timestampFormats); serializedSize = 0; stats = new SerDeStats(); @@ -324,6 +338,12 @@ public class LazySimpleSerDe extends Abs serdeParams.extendedBooleanLiteral = job == null ? false : job.getBoolean(ConfVars.HIVE_LAZYSIMPLE_EXTENDED_BOOLEAN_LITERAL.varname, false); + + String[] timestampFormatsArray = + HiveStringUtils.splitAndUnEscape(tbl.getProperty(serdeConstants.TIMESTAMP_FORMATS)); + if (timestampFormatsArray != null) { + serdeParams.timestampFormats = Arrays.asList(timestampFormatsArray); + } return serdeParams; } Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyTimestamp.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyTimestamp.java?rev=1656519&r1=1656518&r2=1656519&view=diff ============================================================================== --- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyTimestamp.java (original) +++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyTimestamp.java Mon Feb 2 17:04:41 2015 @@ -72,7 +72,7 @@ public class LazyTimestamp extends LazyP logExceptionMessage(bytes, start, length, "TIMESTAMP"); } else { try { - t = Timestamp.valueOf(s); + t = oi.getTimestampParser().parseTimestamp(s); isNull = false; } catch (IllegalArgumentException e) { isNull = true; Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyListObjectInspector.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyListObjectInspector.java?rev=1656519&r1=1656518&r2=1656519&view=diff ============================================================================== --- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyListObjectInspector.java (original) +++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyListObjectInspector.java Mon Feb 2 17:04:41 2015 @@ -23,6 +23,8 @@ import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.serde2.lazy.LazyArray; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParameters; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParametersImpl; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.io.Text; @@ -40,9 +42,7 @@ public class LazyListObjectInspector imp private ObjectInspector listElementObjectInspector; private byte separator; - private Text nullSequence; - private boolean escaped; - private byte escapeChar; + private LazyObjectInspectorParameters lazyParams; protected LazyListObjectInspector() { super(); @@ -51,12 +51,10 @@ public class LazyListObjectInspector imp * Call ObjectInspectorFactory.getLazySimpleListObjectInspector instead. */ protected LazyListObjectInspector(ObjectInspector listElementObjectInspector, - byte separator, Text nullSequence, boolean escaped, byte escapeChar) { + byte separator, LazyObjectInspectorParameters lazyParams) { this.listElementObjectInspector = listElementObjectInspector; this.separator = separator; - this.nullSequence = nullSequence; - this.escaped = escaped; - this.escapeChar = escapeChar; + this.lazyParams = lazyParams; } @Override @@ -116,15 +114,18 @@ public class LazyListObjectInspector imp * Returns the NullSequence for this array. Called by LazyArray.init(...). */ public Text getNullSequence() { - return nullSequence; + return lazyParams.getNullSequence(); } public boolean isEscaped() { - return escaped; + return lazyParams.isEscaped(); } public byte getEscapeChar() { - return escapeChar; + return lazyParams.getEscapeChar(); } + public LazyObjectInspectorParameters getLazyParams() { + return lazyParams; + } } Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyMapObjectInspector.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyMapObjectInspector.java?rev=1656519&r1=1656518&r2=1656519&view=diff ============================================================================== --- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyMapObjectInspector.java (original) +++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyMapObjectInspector.java Mon Feb 2 17:04:41 2015 @@ -23,6 +23,8 @@ import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.serde2.lazy.LazyMap; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParameters; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParametersImpl; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.io.Text; @@ -42,12 +44,11 @@ public class LazyMapObjectInspector impl private ObjectInspector mapValueObjectInspector; private byte itemSeparator; private byte keyValueSeparator; - private Text nullSequence; - private boolean escaped; - private byte escapeChar; + private LazyObjectInspectorParameters lazyParams; protected LazyMapObjectInspector() { super(); + lazyParams = new LazyObjectInspectorParametersImpl(); } /** * Call ObjectInspectorFactory.getStandardListObjectInspector instead. @@ -61,9 +62,19 @@ public class LazyMapObjectInspector impl this.itemSeparator = itemSeparator; this.keyValueSeparator = keyValueSeparator; - this.nullSequence = nullSequence; - this.escaped = escaped; - this.escapeChar = escapeChar; + this.lazyParams = new LazyObjectInspectorParametersImpl( + escaped, escapeChar, false, null, null, nullSequence); + } + + protected LazyMapObjectInspector(ObjectInspector mapKeyObjectInspector, + ObjectInspector mapValueObjectInspector, byte itemSeparator, + byte keyValueSeparator, LazyObjectInspectorParameters lazyParams) { + this.mapKeyObjectInspector = mapKeyObjectInspector; + this.mapValueObjectInspector = mapValueObjectInspector; + + this.itemSeparator = itemSeparator; + this.keyValueSeparator = keyValueSeparator; + this.lazyParams = lazyParams; } @Override @@ -122,14 +133,18 @@ public class LazyMapObjectInspector impl } public Text getNullSequence() { - return nullSequence; + return lazyParams.getNullSequence(); } public boolean isEscaped() { - return escaped; + return lazyParams.isEscaped(); } public byte getEscapeChar() { - return escapeChar; + return lazyParams.getEscapeChar(); + } + + public LazyObjectInspectorParameters getLazyParams() { + return lazyParams; } } Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java?rev=1656519&r1=1656518&r2=1656519&view=diff ============================================================================== --- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java (original) +++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java Mon Feb 2 17:04:41 2015 @@ -23,6 +23,8 @@ import java.util.List; import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.hive.serde2.avro.AvroLazyObjectInspector; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParameters; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParametersImpl; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions; import org.apache.hadoop.io.Text; @@ -43,6 +45,7 @@ public final class LazyObjectInspectorFa static ConcurrentHashMap<ArrayList<Object>, LazySimpleStructObjectInspector> cachedLazySimpleStructObjectInspector = new ConcurrentHashMap<ArrayList<Object>, LazySimpleStructObjectInspector>(); + @Deprecated public static LazySimpleStructObjectInspector getLazySimpleStructObjectInspector( List<String> structFieldNames, List<ObjectInspector> structFieldObjectInspectors, byte separator, @@ -52,7 +55,8 @@ public final class LazyObjectInspectorFa structFieldObjectInspectors, null, separator, nullSequence, lastColumnTakesRest, escaped, escapeChar, ObjectInspectorOptions.JAVA); } - + + @Deprecated public static LazySimpleStructObjectInspector getLazySimpleStructObjectInspector( List<String> structFieldNames, List<ObjectInspector> structFieldObjectInspectors, byte separator, @@ -63,6 +67,7 @@ public final class LazyObjectInspectorFa lastColumnTakesRest, escaped, escapeChar, option); } + @Deprecated public static LazySimpleStructObjectInspector getLazySimpleStructObjectInspector( List<String> structFieldNames, List<ObjectInspector> structFieldObjectInspectors, List<String> structFieldComments, @@ -72,39 +77,49 @@ public final class LazyObjectInspectorFa structFieldComments, separator, nullSequence, lastColumnTakesRest, escaped, escapeChar, ObjectInspectorOptions.JAVA); } - + + @Deprecated public static LazySimpleStructObjectInspector getLazySimpleStructObjectInspector( List<String> structFieldNames, List<ObjectInspector> structFieldObjectInspectors, List<String> structFieldComments, byte separator, Text nullSequence, boolean lastColumnTakesRest, boolean escaped,byte escapeChar, ObjectInspectorOptions option) { + + return getLazySimpleStructObjectInspector(structFieldNames, structFieldObjectInspectors, + structFieldComments, separator, + new LazyObjectInspectorParametersImpl( + escaped, escapeChar, false, null, null, nullSequence, lastColumnTakesRest), + option); + } + + public static LazySimpleStructObjectInspector getLazySimpleStructObjectInspector( + List<String> structFieldNames, + List<ObjectInspector> structFieldObjectInspectors, List<String> structFieldComments, + byte separator, + LazyObjectInspectorParameters lazyParams, ObjectInspectorOptions option) { ArrayList<Object> signature = new ArrayList<Object>(); signature.add(structFieldNames); signature.add(structFieldObjectInspectors); signature.add(Byte.valueOf(separator)); - signature.add(nullSequence.toString()); - signature.add(Boolean.valueOf(lastColumnTakesRest)); - signature.add(Boolean.valueOf(escaped)); - signature.add(Byte.valueOf(escapeChar)); + signature.add(lazyParams.getNullSequence().toString()); + signature.add(Boolean.valueOf(lazyParams.isLastColumnTakesRest())); + LazyObjectInspectorFactory.addCommonLazyParamsToSignature(lazyParams, signature); signature.add(option); if(structFieldComments != null) { signature.add(structFieldComments); } - LazySimpleStructObjectInspector result = cachedLazySimpleStructObjectInspector - .get(signature); + LazySimpleStructObjectInspector result = cachedLazySimpleStructObjectInspector.get(signature); if (result == null) { switch (option) { case JAVA: result = new LazySimpleStructObjectInspector(structFieldNames, structFieldObjectInspectors, - structFieldComments, separator, nullSequence, lastColumnTakesRest, escaped, - escapeChar); + structFieldComments, separator, lazyParams); break; case AVRO: result = new AvroLazyObjectInspector(structFieldNames, structFieldObjectInspectors, - structFieldComments, separator, nullSequence, lastColumnTakesRest, escaped, - escapeChar); + structFieldComments, separator, lazyParams); break; default: throw new IllegalArgumentException("Illegal ObjectInspector type [" + option + "]"); @@ -118,20 +133,27 @@ public final class LazyObjectInspectorFa static ConcurrentHashMap<ArrayList<Object>, LazyListObjectInspector> cachedLazySimpleListObjectInspector = new ConcurrentHashMap<ArrayList<Object>, LazyListObjectInspector>(); + @Deprecated public static LazyListObjectInspector getLazySimpleListObjectInspector( ObjectInspector listElementObjectInspector, byte separator, Text nullSequence, boolean escaped, byte escapeChar) { + return getLazySimpleListObjectInspector(listElementObjectInspector, separator, + new LazyObjectInspectorParametersImpl(escaped, escapeChar, false, null, null, nullSequence)); + } + + public static LazyListObjectInspector getLazySimpleListObjectInspector( + ObjectInspector listElementObjectInspector, byte separator, + LazyObjectInspectorParameters lazyParams) { ArrayList<Object> signature = new ArrayList<Object>(); signature.add(listElementObjectInspector); signature.add(Byte.valueOf(separator)); - signature.add(nullSequence.toString()); - signature.add(Boolean.valueOf(escaped)); - signature.add(Byte.valueOf(escapeChar)); + signature.add(lazyParams.getNullSequence().toString()); + LazyObjectInspectorFactory.addCommonLazyParamsToSignature(lazyParams, signature); LazyListObjectInspector result = cachedLazySimpleListObjectInspector .get(signature); if (result == null) { result = new LazyListObjectInspector(listElementObjectInspector, - separator, nullSequence, escaped, escapeChar); + separator, lazyParams); cachedLazySimpleListObjectInspector.put(signature, result); } return result; @@ -140,25 +162,33 @@ public final class LazyObjectInspectorFa static ConcurrentHashMap<ArrayList<Object>, LazyMapObjectInspector> cachedLazySimpleMapObjectInspector = new ConcurrentHashMap<ArrayList<Object>, LazyMapObjectInspector>(); + @Deprecated public static LazyMapObjectInspector getLazySimpleMapObjectInspector( ObjectInspector mapKeyObjectInspector, ObjectInspector mapValueObjectInspector, byte itemSeparator, byte keyValueSeparator, Text nullSequence, boolean escaped, byte escapeChar) { + return getLazySimpleMapObjectInspector(mapKeyObjectInspector, mapValueObjectInspector, + itemSeparator, keyValueSeparator, + new LazyObjectInspectorParametersImpl(escaped, escapeChar, false, null, null, nullSequence)); + } + + public static LazyMapObjectInspector getLazySimpleMapObjectInspector( + ObjectInspector mapKeyObjectInspector, + ObjectInspector mapValueObjectInspector, byte itemSeparator, + byte keyValueSeparator, LazyObjectInspectorParameters lazyParams) { ArrayList<Object> signature = new ArrayList<Object>(); signature.add(mapKeyObjectInspector); signature.add(mapValueObjectInspector); signature.add(Byte.valueOf(itemSeparator)); signature.add(Byte.valueOf(keyValueSeparator)); - signature.add(nullSequence.toString()); - signature.add(Boolean.valueOf(escaped)); - signature.add(Byte.valueOf(escapeChar)); + signature.add(lazyParams.getNullSequence().toString()); + LazyObjectInspectorFactory.addCommonLazyParamsToSignature(lazyParams, signature); LazyMapObjectInspector result = cachedLazySimpleMapObjectInspector .get(signature); if (result == null) { result = new LazyMapObjectInspector(mapKeyObjectInspector, - mapValueObjectInspector, itemSeparator, keyValueSeparator, - nullSequence, escaped, escapeChar); + mapValueObjectInspector, itemSeparator, keyValueSeparator, lazyParams); cachedLazySimpleMapObjectInspector.put(signature, result); } return result; @@ -168,20 +198,26 @@ public final class LazyObjectInspectorFa cachedLazyUnionObjectInspector = new ConcurrentHashMap<List<Object>, LazyUnionObjectInspector>(); + @Deprecated public static LazyUnionObjectInspector getLazyUnionObjectInspector( List<ObjectInspector> ois, byte separator, Text nullSequence, boolean escaped, byte escapeChar) { + return getLazyUnionObjectInspector(ois, separator, + new LazyObjectInspectorParametersImpl(escaped, escapeChar, false, null, null, nullSequence)); + } + + public static LazyUnionObjectInspector getLazyUnionObjectInspector( + List<ObjectInspector> ois, byte separator, + LazyObjectInspectorParameters lazyParams) { List<Object> signature = new ArrayList<Object>(); signature.add(ois); signature.add(Byte.valueOf(separator)); - signature.add(nullSequence.toString()); - signature.add(Boolean.valueOf(escaped)); - signature.add(Byte.valueOf(escapeChar)); + signature.add(lazyParams.getNullSequence().toString()); + LazyObjectInspectorFactory.addCommonLazyParamsToSignature(lazyParams, signature); LazyUnionObjectInspector result = cachedLazyUnionObjectInspector .get(signature); if (result == null) { - result = new LazyUnionObjectInspector(ois, separator, - nullSequence, escaped, escapeChar); + result = new LazyUnionObjectInspector(ois, separator, lazyParams); cachedLazyUnionObjectInspector.put(signature, result); } return result; @@ -190,4 +226,12 @@ public final class LazyObjectInspectorFa private LazyObjectInspectorFactory() { // prevent instantiation } + + private static void addCommonLazyParamsToSignature(LazyObjectInspectorParameters lazyParams, + List<Object> signature) { + signature.add(lazyParams.isEscaped()); + signature.add(lazyParams.getEscapeChar()); + signature.add(lazyParams.isExtendedBooleanLiteral()); + signature.add(lazyParams.getTimestampFormats()); + } } Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java?rev=1656519&r1=1656518&r2=1656519&view=diff ============================================================================== --- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java (original) +++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java Mon Feb 2 17:04:41 2015 @@ -24,6 +24,8 @@ import java.util.List; import org.apache.hadoop.hive.serde2.BaseStructObjectInspector; import org.apache.hadoop.hive.serde2.StructObject; import org.apache.hadoop.hive.serde2.avro.AvroLazyObjectInspector; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParameters; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParametersImpl; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; @@ -42,22 +44,12 @@ import org.apache.hadoop.io.Text; public class LazySimpleStructObjectInspector extends BaseStructObjectInspector { private byte separator; - private Text nullSequence; - private boolean lastColumnTakesRest; - private boolean escaped; - private byte escapeChar; + private LazyObjectInspectorParameters lazyParams; protected LazySimpleStructObjectInspector() { super(); } - protected LazySimpleStructObjectInspector( - List<StructField> fields, byte separator, Text nullSequence) { - init(fields); - this.separator = separator; - this.nullSequence = nullSequence; - } - /** * Call ObjectInspectorFactory.getLazySimpleStructObjectInspector instead. */ @@ -70,6 +62,7 @@ public class LazySimpleStructObjectInspe nullSequence, lastColumnTakesRest, escaped, escapeChar); } + @Deprecated public LazySimpleStructObjectInspector(List<String> structFieldNames, List<ObjectInspector> structFieldObjectInspectors, List<String> structFieldComments, byte separator, Text nullSequence, @@ -78,17 +71,32 @@ public class LazySimpleStructObjectInspe separator, nullSequence, lastColumnTakesRest, escaped, escapeChar); } + public LazySimpleStructObjectInspector(List<String> structFieldNames, + List<ObjectInspector> structFieldObjectInspectors, + List<String> structFieldComments, byte separator, LazyObjectInspectorParameters lazyParams) { + init(structFieldNames, structFieldObjectInspectors, structFieldComments, + separator, lazyParams); + } + protected void init(List<String> structFieldNames, List<ObjectInspector> structFieldObjectInspectors, List<String> structFieldComments, byte separator, Text nullSequence, boolean lastColumnTakesRest, boolean escaped, byte escapeChar) { + LazyObjectInspectorParameters lazyParams = + new LazyObjectInspectorParametersImpl(escaped, escapeChar, + false, null, null, nullSequence, lastColumnTakesRest); + init(structFieldNames, structFieldObjectInspectors, structFieldComments, + separator, lazyParams); + } + + protected void init(List<String> structFieldNames, + List<ObjectInspector> structFieldObjectInspectors, + List<String> structFieldComments, byte separator, + LazyObjectInspectorParameters lazyParams) { init(structFieldNames, structFieldObjectInspectors, structFieldComments); this.separator = separator; - this.nullSequence = nullSequence; - this.lastColumnTakesRest = lastColumnTakesRest; - this.escaped = escaped; - this.escapeChar = escapeChar; + this.lazyParams = lazyParams; } // With Data @@ -142,19 +150,22 @@ public class LazySimpleStructObjectInspe } public Text getNullSequence() { - return nullSequence; + return lazyParams.getNullSequence(); } public boolean getLastColumnTakesRest() { - return lastColumnTakesRest; + return lazyParams.isLastColumnTakesRest(); } public boolean isEscaped() { - return escaped; + return lazyParams.isEscaped(); } public byte getEscapeChar() { - return escapeChar; + return lazyParams.getEscapeChar(); } + public LazyObjectInspectorParameters getLazyParams() { + return lazyParams; + } } Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyUnionObjectInspector.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyUnionObjectInspector.java?rev=1656519&r1=1656518&r2=1656519&view=diff ============================================================================== --- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyUnionObjectInspector.java (original) +++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyUnionObjectInspector.java Mon Feb 2 17:04:41 2015 @@ -24,6 +24,8 @@ import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.serde2.lazy.LazyUnion; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParameters; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParametersImpl; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; @@ -44,19 +46,16 @@ public class LazyUnionObjectInspector im private List<ObjectInspector> ois; private byte separator; - private Text nullSequence; - private boolean escaped; - private byte escapeChar; + private LazyObjectInspectorParameters lazyParams; protected LazyUnionObjectInspector() { super(); } + protected LazyUnionObjectInspector( List<ObjectInspector> ois, byte separator, - Text nullSequence, boolean escaped, - byte escapeChar) { - init(ois, separator, - nullSequence, escaped, escapeChar); + LazyObjectInspectorParameters lazyParams) { + init(ois, separator, lazyParams); } @Override @@ -66,25 +65,9 @@ public class LazyUnionObjectInspector im protected void init( List<ObjectInspector> ois, byte separator, - Text nullSequence, boolean escaped, - byte escapeChar) { + LazyObjectInspectorParameters lazyParams) { this.separator = separator; - this.nullSequence = nullSequence; - this.escaped = escaped; - this.escapeChar = escapeChar; - this.ois = new ArrayList<ObjectInspector>(); - this.ois.addAll(ois); - } - - protected LazyUnionObjectInspector(List<ObjectInspector> ois, - byte separator, Text nullSequence) { - init(ois, separator, nullSequence); - } - - protected void init(List<ObjectInspector> ois, byte separator, - Text nullSequence) { - this.separator = separator; - this.nullSequence = nullSequence; + this.lazyParams = lazyParams; this.ois = new ArrayList<ObjectInspector>(); this.ois.addAll(ois); } @@ -99,17 +82,20 @@ public class LazyUnionObjectInspector im } public Text getNullSequence() { - return nullSequence; + return lazyParams.getNullSequence(); } public boolean isEscaped() { - return escaped; + return lazyParams.isEscaped(); } public byte getEscapeChar() { - return escapeChar; + return lazyParams.getEscapeChar(); } + public LazyObjectInspectorParameters getLazyParams() { + return lazyParams; + } @Override public Object getField(Object data) { if (data == null) { Added: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyObjectInspectorParameters.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyObjectInspectorParameters.java?rev=1656519&view=auto ============================================================================== --- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyObjectInspectorParameters.java (added) +++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyObjectInspectorParameters.java Mon Feb 2 17:04:41 2015 @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive; + +import java.util.List; + +import org.apache.hadoop.io.Text; + +public interface LazyObjectInspectorParameters { + public boolean isEscaped(); + public byte getEscapeChar(); + public boolean isExtendedBooleanLiteral(); + public List<String> getTimestampFormats(); + public byte[] getSeparators(); + public Text getNullSequence(); + public boolean isLastColumnTakesRest(); +} Added: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyObjectInspectorParametersImpl.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyObjectInspectorParametersImpl.java?rev=1656519&view=auto ============================================================================== --- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyObjectInspectorParametersImpl.java (added) +++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyObjectInspectorParametersImpl.java Mon Feb 2 17:04:41 2015 @@ -0,0 +1,144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive; + +import java.util.Arrays; +import java.util.List; + +import org.apache.commons.lang.ObjectUtils; +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; +import org.apache.hadoop.io.Text; + +public class LazyObjectInspectorParametersImpl implements + LazyObjectInspectorParameters { + protected boolean escaped; + protected byte escapeChar; + protected boolean extendedBooleanLiteral; + protected List<String> timestampFormats; + protected byte[] separators; + protected Text nullSequence; + protected boolean lastColumnTakesRest; + + + public LazyObjectInspectorParametersImpl() { + this.escaped = false; + this.extendedBooleanLiteral = false; + this.timestampFormats = null; + } + + public LazyObjectInspectorParametersImpl(boolean escaped, byte escapeChar, + boolean extendedBooleanLiteral, List<String> timestampFormats, + byte[] separators, Text nullSequence) { + super(); + this.escaped = escaped; + this.escapeChar = escapeChar; + this.extendedBooleanLiteral = extendedBooleanLiteral; + this.timestampFormats = timestampFormats; + this.separators = separators; + this.nullSequence = nullSequence; + this.lastColumnTakesRest = false; + } + + public LazyObjectInspectorParametersImpl(boolean escaped, byte escapeChar, + boolean extendedBooleanLiteral, List<String> timestampFormats, + byte[] separators, Text nullSequence, boolean lastColumnTakesRest) { + super(); + this.escaped = escaped; + this.escapeChar = escapeChar; + this.extendedBooleanLiteral = extendedBooleanLiteral; + this.timestampFormats = timestampFormats; + this.separators = separators; + this.nullSequence = nullSequence; + this.lastColumnTakesRest = lastColumnTakesRest; + } + + public LazyObjectInspectorParametersImpl(LazyObjectInspectorParameters lazyParams) { + this.escaped = lazyParams.isEscaped(); + this.escapeChar = lazyParams.getEscapeChar(); + this.extendedBooleanLiteral = lazyParams.isExtendedBooleanLiteral(); + this.timestampFormats = lazyParams.getTimestampFormats(); + this.separators = lazyParams.getSeparators(); + this.nullSequence = lazyParams.getNullSequence(); + this.lastColumnTakesRest = lazyParams.isLastColumnTakesRest(); + } + + @Override + public boolean isEscaped() { + return escaped; + } + + @Override + public byte getEscapeChar() { + return escapeChar; + } + + @Override + public boolean isExtendedBooleanLiteral() { + return extendedBooleanLiteral; + } + + @Override + public List<String> getTimestampFormats() { + return timestampFormats; + } + + @Override + public byte[] getSeparators() { + return separators; + } + + @Override + public Text getNullSequence() { + return nullSequence; + } + + @Override + public boolean isLastColumnTakesRest() { + return lastColumnTakesRest; + } + + protected boolean equals(LazyObjectInspectorParametersImpl other) { + return this.escaped == other.escaped + && this.escapeChar == other.escapeChar + && this.extendedBooleanLiteral == other.extendedBooleanLiteral + && this.lastColumnTakesRest == other.lastColumnTakesRest + && ObjectUtils.equals(this.nullSequence, other.nullSequence) + && Arrays.equals(this.separators, other.separators) + && ObjectUtils.equals(this.timestampFormats, other.timestampFormats); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof LazyObjectInspectorParametersImpl)) { + return false; + } + return equals((LazyObjectInspectorParametersImpl) obj); + } + + @Override + public int hashCode() { + return new HashCodeBuilder().append(escaped).append(escapeChar) + .append(extendedBooleanLiteral).append(timestampFormats) + .append(lastColumnTakesRest).append(nullSequence).append(separators).toHashCode(); + } +} Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java?rev=1656519&r1=1656518&r2=1656519&view=diff ============================================================================== --- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java (original) +++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java Mon Feb 2 17:04:41 2015 @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.serde2.la import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.hadoop.hive.serde.serdeConstants; @@ -118,17 +119,28 @@ public final class LazyPrimitiveObjectIn public static AbstractPrimitiveLazyObjectInspector<?> getLazyObjectInspector( PrimitiveTypeInfo typeInfo, boolean escaped, byte escapeChar, boolean extBoolean) { + LazyObjectInspectorParameters lazyParams = new LazyObjectInspectorParametersImpl( + escaped, escapeChar, extBoolean, null, null, null); + return getLazyObjectInspector(typeInfo, lazyParams); + } + + public static AbstractPrimitiveLazyObjectInspector<?> getLazyObjectInspector( + PrimitiveTypeInfo typeInfo, LazyObjectInspectorParameters lazyParams) { PrimitiveCategory primitiveCategory = typeInfo.getPrimitiveCategory(); switch(primitiveCategory) { case STRING: - return getLazyStringObjectInspector(escaped, escapeChar); + return getLazyStringObjectInspector(lazyParams.isEscaped(), lazyParams.getEscapeChar()); case CHAR: - return getLazyHiveCharObjectInspector((CharTypeInfo)typeInfo, escaped, escapeChar); + return getLazyHiveCharObjectInspector((CharTypeInfo)typeInfo, + lazyParams.isEscaped(), lazyParams.getEscapeChar()); case VARCHAR: - return getLazyHiveVarcharObjectInspector((VarcharTypeInfo)typeInfo, escaped, escapeChar); + return getLazyHiveVarcharObjectInspector((VarcharTypeInfo)typeInfo, + lazyParams.isEscaped(), lazyParams.getEscapeChar()); case BOOLEAN: - return getLazyBooleanObjectInspector(extBoolean); + return getLazyBooleanObjectInspector(lazyParams.isExtendedBooleanLiteral()); + case TIMESTAMP: + return getLazyTimestampObjectInspector(lazyParams.getTimestampFormats()); default: return getLazyObjectInspector(typeInfo); } @@ -203,6 +215,25 @@ public final class LazyPrimitiveObjectIn cachedLazyStringTypeOIs.put(signature, result); } return result; + } + + public static LazyTimestampObjectInspector getLazyTimestampObjectInspector( + List<String> tsFormats) { + if (tsFormats == null) { + // No timestamp format specified, just use default lazy inspector + return (LazyTimestampObjectInspector) getLazyObjectInspector(TypeInfoFactory.timestampTypeInfo); + } + + ArrayList<Object> signature = new ArrayList<Object>(); + signature.add(TypeInfoFactory.timestampTypeInfo); + signature.add(tsFormats); + LazyTimestampObjectInspector result = (LazyTimestampObjectInspector) cachedLazyStringTypeOIs + .get(signature); + if (result == null) { + result = new LazyTimestampObjectInspector(tsFormats); + cachedLazyStringTypeOIs.put(signature, result); + } + return result; } private static LazyBooleanObjectInspector getLazyBooleanObjectInspector(boolean extLiteral) { Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyTimestampObjectInspector.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyTimestampObjectInspector.java?rev=1656519&r1=1656518&r2=1656519&view=diff ============================================================================== --- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyTimestampObjectInspector.java (original) +++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyTimestampObjectInspector.java Mon Feb 2 17:04:41 2015 @@ -18,18 +18,30 @@ package org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive; import java.sql.Timestamp; +import java.util.List; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.lazy.LazyTimestamp; import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.hive.common.util.TimestampParser; public class LazyTimestampObjectInspector extends AbstractPrimitiveLazyObjectInspector<TimestampWritable> implements TimestampObjectInspector { - protected LazyTimestampObjectInspector() { + protected List<String> timestampFormats = null; + protected TimestampParser timestampParser = null; + + LazyTimestampObjectInspector() { super(TypeInfoFactory.timestampTypeInfo); + timestampParser = new TimestampParser(); + } + + LazyTimestampObjectInspector(List<String> tsFormats) { + super(TypeInfoFactory.timestampTypeInfo); + this.timestampFormats = tsFormats; + timestampParser = new TimestampParser(tsFormats); } public Object copyObject(Object o) { @@ -39,4 +51,13 @@ public class LazyTimestampObjectInspecto public Timestamp getPrimitiveJavaObject(Object o) { return o == null ? null : ((LazyTimestamp) o).getWritableObject().getTimestamp(); } + + public List<String> getTimestampFormats() { + return timestampFormats; + } + + public TimestampParser getTimestampParser() { + return timestampParser; + } + }
